From 7c80ba0ad4d77b442795df5376195b0b1292b998 Mon Sep 17 00:00:00 2001 From: Nicole van der Hoeven Date: Thu, 13 Nov 2025 13:50:56 +0100 Subject: [PATCH 01/28] Add chapters/timestamps feature and improve API reliability - Add create_chapters() function to generate YouTube-style timestamps using OpenAI - Integrate chapters section into markdown output between summary and transcript - Add comprehensive rate limiting to avoid YouTube API quota issues - Implement get_video_transcript_with_retry() with exponential backoff - Add robust error handling for quota exceeded and API failures - Improve transcript validation and filtering ([Music], [Applause], etc) - Fix Japanese language code from 'jp' to 'ja' - Increase batch size from 10 to 50 for better efficiency - Add progress indicators and better logging throughout --- video-transcripts/transcripts.py | 317 +++++++++++++++++++++++++------ 1 file changed, 256 insertions(+), 61 deletions(-) diff --git a/video-transcripts/transcripts.py b/video-transcripts/transcripts.py index 001d25a..e7fb29f 100644 --- a/video-transcripts/transcripts.py +++ b/video-transcripts/transcripts.py @@ -1,12 +1,16 @@ import os import argparse from googleapiclient.discovery import build +from googleapiclient.errors import HttpError from youtube_transcript_api import YouTubeTranscriptApi +from youtube_transcript_api import TranscriptsDisabled, NoTranscriptFound, VideoUnavailable from dotenv import load_dotenv from slugify import slugify import json import openai import sys +import time +import random load_dotenv() @@ -22,41 +26,73 @@ def get_playlist_videos(playlist_id): video_ids = [] next_page_token = None - # This returns a list of youtube:playlistItem + # This returns a list of youtube:playlistItem with rate limiting while True: - request = youtube.playlistItems().list( - part='snippet', - playlistId=playlist_id, - maxResults=10, - pageToken=next_page_token - ) - response = request.execute() - - for item in response['items']: - video_ids.append(item['snippet']['resourceId']['videoId']) - - next_page_token = response.get('nextPageToken') - if not next_page_token: - break - - # This gets all of the details of each video by ID + try: + # Build request parameters + request_params = { + 'part': 'snippet', + 'playlistId': playlist_id, + 'maxResults': 50 + } + + # Only add pageToken if it's not None + if next_page_token: + request_params['pageToken'] = next_page_token + + request = youtube.playlistItems().list(**request_params) + response = request.execute() + + for item in response['items']: + video_ids.append(item['snippet']['resourceId']['videoId']) + + next_page_token = response.get('nextPageToken') + if not next_page_token: + break + + # Rate limiting between pagination requests + delay = random.uniform(3, 8) # Random delay between 3-8 seconds + print(f"Waiting {delay:.1f} seconds before next page...") + time.sleep(delay) + + except HttpError as e: + if e.resp.status == 403 and 'quota' in str(e).lower(): + print("Quota exceeded while fetching playlist items. Waiting before retry...") + time.sleep(60) # Wait 1 minute + continue + else: + print(f"Error fetching playlist items: {e}") + raise + + # Fetch video details in batches with rate limiting videos = [] - next_page_token = None - while True: - request = youtube.videos().list( - part='snippet', - id=','.join(video_ids), - pageToken=next_page_token - ) - - response = request.execute() - for item in response['items']: - print("Found detail") - print(json.dumps(item, indent=2)) - videos.append(item) - next_page_token = response.get('nextPageToken') - if not next_page_token: - break + batch_size = 50 # YouTube API allows up to 50 IDs per request + + for i in range(0, len(video_ids), batch_size): + batch_ids = video_ids[i:i + batch_size] + + try: + request = youtube.videos().list( + part='snippet', + id=','.join(batch_ids) + ) + response = request.execute() + + for item in response['items']: + print(f"Found video: {item['snippet']['title']}") + videos.append(item) + + # Rate limiting between batches + time.sleep(0.2) # 200ms delay between batches + + except HttpError as e: + if e.resp.status == 403 and 'quota' in str(e).lower(): + print(f"Quota exceeded on video details batch {i//batch_size + 1}. Waiting before retry...") + time.sleep(60) + continue + else: + print(f"Error fetching video details batch {i//batch_size + 1}: {e}") + continue return videos @@ -65,26 +101,50 @@ def get_channel_videos(channel_id, start_date, end_date): videos = [] next_page_token = None + page_count = 0 + max_pages = 1 # Limit to 1 page (50 videos max) to avoid pagination issues - while True: - request = youtube.search().list( - part='snippet', - channelId=channel_id, - maxResults=10, - order='date', - publishedAfter=start_date, - publishedBefore=end_date, - type='video', - pageToken=next_page_token - ) - response = request.execute() - - for item in response['items']: - videos.append(item) - - next_page_token = response.get('nextPageToken') - if not next_page_token: - break + while page_count < max_pages: + try: + # Build request parameters + request_params = { + 'part': 'snippet', + 'channelId': channel_id, + 'maxResults': 50, + 'order': 'date', + 'type': 'video' + } + + # Only add pageToken if it's not None + if next_page_token: + request_params['pageToken'] = next_page_token + + request = youtube.search().list(**request_params) + print(f"Making API request with params: {request_params}") + response = request.execute() + print(f"API request successful, got {len(response.get('items', []))} videos") + + for item in response['items']: + videos.append(item) + + next_page_token = response.get('nextPageToken') + if not next_page_token: + break + + # Rate limiting between pagination requests + delay = random.uniform(5, 15) # Random delay between 5-15 seconds + print(f"Waiting {delay:.1f} seconds before next page...") + time.sleep(delay) + page_count += 1 + + except HttpError as e: + if e.resp.status == 403 and 'quota' in str(e).lower(): + print("Quota exceeded while searching channel videos. Waiting before retry...") + time.sleep(60) # Wait 1 minute + continue + else: + print(f"Error searching channel videos: {e}") + raise return videos @@ -103,7 +163,7 @@ def get_id(video): def fetch_transcripts(args, videos): processed = [] - for video in videos: + for i, video in enumerate(videos): video_id = get_id(video) if video_id == '': @@ -115,15 +175,23 @@ def fetch_transcripts(args, videos): print(f"Skipping video {video_id} because it already has a transcript.") continue - try: - transcript = YouTubeTranscriptApi.get_transcript(video_id, languages=('en','es', 'fr', 'de', 'jp')) - full_text = ' '.join([entry['text'] for entry in transcript]) - video['transcript'] = full_text - processed.append(video) - except Exception as e: - print(f"Could not fetch transcript for video {video_id}: {e}") + print(f"Processing video {i+1}/{len(videos)}: {video_id}") + + # Use improved transcript fetching with retry logic + transcript_text = get_video_transcript_with_retry(video_id) - write_markdown(args, video) + if transcript_text: + video['transcript'] = transcript_text + processed.append(video) + write_markdown(args, video) + else: + print(f"Skipping video {video_id} due to transcript issues.") + + # Rate limiting between transcript fetches + if i < len(videos) - 1: # Don't sleep after the last video + delay = random.uniform(10, 30) # Random delay between 10-30 seconds + print(f"Waiting {delay:.1f} seconds before next video...") + time.sleep(delay) return processed @@ -143,6 +211,82 @@ def file_for_video(args, video): filename = f"{args.path}/{published}-{slug}.md" return filename +def get_video_transcript_with_retry(video_id, max_retries=3): + """ + Fetch transcript with retry logic and exponential backoff. + Uses the new youtube-transcript-api 1.x API with fallback to deprecated static methods. + Returns cleaned transcript text or None if unavailable. + """ + for attempt in range(max_retries): + try: + # Try new API first (youtube-transcript-api 1.x) + try: + transcript_api = YouTubeTranscriptApi() + fetched_transcript = transcript_api.fetch( + video_id, + languages=['en', 'es', 'fr', 'de', 'ja'] # Fixed 'jp' to 'ja' + ) + # Convert to the old format (list of dicts) + transcript = fetched_transcript.to_raw_data() + + except (AttributeError, TypeError): + # Fallback to deprecated static method if new API isn't available + print(f"Using deprecated static API for video {video_id}") + transcript = YouTubeTranscriptApi.get_transcript( + video_id, + languages=('en', 'es', 'fr', 'de', 'ja') + ) + + # Validate transcript structure + if not transcript or not isinstance(transcript, list): + print(f"Invalid transcript format for video {video_id}") + return None + + # Safely extract and clean text + text_parts = [] + for entry in transcript: + if isinstance(entry, dict) and 'text' in entry: + text = entry['text'] + if text and isinstance(text, str): + text = text.strip() + # Filter out common non-content entries + if text and text not in ['[Music]', '[Applause]', '[Laughter]']: + text_parts.append(text) + + if not text_parts: + print(f"No valid text found in transcript for video {video_id}") + return None + + full_text = ' '.join(text_parts) + + # Minimum length validation + if len(full_text.strip()) < 50: + print(f"Transcript too short for video {video_id}: {len(full_text)} characters") + return None + + return full_text + + except TranscriptsDisabled: + print(f"Transcripts are disabled for video {video_id}") + return None + except NoTranscriptFound: + print(f"No transcript found for video {video_id}") + return None + except VideoUnavailable: + print(f"Video {video_id} is unavailable") + return None + except Exception as e: + if attempt < max_retries - 1: + # Exponential backoff with jitter + wait_time = (2 ** attempt) + random.uniform(0, 1) + print(f"Attempt {attempt + 1} failed for video {video_id}. Retrying in {wait_time:.1f}s... Error: {str(e)}") + time.sleep(wait_time) + else: + print(f"All {max_retries} attempts failed for video {video_id}: {str(e)}") + return None + + return None + def openai_cleanup(transcript, video_id): """ Take a messy raw YouTube transcript and return a concise summary + a cleaned up version. @@ -201,6 +345,40 @@ def openai_cleanup(transcript, video_id): return [summary, cleaned_up] +def create_chapters(transcript, video_id): + """ + Create timestamps and chapters for a YouTube video transcript. + @param transcript: The raw transcript of a YouTube video + @param video_id: The YouTube video ID + @return: A string containing formatted timestamps and chapters + """ + if openai_key == "": + print("No OpenAI API key found in .env file; skipping chapter creation.") + return "Chapters not available" + + client = openai.OpenAI(api_key=openai_key) + + chapters_prompt = """ + This is a transcript of a YouTube livestream. Could you please identify up to 10 key moments in the stream and give me the timestamps in the format for YouTube like this?: + 00:00:00 Introductions + 00:01:30 What is structured metadata? + + Always start with 00:00:00 and use simple but descriptive language that makes it easier for users to understand what is being spoken about. + """ + + print(f"Creating chapters for video {video_id}") + response = client.chat.completions.create( + model="gpt-4o-mini", + messages=[ + {"role": "system", "content": chapters_prompt}, + {"role": "user", "content": transcript} + ], + temperature=0.7, + ) + + chapters = response.choices[0].message.content + return chapters + def write_markdown(args, video): video_id = get_id(video) @@ -222,6 +400,7 @@ def write_markdown(args, video): filename = file_for_video(args, video) [summary, cleaned_up] = openai_cleanup(transcript, video_id) + chapters = create_chapters(transcript, video_id) with open(filename, "w") as file: file.write(f"# {title}\n\n") @@ -232,6 +411,9 @@ def write_markdown(args, video): file.write("## Summary\n\n") file.write(f"{summary}\n\n") + file.write("## Chapters\n\n") + file.write(f"{chapters}\n\n") + if cleaned_up: # Don't write a heading because OpenAI was instructed to output markdown. file.write(f"{cleaned_up}\n\n") @@ -247,14 +429,27 @@ def have_transcript_file(args, video): return os.path.isfile(file_for_video(args, video)) def main(args): + # Add initial delay to let API "cool down" + initial_delay = random.uniform(5, 15) + print(f"Starting with {initial_delay:.1f} second delay to avoid rate limiting...") + time.sleep(initial_delay) + videos_to_transcribe = [] if args.playlist: + print("Fetching playlist videos") playlist_videos = get_playlist_videos(args.playlist) videos_to_transcribe = videos_to_transcribe + playlist_videos print(f"Found {len(playlist_videos)} videos in playlist %s" % args.playlist) + + # Add delay between different API calls + if args.channel: + delay = random.uniform(10, 20) + print(f"Waiting {delay:.1f} seconds before fetching channel videos...") + time.sleep(delay) if args.channel: + print("Fetching channel videos") channel_videos = get_channel_videos(args.channel, args.start, args.end) videos_to_transcribe = videos_to_transcribe + channel_videos print(f"Found {len(channel_videos)} videos in the specified date range.") From f616590d480a1c64d278a9ad920b2cff0d0551ac Mon Sep 17 00:00:00 2001 From: Nicole van der Hoeven Date: Thu, 13 Nov 2025 13:54:56 +0100 Subject: [PATCH 02/28] Update README to document chapters feature and improvements - Document new chapters/timestamps generation feature - Add comprehensive usage examples with command-line options - Document output format and file structure - Add Features & Reliability section covering rate limiting and error handling - Clarify OpenAI API key requirements and AI-enhanced features - Document multi-language transcript support --- video-transcripts/README.md | 75 ++++++++++++++++++++++++++++++++++--- 1 file changed, 70 insertions(+), 5 deletions(-) diff --git a/video-transcripts/README.md b/video-transcripts/README.md index b079d9f..1f74109 100644 --- a/video-transcripts/README.md +++ b/video-transcripts/README.md @@ -1,7 +1,11 @@ # YouTube Transcripts -This directory contains a small script which fetches transcripts from the YouTube API, so that we can -keep text/markdown corresponding to the transcripts of the videos that the community produces. +This directory contains a script that fetches transcripts from YouTube videos and generates markdown files with: +- Video metadata (title, description, publish date) +- AI-generated summary of the content +- Chapters/timestamps for easy navigation +- Cleaned up transcript with proper formatting +- Raw YouTube transcript for reference ## Setup @@ -14,18 +18,79 @@ keep text/markdown corresponding to the transcripts of the videos that the commu Edit `.env` file and set `API_KEY` to the correct value for YouTube Optionally, add `OPENAI_API_KEY` if you intend to use the categorizer -## Run & Pull Transcripts! +- **`API_KEY`** (required): YouTube Data API v3 key for fetching video data and transcripts +- **`OPENAI_API_KEY`** (optional): OpenAI API key for AI-enhanced features: + - Generates concise summaries of video content + - Creates YouTube-style chapter timestamps (e.g., `00:00:00 Introduction`) + - Cleans up raw transcripts into readable markdown format -Typical usage will involve a YouTube Channel ID, which is in the [URL for the channel](https://www.youtube.com/channel/UCHZDBZTIfdy94xMjMKz-_MA) +If `OPENAI_API_KEY` is not provided, the script will still fetch and save raw transcripts, but summaries and chapters will be unavailable. -From within this directory, we'd place transcripts in that directory off of the root. +## Usage +### Basic Usage - Channel + +Fetch transcripts from a YouTube channel (defaults to OpenTelemetry channel): + +```bash +python3 transcripts.py --channel UCHZDBZTIfdy94xMjMKz-_MA --path ./transcripts +``` + +### Fetch from Playlist + +```bash +python3 transcripts.py --playlist PLDGkOdUX1UjrEOz4fOB4UZW8m-hx8_mtb --path ./transcripts ``` + +### Command-Line Options + +- `-c, --channel`: YouTube Channel ID (default: OpenTelemetry channel) +- `-p, --playlist`: YouTube Playlist ID (no default) +- `-s, --start`: Start date for videos in ISO 8601 format (default: `2023-01-01T00:00:00Z`) +- `-e, --end`: End date for videos in ISO 8601 format (default: `2030-12-31T00:00:00Z`) +- `-d, --path`: Directory to write markdown files to (default: `./transcripts/`) +- `-l, --limit`: Limit the number of videos to process (useful for testing) + +### Example with Date Range and Limit + +```bash python3 transcripts.py \ --channel UCHZDBZTIfdy94xMjMKz-_MA \ + --start 2024-01-01T00:00:00Z \ + --end 2024-12-31T00:00:00Z \ + --limit 5 \ --path ./transcripts ``` +## Output Format + +Each video generates a markdown file named: `{publish_date}-{slugified-title}.md` + +The generated markdown includes: +- **Title and Metadata**: Video title, publish date, description, and YouTube URL +- **Summary**: AI-generated one-paragraph overview of the video content +- **Chapters**: YouTube-style timestamps identifying key moments (e.g., `00:00:00 Introduction`) +- **Cleaned Transcript**: Readable, formatted transcript with proper paragraphs +- **Raw Transcript**: Original YouTube auto-generated transcript for reference + +## Features & Reliability + +### Rate Limiting +The script includes comprehensive rate limiting to avoid YouTube API quota issues: +- Random delays between API requests (3-8 seconds for pagination) +- Longer delays between video processing (10-30 seconds) +- Initial startup delay to "cool down" the API +- Automatic 60-second wait and retry when quota limits are hit + +### Error Handling +- Exponential backoff with up to 3 retry attempts for transcript fetching +- Graceful handling of videos with disabled transcripts +- Validates transcript quality (minimum length, filters music/applause markers) +- Skips videos that already have transcripts (resumes interrupted runs) + +### Language Support +Attempts to fetch transcripts in multiple languages: English, Spanish, French, German, Japanese + ## Need a YouTube Key? * Have a GCP project From d02dac77c7318f01cc792453e872967fd97e4ff0 Mon Sep 17 00:00:00 2001 From: Nicole van der Hoeven Date: Thu, 13 Nov 2025 14:14:26 +0100 Subject: [PATCH 03/28] Fix pydantic dependency conflict in requirements.txt Update pydantic_core from 2.39.0 to 2.33.2 to match the version required by pydantic 2.11.9, resolving pip installation error. --- video-transcripts/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/video-transcripts/requirements.txt b/video-transcripts/requirements.txt index 1e1c17c..a687ad1 100644 --- a/video-transcripts/requirements.txt +++ b/video-transcripts/requirements.txt @@ -35,7 +35,7 @@ protobuf==5.29.5 pyasn1==0.6.1 pyasn1_modules==0.4.2 pydantic==2.11.9 -pydantic_core==2.39.0 +pydantic_core==2.33.2 pyparsing==3.2.5 python-dateutil==2.9.0.post0 python-dotenv==1.1.1 From 9e0ebb3fa51217baffa82383e35ccf4b53170e4b Mon Sep 17 00:00:00 2001 From: Nicole van der Hoeven Date: Thu, 13 Nov 2025 14:49:08 +0100 Subject: [PATCH 04/28] Add env.example --- video-transcripts/env.example | 11 +++++++++++ 1 file changed, 11 insertions(+) create mode 100644 video-transcripts/env.example diff --git a/video-transcripts/env.example b/video-transcripts/env.example new file mode 100644 index 0000000..9055c75 --- /dev/null +++ b/video-transcripts/env.example @@ -0,0 +1,11 @@ +# YouTube API Configuration +# Get your API key from: https://console.cloud.google.com/ +# 1. Go to a GCP project +# 2. Make sure YouTube Data API v3 is enabled +# 3. Create an API key in that project +API_KEY=your_youtube_api_key_here + +# OpenAI API Configuration (optional) +# Only needed if you want AI-powered summaries and cleanup +# Get your API key from: https://platform.openai.com/api-keys +OPENAI_API_KEY=your_openai_api_key_here From 2afaaa5eeffe64c1f0b8e83d2d0a62e5740dbfe2 Mon Sep 17 00:00:00 2001 From: Nicole van der Hoeven Date: Thu, 13 Nov 2025 14:50:38 +0100 Subject: [PATCH 05/28] Update README with instructions to cp env.example --- video-transcripts/README.md | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/video-transcripts/README.md b/video-transcripts/README.md index 1f74109..d2efceb 100644 --- a/video-transcripts/README.md +++ b/video-transcripts/README.md @@ -15,7 +15,13 @@ This directory contains a script that fetches transcripts from YouTube videos an ## Configure Environment -Edit `.env` file and set `API_KEY` to the correct value for YouTube +Copy the example environment file and configure your API keys: + +```bash +cp env.example .env +``` + +Edit `.env` file and set `API_KEY` to the correct value for YouTube. Optionally, add `OPENAI_API_KEY` if you intend to use the categorizer - **`API_KEY`** (required): YouTube Data API v3 key for fetching video data and transcripts From 41feca69aaad4b444a922689a155363f07376f45 Mon Sep 17 00:00:00 2001 From: Nicole van der Hoeven Date: Mon, 17 Nov 2025 13:59:27 +0100 Subject: [PATCH 06/28] Fix transcript fetching by upgrading youtube-transcript-api Major changes: - Upgrade youtube-transcript-api from 0.6.3 to 1.2.3 (fixes empty response issue) - Update code to use new 1.x API (YouTubeTranscriptApi().fetch()) - Remove deprecated fallback to old static methods - Enhance error handling for 429 rate limits with 60-120s delays - Detect XML parse errors as potential rate limiting - Increase max retries from 3 to 5 attempts Root cause: YouTube changed their API and version 0.6.3 was returning empty responses (not rate limiting). The library can now successfully fetch transcripts and generate chapters. --- video-transcripts/requirements.txt | 2 +- video-transcripts/transcripts.py | 75 +++++++++++++++++++++--------- 2 files changed, 53 insertions(+), 24 deletions(-) diff --git a/video-transcripts/requirements.txt b/video-transcripts/requirements.txt index a687ad1..69aecaa 100644 --- a/video-transcripts/requirements.txt +++ b/video-transcripts/requirements.txt @@ -55,4 +55,4 @@ typing_extensions==4.15.0 tzlocal==5.3.1 uritemplate==4.2.0 urllib3==2.5.0 -youtube-transcript-api==0.6.3 +youtube-transcript-api==1.2.3 diff --git a/video-transcripts/transcripts.py b/video-transcripts/transcripts.py index e7fb29f..c4262d2 100644 --- a/video-transcripts/transcripts.py +++ b/video-transcripts/transcripts.py @@ -4,6 +4,7 @@ from googleapiclient.errors import HttpError from youtube_transcript_api import YouTubeTranscriptApi from youtube_transcript_api import TranscriptsDisabled, NoTranscriptFound, VideoUnavailable +from youtube_transcript_api._errors import YouTubeRequestFailed from dotenv import load_dotenv from slugify import slugify import json @@ -211,31 +212,23 @@ def file_for_video(args, video): filename = f"{args.path}/{published}-{slug}.md" return filename -def get_video_transcript_with_retry(video_id, max_retries=3): +def get_video_transcript_with_retry(video_id, max_retries=5): """ Fetch transcript with retry logic and exponential backoff. - Uses the new youtube-transcript-api 1.x API with fallback to deprecated static methods. + Uses youtube-transcript-api 1.2.3+ API. + Handles YouTube rate limiting (429 errors) with extended wait times (60-120 seconds). Returns cleaned transcript text or None if unavailable. """ for attempt in range(max_retries): try: - # Try new API first (youtube-transcript-api 1.x) - try: - transcript_api = YouTubeTranscriptApi() - fetched_transcript = transcript_api.fetch( - video_id, - languages=['en', 'es', 'fr', 'de', 'ja'] # Fixed 'jp' to 'ja' - ) - # Convert to the old format (list of dicts) - transcript = fetched_transcript.to_raw_data() - - except (AttributeError, TypeError): - # Fallback to deprecated static method if new API isn't available - print(f"Using deprecated static API for video {video_id}") - transcript = YouTubeTranscriptApi.get_transcript( - video_id, - languages=('en', 'es', 'fr', 'de', 'ja') - ) + # Use youtube-transcript-api 1.x API + transcript_api = YouTubeTranscriptApi() + fetched_transcript = transcript_api.fetch( + video_id, + languages=['en', 'es', 'fr', 'de', 'ja'] + ) + # Convert to list of dicts format + transcript = fetched_transcript.to_raw_data() # Validate transcript structure if not transcript or not isinstance(transcript, list): @@ -275,14 +268,50 @@ def get_video_transcript_with_retry(video_id, max_retries=3): except VideoUnavailable: print(f"Video {video_id} is unavailable") return None + except YouTubeRequestFailed as e: + # Check if this is a 429 rate limit error + error_str = str(e) + if '429' in error_str or 'Too Many Requests' in error_str: + if attempt < max_retries - 1: + # Use much longer wait time for rate limiting (60-120 seconds) + wait_time = random.uniform(60, 120) + print(f"⚠️ YouTube rate limit (429) detected for video {video_id}") + print(f" Waiting {wait_time:.0f} seconds before retry {attempt + 2}/{max_retries}...") + time.sleep(wait_time) + else: + print(f"❌ YouTube rate limit persists after {max_retries} attempts for video {video_id}") + print(f" Consider waiting 10-15 minutes before running the script again.") + return None + else: + # Other YouTube API errors + if attempt < max_retries - 1: + wait_time = (2 ** attempt) * 5 + random.uniform(0, 5) + print(f"YouTube API error for video {video_id}. Retrying in {wait_time:.1f}s...") + time.sleep(wait_time) + else: + print(f"YouTube API error persists for video {video_id}: {str(e)}") + return None except Exception as e: - if attempt < max_retries - 1: - # Exponential backoff with jitter + error_str = str(e) + # Check if this is likely a rate limit error disguised as XML parse error + if 'no element found' in error_str or 'line 1, column 0' in error_str: + if attempt < max_retries - 1: + # Use longer wait time as this is likely a rate limit issue + wait_time = random.uniform(60, 120) + print(f"⚠️ Possible YouTube rate limit detected for video {video_id} (XML parse error)") + print(f" Waiting {wait_time:.0f} seconds before retry {attempt + 2}/{max_retries}...") + time.sleep(wait_time) + else: + print(f"❌ Persistent error for video {video_id} after {max_retries} attempts") + print(f" This may be due to YouTube rate limiting. Wait 10-15 minutes and try again.") + return None + elif attempt < max_retries - 1: + # Standard exponential backoff for other errors wait_time = (2 ** attempt) + random.uniform(0, 1) - print(f"Attempt {attempt + 1} failed for video {video_id}. Retrying in {wait_time:.1f}s... Error: {str(e)}") + print(f"Attempt {attempt + 1} failed for video {video_id}. Retrying in {wait_time:.1f}s... Error: {error_str}") time.sleep(wait_time) else: - print(f"All {max_retries} attempts failed for video {video_id}: {str(e)}") + print(f"All {max_retries} attempts failed for video {video_id}: {error_str}") return None return None From 598804bcb6bdc0ce2bba25a35dcccfcc14265d5b Mon Sep 17 00:00:00 2001 From: Nicole van der Hoeven Date: Mon, 17 Nov 2025 14:59:20 +0100 Subject: [PATCH 07/28] Optimize rate limiting delays for faster processing Reduced unnecessary delays now that transcript API is fixed: - Remove initial 5-15s startup delay - Reduce pagination delays from 3-8s/5-15s to 1-3s - Reduce inter-video delays from 10-30s to 2-5s - Reduce API call separation from 10-20s to 2-4s Keep essential protections: - YouTube Data API quota error handling (60s retry) - 429 rate limit detection and handling (60-120s retry) - XML parse error detection - Small delays to avoid API hammering Result: ~3-5x faster processing while maintaining API safety. --- video-transcripts/transcripts.py | 21 ++++++++------------- 1 file changed, 8 insertions(+), 13 deletions(-) diff --git a/video-transcripts/transcripts.py b/video-transcripts/transcripts.py index c4262d2..bfac65f 100644 --- a/video-transcripts/transcripts.py +++ b/video-transcripts/transcripts.py @@ -51,8 +51,8 @@ def get_playlist_videos(playlist_id): if not next_page_token: break - # Rate limiting between pagination requests - delay = random.uniform(3, 8) # Random delay between 3-8 seconds + # Small delay between pagination requests + delay = random.uniform(1, 3) # Random delay between 1-3 seconds print(f"Waiting {delay:.1f} seconds before next page...") time.sleep(delay) @@ -132,8 +132,8 @@ def get_channel_videos(channel_id, start_date, end_date): if not next_page_token: break - # Rate limiting between pagination requests - delay = random.uniform(5, 15) # Random delay between 5-15 seconds + # Small delay between pagination requests + delay = random.uniform(1, 3) # Random delay between 1-3 seconds print(f"Waiting {delay:.1f} seconds before next page...") time.sleep(delay) page_count += 1 @@ -188,9 +188,9 @@ def fetch_transcripts(args, videos): else: print(f"Skipping video {video_id} due to transcript issues.") - # Rate limiting between transcript fetches + # Small delay between transcript fetches if i < len(videos) - 1: # Don't sleep after the last video - delay = random.uniform(10, 30) # Random delay between 10-30 seconds + delay = random.uniform(2, 5) # Random delay between 2-5 seconds print(f"Waiting {delay:.1f} seconds before next video...") time.sleep(delay) @@ -458,11 +458,6 @@ def have_transcript_file(args, video): return os.path.isfile(file_for_video(args, video)) def main(args): - # Add initial delay to let API "cool down" - initial_delay = random.uniform(5, 15) - print(f"Starting with {initial_delay:.1f} second delay to avoid rate limiting...") - time.sleep(initial_delay) - videos_to_transcribe = [] if args.playlist: @@ -471,9 +466,9 @@ def main(args): videos_to_transcribe = videos_to_transcribe + playlist_videos print(f"Found {len(playlist_videos)} videos in playlist %s" % args.playlist) - # Add delay between different API calls + # Small delay between different API calls if args.channel: - delay = random.uniform(10, 20) + delay = random.uniform(2, 4) print(f"Waiting {delay:.1f} seconds before fetching channel videos...") time.sleep(delay) From 869fd08b3af116ebcf352a9ed7ea35be435ed932 Mon Sep 17 00:00:00 2001 From: Nicole van der Hoeven Date: Mon, 17 Nov 2025 18:18:57 +0100 Subject: [PATCH 08/28] Add detection and guidance for YouTube IP blocking - Detect IP block errors specifically (vs rate limiting) - Stop retries immediately when IP is blocked (no point retrying) - Add comprehensive troubleshooting section to README - Provide clear workaround options for users - Import TooManyRequests exception for better error handling IP blocks are different from rate limits and require different solutions like waiting 24-48 hours, switching networks, or using cookie auth. --- video-transcripts/README.md | 29 +++++++++++++++++++++++++++++ video-transcripts/transcripts.py | 18 ++++++++++++++++-- 2 files changed, 45 insertions(+), 2 deletions(-) diff --git a/video-transcripts/README.md b/video-transcripts/README.md index d2efceb..bfbb23c 100644 --- a/video-transcripts/README.md +++ b/video-transcripts/README.md @@ -97,6 +97,35 @@ The script includes comprehensive rate limiting to avoid YouTube API quota issue ### Language Support Attempts to fetch transcripts in multiple languages: English, Spanish, French, German, Japanese +## Troubleshooting + +### YouTube IP Blocking + +If you encounter "YouTube is blocking requests from your IP" errors, this means YouTube has blocked your IP address from accessing their transcript API. This is different from rate limiting and won't clear quickly. + +**Common causes:** +- Repeated testing/debugging over multiple days +- Using a cloud provider IP (AWS, GCP, Azure, etc.) +- Using certain VPN services +- Your ISP's IP range being flagged + +**Solutions:** + +1. **Wait 24-48 hours** - IP blocks usually clear after 1-2 days of no activity + +2. **Use a different network** - Switch to mobile hotspot, different WiFi, or different location + +3. **Cookie-based authentication** (Advanced): + - Export cookies from a logged-in YouTube session in your browser + - Use a browser extension like "Get cookies.txt LOCALLY" (Chrome/Firefox) + - Save cookies.txt in the video-transcripts directory + - Modify the script to use cookies (requires code changes) + - See [youtube-transcript-api documentation](https://github.com/jdepoix/youtube-transcript-api#cookies) for details + +4. **Use a residential proxy** - Cloud/datacenter IPs are often blocked, but residential IPs work better + +5. **Run from a different machine** - If possible, run the script from a home computer instead of a server + ## Need a YouTube Key? * Have a GCP project diff --git a/video-transcripts/transcripts.py b/video-transcripts/transcripts.py index bfac65f..44cdee2 100644 --- a/video-transcripts/transcripts.py +++ b/video-transcripts/transcripts.py @@ -5,6 +5,11 @@ from youtube_transcript_api import YouTubeTranscriptApi from youtube_transcript_api import TranscriptsDisabled, NoTranscriptFound, VideoUnavailable from youtube_transcript_api._errors import YouTubeRequestFailed +try: + from youtube_transcript_api._errors import TooManyRequests +except ImportError: + # Fallback if TooManyRequests doesn't exist in this version + TooManyRequests = YouTubeRequestFailed from dotenv import load_dotenv from slugify import slugify import json @@ -269,9 +274,18 @@ def get_video_transcript_with_retry(video_id, max_retries=5): print(f"Video {video_id} is unavailable") return None except YouTubeRequestFailed as e: - # Check if this is a 429 rate limit error + # Check if this is an IP block error error_str = str(e) - if '429' in error_str or 'Too Many Requests' in error_str: + if 'blocking requests from your IP' in error_str or 'IP has been blocked' in error_str: + print(f"❌ YouTube has blocked your IP address for video {video_id}") + print(f" This is not a temporary rate limit - your IP is blocked.") + print(f"\n Workarounds:") + print(f" 1. Wait 24-48 hours for the block to clear") + print(f" 2. Use a different network/WiFi connection") + print(f" 3. Set up cookie-based authentication (see README)") + print(f" 4. Use a residential proxy or VPN (not cloud-based)") + return None + elif '429' in error_str or 'Too Many Requests' in error_str: if attempt < max_retries - 1: # Use much longer wait time for rate limiting (60-120 seconds) wait_time = random.uniform(60, 120) From bdd392ed079465db4c569cfefdd446352ed0f96a Mon Sep 17 00:00:00 2001 From: Nicole van der Hoeven Date: Mon, 17 Nov 2025 18:21:39 +0100 Subject: [PATCH 09/28] Delete old transcripts and regenerate with timestamps --- ...etry-q-a-feat-iris-dyrmishi-of-farfetch.md | 104 ++++++ ...-whats-an-observability-engineer-anyway.md | 27 +- ...8Z-teaser-observability-is-a-team-sport.md | 25 +- ...o-you-foster-a-culture-of-observability.md | 21 +- ...-a-team-sport-with-iris-dyrmishi-teaser.md | 27 +- ...eam-sport-with-iris-dyrmishi-full-video.md | 121 +++++++ ...ob-aronoff-of-lightstep-from-servicenow.md | 116 +++--- ...10Z-opentelemetry-q-a-feat-hazel-weakly.md | 102 +++--- ...-threading-the-needle-with-hazel-weakly.md | 76 ++-- ...n-distributed-tracing-with-doug-ramirez.md | 115 +++--- ...in-practice-fireside-chat-december-2022.md | 85 ++--- ...-end-user-discussions-amer-january-2023.md | 189 ++++------ ...otel-migration-story-with-jacob-aronoff.md | 125 +++++++ ...he-evolution-of-observability-practices.md | 158 +++------ ...ng-parsing-data-with-the-otel-collector.md | 91 ++--- ...T23:34:44Z-otel-q-a-feat-jennifer-moore.md | 121 ++++--- ...Z-otel-q-a-feat-luiz-aoqui-of-hashicorp.md | 81 +++-- ...:51Z-the-humans-of-otel-kubecon-na-2023.md | 128 ++++--- ...ty-majors-amy-tobey-and-adriana-villela.md | 116 +++--- ...ow-to-train-your-teams-on-observability.md | 84 ++--- ...:24Z-otel-collector-user-feedback-panel.md | 138 ++++---- ...us-interoperability-user-feedback-panel.md | 103 +++--- ...:59Z-the-humans-of-otel-kubecon-eu-2024.md | 164 ++++----- ...aniel-dias-and-oscar-reyes-of-tracetest.md | 136 +++++--- ...ile-apps-with-hanson-ho-and-eliab-sisay.md | 130 ++++--- ...liab-sisay-and-austin-emmons-of-embrace.md | 133 ++++--- ...4T15:54:54Z-otel-q-a-with-steven-swartz.md | 276 +++++++++++---- ...T21:03:07Z-otel-q-a-with-dan-ravenstone.md | 97 ++++-- ...umans-of-otel-live-from-kubecon-na-2024.md | 141 +++++--- ...2:14:29Z-humans-of-otel-kubecon-na-2024.md | 100 +++--- ...d-user-conversation-with-ariel-valentin.md | 269 +++++--------- ...31T05:58:14Z-cfp-writing-q-a-livestream.md | 169 +++++---- ...el-for-beginners-the-javascript-journey.md | 186 +++++++--- ...-otel-me-with-jerome-johnson-cal-loomis.md | 187 ++++------ ...on-with-austin-parker-marylia-gutierrez.md | 123 +++---- ...6:00:52Z-otel-me-with-eromosele-akhigbe.md | 140 +++++--- ...humans-of-opentelemetry-kubecon-eu-2025.md | 117 +++---- ...025-05-spring-starter-for-opentelemetry.md | 146 +++----- ...05-leveraging-ai-for-opentelemetry-data.md | 100 +++--- ...h-oluwatomisin-taiwo-and-andrei-morozov.md | 116 ++++++ ...practice-alibabas-opentelemetry-journey.md | 132 +++++++ ...aled-kafkalog-ingestion-for-otel-by-150.md | 140 ++++++++ .../2025-10-22T04:08:27Z-whats-new-in-otel.md | 329 ++++++++++++++++++ ...umans-of-otel-live-from-kubecon-na-2025.md | 258 ++++++++++++++ 44 files changed, 3662 insertions(+), 2080 deletions(-) create mode 100644 video-transcripts/transcripts/2023-06-01T17:22:03Z-opentelemetry-q-a-feat-iris-dyrmishi-of-farfetch.md create mode 100644 video-transcripts/transcripts/2023-06-12T23:45:52Z-otel-in-practice-presents-observability-is-a-team-sport-with-iris-dyrmishi-full-video.md create mode 100644 video-transcripts/transcripts/2023-10-02T15:08:33Z-otel-in-practice-presents-a-hands-on-otel-migration-story-with-jacob-aronoff.md create mode 100644 video-transcripts/transcripts/2025-06-11T05:54:28Z-otel-me-with-oluwatomisin-taiwo-and-andrei-morozov.md create mode 100644 video-transcripts/transcripts/2025-07-16T20:50:15Z-otel-in-practice-alibabas-opentelemetry-journey.md create mode 100644 video-transcripts/transcripts/2025-09-25T05:32:25Z-otel-in-practice-how-we-scaled-kafkalog-ingestion-for-otel-by-150.md create mode 100644 video-transcripts/transcripts/2025-10-22T04:08:27Z-whats-new-in-otel.md create mode 100644 video-transcripts/transcripts/2025-11-13T08:33:00Z-humans-of-otel-live-from-kubecon-na-2025.md diff --git a/video-transcripts/transcripts/2023-06-01T17:22:03Z-opentelemetry-q-a-feat-iris-dyrmishi-of-farfetch.md b/video-transcripts/transcripts/2023-06-01T17:22:03Z-opentelemetry-q-a-feat-iris-dyrmishi-of-farfetch.md new file mode 100644 index 0000000..abdc363 --- /dev/null +++ b/video-transcripts/transcripts/2023-06-01T17:22:03Z-opentelemetry-q-a-feat-iris-dyrmishi-of-farfetch.md @@ -0,0 +1,104 @@ +# OpenTelemetry Q&A Feat. Iris Dyrmishi of Farfetch + +Published on 2023-06-01T17:22:03Z + +## Description + +Adriana Villela of the OTel End User Working group speaks with Iris Dyrmishi of Farfetch about her organization's ... + +URL: https://www.youtube.com/watch?v=9iaGG-YZw5I + +## Summary + +In this YouTube video, Iris and Edith, both engineers at Farfetch, engage in a Q&A session discussing their experiences with OpenTelemetry and observability practices within the organization. Edith, who works on the observability team, shares her journey from a backend developer to a platform engineer, highlighting the importance of OpenTelemetry in improving their monitoring architecture, which encompasses a complex setup of cloud-native systems and Kubernetes. They discuss the adoption process at Farfetch, emphasizing a supportive culture around observability, the implementation of OpenTelemetry for traces and metrics, and the gradual transition from legacy systems. Edith also shares insights about their current use of tools like Grafana, Tempo, and Prometheus, the challenges faced during implementation, and the importance of collaboration across teams to achieve effective observability. The conversation emphasizes the positive reception of OpenTelemetry and the ongoing efforts to enhance their observability practices. + +## Chapters + +00:00:00 Introductions and Overview +00:02:30 Iris introduces Edis and her role at Farfetch +00:05:00 Edis shares her background and journey to observability +00:09:15 Discussion on how Edis discovered OpenTelemetry +00:12:30 Overview of Farfetch's complex architecture +00:17:00 Importance of OpenTelemetry in their system +00:21:45 Edis explains their CI/CD pipeline and deployment process +00:25:30 Tools and technologies used for observability at Farfetch +00:30:00 Edis discusses the adoption of OpenTelemetry within the organization +00:35:45 Challenges faced in implementing OpenTelemetry +00:40:00 Edis shares experiences with OpenTelemetry's operator and community contributions + +# OpenTelemetry Q&A Session + +Thank you all for joining us today! It's a cozy group, which means we can have a more intimate conversation. I'm excited to introduce our guest, Edis, who works at Farfetch and is passionate about OpenTelemetry. + +## Introduction + +Hello everyone! My name is Edis, and I work as a platform engineer as part of the observability team at Farfetch. My role involves providing tools for all engineering teams across Farfetch to monitor their services, including traces, metrics, logs, and alerting. I'm thrilled to be here and look forward to our discussion. + +## OpenTelemetry Journey + +### How did you get to your current role at Farfetch? + +I started my career as a software engineer, specifically as a back-end developer. I transitioned into a DevOps engineer role, where I began working with monitoring tools, primarily in AWS and Azure. Over time, I developed a passion for observability and started working with more equipped observability platforms. My first exposure to OpenTelemetry was when I created a proof of concept (POC) after hearing about it on LinkedIn. Now, I'm excited to be at Farfetch, where I've been continuously learning and evolving my skills in observability. + +### What does the architecture look like at Farfetch? + +Currently, we have around 3,000 engineers at Farfetch, and our architecture is quite complex. We have a mix of cloud-native services, Kubernetes, and virtual machines running on Ubuntu Linux across three different cloud providers. Each team follows certain guidelines, but there is a lot of legacy processes that still exist, which makes uniformity a challenge. + +For metrics, we previously relied heavily on Prometheus, but some applications found it cumbersome. That's where OpenTelemetry came in, allowing us to collect telemetry signals consistently and uniformly. + +## Deployment Process and CI/CD Pipeline + +Our CI/CD pipeline primarily uses Jenkins, managed by a separate team. My role focuses strictly on observability, overseeing the tools, deployments, maintenance, and the release of new features. + +### What OpenTelemetry tooling are you using? + +We primarily use open-source tools, including Grafana for dashboards, Tempo for tracing, and Prometheus and Thanos for metrics. We also have Jaeger in use to some extent as we transition to OpenTelemetry. + +## OpenTelemetry Implementation at Farfetch + +### What was your organization's OpenTelemetry journey like? + +At Farfetch, we have a strong observability culture. When OpenTelemetry was introduced, it was welcomed with enthusiasm, and we quickly made space in our yearly plans to implement it. The positive reception was heartening, and I’m proud to be part of that culture. + +### What about your team's journey to enable OpenTelemetry? + +When I joined the team, much of the groundwork for implementing observability had already been laid. Most engineers had already embraced the importance of observability, making it easier for us to introduce OpenTelemetry. + +To skill up, we focused on a phased implementation to minimize disruption for engineering teams. I took the lead on research and development, and over time, we successfully transitioned to using OpenTelemetry in production. + +## Challenges and Experiences + +### What challenges did you face during implementation? + +One of the biggest challenges was figuring out how to release OpenTelemetry in manageable parts to avoid negatively impacting the engineering teams. We also faced technical challenges with the OpenTelemetry operator and integration with other systems, but we found solutions and adapted accordingly. + +### How are you currently handling traces and metrics? + +We are currently using a combination of manual and auto instrumentation. Our goal is to have all applications using OpenTelemetry instrumentation over time. We're also working on improving our tracing capabilities, gradually increasing the sampling size to provide better insights. + +### How do you manage the volume of data generated? + +We take a cautious approach to data management. While we have a high volume of data, we monitor and adjust our resources accordingly. We implement guidelines and review processes for significant changes to ensure stability and performance. + +## Community and Contributions + +### Have you made any contributions to OpenTelemetry? + +Yes, we recently contributed to the OpenTelemetry operator regarding certificate management. It was a collaborative effort, and the community was very welcoming, which made the process smooth. + +### Any thoughts on areas for improvement in OpenTelemetry? + +While my experience with OpenTelemetry has been overwhelmingly positive, I believe some documentation could be improved, particularly for certain features and configurations. I plan to contribute to that as well. + +## Audience Questions + +If anyone has questions, feel free to ask! + +--- + +**Thank you all for joining today!** I hope this session has been insightful. If you're interested in future discussions or have stories to share, please reach out. We’ll be providing a blog post summary of today’s Q&A, so stay tuned! + +## Raw YouTube Transcript + +thank you I guess we can get started it's a cozy group today which is cool I like cozy groups means we get to uh we we get to have a more intimate conversation but like I said there are there will be ways of consuming this information as well because what Edis has to say is awesome um she's very passionate about open Telemetry so I'm very excited to have her join us and Edith works at farfetch um yeah do you want do you want to do like a brief little intro for everybody for sure hello everyone again my name is Iris I work as a platform engineer yeah I know this title changes every time so in my LinkedIn it could be something different you know how it is I'm platform engineer part of the observability team currently in farfetch um I belong as a part of the central team that we provide tools for all the engineering teams across farfetch to monitor their services including traces metrics logs and alerting again I'm very excited to be here so we're just gonna do this like just a regular q a um and because we have like such a small audience also if we have time at the end um we can um y'all are more than welcome to ask questions um it's you know the purpose of this is to understand what Edis is doing at farfetch um around open Telemetry observability to you know help help the rest of the community share share use cases across the community so that uh we can all we can all learn from each other right so that is that so uh first first things first um how did you how did you come about um to your current role at farfetch so observability has been a part of a build up for me when I started my career I actually started as a software engineer a back-end developer and because it was a type of company that offered service to different clients and it is a devops engineer and I was like okay I was put there so I started working very small scale with monitoring mostly in AWS and Azure with cloudwatch a little bit with insights and then it started becoming more of a passion for me the more I learned about it and then I changed my the position that I was in and I started working in a more let's say well equipped observability platform and that's what I was like okay I really really like this I heard about open Telemetry for the first time and had my chance to touch it a little bit Prometheus grafana so I saw there was a lot of potential there and then where I currently am of course my previous experience really helped to to come now and it's been one year and a little bit of learning and continuously evolving of observability so now I think I've become pretty good at it started from zero yay that's the best and but so how did you uh hear about open Telemetry specifically is it like one of those things where someone mentioned it to you you saw it on the interweb somewhere like what's what's your story I think it was LinkedIn somewhere I know that I was working we were working with no traces at the time I know a blasphemy but and we were looking into tracing Solutions and somewhere like that I saw on Telemetry so I was like okay I'm gonna give this a try and made a small POC for my manager it never went more than that to the POC it was almost more than one year ago but that's how I heard about it I really liked it I had it at the back of my mind so now that there was an opportunity here in farfetch open Telemetry came off again I was like okay I like that I'm gonna go for it so um at farfetch um what um can you tell us a little bit about like the the architecture of the system that you're that you're working with and why why observability open telemetry are are so important to to that so I guess we'll start with like let's give us give us a sense of of like what what the architecture is so we are around 3 000 Engineers currently in farfetch only in Tech and we have a an extremely complex architecture because we have different types of different sides of the business so we have Cloud native we have kubernetes and we have virtual machines Ubuntu Linux the three types of different Cloud providers uh every team of course we have uniformity and we have guidelines that needed to be followed but it's a lot of years in the process and some things are still like in the past every team decides to do things how their best so it is a lot of information coming from everywhere and it's not uniform so for example we were relying heavily on Prometheus to collect metrics but some of our applications some of our Engineers promises was just not a good idea so we're like okay what what could be better here's open Telemetry the same for the traces uh and uh of course this will help us not only collect this Telemetry signals from places where we couldn't before and from services that were not possible but also it's helping us put everything in a uniform way which is it's amazing very cool and um now what what about like um your like what's your what's your building uh deployment process like uh what do you mean by uh building uh like in in the organization like what's your CI CD pipeline like it's not something that you're you're involved in to any extent or not really involved uh more to the sense that I use it a lot and I'm in class communication but yeah we currently use Jenkins in farfetch and we have a separate team taking care of that which tells again how complex everything is like we have a teams and everything is segregated in uh my position is strictly of sorry observability basically everything with the durability all the tools deployments maintenance and uh releasing new features on top of what we have Okay Okay cool so um what um what uh observability tooling um are you using and you know if you're using like a an observability vendor that you'd rather not divulge like that's totally totally okay um but just give us a sense of like maybe your your open Telemetry stack um how it's set up that kind of thing um yeah currently we're uh mostly open source and some things that have some uh tools that have been created in-house by us but mostly open source we use grafana for dashboards we use tempo as a data source which is also part of grafana uh mostly for as a tracing back-end that was something new sorry for the baby cats in the background we use Prometheus and Thanos for metrics and now we have added open Telemetry there as well we used to have Jaeger and we still do to a certain degree because we still haven't completely moved up on Telemetry when it comes to um the Frameworks that some of the teams are using so it is they're working hand in hand Jaeger and that's pretty much it that's all that I can think right now but these are the main ones Thanos cool and um what about um in terms of um tell us a little bit about like your organization's uh uh open Telemetry journey is it you know we we hear kind of It's Always a mixed bag right like some organizations are like giddy up more open Telemetry observability yeah and others are like what was it like at farfetch well I'm very proud to say actually that we have a very good observability culture in Far Fetch and um it was actually one of the ideas that was welcomed immediately okay up on Telemetry let's do it let's go for it uh it took us a bit more time to make some space on our guns on our yearly plan for it but the moment that it was mentioned everyone was okay let's jump to it we only see positives there we couldn't see any negatives other than the time spent which is necessity and yeah it was very well accepted I was surprised I'm very happy obviously that's amazing yeah so it sounds like it came like from from the top then yeah yeah exactly um I of course I don't want to mention names but some of our senior leaders are very involved in the community and they're like always seeing new technologies and says hey what do you think about this do you like it and of course me for I'm extremely ambitious and I'm always on the internet on LinkedIn seeing for new things to implement and it's a great combination there oh awesome awesome now in terms of like um your team to start you know um enabling observability um practices and open Telemetry like what what did you and your team have to go through in order to make that happen uh well the moment that I've joined the the the team in a point of time I think the the biggest struggle had already passed because observability became a very new thing three years ago in Far Fetch and I think the people that struggled the most were the engineers that worked in the team back then of course it was a new thing observability why do we need it why is it so important but I think at the point that I actually joined um everyone had already embraced how important observabilities for everyone of course that's a overestimation but most of the engineers had already embraced so uh for us saying that hey we are implementing this uh this amazing new thing open Telemetry that everyone had heard about it so it's very popular right now it was embraced immediately and of course knowing the benefits that they were getting for it more metrics more traces more uniform way of collecting everything it was it was a blast immediately and how did you um how did your team um skill up in terms of being able to um Implement like start implementing implementing open Telemetry um because it sounds like some people were kind of familiar with it maybe I'm I'm assuming it might have been newer to other folks um you said actually when when the the the directive came um that um your team was gonna start you know um implementing like observability principles open Telemetry that that was something that you you know there was like work involved so what was like what was the kind of work that was involved in in order to enable that for your team well I think the biggest challenge well what we thought at the time was a challenge was how to release this in in patches and in parts so we could we didn't do any damage or um that the engineering teams that use observability every day didn't feel the change for for the bad of course uh we would like them to see the Improvement so we thought that that was going to be a challenge of course we were going to move from a not one technology to the other uh but actually we of course we always have someone who is a driver for the project and in this case in my team it was me so I did a thorough investigation and every time I was reading something I was like wow okay this is cool wow okay this is amazing because everything managed to fit together very well for us of course the fact that we use open source really really helps with open Telemetry has compatibility with Prometheus with Diego with everything that we were working so it it became easier with the time to just put everything up out there so it sounds like you were you were the primary driver for for getting people like leveled up on on open telemetry yeah I'm proud of that together with one the ones who were the biggest Pusher and supporters of open Telemetry and now we're in production so I think we were pretty successful that is super super cool yeah and I I don't know that we hear like too many um stories of of um organizations using hotel and production so that's I think that's really uh really awesome and and how how long did it take you guys to get to the point where um you know you've got open Telemetry in production so let me see we plan to start implementing open Telemetry around January and we started the first investigation gathering information I think by the mid May mid May sorry mid-march we were already ready in production but again uh we are still not there not 100 we're still relying on a lot of things with Prometheus and Jaeger we're just using open Telemetry to transport we still need to do instrumentation with open Telemetry some parts of of our engineering teams are still not using it but currently open Telemetry is our main transport of our technology signals basically so are you um um like are you relying heavily on the hotel collector then to uh to do that for you yeah yeah yeah um especially with traces traces were one of the neglected parts of the observability stack you know it's it's typical because traces are a bit uh let's say more modern and it takes more time to implement and to actually understand how how good they are now traces are becoming the the eat of observability so it really helped us open until which really helped us um get the best of the tracing in in farfetch we were actually doing some numbers today uh with our architect and we were moving around 1 000 spawns per second in the past and now we have 40 000 that flawlessly without even needing to lift a finger and we're not there yet we still have a lot to do there and so um how are you collecting your traces right now is it like um manual instrumentation Auto instrumentation combination of both like where were y'all out with that It's a combination of both so we're currently doing we have another team uh working with us that helps provide the the instrumentation of the Frameworks because we have a huge variety of languages so we still have the open tracing um framework that teams are still using we also have some teams using manual instrumentation and uh we're also implementing the open Telemetry operator with hot instrumentation mostly for DOT net and Java currently but looking forward for go because have a lot of applications running and go so our main goal is that uh very soon we want to have all applications using open Telemetry instrumentation but it's going to be a process as low and as fast depending on the team space so for the for the teams who are using go which to my understanding there is no Auto instrumentation um what um what kind of is there any kind of support that your team um provides around that in terms of um helping helping these teams instrument yeah of course well one of the main teams that this is go especially because of the open source it's my team but yeah of course we provide documentations we provide guidelines and there is a lot of very good documentations in the open Telemetry as well about manual instrumentation so basically we have sessions with Engineers uh not to train them because I think they can do it better than we can because it's their their code but just to show them the best practices and to introduce them to that and to tell them that hey our tools are are here for you and uh it they take it from there it's a team sport let's say awesome awesome I love that because yeah I I I I think it what you said is really important that you're not instrumenting instrumenting their code for them because it's their code but that you provide the guide guidance guidelines on on the instrumentation which I think is super important also I want to call out uh Ubuntu just shared a um a link on in the chat that indicates that there is auto instrumentation for go so yay something to uh something to look at is that a more reason thing she wants you yeah it's a more recent thing and it's it's still in a work in progress but I came across this last week only and so since you mentioned go I was like okay go take a look if there's something interesting is following no I've been following it as well actually because it's uh of course it's the my stack and the things that I'm it's very close to me that I usually wants to go so I'm like every day is there news is there news it's become like a passion now to that's very cool um now I I want to ask on the um on the auto instrumentation um because you know keeping on that thread um I think you mentioned like your leveraging um Auto instrumentation for Java through the hotel operator um can you tell us can you share a little bit of like your experience around using the hotel operator I came across it like maybe a couple months ago and and for me I was like this thing is amazing so yeah like uh it's cool to to hear someone um who's actually using it is that in production right now for you uh yes it is partly in production but it's very isolated it's not available for for everyone um and yeah operator is amazing uh we loved them when we just discovered we're like amazed uh the thing is that it is a bit getting used to we had some some challenges uh when we started well we still do but especially in the beginning because uh the operator and the instrumentation object and The Collector um well it was my bad obviously I was uh we were having some certificate issues and trying to to rotate certificates and I was trying to delete something and they just delete create it was like a big mess and I would just like leave my computer and come back because of how coupled the the operator and the collector in his orientation is but yeah now we've gotten the hang of it and it is amazing one other challenge that um I didn't think of it that's that's the beauty of it we were having a discussion with another engineer and I was complaining that uh Prometheus cannot Target the operator The Collector and the operator for some reason and we cannot create alerts from it and he told me well have you tried using observing open Telemetry using open Telemetry and then sending everything to Prometheus and I'm like hmm and just like it's the it's the beauty of it you know it's compared compatible yeah I guess there's challenges every day one day it's time that's very cool that's very cool and um now speaking of like you know you mentioned like you're you're ingesting traces um I know the the log signal is like a newer newer player um in the Land of Open Telemetry have um have you and your team or anyone at farfetch started playing around with with um Hotel logging as well uh me uh very very little mostly consuming from from a Kafka topic and see how that goes it's pretty good but we know that this is not there yet and it's not stable so we don't expect it to to going into production or to have it part of the the day-to-day because currently we have a huge volume of Vlogs going through far-fetch more than places so a lot more so yeah it's uh it's not worth risking but we're definitely experimenting and we expect in one year from now open Telemetry will be the only um receiver that we're going to use for everything oh cool cool so um for for you know the little experiment that you've done in in using Hotel logs um have you um have you taken advantage of like the log to trace correlation or is that just the logs in isolation like how's how's that been going actually no that's that's a very good uh very good suggestion because I've been focusing mostly just getting things across and mostly the processing of the obfuscation of the data because that's uh that's something that we would like to use up on Telemetry which is amazing for uh but no that's something that uh it's definitely worth for me to investigate into the next awesome awesome um what about uh what about the metric signal um how are you ingesting the metrics are you like um or is is Prometheus passing the the metrics over to you and are you like using the uh the the Prometheus receiver like I know like from personal experience when I started playing around with the Prometheus receiver it was like barely a thing it was like so unstable there was a disclaimer um I'm just wondering how to use the Prometheus receiver and if so what's been your experience around that Yes actually we use the promoters receiver so throughout the instrumentation just to get this out of the way we do get some otlp metrics as well but the majority of our metrics is Prometheus so the permittance receiver works very good for us because we already had an observability system in place and we use console for Target control so it's it was extremely easy for us to use the receiver and it can handle a huge amount of data and it is the scrape configs are the same as in as in Prometheus so technically nothing changes you're just using another tool to to scrape all these metrics it is very straightforward for us and currently we're using both uh open Telemetry for some scenarios and Prometheus but yeah again in the future the the goal is to use only open Telemetry it's just that it's a process and you think then the Prometheus receiver will like be the way to ingest like all your all of your metrics and so you'll be able to scrap Prometheus all together as like kind of your your end goal or um I would say so but uh it would be maybe the majority because we have a lot of um our services running on Virtual machines as well and we have permit as exporters there so it's best that those let's say stand remain in touch and it's going to be more more difficult uh to adapt them but uh when it comes to our kubernetes I think we're going to go more with a hotel because if everything is Prometheus but we have the kubernetes SD configs for uh for the Prometheus receiver which is amazing and we also have uh the target control Target allocator through the operator which I have been testing and and I really liked it so I think we're going to go completely especially on kubernetes and in the cloud we're going to go completely online cool and um like for going going back to kubernetes like how many clusters are you typically um are involved that you're you're observing I would say maybe a hundred in total for different data centers yes and thousands of virtual machines we have we have a huge stack so your your team is responsible then for ensuring that like um if there's an issue with the operator like you are you and your team are managing um the the hotel operator across all these um kubernetes clusters exactly that's why we're also trained uh in kubernetes all of us that are part of the team because it's a very important part of our job to actually maintain everything oh nice nice okay so you're you're you wear multiple hats so you maintain the Clusters as well is what it sounds like or I don't know okay okay cool cool and I know I seem to recall um I want to say like version 1.26 of kubernetes they they enabled like some Hotel capabilities as like an experimental feature is that something that you've ever dabbled with or heard of just curious not yet actually no I haven't come across it but something that I can note and test it there's always a lot to learn yeah I'll see if I can I'll see if I can find a link around that I feel like it was a very sort of like Niche thing that was not often talked about but I'd I'd love to hear if anyone's played around with that um now um for me one of the one of the things that I always love to hear is like how how are teams um structuring Derek collectors like do you have one collector multiple collector if so like how are you deploying them so different configuration yeah we currently have an agent and Central collector uh type of organization well as I mentioned we have especially uh when it comes to kubernetes classes we have a huge number of them so having just one point of interest uh it's going to be overwhelming so we we are deploying well currently we have one open Telemetry agent in in each of the Clusters and uh it starts we are starting to substitute it with open Telemetry operator which has the collector and auto instrumentation so that's the end goal that's where we're getting it so everything is sent to a central collector when where we do the uh observocation of data and the sampling and everything and all that is sent currently or we're using Tempo but in the future we might use a vendor and basically there's going to be a central collector collecting all that per Data Center okay so you basically like each kubernetes cluster has its own collector and then they feed into like your central collector and then where does your central collector reside does it reside on a VM does it reside on another kubernetes cluster it's a kubernetes cluster and it's yeah we call it the central collector because most of the stock on that cluster it's dedicated to us we have a lot of data so we have a lot of requirements so memories wise so most of the applications running in this cluster are for observability and very few for the platform uh so yeah that's that's where everything is okay cool and then how do you ensure um you know if like because everything's being sent to that Central collector that becomes like a single point of failure so how do you ensure that you know if that goes down what what what's your like backup plan uh well we have fallback clusters with fallback collectors so if uh if this one fails uh it goes to the other one immediately uh we have also implemented well it's not currently running but we also have implemented the you know that the open Telemetry collectors can send to as many uh exporters as possible so we are currently uh using a fallback cluster to send uh for example if one fails sent to the other one and we can immediately enable it with without an issue it's like a background yeah and we have equipped our collectors with very we well we use Auto scaling a lot it's based on our metrics so we will make sure that our collectors have a lot of memory and CPU Liberty so their cues will be big as well if there is a small downtime everything will be saved into q and then sent to the central collector okay awesome awesome and on on that Central collect uh well speak like continue on The Collector thread um like what were some of the challenges that you experienced initially when you started implementing The Collector because I would imagine you know you said that you want to make sure you have like enough memory allocated I'm I'm assuming that might have been perhaps an initial challenge um are there any others or can you talk more about that well yeah I think knowing the collector and how it works is a new technology introduced to us was was the biggest challenge uh we're very fortunate in firefight because we rely heavily on auto scaling so the first thing that we did was to enable auto scaling and but yeah we had to do some tests to see how it was working with a small amount of data because again it was completely new and it took us a while to know uh the the memory and CPU requirements so at the same time we're not wasting a huge amount of money just to have this available but at the same time we need to have this available because it is so crucial so yeah definitely the resources memory and CPU are very important everything else I would say the the helm charts in the community are so good we just needed to uh to use them enable it and of course uh modify the everything the configuration basically the export or receivers and that's it but yeah it was pretty straightforward when it comes to that okay and and in in terms of uh configuration like are you using any um like any processors um like to do any data masking or to like add attributes remove attributes do you have custom processors what's what's your processor story like on the collector well we're currently experimenting with processors um I think we only have the batch processor or whatever it is enabled by default on on the charts but we are playing a lot with uh with the data masking processors because especially for the logging part in tracing and in metrics we're not we do not have any data that could be sensitive but uh for logging that's something that we are relying on heavily and that's what we're testing the most but currently we haven't implemented anything special uh for the Telemetry data that we're currently passing cool cool and um I I I was curious because uh you know um you mentioned like you're using traces you're using metrics playing around with logs in within traces um are you aware of any teams like using um span events for example no not yet and that's something that we are really really planning to to introduce currently because of the limitations that we had with our current with our previous tracing system we had a very low sampling it was 0.1 percent uh the interest in the teams was not big they didn't really they care much about tracing it was very rare finding an engineer that relied on tracing so now that we implemented Tempo open Telemetry we are gradually increasing the sampling size and allowing more information to come through and uh I think we are getting better at it but still not there uh that's part of our package of making traces first class citizens that's what I love to hear and um I I I don't think this is the thing yet now but my understanding and talking to a few of the hotel folks is that the idea is that the logs are going to replace the span events because I mean span events are basically like logs embedded in your in your traces anyway but with the idea that um obviously like that you have that correlation you continue having that correlation but I I believe the understanding my understanding is also you get access to the fact that like the logs uh specification is a lot richer than the span events um specification so you get you know you get to take advantage of like having more information potentially at your disposal so that's kind of a thing that I'm I'm looking forward to personally um I noticed that Sebastian just posted a link on the chat with regards to um the the traces for kubernetes cluster in um version 1.27 so for anyone I'm not sure if that was the one that you were speaking of I was just trying to find a reference yeah yeah yeah I believe that is the one I believe that is the one yeah that's super cool I do seem to remember that um for in for enabling it you have to like go into like deep deep in the bowels of kubernetes configuration to be able to enable that feature so um I think if you were using like a um a cloud provider um do so at your own risk kind of thing I think it was a lot easier to use like if you're doing kubernetes locally on your machine but anyway still a cool feature nonetheless um but I definitely something that is worth exploring as a as it matures um um just going back to uh our discussion uh I had a question also with regards to like um have you have you encountered any folks who were resistant to this whole open Telemetry thing like on development teams or even within your own team like what was what was the vibe around that and if so what what did you do to help that help alleviate their stress to be honest not really um at some it's It's Curious actually I haven't really met anyone that opposes it yeah I've met plenty of people that simply do not care for it like they're like okay we have it's okay we do not have it and uh in this situation it's mostly uh for example someone asks for something related to observability and traces and I'm like hey look at this cool thing that we did with open Telemetry now it is available for you and they're like okay and I just keep sending things that I that I consider that are so cool and it's good for them to to use and that's pretty much it and I know that so they're going to use because it's a but never had someone that was against it or like no we don't need it so we shouldn't it's interesting awesome I think we need more companies like yours where people are like yeah if it's a lemon tree yeah we have a very good observability culture I'm actually really really proud of that I I mean I'm I'm proud because I'm helping continue and build it but the previous team where that worked for that could also them that's awesome yeah and I think like that's where we really see success in in open Telemetry is always having like a group of people who are like amazingly enthusiastic about it and and are just like out there and believe in it and and want to make sure that it happens so um I you know and I I know like shubanchu who's also on the call like he's uh he's done a lot of uh evangelism around open Telemetry in his organization which is super awesome so you know hats off to y'all who uh who do that in in your organizations because I think it's gonna keep helping make open Telemetry awesome now um is are is your organization at the point now where um you've started to like make contributions to open Telemetry or is that still something where where you're like not not there yet um Yes actually we made the contribution recently to the operator because of the certification uh I think the certification was reply uh was relying on search manager and not with custom certificates and that didn't work for us so uh there was a a feature request and then a request to to fix that and that's why we are working so heavily now with operators so we can use our own certificates basically until search manager is available for us as well that is so cool how did it feel like making the contribution it was great well actually it was a joint effort our architect uh was the the main figure let's say after this uh after this but yeah it feels great and the community is super welcoming and like it was approved so fast because when he um submitted the feature request we were thinking that it was going to take like months or like okay uh yeah we will have to wait and then a week later he's like hey guess what it's more let's go forward and and test it it's amazing that is so cool yeah I I I'm always like a huge fan of that you know like don't wait around for the future request just do it yourself so it's it's really cool that um that you you and your team were able to uh to achieve that I think that's like you know I I think it's a great accomplishment because like putting yourself out there for open source is like you have to be vulnerable and you have to be okay with people saying like well that's not really correct scary right so yay congrats that's that's super amazing I hope uh I hope the team can continues to make uh Hotel contributions um one one thing I wanted to Pivot to um is uh you know like I am so happy that you have like awesome things to say about open Telemetry is there anything that you think you and your team have encountered where open Telemetry could improve because that's you know that feedback is also super important so that we can continue to improve um as a whole well to be honest I've had a super positive experience with it uh working with it at every step of the way has been super easy and there's been a lot of support from the community and yeah the only thing that I would say maybe is that it is a bit lacking in the documentation in some in some parts for example um the when I was implementing the Prometheus receiver first it was very confusing to use I was using the console um the console SD config and uh it was it was a mess and I couldn't find information anywhere on how to implement it or some some good documentation that of course I'm planning to to contribute to that but that's all that I would have to say it takes a bit longer to to figure it out or to you have to dig very very deep to find some documentation for certain features exporters or receivers but other than that it's been a an amazing experience cool cool awesome yeah I I have to agree with you sometimes that it is a bit of an archaeological dig so anything that that can be done to uh to to help bring that up to the surface is is most welcome so we we definitely look forward to a contribution to to the docs and also like for anyone on here like who's you know played around with open Telemetry like you can always submit a pull request to the to open telemetry.io repo um if you want to like uh write a blog post about anything Hotel related um because I know the comms folks are always looking um for contributions and so also if like you're looking for a first contribution on open Telemetry that is a great place to start um uh now you know like let's turn uh let's turn the tables around to our lovely audience here today do does anybody have any questions for Edis no burning questions we all good I haven't even wanted to say thank you for doing this appreciate it yeah I will second that to both of you um I think you covered a lot Adriana and Harris give it pretty good insight into her company it's amazing to see how little objections there are to change not just observability but change in general so it's nice to see almost wish like can I have some of it I've been grinding for the last year basically talking observability and hotel Non-Stop and I finally make some inways I think but um yeah it's nice to see that there are easier companies or easier paths not everybody has a struggles the same it's good to know yeah that's so true I think my my experience in the past has also been like it's the uphill battle to open to inflammatory so it is so refreshing to hear like such a lovely positive story that there is light at the end of the tunnel there are people who get it there isn't it but now it's becoming uh it's I think it is the second most contributed project and the one of the most most well-known so more people are are getting uh a smell of it so it's getting more and more accepted I'd say than it was one year ago yeah so true like when I started dabbling around in open Telemetry like trace's specification wasn't even finalized I was pushing the organization I was at at the time to be like open Telemetry is going to be the big thing y'all and they're like uh-huh so it was it was an uphill battle I feel like now at least like a lot of the specs are are like finalized so it makes for a very compelling narrative and you get more and more user stories of people like actually using this in production uh even even if it's like you know a little bit like you know kind of like where where farfetched is at where you're not like fully productionalized but you've got some stuff running in production and I think that's that's a compelling story to tell as well right which is like get it out there start using it right I think um I think that's a really great message to to share with folks um do you have any like parting thoughts uh I was gonna say something as well like we're uh yeah it's not fully there but we're passing a crazy amount of data after those collectives and they're tough trust me they're tough it's worth it it's like it might be difficult to convince uh your your peers to start implementing it but it's worth it they are durable collectors like you can pass like thousands and millions of uh data per second per minute and yeah if you if you do a good job without scaling Eden's gonna manage how much memory are they memory and CPU are they using while doing that kind of load do you know offhand when we were traces around 30 000 spawns per second I think we were like at around eight gigabytes of memory and I don't know how many uh onset don't remember how many instances of The Collector we were running but maybe like four but around eight eight gigabytes cool thank you um one question from your earlier conversation about using open Telemetry metrics and opal to Elementary tracing uh are you guys using correlation in any way in terms of the metric data Trace data or log data at all or at this moment that's still something you are exploring uh something that we're still exploring our first uh traces metrics correlation that we're doing uh we're investigating through open Telemetry but it was let's say easier for us to implement it through Tempo they have a metrics generator uh and uh that's what we were doing which we are generating some metrics from from traces but we're planning to move that to open Telemetry because yeah it's it's best it was just easier for us that way still exploring there's a question from Sebastian yeah I was curious um your organization sounds rather sizable um are you at all concerned about the amount of data that in totality your end up like producing transporting and collecting and I guess a follow-up question to that would be then um how do you advocate for valuable data versus just data for the sake of it comes to the sizing and how much data we're actually processing it was we were doing the same before we but they were going through different routes for example Jaeger collector Prometheus and Thanos and now everything is going through open Telemetry so the amount of data is not very troubling because we've been working with it for so long that we can actually know how to handle and know how much our our stack can handle so that's not really a problem but of course in tracing wise we're increasing a lot and it's a very gradual uh increase because that's where we're scared of adding stuff on top of what we already had in the past so that's very gradual but uh so far it's been going well and we we move with a lot of caution and always have a backup plan in mind so it's it's been okay now I don't want to go to brag and tomorrow we were going to have a huge incident when we come to that but yeah I think we're moving very cautiously when it comes to this you need to knock on wood you know that right if you're saying something like that right now um because I'm on call as well this week so that's gonna be like the the Terry on top but and when it comes to Quality data that's something that we are really really hard working on because in traces and metrics we're actually doing a very good job at it um because it's been especially metrics it's been our main focus so we are constantly cleaning up like having guidelines and uh the teams are very comparative traces we're getting there and because it was such a small amount of choices really people didn't really care much about because it was such small sampling size and now we are being very cautious on what is being increased and what is being passed uh logs where we're actually working a lot but open Telemetry hope we're hoping that it's going to help us and that's why we're investigating it a lot especially the processors um it's mostly publishing guidelines having meetings conferences within the company some of the we have some requirements that for example if you need to do an increased amount a crazy amount of data for example metrics to to three times the metrics that you were sending before you have to open a service request and us as a team go there and check it out and talk together what is necessary and what is not so it has to go through these processes only the big change is the small ones usually or something that our style can handle very well thank you have any other questions or comments for Edith well if you like what you heard today you just will be back for our hotel in practice session on June the 8th and it's gonna be I believe at the same time um so keep an eye out for an invite then um and uh why don't you tell folks what you'll be presenting about well my topic will be observability as a team sport it's something that I'm extremely passionate about and I see that in many companies observability teams are the ones that are making the guidelines instrumenting the code uh creating the alerts and responding to them and I am completely against that I think that observability everyone should do their part Engineers know their code and their product better us as a durability Engineers are there to help them and Empower them and provide the tools but they should be the one taking charge when it comes to to this part so that's what I'm going to be talking about awesome awesome I cannot wait to uh to hear this uh this talk so that'll be on June 8th at uh one o'clock Eastern which uh is 10 o'clock Pacific and plus six in Central European Time I think it's going to be awesome so I hope you all can can join and tell your friends we would love to have more people here what it is has to say because I think it's awesome um I think you're a great champion of open Telemetry so keep on keeping on this is this was great thanks so much for joining us here today and um you know if if you have a friend who or or or or or if you yourself are interested in in being um doing like participating one of these q and A's or even hotel in practice please reach out to either uh to Reese or Rin or me um on the hotel end users slack we are more than happy to uh to hear your stories and help folks share them with the world and we will be providing a blog post summary of today's q a for all to see so thank you so much everyone thank you so much Adriana thank you everyone + diff --git a/video-transcripts/transcripts/2023-06-01T17:22:03Z-whats-an-observability-engineer-anyway.md b/video-transcripts/transcripts/2023-06-01T17:22:03Z-whats-an-observability-engineer-anyway.md index f484eee..494522c 100644 --- a/video-transcripts/transcripts/2023-06-01T17:22:03Z-whats-an-observability-engineer-anyway.md +++ b/video-transcripts/transcripts/2023-06-01T17:22:03Z-whats-an-observability-engineer-anyway.md @@ -10,13 +10,32 @@ URL: https://www.youtube.com/watch?v=KRjYXPS3_so ## Summary -In this video, the speaker discusses the importance of documentation and guidelines in implementing OpenTelemetry for engineers. They emphasize that while engineers are skilled in their own code, the sessions are designed to share best practices and introduce the tools available to them, highlighting that effective instrumentation is a collaborative effort. The speaker underscores the supportive role of their team in facilitating this process. +In this video, the speaker discusses the importance of documentation and guidelines provided by OpenTelemetry for manual instrumentation. They emphasize the collaborative approach taken with engineers, not to train them, but to share best practices and highlight the availability of tools to assist them. The speaker conveys that the responsibility lies with the engineers to implement these practices, framing it as a team effort. -Thank you. +## Chapters -We provide documentation and guidelines, and there is a lot of very good information in OpenTelemetry about manual instrumentation. Essentially, we have sessions with engineers—not to train them, as I believe they can do it better than we can since it’s their code—but to showcase best practices and introduce them to these concepts. We want to remind them that our tools are here to support them, and from there, they take it and run with it. It's a team sport, let's say. Thank you. +Based on your provided excerpt, here are the key moments with timestamps for the livestream: + +00:00:00 Welcome and Overview +00:01:20 Importance of Documentation +00:02:10 Guidelines for Manual Instrumentation +00:03:30 Engaging with Engineers +00:04:00 Best Practices in Instrumentation +00:05:15 Role of Tools in Development +00:06:00 Collaborative Approach to Instrumentation +00:07:30 Conclusion and Next Steps +00:08:00 Q&A Session +00:09:30 Wrap Up and Thank You + +Please adjust the timestamps according to the actual content of the livestream if needed. + +Thank you. We provide documentation and guidelines, and there is a lot of very good information in OpenTelemetry regarding manual instrumentation. + +Basically, we have sessions with engineers—not to train them, because I believe they can do it better than we can since it's their code—but to show them best practices and introduce them to our tools. We want to convey that our tools are here for them, and from there, it's a team sport, let's say. + +Thank you. ## Raw YouTube Transcript -[Music] thank you we provide documentations we provide guidelines and there is a lot of very good documentations in the open Telemetry as well about manual instrumentation so basically we have sessions with Engineers uh not to train them because I think they can do it better than we can because it's their their code but just to show the best practices and to introduce them to that and to tell them that hey our tools are are here for you and it they take it from there it's a team sport let's say thank you +thank you we provide documentations we provide guidelines and there is a lot of very good documentations in the open Telemetry as well about manual instrumentation so basically we have sessions with Engineers uh not to train them because I think they can do it better than we can because it's their their code but just to show the best practices and to introduce them to that and to tell them that hey our tools are are here for you and it they take it from there it's a team sport let's say thank you diff --git a/video-transcripts/transcripts/2023-06-05T15:14:28Z-teaser-observability-is-a-team-sport.md b/video-transcripts/transcripts/2023-06-05T15:14:28Z-teaser-observability-is-a-team-sport.md index a6c4108..db409d3 100644 --- a/video-transcripts/transcripts/2023-06-05T15:14:28Z-teaser-observability-is-a-team-sport.md +++ b/video-transcripts/transcripts/2023-06-05T15:14:28Z-teaser-observability-is-a-team-sport.md @@ -10,19 +10,32 @@ URL: https://www.youtube.com/watch?v=HLJe2RLPlWY ## Summary -In this video, the speaker discusses the concept of observability as a collaborative effort within teams rather than a responsibility solely held by observability specialists. The speaker emphasizes the importance of involving all engineers in the observability process, as they have a deeper understanding of their own code and products. The goal is to empower engineers with the necessary tools and guidance while encouraging them to take charge of observability tasks, such as instrumentation, alert creation, and response. The video highlights the shift from a centralized observability approach to a more inclusive team-oriented strategy. +In this video, the speaker discusses the concept of observability as a collaborative effort within teams, advocating for a shift away from the traditional model where dedicated observability teams handle all aspects of monitoring and alerting. The speaker emphasizes that engineers are most familiar with their own code and products and should take an active role in observability practices. Instead of solely relying on observability teams to create guidelines and respond to alerts, the speaker believes in empowering engineers to lead these efforts, with observability engineers providing support and tools. The video aims to encourage a more inclusive approach to observability in software development. + +## Chapters + +00:00:00 Introductions +00:01:00 Overview of the topic: Observability as a team sport +00:02:30 Importance of shared responsibility in observability +00:04:15 Critique of current observability practices in companies +00:06:00 Empowering engineers to take charge of observability +00:07:45 Role of observability engineers in supporting teams +00:09:00 Tools and resources for effective observability +00:11:30 Real-life examples of successful observability practices +00:14:00 Discussion on the cultural shift needed for observability +00:16:45 Q&A session and audience interaction # Observability as a Team Sport -Today, I want to discuss a topic that I am extremely passionate about: **observability as a team sport**. +The topic I want to discuss today is **observability as a team sport**. This is something I am extremely passionate about. -In many companies, observability teams are tasked with creating guidelines, instrumenting the code, setting up alerts, and responding to incidents. However, I am completely against this approach. I believe that observability should be a shared responsibility. +In many companies, observability teams are responsible for creating guidelines, instrumenting the code, setting up alerts, and responding to them. However, I am completely against this approach. I believe that **observability should be a shared responsibility**. -**Engineers know their code and their products better than anyone else.** As observability engineers, our role should be to help and empower them by providing the necessary tools, but it should ultimately be the engineers who take charge of observability in their projects. +Engineers are the ones who know their code and product best. As observability engineers, our role should be to **help and empower** them by providing the necessary tools, rather than taking charge of this aspect. -That’s the essence of what I want to convey today. +That's the core message I want to convey today. ## Raw YouTube Transcript -foreign [Music] [Music] topic will be observability as a team sport it's something that I'm extremely passionate about and I see that in many companies observability teams are the ones that are making the guidelines instrumenting the code creating the alerts and responding to them and I am completely against that I think that observability everyone should do their part Engineers know their code and their product better us as a durability Engineers are there to help them and Empower them and provide the tools but they should be the one taking charge when it comes to to this part so that's what I'm going to be talking about [Music] and [Music] +foreign topic will be observability as a team sport it's something that I'm extremely passionate about and I see that in many companies observability teams are the ones that are making the guidelines instrumenting the code creating the alerts and responding to them and I am completely against that I think that observability everyone should do their part Engineers know their code and their product better us as a durability Engineers are there to help them and Empower them and provide the tools but they should be the one taking charge when it comes to to this part so that's what I'm going to be talking about and diff --git a/video-transcripts/transcripts/2023-06-08T02:24:12Z-teaser-how-do-you-foster-a-culture-of-observability.md b/video-transcripts/transcripts/2023-06-08T02:24:12Z-teaser-how-do-you-foster-a-culture-of-observability.md index eaf1edf..81e990e 100644 --- a/video-transcripts/transcripts/2023-06-08T02:24:12Z-teaser-how-do-you-foster-a-culture-of-observability.md +++ b/video-transcripts/transcripts/2023-06-08T02:24:12Z-teaser-how-do-you-foster-a-culture-of-observability.md @@ -10,11 +10,26 @@ URL: https://www.youtube.com/watch?v=RxP76ahb29s ## Summary -In this video, the speaker expresses pride in fostering a strong observability culture within their organization. They highlight their role in continuing to build and improve this culture, emphasizing its importance and value. The discussion touches on the significance of observability in enhancing operational efficiency and the benefits it brings to the team. The speaker's enthusiasm reflects a commitment to developing a collaborative and transparent environment that prioritizes data-driven decision-making. +In this video, the speaker shares their pride in fostering a strong observability culture within their organization. They emphasize the importance of contributing to and developing this culture, highlighting how it enhances overall performance and collaboration. The discussion revolves around the key elements that make up a successful observability culture, including transparency, continuous improvement, and teamwork. The speaker expresses enthusiasm for their role in this ongoing process. -Thank you. I have a very good observability culture, and I'm actually really proud of that. I mean, I'm proud because I'm helping to continue and build it. +## Chapters + +Sure! Here are the key moments identified from the transcript: + +00:00:00 Introductions +00:01:15 Discussing the importance of observability culture +00:02:45 Personal pride in contributing to observability +00:03:30 Strategies for building a strong observability culture +00:05:00 Challenges faced in promoting observability +00:06:50 Tools and technologies that support observability +00:08:15 Sharing success stories within the observability framework +00:09:30 Future goals for enhancing observability practices +00:10:45 Q&A session on observability topics +00:12:00 Closing remarks and thank yous + +Thank you! I have a very good observability culture, and I'm actually really proud of that. I mean, I'm proud because I'm helping to continue and build it. ## Raw YouTube Transcript -[Music] thank you I have a very good observability culture I'm actually really really proud of that I I mean I'm I'm proud because I'm helping continue and build it [Music] +thank you I have a very good observability culture I'm actually really really proud of that I I mean I'm I'm proud because I'm helping continue and build it diff --git a/video-transcripts/transcripts/2023-06-12T23:45:47Z-teaser-observability-is-a-team-sport-with-iris-dyrmishi-teaser.md b/video-transcripts/transcripts/2023-06-12T23:45:47Z-teaser-observability-is-a-team-sport-with-iris-dyrmishi-teaser.md index 1bc5182..cf62003 100644 --- a/video-transcripts/transcripts/2023-06-12T23:45:47Z-teaser-observability-is-a-team-sport-with-iris-dyrmishi-teaser.md +++ b/video-transcripts/transcripts/2023-06-12T23:45:47Z-teaser-observability-is-a-team-sport-with-iris-dyrmishi-teaser.md @@ -10,19 +10,32 @@ URL: https://www.youtube.com/watch?v=IHeqL36AlK0 ## Summary -In this video, the speaker discusses the importance of enabling engineering teams to build their own observability tools, such as dashboards and alerts, rather than relying on external parties. The speaker emphasizes that if alerts are created generically without understanding the specific products or code, teams may not respond to incidents effectively or promptly. The discussion highlights the significance of team involvement in the development of monitoring tools to ensure they are tailored to their unique needs. The overall message stresses the value of internal knowledge in creating effective observability solutions. +In this video, the speaker discusses the importance of involving team members in the creation of their own dashboards and alert systems for effective observability. They emphasize that without team involvement, alerts would be overly generic and potentially ineffective, as the creator wouldn't have the necessary context about the products or code. The speaker argues that if team members handle the development of these tools, they would be more tailored and responsive, allowing for quicker incident response. Overall, the video underscores the significance of team ownership in observability processes. -Foreign [Music] +## Chapters -ERS were not involved in building their own dashboard or creating their own alerts. Without knowing what data they're sending for their observability reasons, they will not be able to respond to an incident in time. +Sure! Here are the key moments identified from the provided transcript excerpt: -If I were the one making the alerts for them, it would be something extremely generic, with some thresholds that I thought would be right. However, I don’t know their products or their code. Obviously, the dashboards would be completely generic as well. +00:00:00 Introduction to the topic of observability +00:01:15 Discussion on the involvement of foreign ERS in dashboard building +00:02:30 Importance of team-specific alert creation +00:03:45 Consequences of generic alerts on incident response +00:04:10 The role of product knowledge in effective monitoring +00:05:00 Comparison between generic and tailored dashboards +00:06:30 Significance of team ownership in observability tools +00:07:15 The impact of proper alerting on incident management +00:08:00 Summary of best practices for dashboard creation +00:09:00 Closing thoughts on improving observability in teams -If those were developed by a person within the team itself, it would be much easier for them. +Feel free to adjust the timestamps or descriptions based on the actual content! -[Music] +Foreign ERs were not involved in building their own dashboards or alerts, which means they don't know what data they're sending for their observability needs. As a result, they won't be able to respond to an incident in time. + +If I were the one creating the alerts for them, I would have to make them extremely generic with some thresholds that I thought might be appropriate, since I don't know their products or their code. Consequently, the dashboards would also be completely generic. + +However, if those dashboards and alerts were created by someone within the team, it would be much easier for them to manage and respond effectively. ## Raw YouTube Transcript -foreign [Music] [Music] ERS were not involved in the part of building their own dashboard building their own alert knowing what data they're sending for their observability uh reasons they will not be able to respond to an incident in time simple as that if I was the one making the alerting for them it would be something extremely generic with some thresholds that I just thought would be right because I don't know their products or their code obviously the dashboards would be completely generic but if those were done by a person inside the team itself it would be so easy for them [Music] +foreign ERS were not involved in the part of building their own dashboard building their own alert knowing what data they're sending for their observability uh reasons they will not be able to respond to an incident in time simple as that if I was the one making the alerting for them it would be something extremely generic with some thresholds that I just thought would be right because I don't know their products or their code obviously the dashboards would be completely generic but if those were done by a person inside the team itself it would be so easy for them diff --git a/video-transcripts/transcripts/2023-06-12T23:45:52Z-otel-in-practice-presents-observability-is-a-team-sport-with-iris-dyrmishi-full-video.md b/video-transcripts/transcripts/2023-06-12T23:45:52Z-otel-in-practice-presents-observability-is-a-team-sport-with-iris-dyrmishi-full-video.md new file mode 100644 index 0000000..c999093 --- /dev/null +++ b/video-transcripts/transcripts/2023-06-12T23:45:52Z-otel-in-practice-presents-observability-is-a-team-sport-with-iris-dyrmishi-full-video.md @@ -0,0 +1,121 @@ +# OTel in Practice Presents: Observability is a Team Sport with Iris Dyrmishi [FULL VIDEO] + +Published on 2023-06-12T23:45:52Z + +## Description + +Observability ins't the responsibility of one single team. Join Iris Dyrmishi of Farfetch as she talks about how the collaboration ... + +URL: https://www.youtube.com/watch?v=U1yLXnMONkc + +## Summary + +In this YouTube video, Iris, a platform engineer specializing in observability at Farfetch, leads an open discussion about the importance of observability as a collaborative effort within engineering teams. She emphasizes that observability is not the sole responsibility of a central team but rather a shared duty among all engineers, regardless of their specific roles. Iris outlines what she considers a perfect observability system, highlighting the necessity for centralized access to metrics, optimized data costs, reliable alerting, and team involvement in incident response and monitoring. Throughout the session, she encourages audience participation, discussing challenges such as excessive data, the need for structured telemetry, and the integration of observability into team processes and culture. The conversation also touches on practical tools and frameworks used in observability, such as OpenTelemetry, Prometheus, and Grafana, and the significance of creating a culture that values observability across the organization. The discussion concludes with an invitation for further community engagement and sharing of best practices in observability. + +## Chapters + +00:00:00 Introductions +00:01:30 Overview of Observability Culture +00:05:00 Discussion on Responsibility of Observability +00:10:00 Defining a Perfect Observability System +00:15:30 Importance of Team Involvement in Observability +00:20:00 Role of Observability Engineers +00:25:00 Challenges in Observability and Incident Response +00:30:00 Custom Solutions and Tools for Observability +00:35:00 Addressing Anomaly Detection in Data +00:40:00 Centralizing Observability Tools and Data + +# Observability Culture Discussion + +**Iris**: Thank you for joining us again! We had her for a Q&A last month, and she did a fantastic job, so we asked her to come back. I’ll let her take it away. + +--- + +**Iris**: Hello everyone! Thank you so much for joining today. My name is Iris, and I’m a platform engineer focused on observability at Farfetch. I’m passionate about observability and love to share its culture, not just within my company but across the board. + +Today, I want this to be an open discussion. I would love to hear about the observability culture in your companies as well. I believe that observability is a team sport; it shouldn’t be confined to a single central team, regardless of the company size. Everyone should contribute and be involved in observability. + +Observability has existed for a long time, but it’s becoming a significant focus now, and we have this amazing culture spreading. However, we still have a long way to go. Conversations like this are essential for us to learn from each other and improve our practices. + +### Introduction + +Please feel free to interrupt me if you have questions or opinions about what I'm saying. A spoiler alert: these are mostly insights I've gained from my job, and I welcome any differing opinions for discussion. + +The first question I ask all engineers interviewing for my team is: *Whose responsibility is observability?* Let’s be honest, observability is implemented differently across various companies. I've seen many approaches, and the consensus is that observability is everyone’s responsibility. If an engineer answers that question affirmatively, I know we are on the right path. + +### Defining a Perfect Observability System + +To understand why I believe everyone should be responsible for observability, let me outline what I think a perfect observability system looks like. Of course, this is subjective and may not apply universally, but here are the key characteristics I envision: + +1. **Centralized View**: Engineers should have easy access to traces, metrics, logs, dashboards, and alerts without memorizing multiple URLs. + +2. **Optimized Data**: Sending everything doesn’t mean better observability. It’s crucial to optimize data to control costs while ensuring it remains useful for troubleshooting. + +3. **Reliable Alerting**: Alerts must be dependable and customizable. + +4. **Trace Correlation**: There should be correlation between different telemetry signals. + +My job as an observability engineer at Farfetch is to provide these systems for engineers, but this is a collaborative effort. Many characteristics of a perfect observability system require teamwork from various individuals who understand the nuances of their products. + +I would love to hear if anyone has different thoughts or additional ideas about what makes a perfect observability system. + +### The Importance of Team Involvement + +Observability is crucial in a company, especially as it grows. Many organizations still don’t grasp its importance until they face issues. Here’s why I believe teams should be involved: + +- **Incident Response**: In our company, incident response is vital. With a large user base relying on our product, it’s crucial for all teams to be involved in incident management. If engineers are not engaged in monitoring their systems, it could take hours to respond to incidents instead of minutes. + +- **Custom Alerts and Dashboards**: Custom alerts created by the engineering teams will be more effective than generic ones set by someone outside the team. This ensures that alerts are relevant and actionable. + +- **Auto-scaling**: Effective auto-scaling relies on telemetry data. Engineers who know their systems can set appropriate scaling rules based on their unique needs. + +To summarize, observability isn’t just about having a system; it’s about ensuring everyone involved in building and maintaining the product takes ownership of observability. + +### Challenges and Solutions + +Observability can get messy, especially in large organizations. It’s common for engineers to add custom metrics and logs, leading to a cluttered environment. Here are some challenges I’ve encountered: + +- **Too Many Metrics/Logs**: Engineers may flood the system with unnecessary data, making it hard to sift through and find valuable information. + +- **Lack of Structure**: Unstructured logs can be particularly frustrating. If engineers can’t easily query and analyze data, it hampers their ability to troubleshoot effectively. + +- **Cost Concerns**: Observability can be expensive, and it’s essential to strike a balance between collecting sufficient data and managing costs. + +### Creating a Culture of Observability + +For a successful observability culture, we need to: + +- **Educate and Empower**: Provide engineers with the tools and knowledge to take ownership of observability. It’s crucial to avoid judgment and create a supportive environment where engineers can learn and adapt. + +- **Be Patient**: Transitioning to a robust observability culture takes time, especially for new team members who may have different experiences from previous roles. + +- **Foster Collaboration**: Encourage a collaborative atmosphere where observability is a shared responsibility, and everyone feels empowered to contribute. + +### Conclusion + +In summary, we all need to work together to cultivate a great observability culture and system, making our lives significantly easier. If anyone has questions or would like to share their experiences, please feel free to speak up! + +--- + +**Derek**: One comment related to what you said about observability is the ability to form rich queries against data once it's in your analytics backend. If you can’t query effectively, it leads to frustration. + +**Iris**: Absolutely, I completely agree. + +**Derek**: What was the observability culture like when you joined your company, and how did you make improvements? + +**Iris**: I was fortunate because my company had already laid a strong foundation for observability. We continue to improve it by holding presentations, sharing articles, and being available for questions. + +**Derek**: How have you encouraged application teams to take ownership of observability without overwhelming them? + +**Iris**: We promote ownership by making it clear that each team is responsible for their monitoring and alerting. We’re here to guide them with documentation and support, but the responsibility lies with them. + +**Derek**: What has been the reaction from teams when you enforce this? + +**Iris**: Most teams respond positively. They often ask for guidance, and we have the support of management to reinforce this approach. + +**Iris**: Thank you all for the insightful discussion! If you’re interested in sharing your stories or participating in future sessions, please reach out on Slack. Thank you for having me today! + +## Raw YouTube Transcript + +foreign thank you for joining us again um we had her for the Q a last uh last month a couple weeks ago I guess and she did a bang-up job so we we asked her to come back and uh I'll let you take it away well hello everyone thank you so much for joining today my name is Iris I'm a platform engineer with a focus on observability working at firefits currently uh and I'm a super passionate person when it comes to observability so uh yeah I I like to talk about it as much as I can and I like to share the observability culture all over not just in in my company so today I wanted to have a talk and I would like to have it as an open discussion I would love to hear about the observability culture in your companies as well um how I believe that observability is a team sport and it is not something that is being done just by one Central team in a company no matter how big or small it is I think everyone should put their hands and be involved in the observability but of course it's let's say observability has been there forever but now it's becoming this huge thing and now we have this uh amazing culture that is spreading but we're still not there so I feel like this conversation and these discussions are so important to put out there for us to discuss and then to take them to our companies and to learn obviously from each other so yeah this was just a a small intro uh please interrupt me because I do not have a good visibility so if you have a question please don't don't hesitate to unmute yourself and just uh just speak or if you have any any opinions about something that I'm saying I just want to have a spoiler alert these are mostly things that I've learned from my job and things that I believe in so of course if you have a completely different opinion please let me know let's discuss it it's it would be amazing so to have a conversation about this here okay so uh the first question that I ask all Engineers that are inter being interviewed for a com for a job in my team is what whose responsibility is observability because let's be honest observability is be it is being implemented different in in many different companies I've seen so many ways everywhere that have interviewed Forum that I work it's different so of course observability is everyone's responsibility it's simple and if your engineer says that answer to me and then I'm like okay okay we're on board we're getting somewhere uh and why do I believe this uh first of all in order to get deeper into why I believe that everyone uh is responsible for the durability in their company I want to give an overview of what I think of perfect observability system is and that's perfect let's take it to the grain of salt of course it's perfect for me my company it works in another person's company doesn't work so if you have any more ideas here this was just what came at the top of my mind so for me a perfect obserability system it has a centralized view of systems observability it's data Engineers can go and they can access their traces metrics logs dashboards alerts um everywhere they don't have to memorize a lot of URLs or have to go in many places to find the information so that's the first thing the second one is optimized data will optimize costs because doing observables doesn't necessarily mean that you will send everything that you can and rack up huge bills uh but it needs to be optimized data and some good old data that is actually going to help other teams troubleshooting and monitoring their systems obviously reliable alerting reliable and custom dashboard reach traces correlation between the different Telemetry signals so this is something that makes a perfect observability system for us and my job as an observability engineer especially in perfect is to provide that for the engineers but I cannot do it alone because no matter how much I try many are let's say most of this uh characteristics are a teamwork and something that other people also have to interview which means the engineers that are part of I would say our customers I don't know if anyone has any other ideas about what a perfect observability system it is for them I would love to hear more about it or if you completely agree with with my goals of having this this system but at least I'm with yourself and with them throughout me any moment ous thank you definitely agree with a lot of the things that you you said like it just yeah spot on okay uh sorry I'm not able to see the comments here but if there is something interesting we do let me know please okay so uh the question is okay you have this vision of a perfect observability system how do we get to this observability system well observability Engineers like myself my team with the knowledge to shape observability systems into the correct path uh I'm not saying that um observability is something that a simple engineer that is working that can't get into they can be uh participants and active participants in it but of course it is important that there are Engineers who know observability better than everyone who know what guidelines to implement uh what tools to use or how to optimize data it's very important that these people exist in the company uh because as I said before applicability is one of those fields that are moving so fast that it is impossible for a person that is working let's say uh with databases or in Java and their full-time job is is called getting back-end whatever else to keep up with this so it is important for people like like us to be in the company and to preach let's say these values and to bring all these good tools and good things that we that are being developed and are being shared in the community of course as I mentioned uh observability Engineers that also bring the data to collect and process Telemetry data um but my fourth favorite so far of course is open Telemetry which I'm a huge that's why we're here after all I'm a huge fan we have fans grafana Prometheus that are some of the tools that are open source that are being used so massively right now by uh technibility engineers and now I'm getting to what the rest of the presentation will be about it's not only observative Engineers that are needed when it seems to actively participate in optimizing the data that they're sending and things actively building their monitoring five people 10 people 20 that depending on the size of the durability team cannot do the work of two thousand three thousand depending on the work of so many Engineers everyone needs to have control of their own code so uh some of the things why observability is so important to be done as a team is because how crucial it is in a company I don't in some let's say that in some companies it's still not understood how important it is to have observability but that is well what I believe is because of the small scale but the more that the companies grow they understand how important it is to have highly available and reliable system it is crucial to have optimized performance because you could have you could uh allocate a lot of resources and you are not saving on cost basically you're spending too much money or you're cutting back on resources and you are basically not giving a good performance or a good experience to your users depending on the type of product it is also it is crucial for incident response this is something that is very important uh in my company incident response is very important considering that we have a lot of customers that rely on our product a very expensive products at time so having all the teams involved in the crucial in the incident response is extremely important we have around 2000 Engineers I actually thought that we were three thousand we were actually two thousand that are working completely different areas some of them I don't even know except for what they are so when I hear about and I'm like wow okay because it's so big and so diverse in in our company and uh if this Engineers were not involved in the part of building their own dashboard building their own alert knowing what data they're sending for their observability uh reasons they will not be able to respond to an incident in time simple as that if I was the one making the alerting for them it would be something extremely generic with some thresholds that I just thought would be right because I don't know their products or their code obviously um the dashboards would be completely generic but if those were done by a person inside the team itself it would be so easy for them because they know it inside out their work uh what are some of the issues that they might have so if an incident happens you reduce the the response from hours that it would be if it was for me dealing with their incidental to seconds or minutes if someone from their engineering team is involved also it is crucial for incident detection as I said before um us as Engineers of the rebuilt Engineers we can provide a set of alerting dashboards that are very generic that each team can have as a baseline to implement their monitoring system but I don't feel comfortable in a way to go and to implement custom alerts custom dashboards for these teams again it goes to this even if I wanted to there are thousands and thousands of Engineers Technologies hundreds of teams I would never be able to do that properly that's why it's so important that each team is involved and does it themselves if they need it they change it they don't need our permission they are owners of that solution it's extremely important and also critical for auto scaling um I say this because in perfect we rely heavily on Telemetry data to Auto scale because you never know the amount of traffic that you're going to have in your application but for example let's say for me or for another person in the observability if we were to set some Auto scaling rules for you to be based on CPU on memory to be something that is completely generic but an engineer that knows their system so well they would go and say let's do a custom auto scaling depending on the number of through output or whatever basically my bottom goal here would be to say that all this require people from the teams that are monitoring the system to take part in if it is done by people that are outside um I am an infrastructure engineer I work with kubernetes I work with VMS with Azure with cloud and I'm not very good at back-end coding let's say so if I was the one doing this I wouldn't do it properly no matter how much I tried and how much effort I put so let's not do observability for the sake of it yeah let's have some generic alerts just for having them no let's do it properly let's have the people that are working day to day with the product Implement them make it their baby as well it's very important to to show it to the engineers and I I am very proud to say that in my company we have a pretty good observability culture that has started a few years back so now it is uh widely accepted but I've seen it happen that teams rely heavily on on other people to set up alerting for them and that is something that scares me if something happens if an incident happens and I wouldn't be comfortable working in a team like that if I wasn't the one setting up my own monitoring for the system also observability can be so messy uh depending on the scale of the company and so many people that come and go everyone wants to add the custom metric the custom log it can get extremely messy it can get too too many metrics too many logs especially for example living debugging logs flowing all day you know because it can yeah you just sense I'm an engineer I want to have full obserability of my application it's not affecting my performance so I'm sending everything debugging info warning metrics I create like a bunch of them on top of each other uh or the same metric with different names and yeah what can me as uh as observability when you do about it alone I I detect it I see it and I say okay this is completely wrong but of course I'm going to need the collaboration of the engineers that are working with our product to actually change it otherwise it would never be optimized and it should never be um let's say good observability um practices or too many traces well I'm a firm believer that there is no such thing as too many traces I'm a big advocate of that but yeah it's it's possible that we are sending 100 of traces uh with a mess and completely no information there and watch what what can I do considering that I'm not the one doing the day-to-day developing I have to go and talk to the things that they need to be active participants on that without it for me it's there is no durability or it is something very generic just for the sake of it that happens so much and to be honest it rates me but there are also too many dashboards too many alerts uh if someone else creates the alerts or they're just created uh but automatically you have them rigging back and forth and nobody actually looks at them because they know that it is the spam so it is very important for people to go and for engineers to go there and say hey okay I don't need this alert I need this I need to create this and the same for the dashboards it can be thousands and thousands of dashboards and when you actually have an incident that's at midnight three a.m in the morning you wake up your your eyes cannot open and you have to scroll through thousands of those or like try to find a person that might know what this is it's very important to be about as simple as that uh yeah also it is it is not easy it makes it easy I think they go and uh Hand by hand it's it's let's say mostly the same thing with too many alerts and dashboards and you just cannot get the hang of it and you just give up so basically you do not rely on this to troubleshoot them on what do you rely maybe the application logs or it takes you hours to solve an incident which is not uh not great uh either too few or too many Telemetry signals I talked about too many but there is also one thing is to to feel because it is possible that you don't know how metrics are how logs are generated how traces are generated in this case that same place is considering right now it's this huge movement to to make them first class citizens many Engineers let's say they do not know how important tracing is and not how the great capabilities that they can achieve to that and my job as an engineer would be to advise them and to help them with it but not to do it for them because I don't know the type of information they might need it's simple um and it is not cheap uh absolutely it is not cheap depending on the scale of the information that we are having it could rack up to Millions I was reading some articles uh recently about some companies that were paying a huge bill in observability and the question on those articles were who wants to blame and the stack of different building is a team sport it is also the the blame is to share because the engineers of observability are clearly doing a great job at educating or maybe they're not even there that could be one of the cases or the teams are not participating basically you have this framework that just sends 100 of logging tracing metrics and that is all sent somewhere so yeah it's observability is something that is a very uh delicate topic when it comes to costs because so many Engineers consider it as a not a very important thing until they actually need for example when something happens and they don't receive an alert oh my God it is actually important or oh my God the information on that on the dashboard now would be so nice but at the same time we have to know what to collect how to and and to also advise our team to do it and to Empower that to do it themselves it's not an easy task especially when it's so comfortable for them to have all that information let's say for example logging I have a vision of talking in general but nothing against it but it's used so much uh they say okay I just will have the debug logs and even if if something happens I will I will have it there um yeah it's our job to to convince them to show them and to actually make a plan and follow it through together and that's I would say our mission as observability Engineers is not just to to teach people and to show them because sometimes that sounds like I don't know if it is the right word but like stack UPS oh yeah you do it it's not it's not that it's just um trying to do it in a in a collaborative matter to consider it as a teamwork that's a really team it's not an external team that just uh works completely away from you but it's let's say part of your team and at the same time it's not it's it serves as an advisory um wait let's see so how do we change to this observability culture it is very very difficult um I am because of my passion for observability I talk a lot about it and it is very very difficult to convince an engineer with a higher grading as you so to follow a certain observability culture if sometimes it's not even the engineers itself it could be senior leadership it could be higher ups I just do not believe it and some companies and this is very prevalent and it's very difficult to crack in there to actually implement it but I think that our job is to make a change we need to be ambassadors in our companies and I think what we're doing today and all these sessions are a great way for us to talk and to share opinions and to know how to approach our teams better how to to be better at what we're doing and to actually show the importance of our work we need to show the importance of the observability of the data in fact so you can just go and say hey observability a rainbow is amazing beautiful we have to show data and that's very important to collect and very easy as well considering the amount of incidents that are happening or you just show how easy it is to to get information about one aspect of the application that they never thought about it's important to provide up-to-date tools our job is amazing because we have we can work with some someone in modern technologies that are moving so fast at the same time it is very difficult because you need to provide tools that aren't mature enough you have to provide tools that the engineers need and are looking for and sometimes this can get overwhelming but it is part of the challenge and actually this is my favorite aspect of the job how fast it moves and how fast you have to adapt it's part of being an observability engineer uh yeah we have to provide guidelines uh not everyone knows I I mentioned this before but I'm a firm believer of not judging it might sound like a childish thing to say or kindergarten issue but I know that tech people judge each other oh you don't know that much about kubernetes parts of durability it's very important for us to offer help and to avoid Judgment of that sort because observability is not difficult but at the same time it's not easy as I said before there's it's very fast it's moving it's modern you need to read you need to adapt very fast and the person that is completely detached from that will not be able to do it so there is no place for judgment here and I think that if there is Judgment that doesn't make you a good observability engineer I think we're like the Saints of the engineers uh and also we need to give space and time to adapt um in some companies where the durability cultures exist it's easy obviously and if you have a majority of Engineers that follow these practices let's say uh it becomes very easy for it to spread but when you're starting from zero it needs time people that have been working in other companies that have completely different working mentality it's going to be difficult for them to just land in the company and be like ah okay now I accept observability as this amazing thing that it is no we have to give them space and time to adapt show them be patient be there to help to implement even on an engineer that is excellent his job but has no no idea how observable it works we have to we have to be there for them and I think the human aspect of this job is more important than the technical one because it is good it's important to be technically and to adapt and to build but also the human aspect and um spreading the culture and talking to people and being a people's person professionally you don't have to be an extrovert and have this uh beautiful conversations with it's a very important aspect of observability um these are all the slides that I had uh prepared for this presentation but I just want to say my my summary or let's say the message that I have through this is that we all need to work as a team to have great observability culture and great observability system and that's going to make our lives much easier so that's pretty much it I don't know if you have any questions please let me know it was a comment from Derek actually um I think when you're talking about what is important um uh an observability and they mention ability to form Rich queries against the data once it's in your analytics back end is when you're just asking what's important in the design of like a perfect system so you can pop all the perfect data that you want into your back end but if you don't have a way to quickly and effectively query that to get the answers you want then you're just going to be frustrated I think that's one of the reasons why we get frustrated with unstructured Vlogs so much is because you end up having to craft these complicated regexes and pattern matching of unstructured screens and it's really frustrating to try and find what you want so if we can do our part to make sure the data going in is nicely structured it makes it a lot easier to get the signals you want out of it absolutely absolutely agree I had a question too but I can go to the back if someone else have them oh no go ahead Derek I have one but I can I can go after so um I guess my question would be what was the observability culture like when you joined your company um and kind of like what was Ground Zero where were you starting from and then kind of how did you how did you make improvements to that or how did you kind of watch that culture improve so actually I'm pretty lucky when it comes to that extent where in this company when I joined I think the the previous team uh and the just a few members that are currently part of my team had done most of the heavy listing they had started so this the spreading of the observability culture and how to do it correctly so when I joined I think everything was was put in place uh but of course we are improving that day-to-day because you can have a culture there but you have to let's say you have to water it every day and you have to keep it and to improve it so what do we do we have presentation sessions uh with with other engineers in the company showing them how to use our tools best of or what our tools can do we we are always open the team for just a slack message to to help them on how to um create metrics dashboards even though the simplest questions were always there for them we also sharing sharing articles sharing of presentations for observability and it's actually interesting because many people read them and we're also making sure that we're also always on top of the new technologies in this case for example it was open Telemetry it was something that it was very uh well accepted in in far-fetch and some people were actually requested so we provided and and just saying like always providing with a with a news Technologies with the newest updates so nothing is missing and if let's I don't know if it's called I don't know bribing but for example we make sure that it's one of our Engineers who reads something on the Internet it's like oh this new cool feature in grafana that I saw can we have it we make sure that we've had it before we we always make sure that we're on top of things uh because that makes people want to use the system more and use it well and Implement all the all the new changes so it's great for us yeah that's really good if I can ask one more quick follow-up question then I can give someone else a turn um so in your slide deck you talk about how developing ownership right by everybody of observability is super important for Success because ultimately like application Engineers they're the ones who have the context for how their applications run and for the things they need to look for so how in your experience how have you gotten application teams to kind of take that ownership without making it feel like it's just one more thing you're trying to Chuck over the wall at them right because again like it's a unique spot that we're at as like infrastructure platform people because we're not like the best at writing code like we're not that's not going to be our strongest um skill so it feels weird to me at least in the past saying hey can can y'all go learn this thing and put it into practice and I'll coach you from the sidelines but I can't do it as well as you right yeah I think what helps as I said again we are lucky because we have a very good observability cultural already engraved so in most of the engineers just do it but yeah I think what what really helped was that we have given full ownership to to the teams on building their own dashboards alerting and sending their own Telemetry signals so basically uh when they come to us like hey can you help us set up an alert we're like well say you are full owners of that part of the observability and that actually helps uh the teams knowing that they're owners of that they they want to to make it grow and improve it rather than just living it there and we'll we've also made it very clear that we will not be going there and implementing um anything uh related monitoring so if you need monitoring you have to do it yourself we are here for you we provide guidance of course we have tons of documentation on how to do everything they're not alone but we made it very clear that we're not going to do uh that for you it's simple and considering that uh all our incident is coming from the alerting that is set up on our platform let's say they're kind of forced to to to implement it but I think just the fact that uh they're owners of that and it's part of their team responsibility coming from the higher structures of the company okay you'll have this amazing product that you are building maintaining but you also have observability that is your your full ownership so if it's not good then it is your responsibility not somebody else's so I think that has motivated the teams to to really work on there sometimes it happens that when a junior enters the team they're the ones that have to take care of the all the monitoring without knowing well the code but I think they soon realize how important it is for everyone to to put their hands into to contribute there otherwise so does not work out that's great thank you I have a question for you it is um so you're saying like you guys hold your ground as far as like not being the ones to instrument code for folks um what kind of reactions do you get from teams when you say that like are they like oh okay I'll do it or or did they uh like defensive pushback like what what's it like how do you deal with it and most of the time they're like oh okay is there a framework can you send me to the framework so that I can find it and we usually also have a team that helps uh build this framework so we send it to them but most of the time yeah we just hold our ground if they say that oh but I really need to help me like okay I'm gonna help it I don't know how to do it for yourself I don't know your code and of course there are cases that people are not happy and they could go to a higher structure in the company and complain but considering that this is not just something that as the durability team have decided today that oh we're not going to build anything for anyone because it's their responsibility it's not something that it's of course supported and signed by by upper management they're like sorry but you have to do it it's documented and you have to do it and it's simple as that it's great having support that that's awesome because I think you you've like you've nailed it like that is the key thing because you're right like if you don't have the management supporting that decision to like you know have a visibility team we're not going to instrument your code like yeah you're gonna have like back channeling and I mean I've I've experienced that it's really annoying um and and ends up kind of like ruining the thing that you're trying to build like as far as that observability culture yeah exactly uh as I said yeah in farfetch we are very very happy in this aspect because uh yeah being forced to instrument somebody's code that would be the the breaking point for me as an engineer in believing that observability is lived preaching with everyone and talking about it and then being forced to to do to go against these values that that we have that would be the breaking point for me I have to stay yeah I don't blame you um I have a question um so far-fetch is an e-commerce site um correct and so I was curious and Derek I guess actually this question is um inspired by your question in the hotel end users Channel this morning how do you um or do you like what do you use for a client-side instrumentation are you um well we are using a vendor currently um I'd rather not say uh but yeah we're using a vendor for that for that part which is very very small the rest the the rest of the platform is all instrumented by our own custom framework okay okay do you have that integrated at all or is it kind of like its own view of the client and then you have trouble correlating signals with your back end uh currently it is a bit segregated but we're working on on changing it because it's such a small part of our platform but yeah it's not fully fully integrated uh we're focusing more on the let's say on our own custom solution that we have locally and that one is a bit segregated from it and it has only specific information which is not great but we that's our goal for 2023 and 2024 to integrate everything together what's yours what's your custom solution involve like what what makes it custom I guess is the question I mean it's it's uh using open Telemetry now uh we're using thousands Prometheus grafana alert manager basically all open source and we built it uh let's say we have around 100 kubernetes clusters we have created our agent based on this open source uh and just adding them on each of the classes collecting information that's centralizing it oh so custom solution like you're talking specific specifically around tooling and not like I thought maybe you meant like um like having some like abstraction layer on top of like hotel which I've seen some organizations do no no no no okay okay cool any other thoughts or questions for for edius yeah I I guess I'm a little curious so you said you know there's no such thing as like too many traces and I know some people you know prefer to do some form of tail sampling so they're just they're still getting like a subset of you know randomized 200s and then like you know whatever chases with errors or latency or whatever attributes that they might be interested in um so I was just curious if you could expound on that a little bit more especially like you know when talking about like um increased costs with excessive amounts of data and yeah so tracing is something that we are working a lot because it's not one of the Telemetry signals that it's being used a lot in in the company so we are trying to Advocate it and push it a lot and for your when I say too few traces I would say uh 0.01 sampling like normal sampling let's say in younger so the teams were considering it extremely low and they weren't even caring about it even though some of them you could find very good information there for example I use it a lot to troubleshoot our Thanos queries it's amazing um so where we are right now we increased because we were able to change our our background back-end solution from Cassandra to grafana Temple which is also open source so of course because before we were spending a crazy amount of money for 0.01 tracing we were spending so much money I don't want to say numbers so there was no place for us to grow more it was completely out of budget so now we implemented Tempo and we are able to send more traces and we increase to five percent and increasing that just a few applications and actually uh implemented this change because of course it's a custom framework and they can they can do it however they want and we already went for 1 000 uh spans per second to 40 000 from just one application and uh this application is not even using most of what they're sending and imagine we have hundreds of applications all of this sending all that is going to be millions of spawns per second it's going to require a lot of computation power and it's going to require a lot of storage it's going to be a mess and uh the cost will be out of this world in in conclusion so I think that we really need to go there like which we're in the process of doing and try to see to implement better uh better tracing because just normal sampling is not working yeah we're looking into a tail base that's why open Telemetry was uh brought up in the first place so the perfect to look at the tail base and yeah we were sending too little or basically there was not enough information now we're sending too much so we are trying to find something that is uh in between until sampling is one of them and also we're also making changes to the framework so not everything goes through some of the information is just not usable and the engineers are not needed yeah it's a it's a work in progress that is taking a lot of our time I I'd rather spend more money and send more information obviously but at the same time we could not spend millions of dollars a year for it for information it's not going to be used I see a raised hand I did um I wanted to ask if you're doing anything around anomaly detection especially as you increase the volume of information coming in to avoid having to look at everything to look at those anomalies that are out of the time frame uh you mean anomaly detection in the the amount yeah well so the scenario of you know here's a here's a high volume event but it's normal and here's a high volume at what's normally a low time of day right just to look for anomalies and everything that you're receiving without having to have a human look at it Yeah we actually have some alerting implemented there based on standard deviation and uh it's like a simple prompt ql based alerting for that and there are accurate but so we haven't gone about that because it's a custom solution and we actually need to apply some machine learning there which it's uh we haven't really been able to get to it so it's do some alerts with the standard deviation that are okay uh they they can detect some anomalies and some very big changes but also are highly inaccurate so it's not my favorite but they are there we need to to work better on it but yeah for example if we have a huge anomaly or a huge flow of sudden that is not uh predicted it is okay let's say it's not perfect I know it's you can get into some really um interesting your Solutions with that that can't come with a high cost but there's some really interesting things you can do with that too so I would love to know more if you have any links any suggestions for me please let me know because the more solutions we can implement hahaha like you mentioned cost cost is always a worry obviously but if it's actually a quality solution that we are introducing uh of course it is uh a cost that is needed and that's it you know it's it's really enjoyable when you start to use tools like that to um solve the problems because then you see things you didn't expect to see um we were experimenting with something a few years ago and we intended to point at the top 10 um issuers and you know the engineer mistakenly didn't put the top 10 in so he was actually looking for anomalies across the entire user set so you know thousands of um thousands of customers and but they were in entities so you know you can start to see oh look they're not sending at this time of day when they usually are that's a low volume low Center but they're sending nothing and they usually since 70 per hour um and we could see that before it finally blew up from running out of memory but uh you know you really get to say now what can I do with that for my business so that was interesting we presented to them at the the team who wanted to own that didn't want to develop the expertise and that's tooling even though we could show them what it could do so a 500 gigabyte an ec2 instance was running that before it blew up yeah a challenge that we have currently is with our Matrix as well and uh detecting which applications are sending higher technology more number of times series and we have dashboards and alerts for that but nothing is based on machine learning it's just pure statistical data uh and uh it is being evaluated for standard deviation or simple query so some some more inside there would be amazing yeah that's the part that we always worry about too is you know we've got so much data to look at which one's important so you don't look at any other you put your tools against it um yeah that too much data problem is is very interesting to have especially as um the open celebrity standards evolve and add more Fields so now you have more fields to consider yeah and they're good it's good that those are coming absolutely yeah one thing that I wanted to ask you is um is your team involved at all in the creation of slos foreign yes but also no well let's say we are the ones that are providing the tool for creating slos okay uh so basically the teams can implement the structure or basically create alerting dashboards based on it but we are not the ones that are providing the guidelines and the instructions on how to create them we just provide the tooling is it um is it like an open source tool that you guys are using or is it some like commercials we're actually using uh we have our own alerting solution so we are doing the same uh for slos as well and we are currently using tunnels ruler to send the alerts and to evaluate them uh alert manager to send the alerts and we also have a custom tool that reads the slos that are written in terraform and make them compatible with a s permitus rule satanos roller can read them okay and do you um so the reason why I'm asking specifically about slos is because um one of the I'd say one of the practices a team should aim for is like using observability data um to help generate their slos or not generate to you know create create their their slos based on on observability data so I'm just wondering if that's something that uh your team is planning on providing guidance on um like what's are there any plans around that so basically we are providing the data and the tooling to do it uh most of the guidance are like how the SLO should be built most of them are our business and we're not really involved in that part so we cannot give guidelines so that's completely different but yeah for example for infrastructure we provide some basic guidelines of slos but I would say that yeah we just want to provide the the data they feed off of our metrics and the tooling that they can do it and how they can actually build them but know not really how we can go how they can go and uh or like how to calculate the slos or what is best and what is worse and we haven't gotten there I don't think we have the plans currently to to move into that as we have other teams that give advice or recommendations on that matter there is so much uh so much to do in in observability in in our company I think we've done a pretty good job so far but there is so much more that we can do is there something that like your team is is working on implementing um like that you're excited about in the in the near future well uh we decided that because of the huge amount of data and uh we do not have capability to to process it as as good as we would like we are currently trying to see if we can well let's let me put some precedent we are using different tools for different things we're using permitting styles for Matrix uh we're using open Telemetry for traces and open Telemetry for metrics to a certain degree as well uh grafana for dashboards and we're using a different tool for for logging and currently it's all a mess so what we're trying to do right now is we want to centralize them in one platform what we don't know yet if it's going to be an outside platform that we are feeding the data to our like a vendor or if we're going to to provide it ourselves so that's something that we're working on right now centralizing everything in one place and sending all um Telemetry signals through one transporter that they open Telemetry so we could have better correlation which we are also lacking currently awesome so exciting times it is amazing I love I love the work that you're doing right now and it's going to be the next two years at least I know after that it's going to be a very exciting for us well it's very very awesome um we're coming up on time um but I did want to give folks an opportunity to ask any other final burning questions nope everyone's questioned out thank you everyone so much for for joining and if uh anyone who's uh in in the audience is interested in um in in giving a presentation for hotel and practice or would like to participate in one of our hotel q and A's please reach out on slack we are more than happy to hear your stories share your stories so that we can continue to build this amazing open Telemetry Community thank you so much for having me today + diff --git a/video-transcripts/transcripts/2023-07-02T17:46:37Z-otel-q-a-feat-jacob-aronoff-of-lightstep-from-servicenow.md b/video-transcripts/transcripts/2023-07-02T17:46:37Z-otel-q-a-feat-jacob-aronoff-of-lightstep-from-servicenow.md index 8fa65b7..fc17e4f 100644 --- a/video-transcripts/transcripts/2023-07-02T17:46:37Z-otel-q-a-feat-jacob-aronoff-of-lightstep-from-servicenow.md +++ b/video-transcripts/transcripts/2023-07-02T17:46:37Z-otel-q-a-feat-jacob-aronoff-of-lightstep-from-servicenow.md @@ -10,111 +10,79 @@ URL: https://www.youtube.com/watch?v=dpXhgZL9tzU ## Summary -In this YouTube video, a Q&A session features Jacob Aronoff from Lightstep, who discusses his experience with migrating to OpenTelemetry (otel) from OpenTracing and OpenCensus within their organization. Jacob, a staff engineer on the Telemetry pipeline team, shares insights about the challenges and benefits of this migration, including performance improvements and cost savings from eliminating sidecars for metrics. He explains the approach taken, which prioritized safe and gradual migration strategies, and discusses the implications of using a monorepo versus microservices. The conversation also touches upon various deployment strategies for collectors, performance issues encountered, and the importance of keeping up with the latest versions of OpenTelemetry. The session concludes with Jacob expressing interest in sharing more insights on this topic in future talks. +In this YouTube Q&A session, Jacob Arnoff, a staff engineer at Lightstep, discusses his experience with migrating to OpenTelemetry (otel) from OpenTracing and OpenCensus while working at Lightstep. The conversation, led by the host, covers the challenges and strategies involved in this migration, emphasizing the importance of a safe and efficient transition for observability in software applications. Jacob details the initial use of proxies for metrics collection, the performance issues encountered, and the iterative approach taken to migrate tracing and metrics to otel. He explains the various deployment options for collectors in Kubernetes, including sidecar, daemonset, and stateful set, and highlights the benefits and considerations for each. The discussion also touches on the significance of keeping telemetry systems up to date, the advantages of using tools like Dependabot, and the ongoing developments in the open-source observability community. Overall, the session provides valuable insights for organizations looking to adopt OpenTelemetry and improve their observability practices. -# Hotel Q&A with Jacob Arnoff +## Chapters -[Music] +Here are the key moments from the livestream along with their timestamps: -**Thank you!** Welcome everyone to Hotel Q&A. Thanks for joining us today. We are super lucky to have Jacob Arnoff from Lightstep at ServiceNow with us. It's a great treat because I contacted Jacob after he mentioned he had submitted a CFP to one of the conferences, I believe it was KubeCon, right Jacob? +00:00:00 Introductions and welcome message +00:01:30 Introduction of Jacob Arnoff from Lightstep +00:03:00 Discussion about Jacob's experience with OpenTelemetry +00:05:30 Overview of Lightstep's migration from OpenTracing to OpenTelemetry +00:10:00 Challenges faced during the migration process +00:15:00 Explanation of the metrics migration strategy +00:18:30 Discussion on the performance issues encountered +00:25:00 Insights on using attributes and metrics in OpenTelemetry +00:30:00 Overview of the differences between deployments, stateful sets, and daemon sets +00:40:00 Explanation of the Target Allocator and its benefits +00:45:00 Discussion on keeping dependencies up to date and the importance of regular updates -[Music] +These moments encapsulate the essential discussions and insights shared during the livestream, making it easier for viewers to navigate the content. -**Jacob:** Yeah, it was a CFP on migrating to OpenTelemetry (otel) from our organization. +# Hotel Q&A with Jacob Aronoff -**Host:** That's a pretty cool story to tell. I thought it would be great to have you share that experience during this Q&A. It makes for a compelling narrative when an observability vendor talks about using OpenTelemetry themselves in their own products. So here we are! Welcome, Jacob. Would you like to give a quick introduction? +Thank you for joining us for today's Hotel Q&A session. We are fortunate to have Jacob Aronoff from Lightstep, part of ServiceNow, with us today. Jacob recently submitted a CFP for KubeCon, focusing on migrating to OpenTelemetry (otel) within his organization, which is an exciting topic to discuss. -**Jacob:** Sure! Hi, my name is Jacob Arnoff, and I'm a staff engineer at Lightstep on the Telemetry Pipeline team. I've been working on this for almost two years, and the first year of my journey was solely focused on OpenTelemetry migrations—doing them internally and making them easier for customers. +## Introduction -**Host:** That's interesting! How do you want to proceed? Should we keep it Q&A style, or should I just dive right into the story? +**Jacob Aronoff:** Hi, my name is Jacob Aronoff. I’m a staff engineer at Lightstep on the Telemetry Pipeline team. I have been working on this for almost two years, with the first year dedicated to internal migrations to OpenTelemetry and making the process easier for our customers. -**Jacob:** Let’s see how it evolves organically. +## Migration Story -**Host:** Sounds good! Why don’t you start by setting the scene for us? +When I joined Lightstep, we were still using OpenTracing for tracing, alongside a mix of OpenCensus and some hand-rolled StatsD metrics. This setup required running a proxy on every pod in Kubernetes, which was costly and complex. As we were moving towards OpenTelemetry for metrics, I took on the migration effort. Our internal OpenTelemetry team had been working hard on it and needed feedback for improvement. -**Jacob:** When I joined Lightstep, we were still using OpenTracing for tracing, along with a mix of OpenCensus and some hand-rolled StatsD stuff for metrics. This setup meant we had to run a proxy on every single pod in Kubernetes. Since the proxy acts as a sidecar, it required running an additional application alongside each one of our applications, which reads from StatsD and forwards those metrics. +During this migration, I aimed to make it as safe as possible. Since we were in a monorepo, we could have done an all-in-one migration, but that approach is risky. Instead, I opted for a feature-flag-based migration where we could disable the StatsD sidecar and enable the OpenTelemetry code. -For us, as we were building out a metric solution, that destination was Lightstep. I joined at a time when we knew that OpenTelemetry for metrics was going to be a huge effort for us in the next year or two, and we wanted to reach stability. The internal OpenTelemetry team at Lightstep had been working hard on improvements and really wanted feedback on how to enhance it, so I took on that migration. +### Performance Issues and Discoveries -We also theorized that this migration would save us a good chunk of money, as we would no longer need to run those relatively expensive StatsD sidecars. I initially planned the migration to be as safe as possible. There are different approaches to migrations like this; you can do an all-in-one go, which is possible for us since we are in a monorepo. However, it’s much riskier because pushing a bug could take everything down. +Midway through the migration, I tested everything in our staging environment and found performance issues. I collaborated with the OpenTelemetry team to identify the cause, which was related to the attributes used in our metrics. This led me to theorize that by migrating from OpenTracing to OpenTelemetry, we could also see performance improvements. -For us, it was important not to disrupt application data, as this data is critical for alerting and understanding how our workloads function across all environments. So, we wanted a safe and easy migration. Our initial approach used feature flags to disable the sidecar and enable some code that would swap to OpenTelemetry for metrics, forwarding them to the appropriate destination. +I paused the metrics work and began the tracing migration. This time, I decided to try an all-or-nothing approach, as the OpenTelemetry path was more straightforward and backwards compatible. I managed to complete the migration with minimal issues, and it turned out to be a significant improvement in performance. -Midway through the migration journey, after testing everything in staging, I noticed some significant performance issues. I reached out to the OpenTelemetry team, and we worked together to alleviate some of those concerns. One major blocker we found was that we heavily used attributes on metrics, and it became tedious to figure out which metrics were using these attributes and how to remove them. +## Insights from Migration -I theorized that a particular code path was the issue. We were converting our internal tagging implementation through OpenTelemetry tags, which had a lot of additional logic that was expensive to execute on every call. I decided there was no better time to start another migration from OpenTracing to OpenTelemetry while we waited for the team to push out more performant code for metrics. +This migration story is valuable for organizations at different stages of their observability journey. Those just starting with instrumentation and those who have dabbled in OpenTracing can learn from our experience. -So, I paused my metrics work and began the tracing migration. For this migration, I chose the all-or-nothing approach, as the OpenTracing to OpenTelemetry path was more known. There were small docs and examples available, and they were backwards compatible, so emitting OpenTelemetry alongside OpenTracing wasn’t a big deal. +### Starting Points -I started by ensuring that our propagators were set up correctly. The first step was to make sure all of our plugins worked, which at that time weren’t open-sourced. I executed the migration in one swoop and had to revert a few times in our staging environment, but nothing major. I did encounter a bug regarding in-app sampling, which required implementing a custom sampler. This was actually ten times easier with OpenTelemetry than with OpenTracing, allowing me to eliminate around 1,000 lines of code and some risky hacks. +When migrating, I suggest starting with smaller services that have low traffic but enough consistency to gather meaningful data. By comparing metrics before and after migration, you can identify any discrepancies. -**Host:** That’s a fantastic story! I have so many questions already. Do you mind taking a quick pause? +## Challenges and Solutions -**Jacob:** Sure! +One challenge was ensuring that all attributes remained consistent after the migration. This required close monitoring and sometimes rewriting processors to ensure compatibility. -**Host:** A couple of comments: this is an interesting story because we see two types of organizations—those with zero code instrumented and those that have dabbled in OpenTracing. It’s great to have a real-life migration story to provide advice on what to do if someone finds themselves in this situation. +### Future Directions -You mentioned you’re working in a monorepo. Did you find it more challenging to do the migration in a monorepo than if you were dealing with microservices? +With logs maturing in OpenTelemetry, there are potential plans to replace span events with logs, which is an exciting development. However, my focus has recently been on improving infrastructure metrics in Kubernetes. -**Jacob:** We do use microservices, but we have a repo of microservices. Since we were going for a big-bang approach, I started with small services owned by my team that had low traffic but were constant enough to provide meaningful data. +## Collector Setup -If a service is too low traffic, you might not have enough data to compare against, which is crucial. I wrote a script early on to query different build tags on all metrics. This script would query metrics for service X grouped by release tag. If the standard deviation for the newer build tag is greater than one, there’s likely an issue with your instrumentation library. +We run various types of collectors at Lightstep, including metrics and tracing collectors. Currently, we leverage a mix of deployments and daemon sets depending on the use case. -Another thing I checked was to ensure that all attributes were present before and after migration. It was essential to confirm that all attributes stayed the same during the trace migration since I was particularly focused on that aspect. +Daemon sets are particularly useful for node-level scraping, while deployments are more efficient for stateless collectors. Stateful sets are less common but beneficial for certain scenarios where consistent IDs are necessary. -**Host:** That makes sense. Starting from an existing setup gives you a frame of reference, which is a double-edged sword. You know what has been instrumented, but if something is missing, it raises questions. +## Keeping Up to Date -**Jacob:** Absolutely! One of the tricky parts was dealing with gaps where attributes were missing. Sometimes it was due to a forgotten addition, but other times it stemmed from an upstream library not emitting the necessary data. +To ensure everyone stays updated with the latest versions of OpenTelemetry, we use tools like Dependabot. It’s crucial to keep dependencies current to avoid security vulnerabilities and compatibility issues. -For instance, I did a migration on a gRPC utility package and encountered an issue where there was supposed to be a propagator but it was marked as a "to-do." I reached out to the responsible person to resolve that issue, and it turned out to be a relatively simple fix. +## Conclusion -**Host:** That’s a great example. As logs have matured, do you have plans for conversions related to them? The log specification is evolving to replace span events to some extent. +Thank you all for joining us for this informative session! I hope you found the discussion on the migration process and collector setups helpful. If you have more questions or topics you'd like to explore, feel free to reach out to me or engage in future discussions. -**Jacob:** That’s outside my recent focus, so I’m not completely sure. I think we would change how we collect logs, but currently, we use Google's logging agent in our GKE cluster, which runs Fluent Bit on every node to send logs to GCP. - -**Host:** Speaking of GKE, have you heard about the new telemetry collection feature in Kubernetes? - -**Jacob:** Yes, Kubernetes now has the ability to emit OpenTelemetry traces natively, but I’m not sure if we’re collecting those yet. I want to look into whether we can use those to generate better Kubernetes metrics. - -**Host:** It sounds like you’re focused on infrastructure metrics, which can be quite painful in their current form. - -**Jacob:** Exactly! I prefer using Prometheus APIs for those metrics since they are more ubiquitous in the observability community. However, Prometheus script failures can be a common issue, and dealing with metrics cardinality can be frustrating. - -I once found a bug in GKE related to certificate signing requests that led to Prometheus crashing due to high cardinality metrics. It's an ongoing challenge. - -**Host:** That does sound challenging! Could you elaborate on the target allocator? - -**Jacob:** The target allocator is a component of the Kubernetes operator in OpenTelemetry that dynamically shards targets among a pool of scrapers. This approach helps distribute the load effectively and is especially useful when you have many nodes in your cluster. - -**Host:** That’s a great overview. So, the target allocator helps with efficient data collection, especially in large clusters. - -**Jacob:** Yes! It allows us to scrape metrics from various nodes efficiently without overloading a single collector. - -**Host:** Awesome! Now, I know there are different deployment modes for the OpenTelemetry operator. Which mode do you find works best for your needs? - -**Jacob:** We use all deployment modes depending on the requirements. Sidecars are the least popular since they can be expensive. For most cases, we prefer using deployments for stateless collectors, while daemon sets are great for node-level metrics scraping. - -**Host:** It sounds like you have a solid strategy in place for your collector setup. - -**Jacob:** We run various collectors to handle different telemetry types. We’re constantly experimenting and improving our setup based on internal needs. - -**Host:** That’s great to hear. It sounds like keeping telemetry up to date is crucial. - -**Jacob:** Absolutely! Staying current with library updates and performance improvements is vital for effective migrations and overall system health. - -**Host:** As we wrap up, do you have any parting thoughts? - -**Jacob:** I think it’s important to keep learning and sharing knowledge within the community. If anyone has questions after this session, feel free to reach out. Staying updated with the latest changes and improvements in OpenTelemetry is beneficial for everyone. - -**Host:** Thank you so much for your insights today, Jacob! This has been a fantastic discussion, and I’m sure everyone here has learned a lot. - -**Jacob:** Thank you for having me! I’m glad to share my experiences. - -**Host:** Thanks again, everyone! We appreciate you joining us today. - -[Music] +Thank you again, Jacob, for sharing your insights and experiences today! ## Raw YouTube Transcript -[Music] thank you welcome everyone to Hotel q a thanks for joining um today we are super lucky to have Jacob aaronoff from lightstep from service now join us um it's it's a cool treat because um like I tapped Jacob because he was telling me that he had submitted a cfp to one of the I think it was cubecon right Jacob [Music] yeah it was a cfp on like um migrating to Hotel um from like our or like our organization migrating at hotel right if I if I understand correctly which I thought that's pretty freaking cool story to tell so um so I asked them to to join and and have this q a and um and hear the story because I think it's it's I think it makes for a compelling narrative like when an observability vendor talks about using otel themselves on their own products so here we are um so welcome Jacob um do you want to do like a quick little intro sure um yeah hi my name is Jacob arnoff I'm a staff engineer at lightstep um on the Telemetry pipeline team um I've been to lead stuff for almost two years uh and the first like year of that Journey was solely focused on like Hotel migrations um making doing them internally and making them easier for customers um sort of that whole process um so yeah I could get into this anyway that is interesting how do you think do we want to do this very q a style or should I just go right into the story talk about um we can we'll we'll let it evil organically about um cool yeah I like that why don't we start like um I mean I think you're you're rearing to go so maybe why don't you describe like set the scene for us so um when I joined lightstep uh we were still on open tracing for uh tracing um and then a mix of open census and some hand rolled stats key stuff or metrics so this meant that we had to run a proxy on every single pod that we ran in kubernetes and a proxy since it's a sidecar on every pod which means that every single time you run it one of your applications you have to run another little application that's going to read from statsd and then forward those metrics off um and you know for us as we're building out a metric solution that destination was light step so I came in and this was sort of at the beginning of like metrics Alpha I think um and I was like hey it would be great for us to we know that um open uh we know that like hotels hotel for metrics is going to be a huge effort for us in the next like year or two we wanted to reach stability um the hotel team had been we have like an Hotel team internal light step they've been working on it a lot and really wanted some um immediate feedback on how to improve it so I took on that migration for us we also had the theory that doing so would save us you know good chunk of money because we would no longer need to run these relatively expensive stats key um uh stats the sidecars so I planned it initially um to be sort of as safe as possible I'd done some migrations like this in the past um and there are a few different ways that you can do migrations like this you can do the all-in-one go uh which for us would have been possible we're in a mono rebuild um but it's much more dangerous because you you worry about you know am I going to push a bug that's going to take everything down right um obviously this is application data that it's data about your applications which we use for alerting we used to understand you know how our workloads are functioning um in all of our environments and so it's important that we don't take that down because that would be disastrous for us but obviously for you know an end user is going to be the same story they don't they want the comfort that if they migrate to Hotel they're not going to lose all of their alerting capabilities immediately they're not like you want a safe and easy migration so that's the only one with our initial approach for doing a sort of like feature flag based um just like part of that can part of the configuration that you run in kubernetes um it would disable this sidecar enable some code that would then swap to otel for metrics and then forward it off you know where it's supposed to go um so that was sort of the path there um Midway through this journey of doing these migrations I had tested it all out and staging looks pretty good I tested the container in our meta environment so we use to monitor our public environment um I noticed some pretty large performance issues um in those 20 2021 and I had reached out to the hotel team and we had worked together to sort of alleviate some of those concerns one of the ones that we found that was a big blocker was we heavily use attributes on metrics right now and it was incredibly tedious to go in and figure out why you know figure out which metrics are using all these attributes and getting rid of them so um well I had a theory that like really this one code path was the problem where we're doing the conversion from our internal tagging implementation through Hotel tags which had a lot of other logic with it that was pretty expensive to do on pretty much every call so I was like you know what this there's no better time than now to begin another migration for her open tracing to otel basically while we wait on the hotel folks on the metric side to push out um more performant code more performant implementations for us we can also test out this theory that if we migrate the hotel entirely we're actually going to see even more performance benefits so at that point I said okay I'm going to put a pause in the metrics work while we wait for hotel and I'm going to begin on this tracing migration nutrition migration however um I decided to try a different approach which is the All or Nothing approach basically um the open tracing to open Telemetry path was a bit more known there were a few really like small docs and examples and they are backwards compatible like you're able to use them in conjunction with each other so one thing admitting Hotel one thing emitting um open tracing is not the end of the world so you can mix those as long as you have the propagators set up correctly so first step so propagators Second Step um make sure that all of our plugins worked which at the time they weren't open sourced uh now they are open source so people can just use them um and I just did it all in one swoop maybe I had to revert like three times from our staging environment nothing really major um and then there was one bug that I missed where um we were previously doing a lot of in-app sampling because we had a really noisy function call um so I had to implement the custom sampler which is actually like 10 times easier with otel than it was with open tracing I was able to get rid of a good like 1000 lines of code or so and some really dangerous hacks so that was a really good thing um and yeah so then that went out very happy um also stopping at any time if this if we want to get back to uh like more q a stuff but I I have so many questions from this already but do you mind do you mind taking like a quick pause because I I'm like sure um okay so a couple comments so one thing that that came to my mind as you're saying this I'm like this is actually really freaking cool story because like I think for there we we tend to see like two different types of organizations right we see the ones where there's like zero code instrumented like they are this is like their first foray into instrumenting their code and then we see the the organizations that I think have either that have like dabbled in open tracing so I think it's a really cool story because this is like a real life migration story where you can actually provide advice on this is what you can do if you find yourself in this situation which is really cool um so I wanna I wanna call that out because I I think um I think that's a really important thing um especially if you're like starting to get into open Telemetry um the other thing that I wanted to ask you about because uh you said that it's um that it's a monorepo um so in that case did you find it and especially since you did the All or Nothing approach um did you find having a mono repo more challenging than if you'd been dealing with microservices instead um yeah well so we we do use microservices it's just that we have like um repo of microservices sorry my my bad about you guys um in that case yeah then um how do you um how do you know like because I mean yes you're going for like a big bang approach but you got to start somewhere so then like where where do you uh where do you start so I started with um and it was the same with how I started the metrics migration um I started with really small services that um my team owned that were really low traffic but enough for it to be constant um so the reason that you have to that you want to pick a service like that is if it's too low traffic if you're only getting like one request every minute one request every like 10 minutes um you have to worry about sample rates um you might not have a lot of data to compare against really that's like the big um thing that you need to have is some data to compare against I wrote a script early on for the metrics migration that um just queried uh different build tags um that are on all of our metrics so you would say you know query all of the metrics for service X um grouped by release tag and if you see that the standard deviation for the newer build tag is like you know greater than one right so if it's one or more standard deviations away from the previous release then there's probably something going wrong in your instrumentation Library right if you assume that your metrics are relatively stable then if they're not it's important to know um the other thing I had to check for was that all of the attributes were still present before and after migration which is another thing that matters sometimes they weren't because something might be something that stats T just adds automatically that we don't really care about and so those were acceptable I just like hand waved instead those are fine we don't care um for tracing is sort of the same deal where I picked a service that had um both internal only traces traces that stayed within a single service and then traces that um span multiple services with different types of instrumentation so coming from like Envoy to hotel to open tracing and what you want to see is that the trace before has the same structure as the trace Factor so I made another script that checked that those structures were relatively similar and that all of them had the same attributes as well right right um because tracing attributes again I was doing an attribute migration that was really the point of doing the tracing one so what matters that all the attributes um stayed the same right yeah it's interesting too because like because you're you're starting from the point where you were migrating from an existing thing like you have that frame of reference which I guess is kind of a double-edged sword right because on the one hand it's like you know you pretty much know that you've instrumented the things hopefully maybe maybe you'll discover as you go along that there's like more stuff to instrument but at least like you have a baseline to start from but then I guess on the other hand if something's missing you're like oh damn why is that missing right yeah and those you know why is this missing stories or the really complicated ones because of course sometimes it's easy to just like you know oh I forgot to add this thing in this place and that's usually pretty simple but sometimes it's like oh there's an upstream library that doesn't admit the thing or hotel and now I need to like again this is like early stuff most of these have all been upgraded and are fine now um but there is an example with like our grpc um actually this is like an interesting one um I had done a grpc I migrated on a grpc util package which I think is now in like token trip um the like Hotel goken trip um and there was an issue with propagation I was trying to understand you know what's what's going wrong here and when I looked at the code and it just tells you how early in this story I was doing this migration um where there was supposed to be a propagator there was just a to do oh no um so and the city was from someone on it was from Alex book which I'll call him out um and so I I sent it to Alex and I was like hey this is a funny to do because I just you know took down um an entire Services traces and staging um so I spent some time to like fix that to do so it wasn't that difficult it was just that they were waiting on another thing I mean that's how it goes you're waiting on someone else which you know another person it's just endless cycles of that type of thing yeah um but then I got it working so that was like one of the main uh blockers for us nice nice and was able to Upstream it as well it wasn't just a fixed for ourselves it was a fix for the community and so that that was yeah there were a few things like that um a lot of the metrics work um actually resulted in big performance boosts for um Hotel metrics like Hotel go metrics and um it also has given the specs folks like some ideas about how descriptive the API should be or various features so things like um uh views and the use of views is something that we did heavily in that early migration because we were worried about can you just yeah definitely just tell folks what you mean by that yeah so um of metrics View is something that's run inside of the your metrics provider in otel your media provider in hotel so what it's doing is it's saying you can configure it to do kind of a lot it could just be um drop this attribute whenever you see it if some if you're a centralized SRE and you don't want anybody to instrument code with any user ID attributes because that's a super high card analogy thing it's going to explode your metrics cost right so you could just make a view that gets added to your instrumentation that says just don't let this attribute from being recorded just deny it um so that's like it's probably most common use case um there are other ones though more advanced use cases for dynamically changing things like the temporality or the aggregation of your Matrix so temporality being cumulative or Delta for like a counter um I you know am I recording um zero one three I'm trying to two crazy zero one three or am I recording one and two right right um and then your uh aggregation is going to be about how do you um I these things are so I always struggle to explain all of them and I'm trying to like come back to uh what I had done in that moment um talk about temporality oh aggregation is like um oh man being on the spot is so much harder because it's like I want to look it up but feel free to look it up that's totally cool well aggregation is like how you um like send off these metrics basically we had an aggregation that instead of doing um well for histograms this is like most useful when you record a histogram there are a few different types of histograms um datadog's histograms stats of these histogram is not a true histogram because what they're recording is um uh like aggregation samples so they give you a min max sum count average um and so I actually don't even think they give you some I looked at this last night they don't give you some they give you like min max count to average and like P95 or something um and so the problem with that is in distributed computing if you had multiple applications that um are reporting of P95 the there's no way that you could get a true P95 from that observation with that um aggregation um the reason for that is that in order to get P95 like you you can't if you have five p95s oh five P95 observations there's not an aggregation to say give me the overall P95 from that right you need to have something about the original data to actually recalculate it you could get the average of the p95s but that's not a great um metric that's not really like it doesn't really tell you much it's not really accurate um and if you're going to alert on something if you're going to page someone at night you should be paging on accurate measurements yeah um so initially though we did have a few people who relied on this min max some counter instrument so we used um Hotel views in the metrics SDK to configure custom aggregation for our histograms to do emit what some would call a distribution um what Oto calls an exponential histogram um or the min max and the Min Maxim count so we were dual emitting this works because they're different metric names that we were emitting so there was no overlap between them so what we did was we migrated after we did the metrics migration we were able to then go back and say any dashboard any alert anything that had was using a min max some count metric um just change it to be a distribution instead and because we had enough data in the past like you know a few weeks months of running Hotel metrics in our public environment that was possible to do um so that that was like one of the key features that because we had it it was 10 times easier um and we were able to do it from the application uh we didn't have to introduce any other components which is pretty neat right right cool um um another question that I had for you um so like when you were doing this migration it was traces and metrics were in existence logs I believe would not have been like the specification would not have been ready possibly not even in the works no it was still really early for that so um but I guess in lieu of logs there's span events that you could use so it's not something like that was leveraged as well definitely um we've heard a long time have you used span events analogs um or a lot of things um internally um I'm a big fan of them I am not a huge fan of vlogging I find it to be really cumbersome and really expensive um and iops for like tracing and Trace logs whenever possible I find it like easier for myself to reason about there are other people who are like logging first and that's great but um that's just not who I am um I like I like logging for local development and tracing for distributed elements that makes sense um but we use this heavily that was one of the first things that I checked worked um It's actually an interesting bug where we had some custom code or open tracing that allowed us to serialize like Json blobs in the span events um and that stopped working because we didn't emit them in the same way uh it's like a little hazy but um I had to like rewrite a processor to make that work and then update some Downstream code like in lightstep as platform to Facebook cool so now how about um keeping that in mind like now that logs are more mature is there are there any plans to do any conversions like and and please correct me if I'm wrong but my understanding too is that like with the log specification like maturing more and more that um like span events are going to be replaced by logs in some form like it's going to be the log specification for span events have you heard anything around that like no this is a bit outside of um where my recent Focus has been so I'm not positive um I think right now the way that we do I think the thing that we would change is how we collect those logs potentially um right now we use um uh how do we do this right now it changed recently I don't want to say something incorrects but um we previously did it by just using like Google's logging agent where they basically are running like fluent bit on every um node in a in the gke cluster yeah and then they send it off to like gcp and they just like tail it there um I think this changed though and I'm not sure what we do now okay cool cool um speaking of kcp of many questions on gke specifically like so um do you um because I I believe there's like a feature now in like newer versions of of kubernetes where there's like some um I think there's there's like some Telemetry collection do you know if that's been enabled in any of the uh in any of the Clusters yeah so I think that kubernetes now has the ability to emit like the hotel traces natively yeah yeah yeah um I'm not sure if we're collecting those yet um I don't know what version that's more of a like a question for this sres I I don't um kubernetes I came out like I think even last year starting whatever like last fall kind of thing yeah that's a really good question um that I want to look into because I want to see if uh really what I would like to do is see if we can collect the if the traces that we get from those are enough to use the spend metrics processor to generate like better kubernetes metrics from those traces [Music] um I'm very focused on like infrastructure metrics like kubernetes infrastructure metrics um and I find them to be very painful in their current form um and it would be really cool right now um I prefer to use the Prometheus apis for them currently um it's just a bit more ubiquitous in the like observability Community to use Prometheus to do that [Music] um just because like that's that's what kubernetes like natively emits right right go ahead oh uh no go ahead I'll let you complete the thought maybe it answers my questions um and so that's what we do right now and I use the target allocator which is you know nutshell component that I work on um to distribute those targets which is you know pretty efficient way of getting all that data um we also use like demon sets as well that we run in our clusters to get that data um in addition to that so that works pretty effectively the thing that's frustrating is just Prometheus um Prometheus script failures can be um a super common problem and it gets really annoying when you have to like worry about metrics cardinality um as well because it can explode yeah I actually found a bug in gke maybe six to eight months ago six seven months ago but they've since fixed where they weren't deleting uh they weren't reconciling certificate signing requests in their kubernetes cluster which meant that for Kube State metrics which reports on cluster State um it was oming because there were so many um certificate signing requests left from these abandoned nodes and so something on the magnitude of like six hundred thousand for like a single metric which is huge and so then Prometheus the Prometheus that I was running fell over because of that um and that's like a thing that happens constantly in this Prometheus realm which is just like someone admits a high cardinality metric Prometheus goes to scrape it and then it just like crashes oh wow um um which isn't Fun yeah that does not sound fun um I want to just take a step back because you you mentioned the target allocator I was wondering if you could expand a little bit on that because I know like we we actually had one of our previous q a folks also mentioned the target allocator um that was the first time I had heard of it so I think it'd be like super helpful to just get a little overview sure yeah so apparent allocator is component um part of the kubernetes operator um in hotel that does something that Prometheus can't do um which is uh dynamically Shard targets amongst a pool of scrapers so previous has some experimental functionality for sharding but you still have a problem for um querying because Prometheus is a um database not just a scraper yeah um if you Shard your targets you don't necessarily you have to do some amount of coordination within those Prometheus instances which gets expensive it's like a very experimental feature um or you could scale Prometheus with something like Thanos or cortex which is um grafana's Prometheus scaling solution I think right yeah which works but you just then have to run like six more components that you then need to Monitor and then if those go down how do you met a monitor it's all these other problems right right um an hotel we just basically like uh tack on this Prometheus receiver to get all this data but um because we want to be more efficient than Prometheus because we don't need to store the data we tell we have this component the target allocator which goes to do the service discovery from Prometheus so it says give me all of the targets that I need to scrape and then the target allocator says um with those targets distribute them evenly amongst the set of collectors that's running oh okay um so that that's the main thing um that it is doing it does some more stuff around um job Discovery now or if you're using Prometheus service monitors which is part of the Prometheus operator which is a very popular way of like running Prometheus in your cluster um it's what a lot of like vendors use as well so if you're on GAE or openshift I think both of those natively used service monitors and pod monitors oh so the target allocator can also pull those service monitors and pod monitors and uh update the collectors scrape configs to do that oh cool it's awesome um and so one related to the Prometheus thread um are you running like Prometheus itself or are you just scraping the Prometheus metrics and pumping them through to the collector then exactly right just um no Prometheus instances just um The Collector running Prometheus receiver and then sending them off to light step oh living the dream that that was always like that was always my dream that's awesome that's nice um do you use like because I I remember like lights up has like a Prometheus um like a Prometheus operator that uh helps facilitate that so we used to have this thing yeah we used to have this thing called the Prometheus sidecar which you might run yeah you would run it as part of your Prometheus installation which would then sit on the um on the same pod as your Prometheus instance and read the uh write ahead log that Prometheus has for like um persistence and uh batching and all these other things yeah and so we would read the right ahead log and then forward those metrics but if your Prometheus is very noisy as many customers have very noisy Prometheus statistics yeah um it's not really efficient it can get really noisy and it not that uh what's the word it's not the best thing to run the collector is like the fact the best way to run okay so and and it sounds like this thing still requires um to have Prometheus installed and yeah you would still need to be running a whole computer system oh okay I I thought it was I was under the impression it was like a replacement for Prometheus and that it was um maybe I'm thinking of something else his there was a thing that I knew it was like a replacement for for like needing Prometheus um and it was like vendor neutral so it wasn't like oh you have to use lightstep to use this thing um I think I might just be a hotel operator collector Target allocator Trio oh oh okay okay but maybe there's another thing out there oh unless unless maybe that got integrated into like the target allocator as part of anyway it is a mystery yeah okay cool cool oh that's awesome um so then okay since we're we're talking collectors now um I for me like the two million dollar question the one that I'm always curious about is collector setup so what what is the collector setup that you have that y'all have chosen has like what works for for now yeah it's hard to say because we run a lot of different types of collectors yeah headlight stuff we run like metrics things tracing things internal ones external ones there are a lot of different collectors that are running at all times you have like a separate one that just collects metrics and one that just collects traces and um right now we don't um it's all varying flux okay right now we're changing this a lot um to run like experiments and stuff um basically like the the best way for us to be able to make features for customers and end users is by running them ourselves and then using them internally making sure that they work and then sending them for the open source realm and so that's what we're trying to do even more of like we're kind of reaching a point where uh we dog food everything which gets really um confusing because you have to like yeah I can imagine yeah we're running like in a single path there could be like I think two different two collectors in two environments that could be running two different images in two different versions like it's it gets really meta and very confusing to talk about yeah yeah I can imagine and then you know if you're sending from collector eight across an environment to collector B um collector B also emits Telemetry about itself which is then collected by collector C yeah and so it like it's just chains like you basically ensure that that you have to like make sure that the collectors actually working yeah you have to be sure that everything along this path yeah you just have to know which thing has the data right well you shouldn't have to we do that for you but like right um yeah and we make like dashboards to help with that but um that's like the problem when it's like we're debugging this stuff is when there's a problem you have to think about like where's the problem actually is it in how we collect the data is it and how we emit the data is it in um you know the source of the how the data was generated it's it's you know one of like a bunch of things yeah yeah um now like you need to work on the hotel operator so and and I I've been reading up on the operator recently and there's like I think four different deployment modes right there's sidecar deployment demon set and um what's the other one um yeah nice yeah um yeah um so my question is um which mode um or is it like all of the above depending on the thing that you need to do yeah it's all the above depending on what you need to do um and you're neat like your general needs and uh like how you like to run applications for like reliability and stuff yeah so um sidecar is the one that we use the least and is probably used the least if I were to make just like a bet um sidecars are really useful like across the industry you would think yeah across the industry I'd be willing to bet that those are the least popular that's the least popular method sidecares are just expensive and if you're not using them if you don't really need them then you shouldn't use them and you'll really only need them like something that's run as a sidecar it's like istio which yeah yeah makes a lot of sense to run as a sidecar because it's doing like traffic proxy like hooks into your um you know container Network to change how that all does its thing yeah and you get a performance hit uh if you sidecar your collectors right for all your services you just get like a cost it would just cost you a lot more um and you also wouldn't be able to do as much with like um if you're making like kubernetes API calls for attribute enrichment that's like the thing that would get exponentially more expensive if you're running it as a sidecar right but as like a staple set of like you know five pods that's not that expensive but if you have a sidecar on like 10 000 pods then that's 10 000 API calls made to the kubernetes API right yeah yeah that's exclusive what um what would be the advantage of running um your collector as a full set versus a deployment like where I guess what's what's the state that you would want to persist yes this is um like I don't know the right word is an unknown stateful sets aren't only used for their ability to mount volumes um there are a few other things that are inherent to how staple sets run that are really valuable in distributed computing this is an important thing to know for not just like how The Collector runs as an application but how like your applications can run right um stateful sets have um consistent IDs so if you have a staple set with 10 replicas they're all going to be um the staple set name Dash um um counter number so it goes from like zero to n so okay that's a really valuable thing when you want consistent IDs right as opposed to like with with deployments like when you like your your pods are all like random random crap right yeah so the Pod IDs are done where you it's um deployment name Dash replica set ID Dash pod ID right and so with staple sets because we have this consistent IDs we can actually do some extra work with uh for the Target allocator which is why we require that and so the other thing that stateful sets guarantee is what's called a um In-Place deployment which is what demon sets do as well where you have you take the replica you take the Pod down before you create a new one um right so the reason that this is important is that in the deployment you normally do a one up one down right um or what's called a rolling deployment a rolling update and so if we were to do this for um uh if we were to do this for the uh with the target allocator um we would probably get much more unreliable scrapes because you would and someone actually just asked this question in the operator Channel I mean I'm going to give them this exact response um when a new replica comes up um you have to redistribute all the targets because your hash ring that you place these on is changed so if you're doing a rolling deployment if you're doing one up one down that's a really expensive operation um because then you have to recalculate all these hashes that you assign um so if you were to do a one down one up you would still have to redo this whole thing because um you would lose a pod which means it's taken out of the ring redistribute you would gain a new ID and then you'd have to redistribute again right um whereas stateful sets because it's a consistent ID range you don't have to do that at all and so this means that when we do a one down one up it keeps the same targets each time right right so it's like it's almost like a placeholder for it like you don't have to recalculate the ring basically because yeah it's just sort of like a little um uh what's it called uh yeah yeah yeah yeah I can't think of the word cool that's really neat I didn't know that yeah it was funny because I was reading about like pause being deployed a stateful sets I'm like straight or sorry not uh collectors I'm like I straight up do not understand what the use case would be but this makes a lot of sense so that's uh that's really cool and so it's not as useful um or this is really only useful for like a tracing use case I would say or sorry metrics use case where you're like doing complete test scores and we would probably run it as a deployment for anything else um because a deployment gives you everything that you need pretty much um because the collectors are stateless they don't need to hold on to anything yeah um deployments are much more lean as a result yeah yeah um they can just run and roll out and everybody's happy and that's how we run most of our collectors deployment and then at what point would a demon set be useful yeah so demon sets are really good for um things like her node scraping um which we do a lot of so um this allows you to scrape like the kublet that's run on every node um it allows you to scrape the uh node exporter that's also run on every node which is another Prometheus demon set that most people run right right yes the demon sets guarantee that you've got odds running on every node right exactly every node that matches its selector right right right yeah um and so that's really useful for like scaling out so if you have like a cluster of like 800 plus nodes um it's more reliable to run like um a bunch of little collectors that get those tiny metrics rather than a few bigger staple set pods right because your blast radius is much lower so if like one pod goes down you lose like just a tiny bit of data but remember like with all this cardinality stuff that's a lot of memory so um if you're doing like a staple set scraping all these nodes that's a lot of targets that's a lot of memory it can go down much more easily and you lose more data luckily the collector isn't like Prometheus where we don't care about that state so if a collector goes down it comes back up super fast so usually the blip is low but it does mean that the blip is more flappy right where like it could go up and down pretty quickly if you if you're past the point of saturation that's why it's good to have like a HPA horizontal Auto scaler right right on that stuff but still demon said is a bit more reliable right and it sounds like it would be useful again like from a metric standpoint yeah yeah tracing um you could do it for tracing um and just send it on like a node port but tracing workloads again because it's all push based they are much easier to scale on um and you can distribute targets you can load balance like there are all these other benefits that we get from push-based workloads pull based is like the reason that Prometheus is so ubiquitous in my opinion is just because it makes local development really easy um where you just can scrape your local endpoint and that's what most back-end development is anyway so you could like hit end point a and then hit your metric set point and then hit end point a again metric standpoint you can just like check that it's like a very easy like developer Loop no it also means that you don't have to reach out side of the network so if you're a really strict like proxy requirements to send data local Dev is much easier for that right just why like hotel now has like a really good Prometheus exporter so you could do both right right right if if you have that hankering for for running Prometheus yeah um and and then um I'm assuming there's a centralized Gateway somewhere or um or on flights um this is part of the like collector chain that I was talking about um again we're running a lot of experiments cool um I can like half talk about it okay um I can be vague um a big effort within hotel right now is um around Arrow which you might have been hearing about some um the there's been some work done by lightstep and F5 to improve the um processing speed and egress and Ingress costs of otel data um by using Apache Arrow which is a project for columnar based data representations um and so we're just like doing some group of Concepts or like proof of implementation work to to um see what the actual performance of this stuff looks like right um and also like you know check that everything works as expected yeah yeah as well which it is but that's you always have to check yes absolutely well I'd say the main takeaway from from this whole story on collectors is like it sounds like it's always going to be an evolving game which is not a terrible thing to do no it's important that you keep your telemetry up to date yeah um I think that like Library authors and maintainers are like constantly working on new performance features and new ease of use like quality of life stuff as well yeah yeah um especially with an Hotel like we talk about quality of life a lot yeah um yeah and so that is definitely a focus and that's why it's important to keep up to date it makes migrations easier as well trying to migrate from like an ancient version of something to a latest version probably missing a lot of breaking changes potentially yeah and you have to be careful of that and and on that vein then how do you ensure that everyone's keeping up to date with the latest versions of hotel across the org um I think like stuff like depend about is pretty good um we use it internally or not internally we use it um in a hotel we're keeping up to date with Hotel stuff so I find it to be really helpful it's very frustrating sometimes because it's like hotel packages all update and like lockstep pretty much that means you have to update update fair amount of packages at once but it does do it for you which is pretty nice that's nice that's nice um but you should be doing this not just for hotel but like any dependency yeah right like CVS happen in the industry constantly and if you're not staying up to date with um vulnerability fixes then you're opening yourself up to security attacks which you don't want so um yeah yeah do something about it is is my recommendation that's fair that's fair um I know we've got like four minutes left um do you want to give us any like parting thoughts as we as we wrap up um I'm interested to hear if there are any questions from the group that we've we've missed in our discussion here any takers with burning questions Now's the Time if not I'm going to call on uh Rhys but I this was great um I feel like uh I was like rapidly trying to take notes um I probably will have more as I try to like go back and tidy up some of my notes um but yeah honestly I was just trying to like keep up with people with so much information I feel like because I've been reading up on the on on the operator like the last week and so like more questions than answers and I feel like I have some answers now um I I've definitely learned a lot um I hope folks on this call have learned a lot as well I think this is like really great information I think it gives it gives folks like an idea of like what's involved in a migration things to consider like when you're setting up your your collectors um things to avoid doing keeping up with those latest versions of Hotel y'all always a good thing yeah that's the big one is that like new fixes um really often yeah yeah like new versions every two weeks for The Collector I'd say it's like a pretty frequent Cadence there are some Prometheus libraries that like don't update like ever like Thanos hasn't released since March right which is absurd to me because there's like a bug in compatibility between Prometheus and Thanos right now oh really how cheap yeah so you can use them together if you're like coding so not fun yeah damn all right last chance for for burning questions y'all Erica said thanks for the info on your time Jacob thank you Erica for hopping on um yeah this was really awesome thank you so much um because like for real this is a dope dope topic so yeah we'll you know reach out to your uh friendly neighborhood uh cfp approvers no thank you so much I feel like there was so much more we could have chatted about as well so we might um yeah yeah maybe I can um if I get accepted for the talk then I can give a sneak peek at least I can get some feedback on it actually you know what like if you're if you're interested because we have hotel in practice which is kind of like giving like a little talk so if you're interested in doing that even like before you find out whether or not you get accepted like we are happy to have you yeah it sounds great cool I'm in cool cool we'll uh we'll figure out the behind the scenes on like when to schedule you and um sounds good yay cool all right and Daniel said thanks Jacob as well and um yeah we're at the top of the hour thanks for joining us thank you all thank you yeah thank you everyone bye [Music] +thank you welcome everyone to Hotel q a thanks for joining um today we are super lucky to have Jacob aaronoff from lightstep from service now join us um it's it's a cool treat because um like I tapped Jacob because he was telling me that he had submitted a cfp to one of the I think it was cubecon right Jacob yeah it was a cfp on like um migrating to Hotel um from like our or like our organization migrating at hotel right if I if I understand correctly which I thought that's pretty freaking cool story to tell so um so I asked them to to join and and have this q a and um and hear the story because I think it's it's I think it makes for a compelling narrative like when an observability vendor talks about using otel themselves on their own products so here we are um so welcome Jacob um do you want to do like a quick little intro sure um yeah hi my name is Jacob arnoff I'm a staff engineer at lightstep um on the Telemetry pipeline team um I've been to lead stuff for almost two years uh and the first like year of that Journey was solely focused on like Hotel migrations um making doing them internally and making them easier for customers um sort of that whole process um so yeah I could get into this anyway that is interesting how do you think do we want to do this very q a style or should I just go right into the story talk about um we can we'll we'll let it evil organically about um cool yeah I like that why don't we start like um I mean I think you're you're rearing to go so maybe why don't you describe like set the scene for us so um when I joined lightstep uh we were still on open tracing for uh tracing um and then a mix of open census and some hand rolled stats key stuff or metrics so this meant that we had to run a proxy on every single pod that we ran in kubernetes and a proxy since it's a sidecar on every pod which means that every single time you run it one of your applications you have to run another little application that's going to read from statsd and then forward those metrics off um and you know for us as we're building out a metric solution that destination was light step so I came in and this was sort of at the beginning of like metrics Alpha I think um and I was like hey it would be great for us to we know that um open uh we know that like hotels hotel for metrics is going to be a huge effort for us in the next like year or two we wanted to reach stability um the hotel team had been we have like an Hotel team internal light step they've been working on it a lot and really wanted some um immediate feedback on how to improve it so I took on that migration for us we also had the theory that doing so would save us you know good chunk of money because we would no longer need to run these relatively expensive stats key um uh stats the sidecars so I planned it initially um to be sort of as safe as possible I'd done some migrations like this in the past um and there are a few different ways that you can do migrations like this you can do the all-in-one go uh which for us would have been possible we're in a mono rebuild um but it's much more dangerous because you you worry about you know am I going to push a bug that's going to take everything down right um obviously this is application data that it's data about your applications which we use for alerting we used to understand you know how our workloads are functioning um in all of our environments and so it's important that we don't take that down because that would be disastrous for us but obviously for you know an end user is going to be the same story they don't they want the comfort that if they migrate to Hotel they're not going to lose all of their alerting capabilities immediately they're not like you want a safe and easy migration so that's the only one with our initial approach for doing a sort of like feature flag based um just like part of that can part of the configuration that you run in kubernetes um it would disable this sidecar enable some code that would then swap to otel for metrics and then forward it off you know where it's supposed to go um so that was sort of the path there um Midway through this journey of doing these migrations I had tested it all out and staging looks pretty good I tested the container in our meta environment so we use to monitor our public environment um I noticed some pretty large performance issues um in those 20 2021 and I had reached out to the hotel team and we had worked together to sort of alleviate some of those concerns one of the ones that we found that was a big blocker was we heavily use attributes on metrics right now and it was incredibly tedious to go in and figure out why you know figure out which metrics are using all these attributes and getting rid of them so um well I had a theory that like really this one code path was the problem where we're doing the conversion from our internal tagging implementation through Hotel tags which had a lot of other logic with it that was pretty expensive to do on pretty much every call so I was like you know what this there's no better time than now to begin another migration for her open tracing to otel basically while we wait on the hotel folks on the metric side to push out um more performant code more performant implementations for us we can also test out this theory that if we migrate the hotel entirely we're actually going to see even more performance benefits so at that point I said okay I'm going to put a pause in the metrics work while we wait for hotel and I'm going to begin on this tracing migration nutrition migration however um I decided to try a different approach which is the All or Nothing approach basically um the open tracing to open Telemetry path was a bit more known there were a few really like small docs and examples and they are backwards compatible like you're able to use them in conjunction with each other so one thing admitting Hotel one thing emitting um open tracing is not the end of the world so you can mix those as long as you have the propagators set up correctly so first step so propagators Second Step um make sure that all of our plugins worked which at the time they weren't open sourced uh now they are open source so people can just use them um and I just did it all in one swoop maybe I had to revert like three times from our staging environment nothing really major um and then there was one bug that I missed where um we were previously doing a lot of in-app sampling because we had a really noisy function call um so I had to implement the custom sampler which is actually like 10 times easier with otel than it was with open tracing I was able to get rid of a good like 1000 lines of code or so and some really dangerous hacks so that was a really good thing um and yeah so then that went out very happy um also stopping at any time if this if we want to get back to uh like more q a stuff but I I have so many questions from this already but do you mind do you mind taking like a quick pause because I I'm like sure um okay so a couple comments so one thing that that came to my mind as you're saying this I'm like this is actually really freaking cool story because like I think for there we we tend to see like two different types of organizations right we see the ones where there's like zero code instrumented like they are this is like their first foray into instrumenting their code and then we see the the organizations that I think have either that have like dabbled in open tracing so I think it's a really cool story because this is like a real life migration story where you can actually provide advice on this is what you can do if you find yourself in this situation which is really cool um so I wanna I wanna call that out because I I think um I think that's a really important thing um especially if you're like starting to get into open Telemetry um the other thing that I wanted to ask you about because uh you said that it's um that it's a monorepo um so in that case did you find it and especially since you did the All or Nothing approach um did you find having a mono repo more challenging than if you'd been dealing with microservices instead um yeah well so we we do use microservices it's just that we have like um repo of microservices sorry my my bad about you guys um in that case yeah then um how do you um how do you know like because I mean yes you're going for like a big bang approach but you got to start somewhere so then like where where do you uh where do you start so I started with um and it was the same with how I started the metrics migration um I started with really small services that um my team owned that were really low traffic but enough for it to be constant um so the reason that you have to that you want to pick a service like that is if it's too low traffic if you're only getting like one request every minute one request every like 10 minutes um you have to worry about sample rates um you might not have a lot of data to compare against really that's like the big um thing that you need to have is some data to compare against I wrote a script early on for the metrics migration that um just queried uh different build tags um that are on all of our metrics so you would say you know query all of the metrics for service X um grouped by release tag and if you see that the standard deviation for the newer build tag is like you know greater than one right so if it's one or more standard deviations away from the previous release then there's probably something going wrong in your instrumentation Library right if you assume that your metrics are relatively stable then if they're not it's important to know um the other thing I had to check for was that all of the attributes were still present before and after migration which is another thing that matters sometimes they weren't because something might be something that stats T just adds automatically that we don't really care about and so those were acceptable I just like hand waved instead those are fine we don't care um for tracing is sort of the same deal where I picked a service that had um both internal only traces traces that stayed within a single service and then traces that um span multiple services with different types of instrumentation so coming from like Envoy to hotel to open tracing and what you want to see is that the trace before has the same structure as the trace Factor so I made another script that checked that those structures were relatively similar and that all of them had the same attributes as well right right um because tracing attributes again I was doing an attribute migration that was really the point of doing the tracing one so what matters that all the attributes um stayed the same right yeah it's interesting too because like because you're you're starting from the point where you were migrating from an existing thing like you have that frame of reference which I guess is kind of a double-edged sword right because on the one hand it's like you know you pretty much know that you've instrumented the things hopefully maybe maybe you'll discover as you go along that there's like more stuff to instrument but at least like you have a baseline to start from but then I guess on the other hand if something's missing you're like oh damn why is that missing right yeah and those you know why is this missing stories or the really complicated ones because of course sometimes it's easy to just like you know oh I forgot to add this thing in this place and that's usually pretty simple but sometimes it's like oh there's an upstream library that doesn't admit the thing or hotel and now I need to like again this is like early stuff most of these have all been upgraded and are fine now um but there is an example with like our grpc um actually this is like an interesting one um I had done a grpc I migrated on a grpc util package which I think is now in like token trip um the like Hotel goken trip um and there was an issue with propagation I was trying to understand you know what's what's going wrong here and when I looked at the code and it just tells you how early in this story I was doing this migration um where there was supposed to be a propagator there was just a to do oh no um so and the city was from someone on it was from Alex book which I'll call him out um and so I I sent it to Alex and I was like hey this is a funny to do because I just you know took down um an entire Services traces and staging um so I spent some time to like fix that to do so it wasn't that difficult it was just that they were waiting on another thing I mean that's how it goes you're waiting on someone else which you know another person it's just endless cycles of that type of thing yeah um but then I got it working so that was like one of the main uh blockers for us nice nice and was able to Upstream it as well it wasn't just a fixed for ourselves it was a fix for the community and so that that was yeah there were a few things like that um a lot of the metrics work um actually resulted in big performance boosts for um Hotel metrics like Hotel go metrics and um it also has given the specs folks like some ideas about how descriptive the API should be or various features so things like um uh views and the use of views is something that we did heavily in that early migration because we were worried about can you just yeah definitely just tell folks what you mean by that yeah so um of metrics View is something that's run inside of the your metrics provider in otel your media provider in hotel so what it's doing is it's saying you can configure it to do kind of a lot it could just be um drop this attribute whenever you see it if some if you're a centralized SRE and you don't want anybody to instrument code with any user ID attributes because that's a super high card analogy thing it's going to explode your metrics cost right so you could just make a view that gets added to your instrumentation that says just don't let this attribute from being recorded just deny it um so that's like it's probably most common use case um there are other ones though more advanced use cases for dynamically changing things like the temporality or the aggregation of your Matrix so temporality being cumulative or Delta for like a counter um I you know am I recording um zero one three I'm trying to two crazy zero one three or am I recording one and two right right um and then your uh aggregation is going to be about how do you um I these things are so I always struggle to explain all of them and I'm trying to like come back to uh what I had done in that moment um talk about temporality oh aggregation is like um oh man being on the spot is so much harder because it's like I want to look it up but feel free to look it up that's totally cool well aggregation is like how you um like send off these metrics basically we had an aggregation that instead of doing um well for histograms this is like most useful when you record a histogram there are a few different types of histograms um datadog's histograms stats of these histogram is not a true histogram because what they're recording is um uh like aggregation samples so they give you a min max sum count average um and so I actually don't even think they give you some I looked at this last night they don't give you some they give you like min max count to average and like P95 or something um and so the problem with that is in distributed computing if you had multiple applications that um are reporting of P95 the there's no way that you could get a true P95 from that observation with that um aggregation um the reason for that is that in order to get P95 like you you can't if you have five p95s oh five P95 observations there's not an aggregation to say give me the overall P95 from that right you need to have something about the original data to actually recalculate it you could get the average of the p95s but that's not a great um metric that's not really like it doesn't really tell you much it's not really accurate um and if you're going to alert on something if you're going to page someone at night you should be paging on accurate measurements yeah um so initially though we did have a few people who relied on this min max some counter instrument so we used um Hotel views in the metrics SDK to configure custom aggregation for our histograms to do emit what some would call a distribution um what Oto calls an exponential histogram um or the min max and the Min Maxim count so we were dual emitting this works because they're different metric names that we were emitting so there was no overlap between them so what we did was we migrated after we did the metrics migration we were able to then go back and say any dashboard any alert anything that had was using a min max some count metric um just change it to be a distribution instead and because we had enough data in the past like you know a few weeks months of running Hotel metrics in our public environment that was possible to do um so that that was like one of the key features that because we had it it was 10 times easier um and we were able to do it from the application uh we didn't have to introduce any other components which is pretty neat right right cool um um another question that I had for you um so like when you were doing this migration it was traces and metrics were in existence logs I believe would not have been like the specification would not have been ready possibly not even in the works no it was still really early for that so um but I guess in lieu of logs there's span events that you could use so it's not something like that was leveraged as well definitely um we've heard a long time have you used span events analogs um or a lot of things um internally um I'm a big fan of them I am not a huge fan of vlogging I find it to be really cumbersome and really expensive um and iops for like tracing and Trace logs whenever possible I find it like easier for myself to reason about there are other people who are like logging first and that's great but um that's just not who I am um I like I like logging for local development and tracing for distributed elements that makes sense um but we use this heavily that was one of the first things that I checked worked um It's actually an interesting bug where we had some custom code or open tracing that allowed us to serialize like Json blobs in the span events um and that stopped working because we didn't emit them in the same way uh it's like a little hazy but um I had to like rewrite a processor to make that work and then update some Downstream code like in lightstep as platform to Facebook cool so now how about um keeping that in mind like now that logs are more mature is there are there any plans to do any conversions like and and please correct me if I'm wrong but my understanding too is that like with the log specification like maturing more and more that um like span events are going to be replaced by logs in some form like it's going to be the log specification for span events have you heard anything around that like no this is a bit outside of um where my recent Focus has been so I'm not positive um I think right now the way that we do I think the thing that we would change is how we collect those logs potentially um right now we use um uh how do we do this right now it changed recently I don't want to say something incorrects but um we previously did it by just using like Google's logging agent where they basically are running like fluent bit on every um node in a in the gke cluster yeah and then they send it off to like gcp and they just like tail it there um I think this changed though and I'm not sure what we do now okay cool cool um speaking of kcp of many questions on gke specifically like so um do you um because I I believe there's like a feature now in like newer versions of of kubernetes where there's like some um I think there's there's like some Telemetry collection do you know if that's been enabled in any of the uh in any of the Clusters yeah so I think that kubernetes now has the ability to emit like the hotel traces natively yeah yeah yeah um I'm not sure if we're collecting those yet um I don't know what version that's more of a like a question for this sres I I don't um kubernetes I came out like I think even last year starting whatever like last fall kind of thing yeah that's a really good question um that I want to look into because I want to see if uh really what I would like to do is see if we can collect the if the traces that we get from those are enough to use the spend metrics processor to generate like better kubernetes metrics from those traces um I'm very focused on like infrastructure metrics like kubernetes infrastructure metrics um and I find them to be very painful in their current form um and it would be really cool right now um I prefer to use the Prometheus apis for them currently um it's just a bit more ubiquitous in the like observability Community to use Prometheus to do that um just because like that's that's what kubernetes like natively emits right right go ahead oh uh no go ahead I'll let you complete the thought maybe it answers my questions um and so that's what we do right now and I use the target allocator which is you know nutshell component that I work on um to distribute those targets which is you know pretty efficient way of getting all that data um we also use like demon sets as well that we run in our clusters to get that data um in addition to that so that works pretty effectively the thing that's frustrating is just Prometheus um Prometheus script failures can be um a super common problem and it gets really annoying when you have to like worry about metrics cardinality um as well because it can explode yeah I actually found a bug in gke maybe six to eight months ago six seven months ago but they've since fixed where they weren't deleting uh they weren't reconciling certificate signing requests in their kubernetes cluster which meant that for Kube State metrics which reports on cluster State um it was oming because there were so many um certificate signing requests left from these abandoned nodes and so something on the magnitude of like six hundred thousand for like a single metric which is huge and so then Prometheus the Prometheus that I was running fell over because of that um and that's like a thing that happens constantly in this Prometheus realm which is just like someone admits a high cardinality metric Prometheus goes to scrape it and then it just like crashes oh wow um um which isn't Fun yeah that does not sound fun um I want to just take a step back because you you mentioned the target allocator I was wondering if you could expand a little bit on that because I know like we we actually had one of our previous q a folks also mentioned the target allocator um that was the first time I had heard of it so I think it'd be like super helpful to just get a little overview sure yeah so apparent allocator is component um part of the kubernetes operator um in hotel that does something that Prometheus can't do um which is uh dynamically Shard targets amongst a pool of scrapers so previous has some experimental functionality for sharding but you still have a problem for um querying because Prometheus is a um database not just a scraper yeah um if you Shard your targets you don't necessarily you have to do some amount of coordination within those Prometheus instances which gets expensive it's like a very experimental feature um or you could scale Prometheus with something like Thanos or cortex which is um grafana's Prometheus scaling solution I think right yeah which works but you just then have to run like six more components that you then need to Monitor and then if those go down how do you met a monitor it's all these other problems right right um an hotel we just basically like uh tack on this Prometheus receiver to get all this data but um because we want to be more efficient than Prometheus because we don't need to store the data we tell we have this component the target allocator which goes to do the service discovery from Prometheus so it says give me all of the targets that I need to scrape and then the target allocator says um with those targets distribute them evenly amongst the set of collectors that's running oh okay um so that that's the main thing um that it is doing it does some more stuff around um job Discovery now or if you're using Prometheus service monitors which is part of the Prometheus operator which is a very popular way of like running Prometheus in your cluster um it's what a lot of like vendors use as well so if you're on GAE or openshift I think both of those natively used service monitors and pod monitors oh so the target allocator can also pull those service monitors and pod monitors and uh update the collectors scrape configs to do that oh cool it's awesome um and so one related to the Prometheus thread um are you running like Prometheus itself or are you just scraping the Prometheus metrics and pumping them through to the collector then exactly right just um no Prometheus instances just um The Collector running Prometheus receiver and then sending them off to light step oh living the dream that that was always like that was always my dream that's awesome that's nice um do you use like because I I remember like lights up has like a Prometheus um like a Prometheus operator that uh helps facilitate that so we used to have this thing yeah we used to have this thing called the Prometheus sidecar which you might run yeah you would run it as part of your Prometheus installation which would then sit on the um on the same pod as your Prometheus instance and read the uh write ahead log that Prometheus has for like um persistence and uh batching and all these other things yeah and so we would read the right ahead log and then forward those metrics but if your Prometheus is very noisy as many customers have very noisy Prometheus statistics yeah um it's not really efficient it can get really noisy and it not that uh what's the word it's not the best thing to run the collector is like the fact the best way to run okay so and and it sounds like this thing still requires um to have Prometheus installed and yeah you would still need to be running a whole computer system oh okay I I thought it was I was under the impression it was like a replacement for Prometheus and that it was um maybe I'm thinking of something else his there was a thing that I knew it was like a replacement for for like needing Prometheus um and it was like vendor neutral so it wasn't like oh you have to use lightstep to use this thing um I think I might just be a hotel operator collector Target allocator Trio oh oh okay okay but maybe there's another thing out there oh unless unless maybe that got integrated into like the target allocator as part of anyway it is a mystery yeah okay cool cool oh that's awesome um so then okay since we're we're talking collectors now um I for me like the two million dollar question the one that I'm always curious about is collector setup so what what is the collector setup that you have that y'all have chosen has like what works for for now yeah it's hard to say because we run a lot of different types of collectors yeah headlight stuff we run like metrics things tracing things internal ones external ones there are a lot of different collectors that are running at all times you have like a separate one that just collects metrics and one that just collects traces and um right now we don't um it's all varying flux okay right now we're changing this a lot um to run like experiments and stuff um basically like the the best way for us to be able to make features for customers and end users is by running them ourselves and then using them internally making sure that they work and then sending them for the open source realm and so that's what we're trying to do even more of like we're kind of reaching a point where uh we dog food everything which gets really um confusing because you have to like yeah I can imagine yeah we're running like in a single path there could be like I think two different two collectors in two environments that could be running two different images in two different versions like it's it gets really meta and very confusing to talk about yeah yeah I can imagine and then you know if you're sending from collector eight across an environment to collector B um collector B also emits Telemetry about itself which is then collected by collector C yeah and so it like it's just chains like you basically ensure that that you have to like make sure that the collectors actually working yeah you have to be sure that everything along this path yeah you just have to know which thing has the data right well you shouldn't have to we do that for you but like right um yeah and we make like dashboards to help with that but um that's like the problem when it's like we're debugging this stuff is when there's a problem you have to think about like where's the problem actually is it in how we collect the data is it and how we emit the data is it in um you know the source of the how the data was generated it's it's you know one of like a bunch of things yeah yeah um now like you need to work on the hotel operator so and and I I've been reading up on the operator recently and there's like I think four different deployment modes right there's sidecar deployment demon set and um what's the other one um yeah nice yeah um yeah um so my question is um which mode um or is it like all of the above depending on the thing that you need to do yeah it's all the above depending on what you need to do um and you're neat like your general needs and uh like how you like to run applications for like reliability and stuff yeah so um sidecar is the one that we use the least and is probably used the least if I were to make just like a bet um sidecars are really useful like across the industry you would think yeah across the industry I'd be willing to bet that those are the least popular that's the least popular method sidecares are just expensive and if you're not using them if you don't really need them then you shouldn't use them and you'll really only need them like something that's run as a sidecar it's like istio which yeah yeah makes a lot of sense to run as a sidecar because it's doing like traffic proxy like hooks into your um you know container Network to change how that all does its thing yeah and you get a performance hit uh if you sidecar your collectors right for all your services you just get like a cost it would just cost you a lot more um and you also wouldn't be able to do as much with like um if you're making like kubernetes API calls for attribute enrichment that's like the thing that would get exponentially more expensive if you're running it as a sidecar right but as like a staple set of like you know five pods that's not that expensive but if you have a sidecar on like 10 000 pods then that's 10 000 API calls made to the kubernetes API right yeah yeah that's exclusive what um what would be the advantage of running um your collector as a full set versus a deployment like where I guess what's what's the state that you would want to persist yes this is um like I don't know the right word is an unknown stateful sets aren't only used for their ability to mount volumes um there are a few other things that are inherent to how staple sets run that are really valuable in distributed computing this is an important thing to know for not just like how The Collector runs as an application but how like your applications can run right um stateful sets have um consistent IDs so if you have a staple set with 10 replicas they're all going to be um the staple set name Dash um um counter number so it goes from like zero to n so okay that's a really valuable thing when you want consistent IDs right as opposed to like with with deployments like when you like your your pods are all like random random crap right yeah so the Pod IDs are done where you it's um deployment name Dash replica set ID Dash pod ID right and so with staple sets because we have this consistent IDs we can actually do some extra work with uh for the Target allocator which is why we require that and so the other thing that stateful sets guarantee is what's called a um In-Place deployment which is what demon sets do as well where you have you take the replica you take the Pod down before you create a new one um right so the reason that this is important is that in the deployment you normally do a one up one down right um or what's called a rolling deployment a rolling update and so if we were to do this for um uh if we were to do this for the uh with the target allocator um we would probably get much more unreliable scrapes because you would and someone actually just asked this question in the operator Channel I mean I'm going to give them this exact response um when a new replica comes up um you have to redistribute all the targets because your hash ring that you place these on is changed so if you're doing a rolling deployment if you're doing one up one down that's a really expensive operation um because then you have to recalculate all these hashes that you assign um so if you were to do a one down one up you would still have to redo this whole thing because um you would lose a pod which means it's taken out of the ring redistribute you would gain a new ID and then you'd have to redistribute again right um whereas stateful sets because it's a consistent ID range you don't have to do that at all and so this means that when we do a one down one up it keeps the same targets each time right right so it's like it's almost like a placeholder for it like you don't have to recalculate the ring basically because yeah it's just sort of like a little um uh what's it called uh yeah yeah yeah yeah I can't think of the word cool that's really neat I didn't know that yeah it was funny because I was reading about like pause being deployed a stateful sets I'm like straight or sorry not uh collectors I'm like I straight up do not understand what the use case would be but this makes a lot of sense so that's uh that's really cool and so it's not as useful um or this is really only useful for like a tracing use case I would say or sorry metrics use case where you're like doing complete test scores and we would probably run it as a deployment for anything else um because a deployment gives you everything that you need pretty much um because the collectors are stateless they don't need to hold on to anything yeah um deployments are much more lean as a result yeah yeah um they can just run and roll out and everybody's happy and that's how we run most of our collectors deployment and then at what point would a demon set be useful yeah so demon sets are really good for um things like her node scraping um which we do a lot of so um this allows you to scrape like the kublet that's run on every node um it allows you to scrape the uh node exporter that's also run on every node which is another Prometheus demon set that most people run right right yes the demon sets guarantee that you've got odds running on every node right exactly every node that matches its selector right right right yeah um and so that's really useful for like scaling out so if you have like a cluster of like 800 plus nodes um it's more reliable to run like um a bunch of little collectors that get those tiny metrics rather than a few bigger staple set pods right because your blast radius is much lower so if like one pod goes down you lose like just a tiny bit of data but remember like with all this cardinality stuff that's a lot of memory so um if you're doing like a staple set scraping all these nodes that's a lot of targets that's a lot of memory it can go down much more easily and you lose more data luckily the collector isn't like Prometheus where we don't care about that state so if a collector goes down it comes back up super fast so usually the blip is low but it does mean that the blip is more flappy right where like it could go up and down pretty quickly if you if you're past the point of saturation that's why it's good to have like a HPA horizontal Auto scaler right right on that stuff but still demon said is a bit more reliable right and it sounds like it would be useful again like from a metric standpoint yeah yeah tracing um you could do it for tracing um and just send it on like a node port but tracing workloads again because it's all push based they are much easier to scale on um and you can distribute targets you can load balance like there are all these other benefits that we get from push-based workloads pull based is like the reason that Prometheus is so ubiquitous in my opinion is just because it makes local development really easy um where you just can scrape your local endpoint and that's what most back-end development is anyway so you could like hit end point a and then hit your metric set point and then hit end point a again metric standpoint you can just like check that it's like a very easy like developer Loop no it also means that you don't have to reach out side of the network so if you're a really strict like proxy requirements to send data local Dev is much easier for that right just why like hotel now has like a really good Prometheus exporter so you could do both right right right if if you have that hankering for for running Prometheus yeah um and and then um I'm assuming there's a centralized Gateway somewhere or um or on flights um this is part of the like collector chain that I was talking about um again we're running a lot of experiments cool um I can like half talk about it okay um I can be vague um a big effort within hotel right now is um around Arrow which you might have been hearing about some um the there's been some work done by lightstep and F5 to improve the um processing speed and egress and Ingress costs of otel data um by using Apache Arrow which is a project for columnar based data representations um and so we're just like doing some group of Concepts or like proof of implementation work to to um see what the actual performance of this stuff looks like right um and also like you know check that everything works as expected yeah yeah as well which it is but that's you always have to check yes absolutely well I'd say the main takeaway from from this whole story on collectors is like it sounds like it's always going to be an evolving game which is not a terrible thing to do no it's important that you keep your telemetry up to date yeah um I think that like Library authors and maintainers are like constantly working on new performance features and new ease of use like quality of life stuff as well yeah yeah um especially with an Hotel like we talk about quality of life a lot yeah um yeah and so that is definitely a focus and that's why it's important to keep up to date it makes migrations easier as well trying to migrate from like an ancient version of something to a latest version probably missing a lot of breaking changes potentially yeah and you have to be careful of that and and on that vein then how do you ensure that everyone's keeping up to date with the latest versions of hotel across the org um I think like stuff like depend about is pretty good um we use it internally or not internally we use it um in a hotel we're keeping up to date with Hotel stuff so I find it to be really helpful it's very frustrating sometimes because it's like hotel packages all update and like lockstep pretty much that means you have to update update fair amount of packages at once but it does do it for you which is pretty nice that's nice that's nice um but you should be doing this not just for hotel but like any dependency yeah right like CVS happen in the industry constantly and if you're not staying up to date with um vulnerability fixes then you're opening yourself up to security attacks which you don't want so um yeah yeah do something about it is is my recommendation that's fair that's fair um I know we've got like four minutes left um do you want to give us any like parting thoughts as we as we wrap up um I'm interested to hear if there are any questions from the group that we've we've missed in our discussion here any takers with burning questions Now's the Time if not I'm going to call on uh Rhys but I this was great um I feel like uh I was like rapidly trying to take notes um I probably will have more as I try to like go back and tidy up some of my notes um but yeah honestly I was just trying to like keep up with people with so much information I feel like because I've been reading up on the on on the operator like the last week and so like more questions than answers and I feel like I have some answers now um I I've definitely learned a lot um I hope folks on this call have learned a lot as well I think this is like really great information I think it gives it gives folks like an idea of like what's involved in a migration things to consider like when you're setting up your your collectors um things to avoid doing keeping up with those latest versions of Hotel y'all always a good thing yeah that's the big one is that like new fixes um really often yeah yeah like new versions every two weeks for The Collector I'd say it's like a pretty frequent Cadence there are some Prometheus libraries that like don't update like ever like Thanos hasn't released since March right which is absurd to me because there's like a bug in compatibility between Prometheus and Thanos right now oh really how cheap yeah so you can use them together if you're like coding so not fun yeah damn all right last chance for for burning questions y'all Erica said thanks for the info on your time Jacob thank you Erica for hopping on um yeah this was really awesome thank you so much um because like for real this is a dope dope topic so yeah we'll you know reach out to your uh friendly neighborhood uh cfp approvers no thank you so much I feel like there was so much more we could have chatted about as well so we might um yeah yeah maybe I can um if I get accepted for the talk then I can give a sneak peek at least I can get some feedback on it actually you know what like if you're if you're interested because we have hotel in practice which is kind of like giving like a little talk so if you're interested in doing that even like before you find out whether or not you get accepted like we are happy to have you yeah it sounds great cool I'm in cool cool we'll uh we'll figure out the behind the scenes on like when to schedule you and um sounds good yay cool all right and Daniel said thanks Jacob as well and um yeah we're at the top of the hour thanks for joining us thank you all thank you yeah thank you everyone bye diff --git a/video-transcripts/transcripts/2023-09-13T19:26:10Z-opentelemetry-q-a-feat-hazel-weakly.md b/video-transcripts/transcripts/2023-09-13T19:26:10Z-opentelemetry-q-a-feat-hazel-weakly.md index a2bd27b..bf898a4 100644 --- a/video-transcripts/transcripts/2023-09-13T19:26:10Z-opentelemetry-q-a-feat-hazel-weakly.md +++ b/video-transcripts/transcripts/2023-09-13T19:26:10Z-opentelemetry-q-a-feat-hazel-weakly.md @@ -10,97 +10,93 @@ URL: https://www.youtube.com/watch?v=wMJEgrUnX7M ## Summary -In this episode of the OpenTelemetry Q&A series, host Hazel Weekly discusses her experiences with implementing observability through OpenTelemetry within organizations. She shares insights on the challenges faced by novice users, including the initial overwhelming volume of telemetry data collected without meaningful queries, and the struggle to align observability efforts with organizational needs. Hazel emphasizes the importance of asking targeted questions to derive useful insights from telemetry data, rather than simply collecting vast amounts of it. She also highlights the difficulties of implementing OpenTelemetry across different programming languages, the ergonomic challenges of manual instrumentation, and the complexities of managing the OpenTelemetry collector. Additionally, Hazel discusses making the case for observability to executives, focusing on the need for tangible benefits and the alignment of technical and business lifecycles. The conversation provides valuable lessons on the practicalities of using OpenTelemetry effectively, the need for thoughtful instrumentation, and the potential pitfalls of over-instrumentation. +In this YouTube video, Hazel Weekly shares her experiences and insights on observability and the implementation of OpenTelemetry in organizations. She discusses her journey as a novice user approaching OpenTelemetry, the challenges of over-instrumentation, and the importance of asking meaningful questions to utilize telemetry effectively. Hazel explains how she navigated obstacles in organizations, such as managing excessive data volume and ensuring meaningful instrumentation, often finding herself as a one-person team tackling complex issues. She highlights the cultural and technical hurdles in convincing teams and executives to adopt observability practices while emphasizing the need for thoughtful implementation of telemetry tools. The conversation covers the intricacies of sampling, the challenges of using various programming languages, and the significance of aligning technical capabilities with business needs. Hazel also touches on the complexities of regulatory compliance, particularly in relation to government standards like FedRAMP. Overall, the discussion provides valuable insights into the practical application of OpenTelemetry and the importance of a strategic approach to observability. -# Observability Q&A with Hazel Weekly +## Chapters -[Music] +Sure! Here are the key moments from the livestream along with their timestamps: -**Welcome everyone to our OpenTelemetry Q&A!** We have the pleasure of having our good friend Hazel Weekly here to discuss her experiences with observability. Welcome, Hazel! +00:00:00 Introductions and welcome to Hazel +00:02:30 Hazel shares her initial experiences with OpenTelemetry +00:05:00 Definition of observability from different perspectives +00:08:15 Discussion about the challenges of over-instrumentation +00:12:45 How to encourage teams to focus on meaningful telemetry +00:15:30 The importance of asking the right questions in observability +00:20:00 Sampling methods and their impact on data collection +00:27:00 Challenges with asynchronous programming in OpenTelemetry +00:35:00 Insights into convincing executives about observability needs +00:42:00 The complexities of running OpenTelemetry in production environments +00:46:30 Wrap-up and next steps for Hazel's future talks -**Hazel:** Glad to be here! I'm very excited for this. +Feel free to reach out if you need more information or assistance! -Let’s start with some foundational concepts, especially for our novice users who are just getting into OpenTelemetry. From your perspective as someone new to OpenTelemetry, what was your experience like? What kind of landscape were you entering? +# Observability Q&A with Hazel -**Hazel:** First, I want to share my definition of observability, which is slightly different from the traditional control theory definition. Control theory defines observability as the ability to understand a system from its inputs and outputs. Fred Herbert has a definition from cognitive safety systems engineering that focuses on the process required for a group to understand everything about a system. +Thank you for joining us for our observability Q&A session. We have the pleasure of having our good friend Hazel Weekly here to discuss her experiences with OpenTelemetry. Welcome, Hazel! -My definition centers on the process through which you develop the capability of asking meaningful questions and getting useful answers. It’s evolutionary; the questions must be meaningful to you, and the answers don’t always have to be correct, but they need to be useful. +**Hazel:** I'm glad to be here! I'm very excited for this. -When I look back at my company’s initial approach to observability, I consider the types of questions being asked from various perspectives—engineers, managers, and executives. Initially, the engineers had instrumented their systems, recognizing they needed observability, but they lacked motivation. They weren’t asking sophisticated questions that would drive them to add their own telemetry. They had a lot of telemetry data but were not querying it effectively. +Let's start with the basics, since we have a lot of novice users just getting into OpenTelemetry. From your perspective as someone who was new to OpenTelemetry, can you share what that experience was like? -I was able to turn down the volume of that data significantly and encourage them to focus on asking meaningful questions. They had to understand their system to collect only what they actually needed. It's essential to have that second layer of understanding; without it, they were just instrumenting for the sake of instrumenting. +### The Initial Experience with OpenTelemetry -**Interviewer:** It sounds like they were instrumenting everything without a clear purpose. How did you manage to convince them to focus on more directed instrumentation? +**Hazel:** Sure! When I first joined the company, I would say that the definition of observability I hold is slightly different from the traditional one. From control theory, it’s about understanding a system from its inputs and outputs. In cognitive safety systems engineering, it’s about the processes required for a group of people to understand everything about the system. My definition focuses on the capability of asking meaningful questions and getting useful answers. This process is evolutionary, and the questions must be meaningful to the people involved. -**Hazel:** Convincing them to turn down the volume was relatively easy, primarily due to cost concerns. They were over budget with their vendor, and I pointed out that I needed to help them reduce costs. +Looking back, the company was at a stage where they were starting to think about observability, but they weren't quite ready for it. They had a lot of instrumentation but lacked the motivation to ask sophisticated questions. They had a high volume of telemetry data, but no one was actually querying that information. I managed to turn down the volume of telemetry significantly and encouraged them to only add instrumentation thoughtfully and intentionally. -At one point, their ingestion was about 300 million events per day. I figured out some misconfigurations in their sampling and got it down to about 3 to 5 million events per day. Once we accomplished that, they became more comfortable asking meaningful questions. +### The Hurdles of Over-Instrumentation -Then we could enter a useful dialogue: now that you have a specific question to ask, build what you need to get that answer. This feedback loop is crucial in observability. +It seemed they were instrumenting just for the sake of it, throwing everything at the wall and hoping something would stick. How did you convince them to dial it back and make it more directed? -**Interviewer:** That’s a great approach! Knowing what questions to ask certainly makes for a more meaningful experience. Can you tell us about your sampling methods? +**Hazel:** Convincing them to reduce the volume was easier than expected, largely due to cost concerns. The company was over budget with their vendor, and I emphasized the need to get under budget. I managed to reduce the ingestion from hundreds of millions of events per day down to about three to five million. Once we established that they needed to ask specific questions, we could start building what was needed to get those answers. -**Hazel:** We used two methods for sampling. Initially, we relied on the OpenTelemetry collector but later transitioned to using Honeycomb Refinery. My preference is to use both because the OpenTelemetry collector has a more open-source-friendly configuration, which allows us to send data to multiple sources and sinks. +### The Role of Sampling -However, running both can be complicated for some organizations. For my current company, which uses DataDog, one challenge is that they're not yet at the point of asking sophisticated questions about their infrastructure—they’re still at the stage of figuring out if things are running. +You mentioned sampling earlier. Can you elaborate on that? -**Interviewer:** Going back to your previous organization, after you convinced them to focus their instrumentation, what hurdles did you encounter next? +**Hazel:** We used two methods for sampling: the OpenTelemetry collector and Honeycomb's Refinery. My preference is to use both, but running two pieces of information can be complicated. At my current company, we use Datadog, and they’re still not at the point of asking sophisticated questions about their infrastructure. Transitioning them to OpenTelemetry would require running collectors in multiple places, which complicates the setup. -**Hazel:** One notable hurdle was the organization’s structure. They had a developer productivity team but lacked a dedicated platform team. The engineers were primarily focused on writing code and not on infrastructure. +### Challenges with Instrumentation -To adopt OpenTelemetry successfully, we needed to build libraries and write code that allowed engineers to implement their own instrumentation. However, this required a level of abstraction that was difficult to achieve without proper support. +After convincing the first organization to streamline their instrumentation, what hurdles did you face next? -For example, setting up baggage and propagating that information was challenging. If someone used an async function, that context might get lost. Engineers needed to understand the call stack, not just the functionality, which was a significant barrier. +**Hazel:** One major hurdle was the way the company structured its work. They had developer productivity teams but no dedicated platform team. Engineers weren't responsible for infrastructure, which made adopting OpenTelemetry challenging. Many engineers added instrumentation without understanding the full context of where it was implemented, which led to a lot of dropped information. I spent a lot of time manually tracing errors back to the source code and fixing small issues. -**Interviewer:** How did you navigate those hurdles? +### Navigating Team Dynamics -**Hazel:** I wasn’t able to fully overcome those challenges, but I made improvements. I manually traced down errors and fixed a lot of small things. Even though I improved the situation, it was still difficult for engineers to grasp the full context of their instrumentation. +Were you working alone during this process? -I found that many engineers were not able to connect their code to the broader observability framework we were trying to establish. They needed to be more engaged in the process to understand how their contributions fit into the bigger picture. +**Hazel:** For a period, yes. I was the only one tackling these issues while my colleague was on parental leave. It was tricky, but I had a good intuition for what the code was doing at runtime. I had to engage in a lot of dialogue with the team to ensure they understood the issues and could trust my judgment. -**Interviewer:** You mentioned you were essentially a one-woman show during this time. That sounds quite overwhelming. How did you manage the workload? +### Building Trust with Predictions -**Hazel:** It was tricky, but I found that one of the benefits of OpenTelemetry is that it provides a more intuitive sense of what the code is doing at runtime. I spent a lot of time experimenting and asking questions, but I was the only one actively doing that, aside from a couple of power users. +How did you build trust with your team regarding the issues you identified? -I would predict issues and offer solutions. When my predictions turned out to be correct, people started to believe me more. However, for some fundamental architectural issues, I couldn’t convince them before I eventually left the company. +**Hazel:** I started making predictions about system behavior, and when they turned out to be correct, I gained their trust. For example, I predicted that a spike in database activity would cause performance issues, and when I provided a solution, it worked. However, I struggled to convince them of more fundamental architectural issues before I left the company. -**Interviewer:** By the time you left, did you notice any improvements in their observability practices? +### Observability Improvements -**Hazel:** Yes, there was a better understanding of observability among the team. They had a clearer sense of how to use it effectively. I fixed several issues that allowed them to reduce their data usage significantly. +By the time you left, did you see any positive results from your efforts in observability? -They became more comfortable adding instrumentation without fear of exceeding their budget. However, they still had challenges with their existing auto-instrumentation. The manual instrumentation that was done often didn’t yield useful questions. +**Hazel:** Yes, there was a better understanding of observability and how to use it. I fixed a number of issues that led to more accurate error tracking, which made people more willing to utilize the tools we had. Although they had over-instrumented through auto-instrumentation, the manual instrumentation was often useless. I helped them understand how to structure their data better to facilitate useful querying. -**Interviewer:** It sounds like a classic case of over-instrumentation without a clear strategy. Did you find that organizations often create libraries around OpenTelemetry for better management? +### Observability Challenges in Organizations -**Hazel:** Yes, creating libraries is a great way to streamline instrumentation efforts, especially in a platform engineering context. It helps accelerate developers by providing standard templates and handling the complexities of instrumentation. +You mentioned that many organizations struggle with OpenTelemetry. What do you think is the main issue? -In my experience, while I could set up some libraries that abstracted OpenTelemetry's complexities, achieving a truly ergonomic solution was challenging. The framework often assumes a certain level of understanding and control over the code lifecycle that not all developers possess. +**Hazel:** A lot of times, organizations face cultural challenges. Engineers can be very self-directed, leading to incomplete implementations. They might finish 80% of the work and then move on, leaving things half-done. This leads to a lot of noise in the telemetry data without meaningful insights. -**Interviewer:** As you transitioned between organizations, did you find that any of them successfully implemented these kinds of libraries? +### Making the Case for OpenTelemetry -**Hazel:** I was able to create wrappers around functions that made it easier for developers to get started with OpenTelemetry. This included setting up environment variables and providing a simplified API. +Can you share your experience in making the case for OpenTelemetry to executives? -However, the level of success varied. Each organization has different challenges and levels of expertise, which impacts their ability to effectively implement observability. +**Hazel:** Making the case involves two migrations: a technical migration and a social migration. You need to show that the time and effort saved by migrating to OpenTelemetry will pay for itself in a reasonable timeframe, typically around four months. If you can demonstrate tangible benefits, executives are more likely to invest in it. -**Interviewer:** How was your experience managing the infrastructure side of OpenTelemetry? +### Conclusion -**Hazel:** Running OpenTelemetry can be complex. You need to properly configure the OpenTelemetry collector, and if you deviate from the documentation, you're often left reading the RFCs or technical design documents. - -For instance, the configuration for tail sampling can be particularly daunting. In contrast, Honeycomb Refinery is easier to use but comes with its own set of challenges. I found that many edge cases arose, especially with asynchronous jobs that generated an overwhelming number of events. - -**Interviewer:** Finally, how did you navigate making the case for OpenTelemetry to executives? - -**Hazel:** Making the case for observability requires demonstrating a clear value proposition. You need to show that the time and effort saved through migration will pay for itself in a reasonable timeframe—ideally within four months. - -When convincing executives, it's crucial to align the developer lifecycle with business value delivery. By illustrating how better observability can enhance product delivery and improve developer productivity, you can build a compelling case for adoption. - -**Interviewer:** Thank you, Hazel! Your insights have been incredibly valuable. We look forward to having you back on September 14th for our next session. - -**Hazel:** Thank you! I’m excited to continue the conversation! - -[Music] +Thank you again, Hazel, for sharing your insights! This has been a fantastic discussion, and I look forward to having you back for the next session on September 14th. We appreciate your expertise and the valuable lessons you've provided on navigating the complexities of OpenTelemetry! ## Raw YouTube Transcript -[Music] thank you welcome everyone to otel q a we have the pleasure of having our good friend Hazel weekly come talk to us about um her experiences with observability welcome Hazel glad to be here very excited for this yay I guess let's start first things first because I know we still have a bunch of Hotel novice users like just getting into Hotel folks um who who join our um end user working group so from from the perspective of somebody who's new to open Telemetry can you share what that experience was like what kind of landscape you were coming into um and I guess for starters like drawing back on that experience um was the company at that point ready for observability like did that what what kind of what what kick started the conversation into into open Telemetry in the first place I would say um so I'm going to start off with my definition of observability because it's slightly different um the definition from control theory would be like can you understand this daily system from the input in the outputs and um Fred Herbert might should talk about the definition from cognitive safety systems engineering and that one is much more about the work required and the process required for a group of people to be able to understand everything and actually understand the system and that's how they like discovered and think about that and mine is the process through which you develop the capability of asking meaningful questions and getting useful answers and so it's a process because it's evolutionary and the questions have to be meaningful to you whatever that means and the answers don't have to be correct but they have to be useful so when I look back at the company when we're starting to think about observability and or rather like distribution literally what are the questions that the company is starting to ask from the perspective of the engineers and one of the questions of the company is asking for the perspective of the managers and what is it asking from the perspective of the executives and show from the engineers at one of the companies that are the early hand open telemetry they had it and they had instrumented things with it but more from the perspective of we know we need this but they didn't necessarily have that motivation yet the action weren't asking the types of sophisticated questions that motivate people to add their own telemetry they had a bunch of telemetry and they had a huge amount of instrumentation and they had just the volume turned up to 11 and no one was actually querying that information and so what I was able to do was turn down that volume significantly and then what I did was essentially say you're you're not using this and they're not asking any questions from your system so when you do don't turn the volume back up like add that instrumentation in there with thought and intent and that was an interesting hurdle to happen because in a way people would ask questions is to ask these types of semi's physical questions asking them to understand the system they're asking them to solve a problem and so consequently they just wanted every bit of information ever so that whenever they needed it they could just like go through a giant stack of noise and find a needle right and that's not really what open Telemetry is for online distributation it's for can you understand the state of your system to the point that you only collect what you actually need and that is it in of itself a second layer of understanding the system and they didn't have that second layer right so it sounded like they were instrumenting for the sake of instrumenting and just sort of throwing everything at the wall and hoping something would stick yeah that also sounds like a very I came over from Vlogs and I'm used to being able to search everything ever approach they also still had laws which is a surprise basically nobody of course so um in terms of like how did you convince them to kind of Tamp it down and direct it make it more directed so that it could actually work for them so convincing them to Champion it down and turn the volume down was actually pretty easy that came down to cost they were about 300 over budget from the vendor and so I said I need to get you under budget and so I actually uh in the honeycomb Pond eaters I have a very funny image from that time period where you see the ingestion at about like two to three hundred million events per day and you know I'm trying to tweak in and get it down to about like 200 250 or like you know 180 to 250 and that was with very very aggressive sampling and finally I figured out a bunch of misconfigurations in the sampling information and dropped it to about like three to five million events per day and um and once I had done with that they like had concerns about oh what if I can't find something I said well do you need to ask that question and then we were able to actually start the useful dialogue of now that you have a question you need to ask go and and build what you need in order to get that answer and then naturally that sort of feedback loop that you kind of need with observability which whether or not you need that feedback loop end to end is a different question but that is currently what open Telemetry requires I think that's really great because being able to know what questions to ask makes it for a much more uh meaningful um experience and so you know in your words not not searching for that needle in in the haystack I wanna um go back to a sampler for a second do sampling to a sampling was we did the sampling through two methods we had the open Telemetry collector and I'll show you now but at this company we only had the honeycomb Refinery set up until we have that set up um at other companies we've had the open Telemetry collector setup and I've used that my preference is actually to use both regardless of whether or not using honeycomb and the reason for that is because the open Telemetry collector doesn't have the best steel sampling configuration and the open Telemetry collector has like the most open source line economic sort of configuration setup to consent things to multiple sources and multiple syncs so that is actually really useful so I like to use that one and then aggregate everything and send it to Refinery and then do useful tail sampling um but running two pieces of information sorry is kind of complicated for a lot of people um it's not going to be a huge hurdle of adoption to like my current company they use um data dog and one of the challenges there is they're not at the point where they can ask sophisticated questions of their infrastructure and I'm really still more at the point of isn't on and so the question there is well even if I set things up for success by switching us to the open Telemetry line instrumentation tooling and then still send it to the same place we now need to run like the collector in multiple places or run like a collector and Refinery just in order to do the same thing that they already do with their built-in tooling so that can be a bit of a bit of a lift because rather than saying install Library it's install a library and have like five things also all right now um going back to um the the first organization where you're you know where they were sending like way too much data after you um after you convince them to like kind of chill do more directed um instrumentation um what was the next sort of hurdle that you experienced there's a notch Turtle there was that the way that company instruction work was very much the over platformed thing this little towards unusual but they had like a developer productivity teams and they didn't really have a platform team what they had Engineers that didn't do work so the engineers didn't do infrastructure stuff they additional code and they would have a bunch of people working on like build tooling and build other stuff like that and so the question was in the way that they do work in order to adopt open Telemetry successfully you would need to essentially build libraries for things or write stuff into the code in a way that Engineers weren't necessarily writing their own instrumentation and that requires working at a level of abstraction that open television is really really difficult so like you can't propagate certain types of information down to challenge fans very easily or really at all um setting up baggage is still very difficult I'm working with contacts can be really tricky if somebody uses an async function then you might drop that and have to reconnect it somehow and if the lifetimes aren't like already set up nicely people can even add their own instrumentation and then it'll get dropped from something because they don't actually understand the full context of where it is they're just random cone and they add something but an open Telemetry you have to understand the call stack not just the functionality and those aren't always the same interesting so how do you um how do you get over that hurdle so at that company I was never able to fully get over that hurdle I just made it better one of the ways that I made it better was essentially following through all of the um fans manually and looking for errors and when I found errors I would like traced it down to the source code and I would do that and um so I ran into like a bunch of very small things of like propagating information wasn't corrupt so it was probably the information of like the helper functions rather than the actual cost sides to activate that and then I had to fix like 500 small things and even when I did that I found that there are a bunch of places where he's looking as contacts weren't being complicated so functions are just being launched or for example calling uh tracing and that was being disconnected from the actual whatever or sometimes things had to end before they started or start before they ended and that got confusion because the language that they're using it's very easy to write code in that logic and it actually is more correct to do it that way but open Telemetry in its design is very much a chicken is called stack a tree shaped um which is really really ergonomic for a language like Java it's not necessarily ergonomic for languages that aren't like that so like react is a really common example because it's a runtime that's asynchronous and it's really difficult to write code in react to the deadline nicely um I suppose a good one closer is a good example of one that doesn't work very well um watch does work but for a very interesting reason and that is because the Russian language cares about lifetimes and so you think about the lifetime of your functions and so you always know what those are and those end up being the natural tracing Point protocol stack based uh tradition Library um but in terms of what we're able to do and how I was fixing it was really mostly in guacamole and doing some weird clever things to pack into the language runtime in order to make things um hit the semantic model differences between open Telemetry and other language things about calling code so were you like a one-woman show doing all of this um so the other person on this was off on a parental leave so for the duration then I was doing this I was actually one person so and that is that is mad impressive um did you want to like pull your hair out some days it was tricky um a lot of it was that one of the benefits of using open Telemetry is that you get like this intuitive sense of what the code is doing at runtime if you play with it enough they had to sit there and kind of like doodle with it and like look things up and I ask questions and get answers and like you have to have that dialogue running and I knew how to do that and no one else at the company was doing that except like one or two people wow and so you had like a very small select power users favorite people and then you had everyone else who may not have even like used open symmetry at all or even news like a vendor in any way whatsoever and so I would get an intuitive sense if one was broken and what was Iran or something and I would say hey like this is an issue and then people wouldn't necessarily believe me oh interesting so how how do you end up convincing them that it is an issue or or could you convince them I guess is the question so when I was able to I was able to take advantage certain people of the things and one of the ways that I did that was essentially my predicting things and then my predictions would turn out to be correct and then I will give a solution to the position and then the solution will work so like one was the database was very very spiky and I mentioned that this was going to cause my some sort of images and one of the reasons that the database was Frankie was because we had like basically uh the disk would slow on a database and so we spent up the disk on the database quite a bit and when that happened like despite this level down these sets were still very spiky in general but despite going high enough to lock things down and um there's small things like that added up over time and people started to believe me a lot more when it came to certain issues but other ones that were more fundamental in the architecture I never actually got around to being able to convince people of that and I actually ended up leaving that company mainly due to that interesting so did you by the time you left did you find there was at least like they're they were getting more out of their observability at that point like because of your efforts like did you at least see some positive results because of that yeah I did so there was a lot better understanding of where they were with the observability people have more of an understanding of how to use it um I had like a couple of things so that it was more useful like they had a lot of it but some things were wrong like um if the air stack was there the air stack died at the wrong spot so it wouldn't actually tell you where the era happened it would tell you where you defined the open Telemetry like the tradition helper which is which is pointless so I fixed that and then all of a sudden the earrings were correct again and people were happy about that then they would actually use it so I hadn't fixed a bunch of things and I had reduced the usage from like two to three hundred events per day or even maybe 50 million events per day to like right and when that happened then people could continue to add open Telemetry and add instrumentation everywhere without running into the physical they were going to blow their budget even more than their own work and that allows people to continue to add more instrumentation so then I guess there it seems like there wasn't an issue per se as far as like getting people to add the instrumentation at that point because they were obviously they had already over instrumented the system previously so now you put them in it was interesting they had over instrumented it but only via Auto instrumentation so there is very little manual stuff done and the manual stuff that was done was often done in a way that made the questions you can ask relatively useless so like one example was there's like at the top of her watched halfway down is when in the life cycle of the call stack is when they would so you get the user ID associated with it because of how the database calls worked but not user ID was never propagated to the top to it was actually impossible to figure out um like how many pages a user was visiting like um like procession like these types of questions whether or not I usually like it's on the same thing over and over because the question we wanted to ask was how effective would database account should be but would it be effective to put radish there somewhere and like probably don't have a 60 second something TTL we we had no idea because we couldn't actually ask that question all the information was there but not actually in a way that made anything possible to look for right because you couldn't look halfway down the um Span in order to get this span of correlated with this one so that tree you can only ask for things on the same level level and down and like filter that way and so they need to fix that but no one necessarily knew how to and no one necessarily knew that they wanted to because they weren't asking that type of question so I bought that up and then people started to get more of a sense of oh here's kind of where we put the information in a way that's visible and to this day I still don't have like a really good way to like explain to people how to think about that other than you do kind of need to display the system I couldn't understand where the data needs to be in order to be useful for asking questions right yeah that makes a lot of sense um so yeah that's so interesting like I find this such an interesting um sort of I don't want to well I guess use case of sorts uh scenario if you will um because there oftentimes we hear we hear the stories of people just struggling with with getting bringing open Telemetry into the organization um and and then starting to instrument and this this feels like it was like one giant anti-pattern of one giant open Telemetry anti-pattern which I think it's a very important thing to be able to discuss because you know like all tools open till Elementary can be grossly misused and then you end up with with scenarios like this where you're instrumenting but you're not getting anything out of it mm-hmm it was really interesting to see how this happened um because it's a lot of what happened is due to how the company works and how it thought about working so it was very anarchic in the sense that Engineers are very very self-directed and they would do things and so the company had a huge cultural issue of intricate 80 percent done with Instagram maybe like 90 percent done and then they would just drop off or someone like outside traffic and had to do something else and no one else would pick it up and no one would like sit there and make sure the things you got like fully finished and so that essentially happened where we had one employee set up the main amount of instrumentation and then one or two people added like some things to it and one or two people added some things to it but no one did like that last 20 percent of actually tuning things down right it is you know they turn everything on they turned it all up they add a bunch of things and the only instrumentation and then no one actually necessarily used it So like um for example the television instrumentation library was something they had to write because they used high school as a language and um you know what that meant was they had each database query they had four different fans it would make a span from the start of the query instead of a transaction an end of a transaction end of a query and so you had and then you have the command of an aquarium itself so every single database call cost at least four to five spans to appear oh wow and there's no way to configure that or turn that down or uh cool unless things or anything like that and what happened was that word to create and then anytime you had like a n plus one pattern just appear instantly in the database you would have given millions of events it's like millions of spans all over the place and then they were like oh well what do we use uh span events instead and most vendors I either charge per ingestion or they charge for like until then actually doesn't shift the cost anywhere it sometimes meets the user interface slightly better but it doesn't shift the cost um which is one of the reasons why people say like have one wide event when you put things in there then like you know have time stamps and don't be afraid to use that um but they were they were using the SDK essentially as the time snap functionality do a Korean event anytime at any time Mark that something happened whether you're a crazy fan anytime something happened a lab formated all the way down to the auto instrumentation huh so that was tricky that was probably like 80 percent of like now we have instrumentation in the database but no one actually said well it turns out literally every other database SDK lets you turn 90 of the auto instrumentation off and for a good reason um same for like instrumenting the um my HTTP server like the web server you highlight the web server and then you have the web framework and both of them had almost identical levels of instrumentation so depending on which endpoint handler was in place you either got one top level or the other top level or both and so you had like a massive amount of certifications of the amount of spans that you might not need and it turns out it's largely because there wasn't really an ergonomic way to say at this to a span if it exists otherwise create your own span and do that like that pattern is really necessary for libraries and it's not actually really relevant in the SDK it's not really mentioned anywhere and it's not really possible for a lot of languages to do right right yeah that is an interesting problem to have so now when you were helping this organization with open Telemetry how well versed were you an open Telemetry at the time so I um was mountable of it and I had I thought about it that I had done some things with that and there have been like a previous company where I had really played with open Telemetry and react and I had set up like some very interesting and overly clever uh typescript helpers the amazing really easy for people and um I had familiarity there but this company is where I had to really dig into the refinery into configuration into operationalizing it and to controlling the cost and understanding it from like that to the perspective of the operator rather than the end user and so that was an interesting change perspective and I ended up reading pretty much the entire RFC for everything most of the SDK code most of like a whole bunch of other things and so I dug really really deeply into it and um to this day it will don't necessarily understand baggage like I understand it but it's kind of not well implemented yeah you were you were mentioning earlier that um it's funny because I was reading up a baggage today um and you were mentioning earlier that baggage is still kind of wonky to to to work with um specifically that um that that what were some of the challenges in working with baggage that you experienced because I I like this would definitely be good feedback for um for the uh Hotel folks the maintainers it comes down to baggage is a really generic tool that ends up being used for like five different things and they mostly come up one you want to stitch together multiple services or when you want to write a library or do something like that or write like um a platform for people so that they can instrument things very easily and not have to worry about certain underlying implementation details you can also use baggage to paper over the API of a lot of things and do what you want to even if you shouldn't necessarily do that to like for example um package or just contact application in general is more or less the only way you can have a directed encyclical graph otherwise you only really have like a linear constant and you have like some ability to form links but not like even that is pretty under economic but you need to know where you're linking to and where or what you're linking from um but baggage will let you like put things in a header so that you have um cost service cost registration the actual distributive part the markets is also the thing that you need if you want to navigate information down to uh public information upper chain and there's also what you need if you want to write some sort of demand processor as a way to work around issues so if you want to have like something added to multiple Spanish down then the only way to do that really as far as I can tell is to write a span processor and use this package to pull that information out so you would add something to the package and then ask the span processor is really thrown the spams it'll reconstrate to that whole tree and figure everything out and then add that in there see more or less rewrite refinery in your code base we'll rewrite the open television collector in your groupies in order to use packets in order to add certain features that aren't in the SDK yet oh interesting did you did you find that uh with with other aspects of open Telemetry as well that were kind of limiting for you when you were working with it when you were digging deep it was I think the main thing that was unlimiting um outside of that aspect was that um it's really hard to make adding instrumentation that's manual ergonomic like um it's just difficult and a lot of that comes down to open Telemetry really wants you to like have written the code right there and you need to kind of know how to do that and it's very hard to wrap the libraries themselves without adding like that extra layer of interaction that makes things hard so unless your language has a very it's massive macro system and you can use that ergonomically it's difficult to do that and so like Ruby gets away with it because it can monkey patch itself and that's how you get that one layer of interaction removed so if you get around that um languages like uh typescript don't and so in typescript like what you really want is something like a decorator you can't get that and um the other issue that I really ran into ergonomically with open Telemetry would have been a based around Asian Guinness world the ancientist one was probably one of the larger beams of my existence and the reason for that is there's multiple different ways you can write and especially as you're going to smoke and there is one way that open television works one with and that would be if the parent function follows a child function is you're going to sleep passes the context into it and then Waits and outlives the challenge function and then end to tone span and so you have like a circuit historic but actually is this only inside the lifetime of the parents span anything other than that use case gets really weird what about uh what about spam links aren't they supposed to kind of alleviate that though this band links do work but you have to know how to use them and so the parent still has to pull the child and then the child has to have the parent ID in order to link to that or I think vice versa I don't know if you can do that but um essentially I couldn't write a function generically so I was unaware that it was being called from the parent and then had to link up to the parent and so that was tricky because a lot of languages you want to write like a library or something and the library can be very agnostic so I don't necessarily want to say this function is called pharmaceutical contest all the time I want to say this function is called basically in this context that's relevant to the application domain and sometimes that's interesting context and in which case it's been called asynchronously but sometimes it's more synchronously and sometimes it's called a different context altogether and I can't express that with open Telemetry very easily if really at all so 19 was what you do before doing sort of instant patterns but you can't do that like very easily it's very unorganomic in a lot of languages got it got it and and speaking of of languages like what what languages were part of that landscape in that organization um so this organization had a typescript and high school as its main languages but I've also done open Telemetry with um I've done it with react I don't know what typescript done with the hospital done it with a little bit of your name um and I thought which one was the least annoying to implement Unleashed annoying to instrument would be probably python because so many other people use Python and python is used in more of a service sound contests and so open Telemetry is often used there um that that was actually more ergonomic python also has like more like decorators and meta programs how many things like that to make it easier to work with yeah um yeah is interesting because it's both a front end and a back-end um and so a lot of things that you can do with it may have to run both in a voucher context and server contest and open television works really well Riverside and very not great on the client side and so dealing with all of those issues can't be really tricky and it may surviving an API that works while I'm bullet is really really hard to like on the client side you want to send as little as data as possible on the service side you want to send as much data as possible to the collectors of The Collector can filter it um and so you have completely need to there you also have the issue of like how do I actually send data to the cluster on the client side which is kind of solved but not really and you end up like writing an API endpoint that supports things to collector so that you can transparently authenticate or not authenticate in a way that light actually works and you can start to try and filter out noise signal I guess someone calling the endpoint or are they just like because of where did they answer something I was reacting the the most annoying one to instrument or was there another language that you've worked with that was even more annoying high school was the most annoying one to instrument how how big is like the the the the hotels support around around High School um the open Telemetry support for high school is actually pretty decent which is really funny because Haskell has a bigger usage Community wise than Master does for open telemetry um yeah and uh so high school is an ergonomic language it's very expressive it's very interesting but the one time of high school is absolutely absurd and makes um open Telemetry extremely difficult so high school is a funny lazy language which means that if you write the code you don't have any control over the lifetime of one the code is executed or actually evaluated or How Deeply it's evaluated and open Telemetry really really wants you to explicitly call things have it starts then and have that happen and turn like point and so in household and laziness makes it interesting because you end up peppering the laziness with the ones of strict functions which are the tracing stuff and to that can mess with your memory usage you can mention your performance you're interested it can match the couple other things and it's very very difficult in language to have something include transparent would you want the tradition to be transparent otherwise the tracing is really tricky and it brings a bunch of other things and so making it transparent actually require the use of unstable Primitives exposed by the internal implementation details of the compiler and the runtime interesting I feel like I learned something new today um I want to go back to another point that you had made earlier on and hopefully I I understood it correctly I think he made a point saying that generally organizations do tend to like create libraries around open Telemetry did I understand you correctly yeah and it's not necessarily the organizations tend to do that it's that that is one of the best ways to multiply efforts when you have like a platform team when you're thinking of things in a platform engineering manner so in general if you have like something people need to care about life testing or instrumentation or runtime performance or some aspect of the code that isn't necessarily functionality getting everyone to care about that equally is really hard it'll live on to have the same level of knowledge it's really hard um in fact it's kind of not even so much as impossible but I think it's I think oh people try too hard to hit and consequently um it ends up being very very useful from a um organization perspective to accelerate your developers by having like libraries built around things or my platforms putting on things so you have like maybe a standard template and then your instrumentations just don't work and your cicds just dealt with and a whole bunch of other things you're done with it and you can use it but these structures are already set up percent of the structure giving people capabilities for handling things taking care of like distribution so that cross service instrumentation to support the Box all those things would be things that I would want to deal with from a platform team perspective of making Telemetry easier to do yeah that makes a lot easier at home really hard so was there um at this previous organization even other org even subsequent organizations is that something that um it's one thing too I I think have have the aspiration to to like abstract that stuff from from folks to you know make sure that they're at the same level when it comes to instrumenting um it's another to like be able to achieve that goal do you feel that in any of the organizations where you worked that that has been actually achieved with with these kinds of libraries I've been able to get things to a point where it was achievable whether or not I actually achieved it uh it's a different question um but so what I mean by that is I was able to figure out ways to um like wrap an abstract away the vast majority of the setup for open Telemetry when I uh typescript or Hassle and I was able to write um rappers around functions in a way that the rappers are transparent so we could have like a bunch of environment variables properly in and a bunch of like sort of stand up for things dealt with in a way that people just needed to set up their application in a certain way and then you got like the very base skeleton of the Tracer and all those things she's just ready to go right and they can like um get the you know without the span sort of function and have like a more limited understandable API where you didn't need to pass in certain types of contents manually they've been able to do that for both and I've even been able to do that in a way that works um isometrically in JavaScript or lifetime script and what what that means is you can have the same code running in server and the browser and it works and that involved a mild amount of crimes it involved a lot of crimes it involves a lot of crimes um you abused how node modules cached a bunch of things and then in chapter return globals and then the Iran certain stable code here and there and then if you import everything in the right order in the right place and then you rely on trade shaking then your bundle on the server can be different than your bundle on the client and you can end up splitting that out in a way that doesn't break the global Singleton pattern of observability of the open Telemetry stuff and also not explained anywhere Switching gears a bit um to uh more managing the infrastructure side of open Telemetry how was that experience for you in contrast to having to I guess to previous rules of of being more I guess who I guess holistically focused or or having to like lean up the clean up the hotel mess how how was how is that in contrast so running open Telemetry is interesting because like you have to open Telemetry collector to run and The Collector like then you have to set it up and if you go outside of essentially any of the very small documentation examples you end up having to read the rfcs for things or you end up having to read the technical like design document and they are not obtuse but the they're understandable by like a select amount of people and you have to really dig in to understand that to like even for the um tail sampling configuration of the open Telemetry collector like an entire document of how to do it is the most difficult thing I've ever seen um it has like different layers of things each one has its own entire language and specification setup and it's wild the honeycomb Refinery is much better in that regard but running Refinery and production is hey janky mouse in a lot of ways and that largely comes down to Refinery is really a tool that they built for themselves and then they made it available to other people and so it here's Refinery if you know how it works great you just run it but otherwise you can run Refinery successfully if you're an organization with high performance cicd the ability to implement things and look at them and you have that ability to like dial things down very rapidly otherwise Refinery is not going to work super well for you and to the organization that I ran refinery in split the difference they had some amount of rapid cicd and some men of that and I was able to look into um laws and stuff like that and get some things down but they use its pattern and Refinery due to how broken the open Telemetry stuff was we ran into essentially every error case and Edge case of it so for example one of the edge cases that we had was many spans were not like you know a minute or two long finished fans were hours long or even days and we had some Spanish door multiple weeks in length and we also had some Spanish that had over three to five million events in them and without ended up being was you would have something that would go you know I started to ask and then so that starts to span and so this would be like an asynchronous job into the asynchronous job would for every single user in the database do a whole bunch of validation stuff and so that would be about like 1000 demands per user times 500 000 users or twenty thousand Spanish per user times forty thousand users or and so like it's it's ridiculous and um or like for every single item in the database check its consistency in one span so that would take like you know 20 hours and uh that would break every possible configuration of Refinery you just can't have this fan that's not broken and also have Refinery store 50 million things in memory so uh I got things improved and uh but ultimately I ended up saying this entire way we do open Telemetry in like this with an asynchronous job is actually now close enough to streaming services that it doesn't work that way and so the wheel dealer streaming services is to do it the way honeycomb does which is you have like um you just send a snapshot of his fan every like minute show any rule of things in the code and that requires you to have written your code in a way that you can roll things up and then send it every minute so you need to completely rewrite how you do most of your task handling to have some sort of like task handling manager that sits there and can collect a bunch of things in every minute do that and start a new span and that is not really possible the way most people write as you're going to stats and fixing that is really tricky because there's not really a wheel if it's now we're not actually just rewriting the code it requires you to really just um start a million different tasks and each task is linked together casually by like span information and then the collector is sort of like does some things in there or maybe your spam processor can roll that up but otherwise you can't actually do that now you should do your tasks in that manner anyway because then it allowed to have a right hand lock and then you can ensure your task consistency is a longer think it's not interested in the middle you should do that anyway because that's how you do it if you understand Distributing systems but you run into this thing in open Telemetry uh it was written by systems-minded people and so a lot of design decisions that they make make sense if you know how to write a standard stability system like of course you use a writer had a lack of question Journal things a question like you want to just spawn an event from like respond one task somewhere for like 20 hours who does that um everyone does that until they know better that's a really good point um we've got about six minutes left um but before we wrap up um I did want um I would like to talk briefly about um uh some of your experience in trying to make the case for open Telemetry to Executives because that's that that can be challenging yeah to open Telemetry um they're making me so there's there's two sides of it there's making the case for any sort of monitoring or observability in general and I was making the case for more open Telemetry specifically yeah potential um and so the second case usually happens when you have some pre-existing stuff and your article and we should not use the existing stuff when we should use this other thing instead and so what you're arguing for there is a migration two migrations a technical migration and the social migration and those are very tricky and so you want those both migrations to happen and so you need to be able to essentially show that the time and effort saved my via this migration will pay for itself in about four months and if you can't show that it will pay for itself in about four months then you probably shouldn't actually do it and you should figure out how to make it pay for itself in four months and then do it otherwise you'll end up with a migration that takes life a year or something and it has no perceivable difference in benefit and then people are not going to be sold on it people need to actually tangibly feel the benefit or the migration is going to feel on the social aspected technically um for a convention organizations that you want to get like any sort of monitoring in general that relies on being able to convince people that the life cycle leads to Encompass more at the life cycle and the easiest way to do that is in my opinion you tie the code lifecycle to the business value delivery life cycle so what can happen in organizations is you have like the developer life cycle which is over here and it's I write code I commit code I merge code and and you have the business lifecycle which is we could um like we do some market research or we get like some seamless research and then we design a product and we get the final features and then that final feature is implemented and then we see what the feedback is and at no point to do those two overlap or cross so that leads to people to my product owners handing features over to developers and developers writing tune and just handing lots of production and instead of playing frisbee you need to merge those and when those get merged then essentially the developers have to care about things all the way to the point where it delivers value to the customer which is absolutely what every executive already wants to have happen the state they have within separated it's what they think is like has to be that way and the second is a it's possible and even desirable to get developers to care about like the uh so outcomes from the customers get them talking to sales people and get them talking to like products people and that will not only improve things for the product improve the internet but it'll actually improve the productivity of the developers and their experience and their ability to do this that usually sounds Executives and you need this the next uh checking after that is once they have you need this if they just look for like observability they're going to get one of like three or four different vendors and probably going to end up an expensive vendor and you need to pick the vendor based on what the business needs and not necessarily what you like the best like for example my current company we have advanced Monument compliance requirements we have it for only one environment I'm not that provides more than one code base and so because of that no one in the company can use anything that isn't fedramp compliant which significantly limits the amount of Advantage we can have to basically one and I don't necessarily want to use that vendor necessarily but I have no plans to migrate off of it if I were to migrate off of it it would be in a very interesting manner what I would do is I would split the environment into two environments and have them be identical one fan of certified one not federam certified and then essentially would have a fenramp program but that would be like a separate sales funnel and anyone who actually needs it can have that because the Phantom amp is valuable in two aspects for the business one is that it gets them certain customers and one is that it gets some certain conversations and for the conversations those people may not be on federal but they make one eventually when it means that I can run the same code base in both locations and use a different vendor and the one that isn't vetramp and if n Ram verified vendor and that one but that whole migration for the current company would be allowed two years of work so are we gonna do that who knows maybe but there's a lot of other questions to ask first and a lot more that we need to do before we can even get to that point yeah speaking as a vendor we've had a lot of conversations about Feb ramp certification and being that not 100 of our employees are in the us we'd have to do exactly what you described and we tried to do that before for HIPAA and it created kind of a weak experience for some customers so yeah it's a tricky problem interesting yeah like the divide into two environments pushed to both environments eventually we figured out that it was hurting our whole development velocity to push to those two different environments and even though we thought it would be very straightforward now uh because you know the environments aren't identical and if you turn one thing off in one environment and keep it on a different environment you're going to get like a completely different experience and so what if you have an outage in one versus the enter yeah there's no way it doesn't hurt development velocity the question is whether it doesn't hurt development velocity more than the benefit from having a different vendor for something else does like if having one vendor would be extremely beneficial and that benefit is more than the harm you get from Harmony developer velocity by splitting your production environments then maybe it makes sense to do that but realistically it probably doesn't because having homogenized too lean that you can build on top of for yourself often end of having a higher reward than having like a quote technically Superior vendor but you have to invest in that yourself exactly yeah and in the case of the HIPAA stuff it didn't because we were able to bring support into HIPAA and there were only like four customers on that sort of private honeycomb instance that had more support for hippo greater support for security and they were large customers but it was like we can't sort of the fact that they were large customers meant that they made requests and that eventually work the two environments to be significantly different okay in my experience one of the most interesting things about HIPAA is that any sort of regulatory environment there's like there's one regulatory environment and then there's like Federal amp and the overlap between people that need any regulatory environment and people that need or really want fun wrap is basically a circle yeah so if you're going to go like oh we're gonna have like a HIPAA environment we're gonna have a fips environment we could have like you know a shock to like it used to be stocked too anyway whatever um but if you if you have my one of those you probably would actually get as much benefit if not more if I'm going straight to fin ramp even though it's like three times as expected to do that at least and the ongoing maintenance burden of that is ridiculous but it gets you on the other customers you need like HIPAA plus fan ramp or fips plus van ramp and that is a really interesting like business thing that's non-obvious any type of regulatory environment completely changes the game for open Telemetry what you can do with it how it works your vendors in general and even like your hiring practices and it's really weird that you can't really have asset but you can't just get HIPAA you really should probably go all the way to a fat ramp just because of who the market is the needs one or both yeah I totally agree with you based on like our experience with that HIPAA server because it turned out there were companies camped on it that just had who went for that offering that just had a desire for like more privacy stricter controls because of how their company culture and their data worked without actually having the need for the Federal Regulation and I think you're right that it makes it easier to sell I think probably we wouldn't end up going in that direction again unless we managed to partner with somebody who was actively like bringing in fedramp customers like some agency that worked with the government or whatever like would have to be but I I wish we had known that at the start and we didn't one thing that would be interesting there would be to have open Telemetry have some sort of like agency that and that agency gets van ramp compliance and then what you can do with that is you can have that agency essentially facilitate the onboarding of open Telemetry vendors or just other observability vendors into the space and you can use that to sort of start Bridging the Gap and giving people more capabilities and sharing that my fan Ram specific architecture concerns in the way that spreads their compliance load amongst multiple companies and if you could do that oh sorry I was very excited by that idea that makes a ton of sense we could maybe make it work through the project and I would say for us as a small vendor that really is a problem that if we are not allowed to have our Engineers who are in other countries touch the system we may only have one engineer working in a particular area and so we have to hire a second person in the U.S which is undoable there are certain ways um and you would need somebody to talk to someone who's experience in Finland like negotiate that would happen like 100 U.S citizenship is in not see her own requirements it's it gets nuanced and it gets tricky oh interesting well I think we have to leave it at that before we got into a funny side um thank you again Hazel for sharing your insights this has been really cool um I I don't think we get to talk to too many people who uh who have had like such Advanced use cases for open Telemetry and and um being able to share that with the community is is really good because I I think a lot of us get you know we we do like those intro tutorials we're like hey that's pretty easy and then you get into like the the guts of it and then that's when you start uh getting into some of those gnarly use cases can be really tricky in life yeah yeah definitely so definitely appreciate that that feedback and and the Viewpoint and you will be back um for hotel in practice I believe on September 14th looking forward to that um do you know what uh what topic you'll be discussing no idea I'm gonna figure it out but it's gonna be a lot of fun so awesome really looking forward to that um thank you again as always super super awesome insights um yeah and uh we will hopefully see everyone um for hotel and practice with hazel on September 14th [Music] thank you +thank you welcome everyone to otel q a we have the pleasure of having our good friend Hazel weekly come talk to us about um her experiences with observability welcome Hazel glad to be here very excited for this yay I guess let's start first things first because I know we still have a bunch of Hotel novice users like just getting into Hotel folks um who who join our um end user working group so from from the perspective of somebody who's new to open Telemetry can you share what that experience was like what kind of landscape you were coming into um and I guess for starters like drawing back on that experience um was the company at that point ready for observability like did that what what kind of what what kick started the conversation into into open Telemetry in the first place I would say um so I'm going to start off with my definition of observability because it's slightly different um the definition from control theory would be like can you understand this daily system from the input in the outputs and um Fred Herbert might should talk about the definition from cognitive safety systems engineering and that one is much more about the work required and the process required for a group of people to be able to understand everything and actually understand the system and that's how they like discovered and think about that and mine is the process through which you develop the capability of asking meaningful questions and getting useful answers and so it's a process because it's evolutionary and the questions have to be meaningful to you whatever that means and the answers don't have to be correct but they have to be useful so when I look back at the company when we're starting to think about observability and or rather like distribution literally what are the questions that the company is starting to ask from the perspective of the engineers and one of the questions of the company is asking for the perspective of the managers and what is it asking from the perspective of the executives and show from the engineers at one of the companies that are the early hand open telemetry they had it and they had instrumented things with it but more from the perspective of we know we need this but they didn't necessarily have that motivation yet the action weren't asking the types of sophisticated questions that motivate people to add their own telemetry they had a bunch of telemetry and they had a huge amount of instrumentation and they had just the volume turned up to 11 and no one was actually querying that information and so what I was able to do was turn down that volume significantly and then what I did was essentially say you're you're not using this and they're not asking any questions from your system so when you do don't turn the volume back up like add that instrumentation in there with thought and intent and that was an interesting hurdle to happen because in a way people would ask questions is to ask these types of semi's physical questions asking them to understand the system they're asking them to solve a problem and so consequently they just wanted every bit of information ever so that whenever they needed it they could just like go through a giant stack of noise and find a needle right and that's not really what open Telemetry is for online distributation it's for can you understand the state of your system to the point that you only collect what you actually need and that is it in of itself a second layer of understanding the system and they didn't have that second layer right so it sounded like they were instrumenting for the sake of instrumenting and just sort of throwing everything at the wall and hoping something would stick yeah that also sounds like a very I came over from Vlogs and I'm used to being able to search everything ever approach they also still had laws which is a surprise basically nobody of course so um in terms of like how did you convince them to kind of Tamp it down and direct it make it more directed so that it could actually work for them so convincing them to Champion it down and turn the volume down was actually pretty easy that came down to cost they were about 300 over budget from the vendor and so I said I need to get you under budget and so I actually uh in the honeycomb Pond eaters I have a very funny image from that time period where you see the ingestion at about like two to three hundred million events per day and you know I'm trying to tweak in and get it down to about like 200 250 or like you know 180 to 250 and that was with very very aggressive sampling and finally I figured out a bunch of misconfigurations in the sampling information and dropped it to about like three to five million events per day and um and once I had done with that they like had concerns about oh what if I can't find something I said well do you need to ask that question and then we were able to actually start the useful dialogue of now that you have a question you need to ask go and and build what you need in order to get that answer and then naturally that sort of feedback loop that you kind of need with observability which whether or not you need that feedback loop end to end is a different question but that is currently what open Telemetry requires I think that's really great because being able to know what questions to ask makes it for a much more uh meaningful um experience and so you know in your words not not searching for that needle in in the haystack I wanna um go back to a sampler for a second do sampling to a sampling was we did the sampling through two methods we had the open Telemetry collector and I'll show you now but at this company we only had the honeycomb Refinery set up until we have that set up um at other companies we've had the open Telemetry collector setup and I've used that my preference is actually to use both regardless of whether or not using honeycomb and the reason for that is because the open Telemetry collector doesn't have the best steel sampling configuration and the open Telemetry collector has like the most open source line economic sort of configuration setup to consent things to multiple sources and multiple syncs so that is actually really useful so I like to use that one and then aggregate everything and send it to Refinery and then do useful tail sampling um but running two pieces of information sorry is kind of complicated for a lot of people um it's not going to be a huge hurdle of adoption to like my current company they use um data dog and one of the challenges there is they're not at the point where they can ask sophisticated questions of their infrastructure and I'm really still more at the point of isn't on and so the question there is well even if I set things up for success by switching us to the open Telemetry line instrumentation tooling and then still send it to the same place we now need to run like the collector in multiple places or run like a collector and Refinery just in order to do the same thing that they already do with their built-in tooling so that can be a bit of a bit of a lift because rather than saying install Library it's install a library and have like five things also all right now um going back to um the the first organization where you're you know where they were sending like way too much data after you um after you convince them to like kind of chill do more directed um instrumentation um what was the next sort of hurdle that you experienced there's a notch Turtle there was that the way that company instruction work was very much the over platformed thing this little towards unusual but they had like a developer productivity teams and they didn't really have a platform team what they had Engineers that didn't do work so the engineers didn't do infrastructure stuff they additional code and they would have a bunch of people working on like build tooling and build other stuff like that and so the question was in the way that they do work in order to adopt open Telemetry successfully you would need to essentially build libraries for things or write stuff into the code in a way that Engineers weren't necessarily writing their own instrumentation and that requires working at a level of abstraction that open television is really really difficult so like you can't propagate certain types of information down to challenge fans very easily or really at all um setting up baggage is still very difficult I'm working with contacts can be really tricky if somebody uses an async function then you might drop that and have to reconnect it somehow and if the lifetimes aren't like already set up nicely people can even add their own instrumentation and then it'll get dropped from something because they don't actually understand the full context of where it is they're just random cone and they add something but an open Telemetry you have to understand the call stack not just the functionality and those aren't always the same interesting so how do you um how do you get over that hurdle so at that company I was never able to fully get over that hurdle I just made it better one of the ways that I made it better was essentially following through all of the um fans manually and looking for errors and when I found errors I would like traced it down to the source code and I would do that and um so I ran into like a bunch of very small things of like propagating information wasn't corrupt so it was probably the information of like the helper functions rather than the actual cost sides to activate that and then I had to fix like 500 small things and even when I did that I found that there are a bunch of places where he's looking as contacts weren't being complicated so functions are just being launched or for example calling uh tracing and that was being disconnected from the actual whatever or sometimes things had to end before they started or start before they ended and that got confusion because the language that they're using it's very easy to write code in that logic and it actually is more correct to do it that way but open Telemetry in its design is very much a chicken is called stack a tree shaped um which is really really ergonomic for a language like Java it's not necessarily ergonomic for languages that aren't like that so like react is a really common example because it's a runtime that's asynchronous and it's really difficult to write code in react to the deadline nicely um I suppose a good one closer is a good example of one that doesn't work very well um watch does work but for a very interesting reason and that is because the Russian language cares about lifetimes and so you think about the lifetime of your functions and so you always know what those are and those end up being the natural tracing Point protocol stack based uh tradition Library um but in terms of what we're able to do and how I was fixing it was really mostly in guacamole and doing some weird clever things to pack into the language runtime in order to make things um hit the semantic model differences between open Telemetry and other language things about calling code so were you like a one-woman show doing all of this um so the other person on this was off on a parental leave so for the duration then I was doing this I was actually one person so and that is that is mad impressive um did you want to like pull your hair out some days it was tricky um a lot of it was that one of the benefits of using open Telemetry is that you get like this intuitive sense of what the code is doing at runtime if you play with it enough they had to sit there and kind of like doodle with it and like look things up and I ask questions and get answers and like you have to have that dialogue running and I knew how to do that and no one else at the company was doing that except like one or two people wow and so you had like a very small select power users favorite people and then you had everyone else who may not have even like used open symmetry at all or even news like a vendor in any way whatsoever and so I would get an intuitive sense if one was broken and what was Iran or something and I would say hey like this is an issue and then people wouldn't necessarily believe me oh interesting so how how do you end up convincing them that it is an issue or or could you convince them I guess is the question so when I was able to I was able to take advantage certain people of the things and one of the ways that I did that was essentially my predicting things and then my predictions would turn out to be correct and then I will give a solution to the position and then the solution will work so like one was the database was very very spiky and I mentioned that this was going to cause my some sort of images and one of the reasons that the database was Frankie was because we had like basically uh the disk would slow on a database and so we spent up the disk on the database quite a bit and when that happened like despite this level down these sets were still very spiky in general but despite going high enough to lock things down and um there's small things like that added up over time and people started to believe me a lot more when it came to certain issues but other ones that were more fundamental in the architecture I never actually got around to being able to convince people of that and I actually ended up leaving that company mainly due to that interesting so did you by the time you left did you find there was at least like they're they were getting more out of their observability at that point like because of your efforts like did you at least see some positive results because of that yeah I did so there was a lot better understanding of where they were with the observability people have more of an understanding of how to use it um I had like a couple of things so that it was more useful like they had a lot of it but some things were wrong like um if the air stack was there the air stack died at the wrong spot so it wouldn't actually tell you where the era happened it would tell you where you defined the open Telemetry like the tradition helper which is which is pointless so I fixed that and then all of a sudden the earrings were correct again and people were happy about that then they would actually use it so I hadn't fixed a bunch of things and I had reduced the usage from like two to three hundred events per day or even maybe 50 million events per day to like right and when that happened then people could continue to add open Telemetry and add instrumentation everywhere without running into the physical they were going to blow their budget even more than their own work and that allows people to continue to add more instrumentation so then I guess there it seems like there wasn't an issue per se as far as like getting people to add the instrumentation at that point because they were obviously they had already over instrumented the system previously so now you put them in it was interesting they had over instrumented it but only via Auto instrumentation so there is very little manual stuff done and the manual stuff that was done was often done in a way that made the questions you can ask relatively useless so like one example was there's like at the top of her watched halfway down is when in the life cycle of the call stack is when they would so you get the user ID associated with it because of how the database calls worked but not user ID was never propagated to the top to it was actually impossible to figure out um like how many pages a user was visiting like um like procession like these types of questions whether or not I usually like it's on the same thing over and over because the question we wanted to ask was how effective would database account should be but would it be effective to put radish there somewhere and like probably don't have a 60 second something TTL we we had no idea because we couldn't actually ask that question all the information was there but not actually in a way that made anything possible to look for right because you couldn't look halfway down the um Span in order to get this span of correlated with this one so that tree you can only ask for things on the same level level and down and like filter that way and so they need to fix that but no one necessarily knew how to and no one necessarily knew that they wanted to because they weren't asking that type of question so I bought that up and then people started to get more of a sense of oh here's kind of where we put the information in a way that's visible and to this day I still don't have like a really good way to like explain to people how to think about that other than you do kind of need to display the system I couldn't understand where the data needs to be in order to be useful for asking questions right yeah that makes a lot of sense um so yeah that's so interesting like I find this such an interesting um sort of I don't want to well I guess use case of sorts uh scenario if you will um because there oftentimes we hear we hear the stories of people just struggling with with getting bringing open Telemetry into the organization um and and then starting to instrument and this this feels like it was like one giant anti-pattern of one giant open Telemetry anti-pattern which I think it's a very important thing to be able to discuss because you know like all tools open till Elementary can be grossly misused and then you end up with with scenarios like this where you're instrumenting but you're not getting anything out of it mm-hmm it was really interesting to see how this happened um because it's a lot of what happened is due to how the company works and how it thought about working so it was very anarchic in the sense that Engineers are very very self-directed and they would do things and so the company had a huge cultural issue of intricate 80 percent done with Instagram maybe like 90 percent done and then they would just drop off or someone like outside traffic and had to do something else and no one else would pick it up and no one would like sit there and make sure the things you got like fully finished and so that essentially happened where we had one employee set up the main amount of instrumentation and then one or two people added like some things to it and one or two people added some things to it but no one did like that last 20 percent of actually tuning things down right it is you know they turn everything on they turned it all up they add a bunch of things and the only instrumentation and then no one actually necessarily used it So like um for example the television instrumentation library was something they had to write because they used high school as a language and um you know what that meant was they had each database query they had four different fans it would make a span from the start of the query instead of a transaction an end of a transaction end of a query and so you had and then you have the command of an aquarium itself so every single database call cost at least four to five spans to appear oh wow and there's no way to configure that or turn that down or uh cool unless things or anything like that and what happened was that word to create and then anytime you had like a n plus one pattern just appear instantly in the database you would have given millions of events it's like millions of spans all over the place and then they were like oh well what do we use uh span events instead and most vendors I either charge per ingestion or they charge for like until then actually doesn't shift the cost anywhere it sometimes meets the user interface slightly better but it doesn't shift the cost um which is one of the reasons why people say like have one wide event when you put things in there then like you know have time stamps and don't be afraid to use that um but they were they were using the SDK essentially as the time snap functionality do a Korean event anytime at any time Mark that something happened whether you're a crazy fan anytime something happened a lab formated all the way down to the auto instrumentation huh so that was tricky that was probably like 80 percent of like now we have instrumentation in the database but no one actually said well it turns out literally every other database SDK lets you turn 90 of the auto instrumentation off and for a good reason um same for like instrumenting the um my HTTP server like the web server you highlight the web server and then you have the web framework and both of them had almost identical levels of instrumentation so depending on which endpoint handler was in place you either got one top level or the other top level or both and so you had like a massive amount of certifications of the amount of spans that you might not need and it turns out it's largely because there wasn't really an ergonomic way to say at this to a span if it exists otherwise create your own span and do that like that pattern is really necessary for libraries and it's not actually really relevant in the SDK it's not really mentioned anywhere and it's not really possible for a lot of languages to do right right yeah that is an interesting problem to have so now when you were helping this organization with open Telemetry how well versed were you an open Telemetry at the time so I um was mountable of it and I had I thought about it that I had done some things with that and there have been like a previous company where I had really played with open Telemetry and react and I had set up like some very interesting and overly clever uh typescript helpers the amazing really easy for people and um I had familiarity there but this company is where I had to really dig into the refinery into configuration into operationalizing it and to controlling the cost and understanding it from like that to the perspective of the operator rather than the end user and so that was an interesting change perspective and I ended up reading pretty much the entire RFC for everything most of the SDK code most of like a whole bunch of other things and so I dug really really deeply into it and um to this day it will don't necessarily understand baggage like I understand it but it's kind of not well implemented yeah you were you were mentioning earlier that um it's funny because I was reading up a baggage today um and you were mentioning earlier that baggage is still kind of wonky to to to work with um specifically that um that that what were some of the challenges in working with baggage that you experienced because I I like this would definitely be good feedback for um for the uh Hotel folks the maintainers it comes down to baggage is a really generic tool that ends up being used for like five different things and they mostly come up one you want to stitch together multiple services or when you want to write a library or do something like that or write like um a platform for people so that they can instrument things very easily and not have to worry about certain underlying implementation details you can also use baggage to paper over the API of a lot of things and do what you want to even if you shouldn't necessarily do that to like for example um package or just contact application in general is more or less the only way you can have a directed encyclical graph otherwise you only really have like a linear constant and you have like some ability to form links but not like even that is pretty under economic but you need to know where you're linking to and where or what you're linking from um but baggage will let you like put things in a header so that you have um cost service cost registration the actual distributive part the markets is also the thing that you need if you want to navigate information down to uh public information upper chain and there's also what you need if you want to write some sort of demand processor as a way to work around issues so if you want to have like something added to multiple Spanish down then the only way to do that really as far as I can tell is to write a span processor and use this package to pull that information out so you would add something to the package and then ask the span processor is really thrown the spams it'll reconstrate to that whole tree and figure everything out and then add that in there see more or less rewrite refinery in your code base we'll rewrite the open television collector in your groupies in order to use packets in order to add certain features that aren't in the SDK yet oh interesting did you did you find that uh with with other aspects of open Telemetry as well that were kind of limiting for you when you were working with it when you were digging deep it was I think the main thing that was unlimiting um outside of that aspect was that um it's really hard to make adding instrumentation that's manual ergonomic like um it's just difficult and a lot of that comes down to open Telemetry really wants you to like have written the code right there and you need to kind of know how to do that and it's very hard to wrap the libraries themselves without adding like that extra layer of interaction that makes things hard so unless your language has a very it's massive macro system and you can use that ergonomically it's difficult to do that and so like Ruby gets away with it because it can monkey patch itself and that's how you get that one layer of interaction removed so if you get around that um languages like uh typescript don't and so in typescript like what you really want is something like a decorator you can't get that and um the other issue that I really ran into ergonomically with open Telemetry would have been a based around Asian Guinness world the ancientist one was probably one of the larger beams of my existence and the reason for that is there's multiple different ways you can write and especially as you're going to smoke and there is one way that open television works one with and that would be if the parent function follows a child function is you're going to sleep passes the context into it and then Waits and outlives the challenge function and then end to tone span and so you have like a circuit historic but actually is this only inside the lifetime of the parents span anything other than that use case gets really weird what about uh what about spam links aren't they supposed to kind of alleviate that though this band links do work but you have to know how to use them and so the parent still has to pull the child and then the child has to have the parent ID in order to link to that or I think vice versa I don't know if you can do that but um essentially I couldn't write a function generically so I was unaware that it was being called from the parent and then had to link up to the parent and so that was tricky because a lot of languages you want to write like a library or something and the library can be very agnostic so I don't necessarily want to say this function is called pharmaceutical contest all the time I want to say this function is called basically in this context that's relevant to the application domain and sometimes that's interesting context and in which case it's been called asynchronously but sometimes it's more synchronously and sometimes it's called a different context altogether and I can't express that with open Telemetry very easily if really at all so 19 was what you do before doing sort of instant patterns but you can't do that like very easily it's very unorganomic in a lot of languages got it got it and and speaking of of languages like what what languages were part of that landscape in that organization um so this organization had a typescript and high school as its main languages but I've also done open Telemetry with um I've done it with react I don't know what typescript done with the hospital done it with a little bit of your name um and I thought which one was the least annoying to implement Unleashed annoying to instrument would be probably python because so many other people use Python and python is used in more of a service sound contests and so open Telemetry is often used there um that that was actually more ergonomic python also has like more like decorators and meta programs how many things like that to make it easier to work with yeah um yeah is interesting because it's both a front end and a back-end um and so a lot of things that you can do with it may have to run both in a voucher context and server contest and open television works really well Riverside and very not great on the client side and so dealing with all of those issues can't be really tricky and it may surviving an API that works while I'm bullet is really really hard to like on the client side you want to send as little as data as possible on the service side you want to send as much data as possible to the collectors of The Collector can filter it um and so you have completely need to there you also have the issue of like how do I actually send data to the cluster on the client side which is kind of solved but not really and you end up like writing an API endpoint that supports things to collector so that you can transparently authenticate or not authenticate in a way that light actually works and you can start to try and filter out noise signal I guess someone calling the endpoint or are they just like because of where did they answer something I was reacting the the most annoying one to instrument or was there another language that you've worked with that was even more annoying high school was the most annoying one to instrument how how big is like the the the the hotels support around around High School um the open Telemetry support for high school is actually pretty decent which is really funny because Haskell has a bigger usage Community wise than Master does for open telemetry um yeah and uh so high school is an ergonomic language it's very expressive it's very interesting but the one time of high school is absolutely absurd and makes um open Telemetry extremely difficult so high school is a funny lazy language which means that if you write the code you don't have any control over the lifetime of one the code is executed or actually evaluated or How Deeply it's evaluated and open Telemetry really really wants you to explicitly call things have it starts then and have that happen and turn like point and so in household and laziness makes it interesting because you end up peppering the laziness with the ones of strict functions which are the tracing stuff and to that can mess with your memory usage you can mention your performance you're interested it can match the couple other things and it's very very difficult in language to have something include transparent would you want the tradition to be transparent otherwise the tracing is really tricky and it brings a bunch of other things and so making it transparent actually require the use of unstable Primitives exposed by the internal implementation details of the compiler and the runtime interesting I feel like I learned something new today um I want to go back to another point that you had made earlier on and hopefully I I understood it correctly I think he made a point saying that generally organizations do tend to like create libraries around open Telemetry did I understand you correctly yeah and it's not necessarily the organizations tend to do that it's that that is one of the best ways to multiply efforts when you have like a platform team when you're thinking of things in a platform engineering manner so in general if you have like something people need to care about life testing or instrumentation or runtime performance or some aspect of the code that isn't necessarily functionality getting everyone to care about that equally is really hard it'll live on to have the same level of knowledge it's really hard um in fact it's kind of not even so much as impossible but I think it's I think oh people try too hard to hit and consequently um it ends up being very very useful from a um organization perspective to accelerate your developers by having like libraries built around things or my platforms putting on things so you have like maybe a standard template and then your instrumentations just don't work and your cicds just dealt with and a whole bunch of other things you're done with it and you can use it but these structures are already set up percent of the structure giving people capabilities for handling things taking care of like distribution so that cross service instrumentation to support the Box all those things would be things that I would want to deal with from a platform team perspective of making Telemetry easier to do yeah that makes a lot easier at home really hard so was there um at this previous organization even other org even subsequent organizations is that something that um it's one thing too I I think have have the aspiration to to like abstract that stuff from from folks to you know make sure that they're at the same level when it comes to instrumenting um it's another to like be able to achieve that goal do you feel that in any of the organizations where you worked that that has been actually achieved with with these kinds of libraries I've been able to get things to a point where it was achievable whether or not I actually achieved it uh it's a different question um but so what I mean by that is I was able to figure out ways to um like wrap an abstract away the vast majority of the setup for open Telemetry when I uh typescript or Hassle and I was able to write um rappers around functions in a way that the rappers are transparent so we could have like a bunch of environment variables properly in and a bunch of like sort of stand up for things dealt with in a way that people just needed to set up their application in a certain way and then you got like the very base skeleton of the Tracer and all those things she's just ready to go right and they can like um get the you know without the span sort of function and have like a more limited understandable API where you didn't need to pass in certain types of contents manually they've been able to do that for both and I've even been able to do that in a way that works um isometrically in JavaScript or lifetime script and what what that means is you can have the same code running in server and the browser and it works and that involved a mild amount of crimes it involved a lot of crimes it involves a lot of crimes um you abused how node modules cached a bunch of things and then in chapter return globals and then the Iran certain stable code here and there and then if you import everything in the right order in the right place and then you rely on trade shaking then your bundle on the server can be different than your bundle on the client and you can end up splitting that out in a way that doesn't break the global Singleton pattern of observability of the open Telemetry stuff and also not explained anywhere Switching gears a bit um to uh more managing the infrastructure side of open Telemetry how was that experience for you in contrast to having to I guess to previous rules of of being more I guess who I guess holistically focused or or having to like lean up the clean up the hotel mess how how was how is that in contrast so running open Telemetry is interesting because like you have to open Telemetry collector to run and The Collector like then you have to set it up and if you go outside of essentially any of the very small documentation examples you end up having to read the rfcs for things or you end up having to read the technical like design document and they are not obtuse but the they're understandable by like a select amount of people and you have to really dig in to understand that to like even for the um tail sampling configuration of the open Telemetry collector like an entire document of how to do it is the most difficult thing I've ever seen um it has like different layers of things each one has its own entire language and specification setup and it's wild the honeycomb Refinery is much better in that regard but running Refinery and production is hey janky mouse in a lot of ways and that largely comes down to Refinery is really a tool that they built for themselves and then they made it available to other people and so it here's Refinery if you know how it works great you just run it but otherwise you can run Refinery successfully if you're an organization with high performance cicd the ability to implement things and look at them and you have that ability to like dial things down very rapidly otherwise Refinery is not going to work super well for you and to the organization that I ran refinery in split the difference they had some amount of rapid cicd and some men of that and I was able to look into um laws and stuff like that and get some things down but they use its pattern and Refinery due to how broken the open Telemetry stuff was we ran into essentially every error case and Edge case of it so for example one of the edge cases that we had was many spans were not like you know a minute or two long finished fans were hours long or even days and we had some Spanish door multiple weeks in length and we also had some Spanish that had over three to five million events in them and without ended up being was you would have something that would go you know I started to ask and then so that starts to span and so this would be like an asynchronous job into the asynchronous job would for every single user in the database do a whole bunch of validation stuff and so that would be about like 1000 demands per user times 500 000 users or twenty thousand Spanish per user times forty thousand users or and so like it's it's ridiculous and um or like for every single item in the database check its consistency in one span so that would take like you know 20 hours and uh that would break every possible configuration of Refinery you just can't have this fan that's not broken and also have Refinery store 50 million things in memory so uh I got things improved and uh but ultimately I ended up saying this entire way we do open Telemetry in like this with an asynchronous job is actually now close enough to streaming services that it doesn't work that way and so the wheel dealer streaming services is to do it the way honeycomb does which is you have like um you just send a snapshot of his fan every like minute show any rule of things in the code and that requires you to have written your code in a way that you can roll things up and then send it every minute so you need to completely rewrite how you do most of your task handling to have some sort of like task handling manager that sits there and can collect a bunch of things in every minute do that and start a new span and that is not really possible the way most people write as you're going to stats and fixing that is really tricky because there's not really a wheel if it's now we're not actually just rewriting the code it requires you to really just um start a million different tasks and each task is linked together casually by like span information and then the collector is sort of like does some things in there or maybe your spam processor can roll that up but otherwise you can't actually do that now you should do your tasks in that manner anyway because then it allowed to have a right hand lock and then you can ensure your task consistency is a longer think it's not interested in the middle you should do that anyway because that's how you do it if you understand Distributing systems but you run into this thing in open Telemetry uh it was written by systems-minded people and so a lot of design decisions that they make make sense if you know how to write a standard stability system like of course you use a writer had a lack of question Journal things a question like you want to just spawn an event from like respond one task somewhere for like 20 hours who does that um everyone does that until they know better that's a really good point um we've got about six minutes left um but before we wrap up um I did want um I would like to talk briefly about um uh some of your experience in trying to make the case for open Telemetry to Executives because that's that that can be challenging yeah to open Telemetry um they're making me so there's there's two sides of it there's making the case for any sort of monitoring or observability in general and I was making the case for more open Telemetry specifically yeah potential um and so the second case usually happens when you have some pre-existing stuff and your article and we should not use the existing stuff when we should use this other thing instead and so what you're arguing for there is a migration two migrations a technical migration and the social migration and those are very tricky and so you want those both migrations to happen and so you need to be able to essentially show that the time and effort saved my via this migration will pay for itself in about four months and if you can't show that it will pay for itself in about four months then you probably shouldn't actually do it and you should figure out how to make it pay for itself in four months and then do it otherwise you'll end up with a migration that takes life a year or something and it has no perceivable difference in benefit and then people are not going to be sold on it people need to actually tangibly feel the benefit or the migration is going to feel on the social aspected technically um for a convention organizations that you want to get like any sort of monitoring in general that relies on being able to convince people that the life cycle leads to Encompass more at the life cycle and the easiest way to do that is in my opinion you tie the code lifecycle to the business value delivery life cycle so what can happen in organizations is you have like the developer life cycle which is over here and it's I write code I commit code I merge code and and you have the business lifecycle which is we could um like we do some market research or we get like some seamless research and then we design a product and we get the final features and then that final feature is implemented and then we see what the feedback is and at no point to do those two overlap or cross so that leads to people to my product owners handing features over to developers and developers writing tune and just handing lots of production and instead of playing frisbee you need to merge those and when those get merged then essentially the developers have to care about things all the way to the point where it delivers value to the customer which is absolutely what every executive already wants to have happen the state they have within separated it's what they think is like has to be that way and the second is a it's possible and even desirable to get developers to care about like the uh so outcomes from the customers get them talking to sales people and get them talking to like products people and that will not only improve things for the product improve the internet but it'll actually improve the productivity of the developers and their experience and their ability to do this that usually sounds Executives and you need this the next uh checking after that is once they have you need this if they just look for like observability they're going to get one of like three or four different vendors and probably going to end up an expensive vendor and you need to pick the vendor based on what the business needs and not necessarily what you like the best like for example my current company we have advanced Monument compliance requirements we have it for only one environment I'm not that provides more than one code base and so because of that no one in the company can use anything that isn't fedramp compliant which significantly limits the amount of Advantage we can have to basically one and I don't necessarily want to use that vendor necessarily but I have no plans to migrate off of it if I were to migrate off of it it would be in a very interesting manner what I would do is I would split the environment into two environments and have them be identical one fan of certified one not federam certified and then essentially would have a fenramp program but that would be like a separate sales funnel and anyone who actually needs it can have that because the Phantom amp is valuable in two aspects for the business one is that it gets them certain customers and one is that it gets some certain conversations and for the conversations those people may not be on federal but they make one eventually when it means that I can run the same code base in both locations and use a different vendor and the one that isn't vetramp and if n Ram verified vendor and that one but that whole migration for the current company would be allowed two years of work so are we gonna do that who knows maybe but there's a lot of other questions to ask first and a lot more that we need to do before we can even get to that point yeah speaking as a vendor we've had a lot of conversations about Feb ramp certification and being that not 100 of our employees are in the us we'd have to do exactly what you described and we tried to do that before for HIPAA and it created kind of a weak experience for some customers so yeah it's a tricky problem interesting yeah like the divide into two environments pushed to both environments eventually we figured out that it was hurting our whole development velocity to push to those two different environments and even though we thought it would be very straightforward now uh because you know the environments aren't identical and if you turn one thing off in one environment and keep it on a different environment you're going to get like a completely different experience and so what if you have an outage in one versus the enter yeah there's no way it doesn't hurt development velocity the question is whether it doesn't hurt development velocity more than the benefit from having a different vendor for something else does like if having one vendor would be extremely beneficial and that benefit is more than the harm you get from Harmony developer velocity by splitting your production environments then maybe it makes sense to do that but realistically it probably doesn't because having homogenized too lean that you can build on top of for yourself often end of having a higher reward than having like a quote technically Superior vendor but you have to invest in that yourself exactly yeah and in the case of the HIPAA stuff it didn't because we were able to bring support into HIPAA and there were only like four customers on that sort of private honeycomb instance that had more support for hippo greater support for security and they were large customers but it was like we can't sort of the fact that they were large customers meant that they made requests and that eventually work the two environments to be significantly different okay in my experience one of the most interesting things about HIPAA is that any sort of regulatory environment there's like there's one regulatory environment and then there's like Federal amp and the overlap between people that need any regulatory environment and people that need or really want fun wrap is basically a circle yeah so if you're going to go like oh we're gonna have like a HIPAA environment we're gonna have a fips environment we could have like you know a shock to like it used to be stocked too anyway whatever um but if you if you have my one of those you probably would actually get as much benefit if not more if I'm going straight to fin ramp even though it's like three times as expected to do that at least and the ongoing maintenance burden of that is ridiculous but it gets you on the other customers you need like HIPAA plus fan ramp or fips plus van ramp and that is a really interesting like business thing that's non-obvious any type of regulatory environment completely changes the game for open Telemetry what you can do with it how it works your vendors in general and even like your hiring practices and it's really weird that you can't really have asset but you can't just get HIPAA you really should probably go all the way to a fat ramp just because of who the market is the needs one or both yeah I totally agree with you based on like our experience with that HIPAA server because it turned out there were companies camped on it that just had who went for that offering that just had a desire for like more privacy stricter controls because of how their company culture and their data worked without actually having the need for the Federal Regulation and I think you're right that it makes it easier to sell I think probably we wouldn't end up going in that direction again unless we managed to partner with somebody who was actively like bringing in fedramp customers like some agency that worked with the government or whatever like would have to be but I I wish we had known that at the start and we didn't one thing that would be interesting there would be to have open Telemetry have some sort of like agency that and that agency gets van ramp compliance and then what you can do with that is you can have that agency essentially facilitate the onboarding of open Telemetry vendors or just other observability vendors into the space and you can use that to sort of start Bridging the Gap and giving people more capabilities and sharing that my fan Ram specific architecture concerns in the way that spreads their compliance load amongst multiple companies and if you could do that oh sorry I was very excited by that idea that makes a ton of sense we could maybe make it work through the project and I would say for us as a small vendor that really is a problem that if we are not allowed to have our Engineers who are in other countries touch the system we may only have one engineer working in a particular area and so we have to hire a second person in the U.S which is undoable there are certain ways um and you would need somebody to talk to someone who's experience in Finland like negotiate that would happen like 100 U.S citizenship is in not see her own requirements it's it gets nuanced and it gets tricky oh interesting well I think we have to leave it at that before we got into a funny side um thank you again Hazel for sharing your insights this has been really cool um I I don't think we get to talk to too many people who uh who have had like such Advanced use cases for open Telemetry and and um being able to share that with the community is is really good because I I think a lot of us get you know we we do like those intro tutorials we're like hey that's pretty easy and then you get into like the the guts of it and then that's when you start uh getting into some of those gnarly use cases can be really tricky in life yeah yeah definitely so definitely appreciate that that feedback and and the Viewpoint and you will be back um for hotel in practice I believe on September 14th looking forward to that um do you know what uh what topic you'll be discussing no idea I'm gonna figure it out but it's gonna be a lot of fun so awesome really looking forward to that um thank you again as always super super awesome insights um yeah and uh we will hopefully see everyone um for hotel and practice with hazel on September 14th thank you diff --git a/video-transcripts/transcripts/2023-09-16T04:37:50Z-otel-in-practice-presents-context-abstraction-threading-the-needle-with-hazel-weakly.md b/video-transcripts/transcripts/2023-09-16T04:37:50Z-otel-in-practice-presents-context-abstraction-threading-the-needle-with-hazel-weakly.md index e52947a..c4f65dc 100644 --- a/video-transcripts/transcripts/2023-09-16T04:37:50Z-otel-in-practice-presents-context-abstraction-threading-the-needle-with-hazel-weakly.md +++ b/video-transcripts/transcripts/2023-09-16T04:37:50Z-otel-in-practice-presents-context-abstraction-threading-the-needle-with-hazel-weakly.md @@ -10,61 +10,73 @@ URL: https://www.youtube.com/watch?v=1a7GyarlGAQ ## Summary -In this YouTube video, Hazel, who is involved in innovation and developer experience at her company and also serves on a hospital foundation board, presents a detailed discussion on observability, context abstraction, and the challenges associated with distributed tracing and OpenTelemetry. The talk explores the definitions of observability, emphasizing the importance of asking meaningful questions to derive useful insights from systems. Hazel outlines the components of context and how they relate to observability, discussing various types of context, including in-process and cross-process context, and the complexities of manual instrumentation. She highlights challenges such as the lifecycle of traces, context management, and the issues that arise when trying to abstract these concepts in a distributed system. The presentation concludes with a call for collaboration and a deeper understanding of these technical challenges, inviting further discussion and sharing of knowledge in the observability community. The video includes interactions with participants who express appreciation for Hazel's clear explanations and raise questions about post-processing in telemetry data. +In this video, the speaker, Hazel, discusses the complexities of observability in distributed systems, focusing on concepts such as context abstraction and the challenges associated with implementing observability in real-world applications. Hazel provides a primer on observability, exploring its definitions from consumer theory and cognitive systems engineering, and emphasizes the importance of asking meaningful questions to derive useful insights from systems. The discussion covers the role of context in open telemetry, the significance of tracing in understanding system interactions, and the difficulties encountered when trying to instrument code without intrusive methods. Key points include the challenges of managing context propagation, the lifecycle of spans in tracing, and the implications of abstraction on observability practices. The session also includes audience engagement, with questions and reflections on the current state and future of observability practices, highlighting the need for effective collaboration among engineers to enhance system understanding and problem-solving. -# Observability and Context Abstraction +## Chapters -I am having a lot of fun hoping to run Innovation and developer experience in my current company. I also serve on the board of directors of the hospital foundation and have a lot of opinions online. You can find me pretty much everywhere, and I am really excited to be here today to talk about context abstraction and threading the needle. +00:00:00 Introductions +00:02:30 Overview of the topic: Context Abstraction and Observability +00:04:00 Definition of Observability from Consumer Theory +00:06:15 Cognitive Systems Engineering definition of Observability +00:08:45 Personal definition of Observability +00:12:00 The importance of asking meaningful questions in observability +00:17:00 Introduction to Context Propagation in OpenTelemetry +00:22:30 Explanation of different types of context: In-Process and Cross-Process +00:30:00 Discussion on the Life Cycle Problem in tracing +00:36:00 Graph Rewriting Problem in instrumentation +00:45:00 Conclusion and final thoughts on Observability challenges -## Introduction to Observability +# Transcript Cleanup -I'm going to give a primer on observability and discuss how I think about it in the context of historical necessity. We'll talk about why we need distribution and how that ties into observability. Additionally, we'll explore some components of social filtration, mainly context, and abstraction. One of the difficulties you may encounter when trying to build libraries or a platform team is moving beyond manually instrumenting all your code and writing everything from scratch. +**Introduction** +Hello everyone! I’m having a lot of fun hoping to run innovation and developer experience in my current company. I also serve on the board of directors of the hospital foundation, and I have a lot of opinions online. You can find me pretty much everywhere, and I’m really excited to be here today talking about context abstraction and typing the needle. -### Starting with Observability +**Observability Primer** +I’m going to cover a primer on observability and discuss how I think about it, what the context is, and the historical reasons we need distribution. Then, we’ll tie that into observability, along with some components of social filtration, mainly context. We’ll also explore abstraction and some difficulties you may encounter when trying to build libraries or platform teams, rather than just manually instrumenting all your code and writing everything from scratch. -When you start with observability, you have your entire system, and you want to figure out what’s going on. Over time, different groups have defined observability in various ways. One definition comes from consumer theory and states that it's the ability to measure the internal state of a system based on its external outputs. This is a great definition, but it doesn't tell the whole story. +When you start with observability, you have your system, and you want to figure out what’s going on. Over time, people have encountered this problem, and different groups have come up with various definitions of observability. -Another definition originates from cognitive systems engineering, which describes observability as feedback that provides insight into processes and the work required to derive meaning from available data. I appreciate this definition because it emphasizes that observability is a dialogue. You need to know not just how to observe everything but also how to reach that point and what it means. +**Definitions of Observability** +One definition you might hear from Journey Majors and others in the open telemetry and tracing backgrounds is from consumer theory. It defines observability as the ability to measure the internal state of a system based on its external outputs. This is a great definition, but it doesn’t tell the whole story. -### My Personal Definition of Observability +Another definition comes from cognitive systems engineering, which defines it as feedback that provides insight into the process and the work required to derive meaning from available data. This definition highlights that observability is a back-and-forth dialogue. It’s not just about observing everything; it’s about the process of getting there and understanding what it means. -My personal definition blends the two previous definitions: observability is the process through which one develops the ability to ask meaningful questions and receive useful answers. This definition is important because it highlights that observability evolves over time. It may mean something different today than it does tomorrow. +My personal definition, which blends elements from both definitions, is that observability is the process through which one develops the ability to ask meaningful questions and receive useful answers. This means it’s an evolving process; what observability means to you can change over time. -Meaningful questions are crucial to observability. These questions may or may not involve computers or people, and they should be significant to your business. For example, if a CEO wants to know how the company stands in the market and how feasible it is to reach a desired position, that is a meaningful question that an observable system can help answer. +**Meaningful Questions** +A meaningful question is one that maximizes how much you learn about the system you’re participating in. For example, if a CEO wants to know their market position compared to where they want to be, an observable system should enable them to answer that. For engineers, observability questions often relate to system performance and code behavior, which we’ll delve into more today. -For engineers, observability often revolves around questions about the system and code. We should all collaborate to solve these problems and find answers together. +**Early Observability Practices** +In the early days of observability, when you had a web server but no data, you might have resorted to stopping production, launching a debugger, and stepping through the entire system. This method could lead to meaningful questions and useful answers, but it’s not the most efficient approach. -## Context and Abstraction +As we moved forward, the desire to observe without interruption became essential. The introduction of logging led to a more structured approach, but it still required careful consideration to connect the front end and back end of systems for a complete understanding. -Context plays a significant role in observability. The documentation for OpenTelemetry states that context is a propagation mechanism that carries execution state values across API boundaries and between logically associated execution units. This is essential for understanding how to instrument and use context effectively. +**Context in OpenTelemetry** +Context in OpenTelemetry is crucial for managing cross-cutting concerns. It acts as a propagation mechanism, carrying execution context values across API boundaries and between logically associated execution units. Context allows us to unify disparate pieces of information and understand the system as a whole. -When you instrument your code, you will typically create spans that contain context information. Each span will have identifiers such as span ID and trace ID. You can think of traces as graphs, where spans are nodes and links are edges. However, managing these links can be challenging. +For each span, there is a variety of context elements, including span IDs, trace IDs, and trace states, which help identify relationships between spans and facilitate tracing across services. However, there are challenges in managing these contexts, especially when dealing with baggage and how it propagates through systems. -### Types of Context +**Process of Context Management** +When dealing with context, you have in-process context (the normal context created in your code) and cross-process context (how that context is serialized and transmitted over the network). It’s essential to ensure that the context is correctly integrated at various points in your system. -There are different types of context to consider: +When you receive context from another service, you typically deserialize it and make it your current context. However, if you don’t own your systems end-to-end, it can be challenging to ensure that context propagation remains consistent. -1. **In-Process Context**: This is standard and handled automatically by the library or SDK. You typically don't need to think about it unless you explicitly need to manage links between spans. - -2. **Cross-Process Context**: Involves serializing context and attaching it to payloads during inter-service communication. This can be set up to ensure that the context flows seamlessly between services. +**Challenges of Manual Instrumentation** +The biggest issue with manual instrumentation is that it assumes you’re writing all your code and instrumentation in an intrusive manner. This can lead to problems, especially when you have to manage multiple spans and their relationships. -3. **Baggage**: This allows you to carry values downstream but is typically viewed as a questionable practice. It can be powerful but should be used sparingly. +For instance, once a span is created, you can’t add information to it after it has ended. This limitation can lead to confusion and silent failures in your system. Moreover, the life cycle of spans complicates the ability to trace and sample effectively. -### Challenges with Context +**The Graph Rewriting Problem** +The graph rewriting problem arises when you want to manipulate spans in a way that isn’t straightforward. You may want to collapse spans or add attributes without creating additional spans unnecessarily. However, existing instrumentation frameworks often don’t provide a clear way to do this. -When working with context, you may face challenges. For instance, once a span has ended, you cannot add information to it. This limitation can lead to issues where adding information after the fact becomes impossible or inconvenient. +**Conclusion** +In conclusion, observability is about developing the ability to ask meaningful questions and derive useful answers from the system you’re working with. The more you can observe, the more correlations you can infer. However, be cautious about the additional complexity and noise that can come from adding more data. -Additionally, if you want to build reusable abstractions, you may encounter difficulties due to the need for cross-cutting concerns. For example, if you want to collapse spans or manipulate the graph of spans, it can be challenging to do so effectively. +Context plays a vital role in turning disparate data points into a unified trace. Understanding how to effectively manage context and its implications can significantly enhance your observability practices. -## Conclusion - -To sum up, observability is the process of developing the ability to ask meaningful questions and obtain useful answers. The more you can observe your system, the more correlations you can infer from your data. However, be cautious—just because you can find the needle doesn't mean you need to build a haystack. - -Context is the glue that turns disparate data points into a unified trace. Understanding how to use context effectively can enhance your observability efforts. Observability and abstraction can sometimes be at odds, creating challenges in how you instrument your systems and collaborate with others. - -Thank you for your attention today, and I'm open to any questions you might have! +Thank you all for your attention! If anyone has questions or thoughts, I’m here to discuss further. ## Raw YouTube Transcript -foreign [Music] [Music] I am having a lot of fun hoping to run Innovation and developer experience in my current company I also serve on the board of directors of the hospital foundation and I have a lot of opinions online you can find me pretty much everywhere and I am really excited to be here today talking about contact abstraction and typing the needle so I'm gonna go over kind of like a primer of observability and talk to you a little bit about how I think about it and sort of what that contest is and the historical why do we need distributation and how does that tie into observability and then we're going to talk about some components of the social filtration mainly context and then we're going to talk about abstraction and one of the difficulties that you can run into when you're trying to actually in real life build down libraries or build on platform team or build out something other than just manually instrumenting all of your code and writing everything from scratch don't you started with observability you have essentially you have your system you have everything and you want to figure out what's going on and over time people have run into this problem and different groups of people have come up with different definitions for observability so one definition that you've heard Journey majors and a lot of other people um open Telemetry and tracing backgrounds and distribution system background to talk about is the one from consumer Theory and that is the ability to measure the internal state of a system based on its external objects and so this is a definition that you see repeated in a lot of places it's a great definition I like it a lot um but it isn't necessarily the entire story um one other definition comes from a different theory of practice which is cognitive systems engineering and so for them their definition is feedback that provides insight into process and the work required to start to meaning from available data so we like this definition for a lot of reasons mainly that it does a really good job at showing that you that observability is really this back and forth dialogue you need to not just have you know oh you can observe everything well that's cool once everything's built out but like how do you get there what does it mean and the whole process of people are involved too is something that this definition kind of highlights and I love any technical definition that includes people it's my favorite Mr speaking of which my personal definition is I'm not biased but I like listen the most um and this one is I took the other two definitions from cognitive systems and from Contour Theory they're kind of blending them together and so for me my definition of observability is the process through which one develops the ability to ask me equal questions and again useful answers so the way I think about that is it's a process which means it has to be evolved and you start from somewhere but you're not going to stay there you're going to continually figure out what observability means to you and it may mean something different today than it does tomorrow then it will not show you than it did last week and then additionally meaningful questions is really the thing that observability to me hinges around can you ask a meaningful question of your system which may or may not involve computers may or may not around people it may involve like a whole bunch of different things and are those questions meaningful to business meaningful to you meaningful to what you need out of that learning and in asking those questions can you get a useful answer so I like this definition for me because it includes every aspect of observability in my opinion like if a CEO wants to say hey I have this meaningful question which is I want to know where we are in the market compared to where we want to be in the market and how feasible am I going to get there can they answer that to build like what we need in order to approach our problems for the next five years I only said in Los Angeles so I'm keeping them happy and healthy that's a meaningful question an observable system will let you answer that and for engineers a lot of that observability questions it's going to come down to things about the system things about code and we're going to dig a lot more into that today than the other questions and we should all be working together and we should all be able to collaborate in order to actually solve all these problems into these answers so for me a meaningful question is one that maximizes how much you learn about the system that you're participating in to like if you have that system and you want to know more about it what does that mean how do you get there what are you doing right so for programming we have all my camera disappeared brilliant some I love silent problems all right so I'm going to figure out what happened there super briefly and then fix that and then get back and if I can't fix it then I'll just tell everyone what's on the slide um oh well all right let me share my screen again and I'll just tell you what was on the slide sorry about that I debugged everything and we're learning information awesome so imagine on the slide that you have a typical system in which you have someone and they're running a web shiver and they are asking themselves what is this web server doing and should they look at the ones and the lots and once you have no data whatsoever absolutely nothing nothing's there and they're like okay now what um what do I do back in the good old days well you probably would have done was you would have just killed production taking the whole thing down launched the debugger and just steps through the entire system step by step and solving everything you can ask a lot of meaningful questions that way and it can get a lot of useful answers everything's solved and all you had to do is completely change production in order to do it which is you know maybe not maybe not the best approach but it is you know it's in votes that's what we did for a long time and in many ways it still works but one of you have to constraint of to I want to observe without interruption I want to you know keep everything running and ask me to focus into the system and now I'm considering the system running as part of that question so let me try you know adding some French statements which again we're seeing a little bit of lack of observability here in the slides that's fine so if we have a French statement and we've added them everywhere and we now have a real login this is awesome so you can imagine us taking the web server and getting those alarms and looking through and you can see oh hey this one usually locked in this one um I use a middle transaction they're able to do something and then they locked out that's awesome we have everything there we have it done and we're ready to go right is everything is everything there well so you have the web server and you have the back end and unless you have very carefully you know done everything with your logging you'll notice that the front end and the back end nothing's tied together the pen and might say oh you should check down use your uh in the back end might say this database middle transaction this little visual transaction this individual transaction and you have nothing to tie those two disparate pieces of information together to like okay I still can't really understand anything about the system I know when each individual piece is doing but I have nothing to tie it all together contest it really ties the system together so contest in an open Telemetry is for class cutting concerns the documentation for open Telemetry they say that context is a propagation mechanism which carries execution script values across API boundaries and between logically Associated execution units and that is awesome let's break that down just a tiny bit so contest let's skip on the middle bit carries values contest is the blue the terms of macadata into unified trees so what does that mean for you what does that mean but it's just um what does that mean when you're trying to understand it and instrument it so let's break down and decent contest they're going to have for each span they're going to have a whole bunch of context in there you'll have like this fan ID which identifies that the trace ID some Trace flies and a train state they're going to have essentially everything that identifies to spam of which there are many spans in one trace and many traces in one system scan links or another type of context and they say hey this Advantage related to Market so you can think of traces as a graph expands or nodes and then links to the edges so if you have a graph that's a tree and you're like sort of building it down and all of a sudden you need to take this one span over here and it's one span over here and to say but they're kind of links together you build that link and that's an edge in that graph Lynch can be really confusing they can be kind of weird they're not always integrated very well in the whole setup but they are needed for especially in some types and patterns and they are what turn open Telemetry traces from a tree into a graph and so they're the most expensive thing you have in there and also the hardest thing to work with at the same time So currently you can only add them at span pretty soon time but people want to be able to add them anytime and that may happen in the future does you'll be able to have a non-directed graph and a little drag which will be fascinating another type of context is baggage packets want to keep valued parents Downstream it is widely regarded as a questionable thing to use I know many people who might banned it in their code base it's a bit of a on but also very very powerful and flexible it is one of the ways to send information Downstream but you still can't use it to send information upstream and I'll get more into what I mean by that later um the package is there you may eventually need it I kind of hope you don't but it's there and when you have it so when you have all these different types of context and you have these different services and you have the different processes how do you take this contest and actually use it to glue together everything and make it usable you have the in-process contest and then you have the class process and how to complicate things from one thing to another and then you have fun process which is in everything in and actually taking it in the next service and going down that way so I'm going to go over all of them in order so there's some code here sorry about that um I've never even probably known as contest in process it's the normal context it's very typical very standard and this is what you've seen the documentation this is the in process contest you create an octaves fan and then in that disadvantages and work you create another active span and then that context and links the two actually happens pretty much automatically and you never really need to think about it the library and the SDK will handle all of that for you sometimes however you do need to add it explicitly um and this is a invisible example showing you that you need to do it especially one comes to um making links and doing things that way I'm going to one second oh amazing I'm going to watch my screen shoot uh stop sharing and then share something different it turns out I accidentally made my syntax highlight and only work in Firefox that's what that was cool yay there you go you can see here that we have the current span we have these fan contents and we get that context and pull it out manually so then we can create a span with a link we need to do that in order to propagate things manually for this type of context information but typically you don't really need to do that and things through it just naturally words with the links you need to do normally to that that can be a little tricky blush process there's two steps involved you serialize the context foreign it can actually secretly don't tell anyone I told you this but there's no reason context needs to be an adder you can do anything with any transform mechanism that you want to it's just really serializing the context needs to happen somewhere and then that needs to be attached to the payload of your call so I know some people who have actually serialized the context and embedded it into an RPC account as a way of actually instrumenting and tracing embedded systems um some people have done that with their own bootloaders which is actually really cool um it doesn't need to be a matter a little wild if you want to use the library should not write everything yourself down all the open television stuff will stick it in each TV header for you so what that looks like is you have from the sending service the sending service is going to inject what's the publication object the current context into some output and so that will do the serialization process and then you'll have a transparent and a tree State and then output and you can use that over the internet the transparent is actually the only thing that you need the Twisted has a bunch of other stuff but you don't actually kneel in and you may not want it so the transparent is the trace ID and everything required to actually link the span from the net system over the trace state has on the other fun goodies that you may want but may not want some people I know have been using twisted and can't touch specification because of issues it has then I'll get to later the context one process this is the second process we've yielded everything up on the internet and now I'm ready to receive that and actually integrate it you'll have you get the fabricator you ingest everything and you deserialize the contacts and you extract it and then you did that it's not a context and you make it your current contest you don't need to make it the current content but typically that's what people do um sometimes people use the trace state in order to determine whether or not they should make the contents the current context and you can maybe want to do that sometimes if you don't always own your systems end to end and someone may be putting something in the middle somewhere um that can be really tricky I know that some open Telemetry vendors for example have run into this problem of you need to really make sure that you're contacted your contacts before actually making it your current contest um so that's a lot of fun and then once you have that context then you've started it you set your span and just done working from there so that is moving the context wrapping everything together and you have all of that and that is great but that is all really about manual instrumentation that assumes that at every single point in every service you are writing all of your code you're writing all of your instrumentation you're writing everything from scratch and you're writing it right there in line with all the code in an intrusive manner if you don't want that intrusive style and you want to be able to build something for other people to use this is where you'd like to run into a lot of problems and we'll get into this now this is in my opinion one of the weakest aspects of fishing in general and it's a sign of the image journey in the ecosystem I'm gonna have a lot of hope that can be improved and I'm looking forward to seeing what happens there do you remember earlier when I said that context is about to cause any concerns when they have a cross Canadian concern and you tie everything together what you also have is cross-cutting breakage because um one of my favorite um launch out there is Hiram's law which tells them with a sufficient number of users of an API it does not matter what you promise in the contact all observable behaviors of your system will be dependent on by someone which means that the more instrumentation that you add into your system the more things that you're adding that can be observed by yourself but also the rest of the system and so paradoxally when you come on service boundaries in theory you can only depend on the public API and service it's a very standard service oriented architecture and things like that the tracing actually doesn't do that it's out of hand intentionally invisible not apparently API and if you've been with anything anywhere it'll propagate to the rest of the system and you'll end up having a lot of essentially convention and hand holding and little manual pieces put together that will break every time you change something to those out of band information become in band semantics and this problem is something that you see in a lot of different ways more open Telemetry it's actually specifically about this context modification but in form of verification you see this in terms of the internal implementation details of a function changing the proof required to show the scratch verified for property and in distributed systems you can see this in the end-to-end property of networks and in lots of other places you can see it probably one of the easiest ones is different agility of snapshot testing for fun and Design Systems if you've ever worked on those you know that you can meet the tiniest little change everything's the worst and somehow you just broke your entire test Suite because some lines moved here and there in the code itself or something changed here and there in the code itself or in like the HTML but the actual appearance and everything's fine and you end up dealing with this frustration and description of the out-of-man information becoming in man to mantis and that mismatch causing wreckage and so in open telemetry package is not simple enough to cost a lot of honey it's abused one context is key and when you have this uh class process propagation you lose the ability to have your out of band and inbound to mantis match up and be enforced across boundaries additionally in Winter designing options or you're wrapping them that same Alabama information will be very painful because you can't necessarily build anything that actually lets you do this in a reusable way unless you build a shared library of semantic conventions and speaking of which a good options are both transparent and opaque what I mean by that is they're transparent in that you don't have to know that they're there any condition about them at that level of abstraction it's a good way to think about that is for example HTTP requests they just happen and they're just there and you don't really have to know that they're implemented on top of a bunch of wire or glue terrible crimes and weird packet header loss information blah blah blah um there's a lot of trauma behind that but you don't you have to know that with that said you can't know that and the transparency there is that you can reason about everything at that level as if the abstin was in there and you can write things that are aware of the fact that they actually work through it in order to know and understand the underlying system to having that absorption means that you don't need to know the system underneath it but that the system underneath it doesn't make it less understandable of inflammatory and out-of-band information in general meets both of those goals hard to achieve the Louisiana ban information is not actually semantically embedded in the system and becomes relevant to the system and so you can't build an abstraction that lets you operate without knowing the internal implementation details and you don't actually have a way to observe the internal implementation details and know what they are without disapproved forms memorizing those conventions and so that gives you this quandary uh what do you do with that and this can manifest in a couple interesting ways um but a different problem of abstraction is I'll show you this one and this is problem that I run into the most which is what I call it the life cycle problem and that is you have tracing which is a graph but you don't actually have a way to really manipulate the graph very well and that shows up in two ways the livestock of how much the first flame which is that sending information up a graph or back in time so to speak isn't super possible or convenient or really ergonomic in any way and then adding then and then adding things after the fact is not possible either so you can't like pack to things as a girl and like incrementally build up or out of Beyond build up a view of the graph that's more complete so here's an example we have a web server a service and a client and so the web server has you know one event a once fan and then a second span two fish has one span and then a plan has one span and they have overlapping timelines there's problems that I ran into trying to deal with building this so the first Span in the web server can pass information to the second span when creating it that's totally fine that works that's amazing that's awesome however it has no real way to pass information to the second span after creating it so after you've created that a challenge fan the um root doesn't really have any way to continue to add information it doesn't have any way to modify that and doesn't have any way to like dig down into the tree and continue to add things if you have intuitively instruments and everything you do have the reference to this fan and the theory you can work with it one of your building apps into new building libraries you don't have that so you don't have a way to send information down they have baggage but it almost feels like a workaround it doesn't feel like it's embedded in this idea of how do I add things into the system and build this graph conversely the challenge man doesn't have any good way to send information to the parents fan conveniently at all it's possible if you know how to do it it's possible if you know how to build and work around for that you can even build conventions to make it easier but it's not absolutely convenient in any way the lump server is able to pass information to the service and then the client via contact configuration which I talked about earlier and that's possible not to create Networks however the service and the client essentially have no way to pass any information back up to the web server there's test that's not happening forget about it another thing that's interesting here is if you notice the life cycle of the web server watched is actually shorter than the life cycle of the whole trees which means that Toyota sampling is not going to work as you might expect till sampling really wants you to have a full Stitch treat with a scooped corn snack life cycle to the web server or the root span has to be the longest span and everything else has to be inside of it and organized nicely so that you can make a decision once you've collected everything and when you have these Instagram timelines and you don't actually know how long everything is going to be if you don't know how long to wait to make your final decision and you end up discussing until it becomes heuristic faced rather than oh and let's actually do things but you can really really bring you if those heuristics are used as part of filtering your traces and actually trying to sample things so Taylor sampling Distributing choices is actually really really hard um another problem is that once each fan has ended you have no way to add information to it so even though it is technically possible to try and add information to expand from the parent you can add a permission to a challenge fan in some ways that only works if the child span hasn't ended yet and you have no way to know one or not adding that information worked she can end up with code then adds information to expand works great more Express Financial production it's awesome and then someone optimizes the John function I mean it should take half the time and used to and then everything starts breaking silently because you didn't actually know then now you're trying to add things to his fan after it ended you can have a really weird and Rich conditions essentially um speaking of that also happens um yeah she can add information to the service and also the child and parents have the same problem to even in context and uh even in process versus Under severe process you still run into the same issue which means that you have a ton of internal implementation details of the instrumentation the shape of the code how the code works the life cycle of their code the column staff of everything to keep track of which if you are the engineer writing everything interviewing with the whole system and working end to end with it that may not be a huge deal for you and that may actually be there that's one of the reasons why open to my materials so deeply in hand with people owning their systems end to end and having that feedback there's actually in some ways available compensation because you can't build an abstraction unless a lot of people work on top of that so you have to own your system end to end intrusively rather than being able to build these abstracts attention let you instrument things more effectively the second problem there that you run into a lot is what I call the graph rewriting problem so we dig back into this livestock problem you have a bunch of information based on life cycle so when you make lesbians lens fans and and how that interfaces with the constant of the the stimulate system in general need graph rewriting problem is this one the rule of thumb when doing instrumentation is that you have like this final instrumentation that can build this scaffolding uh you would take all of your manual instrumentation and hang it on that so in theory you're not really creating one spans your usual understands that auto instrumentation already set up and you're just adding attributes in my spots which sometimes that doesn't work especially when you're trying to write abstract code you end up wanting to create this fan only abstract couldn't do that but in general the more you can just add attributes to an existing span created by Honor instrumentation but not sad where do libraries go in this and how do they work and how do you actually do this so let's look at this as an example we have a um tree uh it's fast client and then It smash client has some action someone called make request on the client side and then the auto instrumentation has that git endpoint the next step on the client side to half the function call and then you have the honor instrumentation of the actual that response awesome that's great that's cool then we go over into the server and the server has the server endpoint that has three manaway functions that it has a post endpoint uh for the sending uh for an internal call then that has the same chain uh you post on one thing and then you go to the next one then you keep going and keep going and keep going and you can run into the same middleware and you're going to do all that so if you're going to run into a couple problems here the first problem is how do you collapse spans you don't actually have a good one to do that what if I don't actually want all of the middleware to be their own spans and I want to have just one span for everything just add information the development to it with attributes as I go down to me if you want some spam Events maybe I just want the attributes maybe I want things like that and I don't actually need this fan for like the middleware around or maybe I don't need a fan for some other things I don't have a way to like collapse that tree down and that can be is activated by the Fall that's that if you want to search through your system typically the root span is where you search or the same level of a span is when you search when you're trying to like index through your data and so if you have dealer at different span levels you essentially you have no way to correlate those two pieces of datum and do a query in which you aggregate things across those spans so that's one reason why a lot of good practice is to have very very wide events and a very wide spans and less of them so that way you have all the data at the same level and you can do all the correlation and grouping and aggregation and everything else there that you need to in order to effectively look through your data and find something but if you have all of your instrumentation and your honor instrumentation building a nested tree of like 10 or 20 inch fans before yeah as you get to run any of your code where do you stick in so that it's at the level that it needs to be and then on top of that every Library Auto instrumentation um and things like that reinvent their weird set of configuration I want you to try and make it to that they can help solve some of these problems so you'll have um if you look through all the sdks or all the contributors contributing libraries for open Telemetry you'll notice this problem over and over you're like oh how do I use for example the express um server Auto instrumentation it'll save you can just use it and also here's like 15 different options do you want to silence some endpoints do you want to filter some endpoints do you want to silence submit awareness do you want to whatever here's a whole bunch of functions you can use here's some callbacks and we're going to call this compact before and after this we're going to call these to these endpoints and essentially it gives you every possible hook point to you can think of to enter into system and stick your own code there as if you haven't written it yourself but when you don't have this way to write an abstraction and you end up having to sit there and write the code yourself in all these intuition endpoints and you need to have people have having foresight in their libraries to put these uh insertion points for you you have this very very weird platform of really people want to like more or less patch the source to go to the libraries and the other instrumentation and they want to have written all of it themselves graph effectively why can't we deal with that right what's going on I think I cut out for a brief second there did everyone catch the last things that I said awesome so in conclusion there are some problems here there are a lot of things going on there's some advice on what to do and I'm gonna go through all of that now to observability is the process through which one develops the ability to ask meaningful questions and get useful answers meaningful questions for you are going to be about learning about the system how do you collectively get together and Advance your understanding of the system Its Behavior how it interacts with you and everything and how do you get that knowledge as quickly as possible those are meaningful questions continue can use for answers from them if you can your system is becoming observable however observability is interesting because the more you can observe the more correlations you're going to infer with your data and the harder it's going to be to discover contributing factors so just because you can find needle it doesn't mean you need to build a haystack and when you have more and more data and you keep shoving more things in there then you have more and more out of band data and you just add more and more layers of things they can give you observing money into the system you should be really careful to look at everything and go is this actually an intervention or understanding of the system or is this adding noise contacts to contest plus configuration is traces is the glue that turns and back of data points into a unified Trace when working with context it interfaces with everything that has to do with open Telemetry and in many ways remains invisible the entire time using until you're wondering where the entire or how to use it or maybe you're confused but if you know how to use context and know what to think about when using contest it can become your friend one way to map contacts and map different like aspects of open Telemetry together is thinking about the shape of those contacts of those Concepts so traces are a graph expand your nodes and links your edges despite choices being a graph they're often actually mostly thought of and best most ergonomically designed as a cardstock which is a training that is like perhaps multi-branched and that tree is very uh scooped in the top node is always the whitest and the longest and everything is only encapsulated all the way down and that's the most natural and ergonomic way to do open Telemetry and tracing and everything's going to push you towards that but traces are a graph and you can actually access all of that graph power when you need it and sometimes you wish you didn't observability and abstraction are natural enemies they are the chief people in high school that love to take in on each other the entire time they're doing everything and so even though we love open Telemetry and we love observability and we love my other sister meditation the problem of needing a large amount of out-of-band context but also wanting the ability to add and annotate a very complicated problem of your entire system it's business Logic the flow of information throughout the system those are all problems that we don't actually have good answers for in general we don't have good leverage for that we don't have good approaches for that and then it stands Beyond open Telemetry and Beyond observability this is just one of those ways that it becomes a very very obvious problem and two ways of having an average problem are the life cycle problem which we talked about which is where you have this graph you have the whole power of a graph where you don't have a graph that is like convergent you have a graph that you kind of have the satanically know the shape of an Azure building out and you can't really patch it up and picture them later and speaking of that graph reminding it is really difficult if not impossible it's very much convention and ad hoc based there you are does anyone have any questions thoughts or anything else I just want to say hazel that the way that you uh describe these Concepts in in uh distributed tracing and open Telemetry are like I think the clearest I've heard um so thank you for that I I definitely really appreciate that I I think it's uh it's like every everything now like just fully makes sense so yeah I just want to Echo that but that's that's definitely the vibe it's like oh yeah cool now I know what I'm writing about for the next six months it's like the stuff that the stuff that Hazel has put so clearly it's very good really happy to hear that the the question that I had is I'm I'm curious how you feel like um like post-processing fits into that picture like with the collector um a lot of your times revolved around like developers devops people who are working with the code like like how they're able to propagate data but what about that like not so much like tail based sampling where you're deciding right at the moment but like maybe a little further down the line being able to connect data points you know once you're holding five minutes of of Trace data yeah so one thing that's interesting there is that open Telemetry feels in many ways like an event based emitting style of instruments in your code but it's actually not that and the collector and other like um ways of post-processing data is where you start to feel that sort of impedance mismatch in particular um and so because of that um you can run into certain problems like the a great way a great example of that is that links can't be added after expanded there and so even uh as far as I know like once you create this fan you can't add a link to it even in push processing and so if you're trying to do that things are going to be really drinky maybe post-processing can fits that but I don't know that that's now I'm gonna go like research that that's wow okay cool yeah I mean I realize I have not done it I've been like digging through it haven't done it so wow okay so there is interesting historical contents there in that there used to be a method in the open Telemetry uh specification of you can add a link to his fan as long as this man is active then they took that method out several years ago and it's been an open um discussion and the um GitHub tracker for about three or four years now of this is absolutely useful we do end up needing this a lot and can we do that um apparently as far as I know the only way to do that is that's fan Creation in the options of the um for each band you can add links in there and that's the only time which means certain types of Instagram patterns or certain types of like that type of thing really really difficult to do and also means you know when you can approach process and then after that pretty much and then another way to think about the urban Telemetry commentary that's interesting is I actually view the counter and how to set up libraries and baggage and other things like that all as different types of convention-based out-of-band information and handling so when you build up conventions in your company or in your system or in anything like that you can start to say oh yeah if you set this information here then it'll be there or you can actually just assume that we'll have this information because the clutter will add it here so as an example um and when it's metal and a blog post in which they talked about they said of their collectors to actually add to every span what data center that's man came from what machine that span came from what um and then where everything was what cluster it was in and all this actually information so everyone can always clear all that information you don't know that like it's there because you read the documentation in the internal tooling and you can build internal tooling that works on top of that natural convention rather than something you can actually Express in your Telemetry itself anything else well I guess if there are no further questions thank you so much for for coming on for coming on twice and for putting this together I know uh presentations are definitely a lot more work to put together compared to just you know sitting there answering questions so definitely appreciate the time that you took to put this together um and to share your knowledge with folks I think there's uh you know so much are out there on on like the novice Hotel stuff but when it starts to get into the gnarly bits um we're definitely lacking in that information so we definitely appreciate um you sharing that and uh call out to anybody if you if you or you know anybody who is interested in sharing some Hotel goodies um in hotel in practice we would absolutely love to have you um and also if you want to share your hotel story we would love to have you for open a and I think we've got a session coming up later this month right Rhys we do we have a special discussion actually coming up on the 28th um it's going to be about the evolution of observability practices and actually Nika is going to be part of that panel um and we'll do a full announcement so um thank you Hazel for uh for everything and um I guess we'll see y'all um on the internet [Music] thank you [Music] +foreign I am having a lot of fun hoping to run Innovation and developer experience in my current company I also serve on the board of directors of the hospital foundation and I have a lot of opinions online you can find me pretty much everywhere and I am really excited to be here today talking about contact abstraction and typing the needle so I'm gonna go over kind of like a primer of observability and talk to you a little bit about how I think about it and sort of what that contest is and the historical why do we need distributation and how does that tie into observability and then we're going to talk about some components of the social filtration mainly context and then we're going to talk about abstraction and one of the difficulties that you can run into when you're trying to actually in real life build down libraries or build on platform team or build out something other than just manually instrumenting all of your code and writing everything from scratch don't you started with observability you have essentially you have your system you have everything and you want to figure out what's going on and over time people have run into this problem and different groups of people have come up with different definitions for observability so one definition that you've heard Journey majors and a lot of other people um open Telemetry and tracing backgrounds and distribution system background to talk about is the one from consumer Theory and that is the ability to measure the internal state of a system based on its external objects and so this is a definition that you see repeated in a lot of places it's a great definition I like it a lot um but it isn't necessarily the entire story um one other definition comes from a different theory of practice which is cognitive systems engineering and so for them their definition is feedback that provides insight into process and the work required to start to meaning from available data so we like this definition for a lot of reasons mainly that it does a really good job at showing that you that observability is really this back and forth dialogue you need to not just have you know oh you can observe everything well that's cool once everything's built out but like how do you get there what does it mean and the whole process of people are involved too is something that this definition kind of highlights and I love any technical definition that includes people it's my favorite Mr speaking of which my personal definition is I'm not biased but I like listen the most um and this one is I took the other two definitions from cognitive systems and from Contour Theory they're kind of blending them together and so for me my definition of observability is the process through which one develops the ability to ask me equal questions and again useful answers so the way I think about that is it's a process which means it has to be evolved and you start from somewhere but you're not going to stay there you're going to continually figure out what observability means to you and it may mean something different today than it does tomorrow then it will not show you than it did last week and then additionally meaningful questions is really the thing that observability to me hinges around can you ask a meaningful question of your system which may or may not involve computers may or may not around people it may involve like a whole bunch of different things and are those questions meaningful to business meaningful to you meaningful to what you need out of that learning and in asking those questions can you get a useful answer so I like this definition for me because it includes every aspect of observability in my opinion like if a CEO wants to say hey I have this meaningful question which is I want to know where we are in the market compared to where we want to be in the market and how feasible am I going to get there can they answer that to build like what we need in order to approach our problems for the next five years I only said in Los Angeles so I'm keeping them happy and healthy that's a meaningful question an observable system will let you answer that and for engineers a lot of that observability questions it's going to come down to things about the system things about code and we're going to dig a lot more into that today than the other questions and we should all be working together and we should all be able to collaborate in order to actually solve all these problems into these answers so for me a meaningful question is one that maximizes how much you learn about the system that you're participating in to like if you have that system and you want to know more about it what does that mean how do you get there what are you doing right so for programming we have all my camera disappeared brilliant some I love silent problems all right so I'm going to figure out what happened there super briefly and then fix that and then get back and if I can't fix it then I'll just tell everyone what's on the slide um oh well all right let me share my screen again and I'll just tell you what was on the slide sorry about that I debugged everything and we're learning information awesome so imagine on the slide that you have a typical system in which you have someone and they're running a web shiver and they are asking themselves what is this web server doing and should they look at the ones and the lots and once you have no data whatsoever absolutely nothing nothing's there and they're like okay now what um what do I do back in the good old days well you probably would have done was you would have just killed production taking the whole thing down launched the debugger and just steps through the entire system step by step and solving everything you can ask a lot of meaningful questions that way and it can get a lot of useful answers everything's solved and all you had to do is completely change production in order to do it which is you know maybe not maybe not the best approach but it is you know it's in votes that's what we did for a long time and in many ways it still works but one of you have to constraint of to I want to observe without interruption I want to you know keep everything running and ask me to focus into the system and now I'm considering the system running as part of that question so let me try you know adding some French statements which again we're seeing a little bit of lack of observability here in the slides that's fine so if we have a French statement and we've added them everywhere and we now have a real login this is awesome so you can imagine us taking the web server and getting those alarms and looking through and you can see oh hey this one usually locked in this one um I use a middle transaction they're able to do something and then they locked out that's awesome we have everything there we have it done and we're ready to go right is everything is everything there well so you have the web server and you have the back end and unless you have very carefully you know done everything with your logging you'll notice that the front end and the back end nothing's tied together the pen and might say oh you should check down use your uh in the back end might say this database middle transaction this little visual transaction this individual transaction and you have nothing to tie those two disparate pieces of information together to like okay I still can't really understand anything about the system I know when each individual piece is doing but I have nothing to tie it all together contest it really ties the system together so contest in an open Telemetry is for class cutting concerns the documentation for open Telemetry they say that context is a propagation mechanism which carries execution script values across API boundaries and between logically Associated execution units and that is awesome let's break that down just a tiny bit so contest let's skip on the middle bit carries values contest is the blue the terms of macadata into unified trees so what does that mean for you what does that mean but it's just um what does that mean when you're trying to understand it and instrument it so let's break down and decent contest they're going to have for each span they're going to have a whole bunch of context in there you'll have like this fan ID which identifies that the trace ID some Trace flies and a train state they're going to have essentially everything that identifies to spam of which there are many spans in one trace and many traces in one system scan links or another type of context and they say hey this Advantage related to Market so you can think of traces as a graph expands or nodes and then links to the edges so if you have a graph that's a tree and you're like sort of building it down and all of a sudden you need to take this one span over here and it's one span over here and to say but they're kind of links together you build that link and that's an edge in that graph Lynch can be really confusing they can be kind of weird they're not always integrated very well in the whole setup but they are needed for especially in some types and patterns and they are what turn open Telemetry traces from a tree into a graph and so they're the most expensive thing you have in there and also the hardest thing to work with at the same time So currently you can only add them at span pretty soon time but people want to be able to add them anytime and that may happen in the future does you'll be able to have a non-directed graph and a little drag which will be fascinating another type of context is baggage packets want to keep valued parents Downstream it is widely regarded as a questionable thing to use I know many people who might banned it in their code base it's a bit of a on but also very very powerful and flexible it is one of the ways to send information Downstream but you still can't use it to send information upstream and I'll get more into what I mean by that later um the package is there you may eventually need it I kind of hope you don't but it's there and when you have it so when you have all these different types of context and you have these different services and you have the different processes how do you take this contest and actually use it to glue together everything and make it usable you have the in-process contest and then you have the class process and how to complicate things from one thing to another and then you have fun process which is in everything in and actually taking it in the next service and going down that way so I'm going to go over all of them in order so there's some code here sorry about that um I've never even probably known as contest in process it's the normal context it's very typical very standard and this is what you've seen the documentation this is the in process contest you create an octaves fan and then in that disadvantages and work you create another active span and then that context and links the two actually happens pretty much automatically and you never really need to think about it the library and the SDK will handle all of that for you sometimes however you do need to add it explicitly um and this is a invisible example showing you that you need to do it especially one comes to um making links and doing things that way I'm going to one second oh amazing I'm going to watch my screen shoot uh stop sharing and then share something different it turns out I accidentally made my syntax highlight and only work in Firefox that's what that was cool yay there you go you can see here that we have the current span we have these fan contents and we get that context and pull it out manually so then we can create a span with a link we need to do that in order to propagate things manually for this type of context information but typically you don't really need to do that and things through it just naturally words with the links you need to do normally to that that can be a little tricky blush process there's two steps involved you serialize the context foreign it can actually secretly don't tell anyone I told you this but there's no reason context needs to be an adder you can do anything with any transform mechanism that you want to it's just really serializing the context needs to happen somewhere and then that needs to be attached to the payload of your call so I know some people who have actually serialized the context and embedded it into an RPC account as a way of actually instrumenting and tracing embedded systems um some people have done that with their own bootloaders which is actually really cool um it doesn't need to be a matter a little wild if you want to use the library should not write everything yourself down all the open television stuff will stick it in each TV header for you so what that looks like is you have from the sending service the sending service is going to inject what's the publication object the current context into some output and so that will do the serialization process and then you'll have a transparent and a tree State and then output and you can use that over the internet the transparent is actually the only thing that you need the Twisted has a bunch of other stuff but you don't actually kneel in and you may not want it so the transparent is the trace ID and everything required to actually link the span from the net system over the trace state has on the other fun goodies that you may want but may not want some people I know have been using twisted and can't touch specification because of issues it has then I'll get to later the context one process this is the second process we've yielded everything up on the internet and now I'm ready to receive that and actually integrate it you'll have you get the fabricator you ingest everything and you deserialize the contacts and you extract it and then you did that it's not a context and you make it your current contest you don't need to make it the current content but typically that's what people do um sometimes people use the trace state in order to determine whether or not they should make the contents the current context and you can maybe want to do that sometimes if you don't always own your systems end to end and someone may be putting something in the middle somewhere um that can be really tricky I know that some open Telemetry vendors for example have run into this problem of you need to really make sure that you're contacted your contacts before actually making it your current contest um so that's a lot of fun and then once you have that context then you've started it you set your span and just done working from there so that is moving the context wrapping everything together and you have all of that and that is great but that is all really about manual instrumentation that assumes that at every single point in every service you are writing all of your code you're writing all of your instrumentation you're writing everything from scratch and you're writing it right there in line with all the code in an intrusive manner if you don't want that intrusive style and you want to be able to build something for other people to use this is where you'd like to run into a lot of problems and we'll get into this now this is in my opinion one of the weakest aspects of fishing in general and it's a sign of the image journey in the ecosystem I'm gonna have a lot of hope that can be improved and I'm looking forward to seeing what happens there do you remember earlier when I said that context is about to cause any concerns when they have a cross Canadian concern and you tie everything together what you also have is cross-cutting breakage because um one of my favorite um launch out there is Hiram's law which tells them with a sufficient number of users of an API it does not matter what you promise in the contact all observable behaviors of your system will be dependent on by someone which means that the more instrumentation that you add into your system the more things that you're adding that can be observed by yourself but also the rest of the system and so paradoxally when you come on service boundaries in theory you can only depend on the public API and service it's a very standard service oriented architecture and things like that the tracing actually doesn't do that it's out of hand intentionally invisible not apparently API and if you've been with anything anywhere it'll propagate to the rest of the system and you'll end up having a lot of essentially convention and hand holding and little manual pieces put together that will break every time you change something to those out of band information become in band semantics and this problem is something that you see in a lot of different ways more open Telemetry it's actually specifically about this context modification but in form of verification you see this in terms of the internal implementation details of a function changing the proof required to show the scratch verified for property and in distributed systems you can see this in the end-to-end property of networks and in lots of other places you can see it probably one of the easiest ones is different agility of snapshot testing for fun and Design Systems if you've ever worked on those you know that you can meet the tiniest little change everything's the worst and somehow you just broke your entire test Suite because some lines moved here and there in the code itself or something changed here and there in the code itself or in like the HTML but the actual appearance and everything's fine and you end up dealing with this frustration and description of the out-of-man information becoming in man to mantis and that mismatch causing wreckage and so in open telemetry package is not simple enough to cost a lot of honey it's abused one context is key and when you have this uh class process propagation you lose the ability to have your out of band and inbound to mantis match up and be enforced across boundaries additionally in Winter designing options or you're wrapping them that same Alabama information will be very painful because you can't necessarily build anything that actually lets you do this in a reusable way unless you build a shared library of semantic conventions and speaking of which a good options are both transparent and opaque what I mean by that is they're transparent in that you don't have to know that they're there any condition about them at that level of abstraction it's a good way to think about that is for example HTTP requests they just happen and they're just there and you don't really have to know that they're implemented on top of a bunch of wire or glue terrible crimes and weird packet header loss information blah blah blah um there's a lot of trauma behind that but you don't you have to know that with that said you can't know that and the transparency there is that you can reason about everything at that level as if the abstin was in there and you can write things that are aware of the fact that they actually work through it in order to know and understand the underlying system to having that absorption means that you don't need to know the system underneath it but that the system underneath it doesn't make it less understandable of inflammatory and out-of-band information in general meets both of those goals hard to achieve the Louisiana ban information is not actually semantically embedded in the system and becomes relevant to the system and so you can't build an abstraction that lets you operate without knowing the internal implementation details and you don't actually have a way to observe the internal implementation details and know what they are without disapproved forms memorizing those conventions and so that gives you this quandary uh what do you do with that and this can manifest in a couple interesting ways um but a different problem of abstraction is I'll show you this one and this is problem that I run into the most which is what I call it the life cycle problem and that is you have tracing which is a graph but you don't actually have a way to really manipulate the graph very well and that shows up in two ways the livestock of how much the first flame which is that sending information up a graph or back in time so to speak isn't super possible or convenient or really ergonomic in any way and then adding then and then adding things after the fact is not possible either so you can't like pack to things as a girl and like incrementally build up or out of Beyond build up a view of the graph that's more complete so here's an example we have a web server a service and a client and so the web server has you know one event a once fan and then a second span two fish has one span and then a plan has one span and they have overlapping timelines there's problems that I ran into trying to deal with building this so the first Span in the web server can pass information to the second span when creating it that's totally fine that works that's amazing that's awesome however it has no real way to pass information to the second span after creating it so after you've created that a challenge fan the um root doesn't really have any way to continue to add information it doesn't have any way to modify that and doesn't have any way to like dig down into the tree and continue to add things if you have intuitively instruments and everything you do have the reference to this fan and the theory you can work with it one of your building apps into new building libraries you don't have that so you don't have a way to send information down they have baggage but it almost feels like a workaround it doesn't feel like it's embedded in this idea of how do I add things into the system and build this graph conversely the challenge man doesn't have any good way to send information to the parents fan conveniently at all it's possible if you know how to do it it's possible if you know how to build and work around for that you can even build conventions to make it easier but it's not absolutely convenient in any way the lump server is able to pass information to the service and then the client via contact configuration which I talked about earlier and that's possible not to create Networks however the service and the client essentially have no way to pass any information back up to the web server there's test that's not happening forget about it another thing that's interesting here is if you notice the life cycle of the web server watched is actually shorter than the life cycle of the whole trees which means that Toyota sampling is not going to work as you might expect till sampling really wants you to have a full Stitch treat with a scooped corn snack life cycle to the web server or the root span has to be the longest span and everything else has to be inside of it and organized nicely so that you can make a decision once you've collected everything and when you have these Instagram timelines and you don't actually know how long everything is going to be if you don't know how long to wait to make your final decision and you end up discussing until it becomes heuristic faced rather than oh and let's actually do things but you can really really bring you if those heuristics are used as part of filtering your traces and actually trying to sample things so Taylor sampling Distributing choices is actually really really hard um another problem is that once each fan has ended you have no way to add information to it so even though it is technically possible to try and add information to expand from the parent you can add a permission to a challenge fan in some ways that only works if the child span hasn't ended yet and you have no way to know one or not adding that information worked she can end up with code then adds information to expand works great more Express Financial production it's awesome and then someone optimizes the John function I mean it should take half the time and used to and then everything starts breaking silently because you didn't actually know then now you're trying to add things to his fan after it ended you can have a really weird and Rich conditions essentially um speaking of that also happens um yeah she can add information to the service and also the child and parents have the same problem to even in context and uh even in process versus Under severe process you still run into the same issue which means that you have a ton of internal implementation details of the instrumentation the shape of the code how the code works the life cycle of their code the column staff of everything to keep track of which if you are the engineer writing everything interviewing with the whole system and working end to end with it that may not be a huge deal for you and that may actually be there that's one of the reasons why open to my materials so deeply in hand with people owning their systems end to end and having that feedback there's actually in some ways available compensation because you can't build an abstraction unless a lot of people work on top of that so you have to own your system end to end intrusively rather than being able to build these abstracts attention let you instrument things more effectively the second problem there that you run into a lot is what I call the graph rewriting problem so we dig back into this livestock problem you have a bunch of information based on life cycle so when you make lesbians lens fans and and how that interfaces with the constant of the the stimulate system in general need graph rewriting problem is this one the rule of thumb when doing instrumentation is that you have like this final instrumentation that can build this scaffolding uh you would take all of your manual instrumentation and hang it on that so in theory you're not really creating one spans your usual understands that auto instrumentation already set up and you're just adding attributes in my spots which sometimes that doesn't work especially when you're trying to write abstract code you end up wanting to create this fan only abstract couldn't do that but in general the more you can just add attributes to an existing span created by Honor instrumentation but not sad where do libraries go in this and how do they work and how do you actually do this so let's look at this as an example we have a um tree uh it's fast client and then It smash client has some action someone called make request on the client side and then the auto instrumentation has that git endpoint the next step on the client side to half the function call and then you have the honor instrumentation of the actual that response awesome that's great that's cool then we go over into the server and the server has the server endpoint that has three manaway functions that it has a post endpoint uh for the sending uh for an internal call then that has the same chain uh you post on one thing and then you go to the next one then you keep going and keep going and keep going and you can run into the same middleware and you're going to do all that so if you're going to run into a couple problems here the first problem is how do you collapse spans you don't actually have a good one to do that what if I don't actually want all of the middleware to be their own spans and I want to have just one span for everything just add information the development to it with attributes as I go down to me if you want some spam Events maybe I just want the attributes maybe I want things like that and I don't actually need this fan for like the middleware around or maybe I don't need a fan for some other things I don't have a way to like collapse that tree down and that can be is activated by the Fall that's that if you want to search through your system typically the root span is where you search or the same level of a span is when you search when you're trying to like index through your data and so if you have dealer at different span levels you essentially you have no way to correlate those two pieces of datum and do a query in which you aggregate things across those spans so that's one reason why a lot of good practice is to have very very wide events and a very wide spans and less of them so that way you have all the data at the same level and you can do all the correlation and grouping and aggregation and everything else there that you need to in order to effectively look through your data and find something but if you have all of your instrumentation and your honor instrumentation building a nested tree of like 10 or 20 inch fans before yeah as you get to run any of your code where do you stick in so that it's at the level that it needs to be and then on top of that every Library Auto instrumentation um and things like that reinvent their weird set of configuration I want you to try and make it to that they can help solve some of these problems so you'll have um if you look through all the sdks or all the contributors contributing libraries for open Telemetry you'll notice this problem over and over you're like oh how do I use for example the express um server Auto instrumentation it'll save you can just use it and also here's like 15 different options do you want to silence some endpoints do you want to filter some endpoints do you want to silence submit awareness do you want to whatever here's a whole bunch of functions you can use here's some callbacks and we're going to call this compact before and after this we're going to call these to these endpoints and essentially it gives you every possible hook point to you can think of to enter into system and stick your own code there as if you haven't written it yourself but when you don't have this way to write an abstraction and you end up having to sit there and write the code yourself in all these intuition endpoints and you need to have people have having foresight in their libraries to put these uh insertion points for you you have this very very weird platform of really people want to like more or less patch the source to go to the libraries and the other instrumentation and they want to have written all of it themselves graph effectively why can't we deal with that right what's going on I think I cut out for a brief second there did everyone catch the last things that I said awesome so in conclusion there are some problems here there are a lot of things going on there's some advice on what to do and I'm gonna go through all of that now to observability is the process through which one develops the ability to ask meaningful questions and get useful answers meaningful questions for you are going to be about learning about the system how do you collectively get together and Advance your understanding of the system Its Behavior how it interacts with you and everything and how do you get that knowledge as quickly as possible those are meaningful questions continue can use for answers from them if you can your system is becoming observable however observability is interesting because the more you can observe the more correlations you're going to infer with your data and the harder it's going to be to discover contributing factors so just because you can find needle it doesn't mean you need to build a haystack and when you have more and more data and you keep shoving more things in there then you have more and more out of band data and you just add more and more layers of things they can give you observing money into the system you should be really careful to look at everything and go is this actually an intervention or understanding of the system or is this adding noise contacts to contest plus configuration is traces is the glue that turns and back of data points into a unified Trace when working with context it interfaces with everything that has to do with open Telemetry and in many ways remains invisible the entire time using until you're wondering where the entire or how to use it or maybe you're confused but if you know how to use context and know what to think about when using contest it can become your friend one way to map contacts and map different like aspects of open Telemetry together is thinking about the shape of those contacts of those Concepts so traces are a graph expand your nodes and links your edges despite choices being a graph they're often actually mostly thought of and best most ergonomically designed as a cardstock which is a training that is like perhaps multi-branched and that tree is very uh scooped in the top node is always the whitest and the longest and everything is only encapsulated all the way down and that's the most natural and ergonomic way to do open Telemetry and tracing and everything's going to push you towards that but traces are a graph and you can actually access all of that graph power when you need it and sometimes you wish you didn't observability and abstraction are natural enemies they are the chief people in high school that love to take in on each other the entire time they're doing everything and so even though we love open Telemetry and we love observability and we love my other sister meditation the problem of needing a large amount of out-of-band context but also wanting the ability to add and annotate a very complicated problem of your entire system it's business Logic the flow of information throughout the system those are all problems that we don't actually have good answers for in general we don't have good leverage for that we don't have good approaches for that and then it stands Beyond open Telemetry and Beyond observability this is just one of those ways that it becomes a very very obvious problem and two ways of having an average problem are the life cycle problem which we talked about which is where you have this graph you have the whole power of a graph where you don't have a graph that is like convergent you have a graph that you kind of have the satanically know the shape of an Azure building out and you can't really patch it up and picture them later and speaking of that graph reminding it is really difficult if not impossible it's very much convention and ad hoc based there you are does anyone have any questions thoughts or anything else I just want to say hazel that the way that you uh describe these Concepts in in uh distributed tracing and open Telemetry are like I think the clearest I've heard um so thank you for that I I definitely really appreciate that I I think it's uh it's like every everything now like just fully makes sense so yeah I just want to Echo that but that's that's definitely the vibe it's like oh yeah cool now I know what I'm writing about for the next six months it's like the stuff that the stuff that Hazel has put so clearly it's very good really happy to hear that the the question that I had is I'm I'm curious how you feel like um like post-processing fits into that picture like with the collector um a lot of your times revolved around like developers devops people who are working with the code like like how they're able to propagate data but what about that like not so much like tail based sampling where you're deciding right at the moment but like maybe a little further down the line being able to connect data points you know once you're holding five minutes of of Trace data yeah so one thing that's interesting there is that open Telemetry feels in many ways like an event based emitting style of instruments in your code but it's actually not that and the collector and other like um ways of post-processing data is where you start to feel that sort of impedance mismatch in particular um and so because of that um you can run into certain problems like the a great way a great example of that is that links can't be added after expanded there and so even uh as far as I know like once you create this fan you can't add a link to it even in push processing and so if you're trying to do that things are going to be really drinky maybe post-processing can fits that but I don't know that that's now I'm gonna go like research that that's wow okay cool yeah I mean I realize I have not done it I've been like digging through it haven't done it so wow okay so there is interesting historical contents there in that there used to be a method in the open Telemetry uh specification of you can add a link to his fan as long as this man is active then they took that method out several years ago and it's been an open um discussion and the um GitHub tracker for about three or four years now of this is absolutely useful we do end up needing this a lot and can we do that um apparently as far as I know the only way to do that is that's fan Creation in the options of the um for each band you can add links in there and that's the only time which means certain types of Instagram patterns or certain types of like that type of thing really really difficult to do and also means you know when you can approach process and then after that pretty much and then another way to think about the urban Telemetry commentary that's interesting is I actually view the counter and how to set up libraries and baggage and other things like that all as different types of convention-based out-of-band information and handling so when you build up conventions in your company or in your system or in anything like that you can start to say oh yeah if you set this information here then it'll be there or you can actually just assume that we'll have this information because the clutter will add it here so as an example um and when it's metal and a blog post in which they talked about they said of their collectors to actually add to every span what data center that's man came from what machine that span came from what um and then where everything was what cluster it was in and all this actually information so everyone can always clear all that information you don't know that like it's there because you read the documentation in the internal tooling and you can build internal tooling that works on top of that natural convention rather than something you can actually Express in your Telemetry itself anything else well I guess if there are no further questions thank you so much for for coming on for coming on twice and for putting this together I know uh presentations are definitely a lot more work to put together compared to just you know sitting there answering questions so definitely appreciate the time that you took to put this together um and to share your knowledge with folks I think there's uh you know so much are out there on on like the novice Hotel stuff but when it starts to get into the gnarly bits um we're definitely lacking in that information so we definitely appreciate um you sharing that and uh call out to anybody if you if you or you know anybody who is interested in sharing some Hotel goodies um in hotel in practice we would absolutely love to have you um and also if you want to share your hotel story we would love to have you for open a and I think we've got a session coming up later this month right Rhys we do we have a special discussion actually coming up on the 28th um it's going to be about the evolution of observability practices and actually Nika is going to be part of that panel um and we'll do a full announcement so um thank you Hazel for uh for everything and um I guess we'll see y'all um on the internet thank you diff --git a/video-transcripts/transcripts/2023-09-20T19:30:01Z-otel-in-practice-context-propapagtion-in-distributed-tracing-with-doug-ramirez.md b/video-transcripts/transcripts/2023-09-20T19:30:01Z-otel-in-practice-context-propapagtion-in-distributed-tracing-with-doug-ramirez.md index b4e6266..fda5e3b 100644 --- a/video-transcripts/transcripts/2023-09-20T19:30:01Z-otel-in-practice-context-propapagtion-in-distributed-tracing-with-doug-ramirez.md +++ b/video-transcripts/transcripts/2023-09-20T19:30:01Z-otel-in-practice-context-propapagtion-in-distributed-tracing-with-doug-ramirez.md @@ -10,98 +10,115 @@ URL: https://www.youtube.com/watch?v=Of_ygNU_Cbw ## Summary -In this YouTube video, Doug Ramirez, a principal architect at Uplight, presents a session on distributed tracing and context propagation within microservices, part of the OpenTelemetry initiative. He provides a straightforward explanation of distributed tracing, which helps in understanding how various services interact in a microservices architecture. Doug discusses key concepts like spans (units of work) and traces (collections of spans), emphasizing the importance of context propagation for observability across service boundaries. He demonstrates a minimal code example using a fictitious "band database" API and a review service, highlighting how to implement distributed tracing using OpenTelemetry SDKs and W3C recommendations. Doug addresses practical considerations, including privacy concerns and the benefits of integrating logging with tracing. The session concludes with an interactive Q&A where Doug encourages further exploration of the tools available for developers. +In this YouTube video, Doug Ramirez, a principal architect at Uplight, presents on the concepts of distributed tracing and context propagation within the realm of observability, particularly in microservices architecture. He emphasizes the importance of distributed tracing as a means to understand interactions among various services, as traditional monolithic structures provided easier debugging and observability. Doug explains key concepts like spans and traces and provides a minimal code example demonstrating how to implement distributed tracing using OpenTelemetry and adhere to W3C recommendations. He showcases a scenario involving a bands API and a reviews service, illustrating how to send and propagate trace context through HTTP headers to monitor performance issues effectively. Throughout the session, he engages with the audience, answering questions and providing resources for further exploration. The video concludes with a mention of future meetups and the importance of integrating logging with tracing for optimal observability in production environments. -# Hotel in Practice - Distributed Tracing with Doug Ramirez +## Chapters -[Music] +Sure! Here are the key moments from the livestream, along with their timestamps: -Thank you! Hi everyone, thanks for joining us for Hotel in Practice and for being patient and sticking around. We've got Doug Ramirez from Uplight joining us for a second time, this time in the Hotel in Practice capacity. He has joined us previously for the Hotel Q&A session, and in case you missed that, we do have a blog post summarizing that session on the Hotel blog at OpenTelemetry.io under the blog section. +00:00:00 Introductions and guest introduction +00:02:30 Overview of OpenTelemetry and distributed tracing +00:05:00 Explanation of distributed tracing concepts: spans and traces +00:10:15 Importance of context propagation in distributed tracing +00:13:30 Overview of W3C recommendations for trace context +00:18:00 Live coding demonstration of a minimal distributed tracing example +00:25:00 Explanation of the importance of observability in microservices +00:30:45 Debugging and identifying performance issues in service calls +00:35:00 Integration of tracing with logging for better observability +00:40:00 Wrap-up and audience Q&A session -So, I guess without further ado, here's Doug! +Feel free to ask if you need any more information! + +# Hotel in Practice Webinar Transcript + +**Thank you!** Hi everyone, thanks for joining us for Hotel in Practice and for being patient and sticking around. We've got Doug Ramirez from Uplight joining us for a second time, this time in the Hotel in Practice capacity. He has joined us previously for the Hotel Q&A session, and in case you missed that, we do have a blog post summarizing that session in the Hotel blogs at OpenTelemetry.io under the blog section. + +So, I guess without further ado, here’s Doug. --- -**Doug Ramirez:** Hi everyone, I am Doug Ramirez, a principal architect at Uplight. We do cool things to try to save the planet. Just a shameless plug: Uplight.com/careers, we're hiring, and we're a B Corp! But let's move on to OpenTelemetry and distributed tracing. +**Doug:** Hi everyone, I am Doug Ramirez, a principal architect at Uplight. We do cool things to try to save the planet. Just a shameless plug: uplight.com/careers, we're hiring, we're a B Corp, we do cool things. But let's move on to OpenTelemetry and distributed tracing. -Today, I have a few things I want to cover: +I have a few things that I want to cover today. Those are to introduce the concept of distributed tracing and context propagation, share a very minimal example of distributed tracing in action, and provide some resources for people to explore distributed tracing further. -1. Introduce the concept of distributed tracing and context propagation. -2. Share a very minimal example of distributed tracing in action, peeling away a lot of the extraneous stuff and noise to show you how this is implemented in just a few lines of code. -3. Share some resources for further exploration of distributed tracing. +### What is Distributed Tracing? -So, what is distributed tracing? It’s an observability concept that gives us a big picture of how our myriad of services interact. If you think about the three main pillars of observability—logs, metrics, and tracing—they have all been around for a while. Distributed tracing, however, has become increasingly important as microservices architecture gains popularity. +Distributed tracing is an observability concept that gives us a big picture of how our myriad of services interact with each other. If you think about the three main pillars of observability—logs, metrics, and tracing—those have all been around for a while. Distributed tracing has also been around for a while, but as microservices architecture becomes more popular, it really elevates the need for the ability to trace things that span process and service boundaries. -**Key Concepts:** +Monoliths offer certain advantages, but with microservices, it’s crucial to treat distributed tracing as a first-class citizen during implementation. Context propagation is the mechanism that allows us to observe things happening in a service as a transactional request spans multiple services in a call chain of microservices. -- **Spans and Traces:** A span is essentially a unit of operation or work. For example, a database query is a great example of a span. A trace is a collection of these spans and tells the story of how a request traverses various services. Each trace has a unique identifier, which is globally unique and random. +### Key Concepts in Distributed Tracing -- **Trace Context:** This is like the envelope or baggage passed between services to uniquely identify the request. There must be an agreement on the format and specification for the trace context. The W3C provides recommendations on how a trace context should look, which the OpenTelemetry community has codified into SDKs and tools like the OpenTelemetry collector. +When thinking about distributed tracing and context propagation, two important concepts are **spans** and **traces**. A span is essentially a unit of operation, like a database query. It represents a unit of work that a service performs. A trace is a collection of spans that tells the story of how a request traverses through different services to enable a feature or capability. Each trace has a globally unique identifier. -The W3C recommends using HTTP headers to send information from one service to another. The two headers specified are: +A **trace context** is like the envelope or baggage passed between services to uniquely identify a request. For this to work, there needs to be an agreement on the format and specification for a trace context. The W3C provides recommendations on what a trace context should look like, and the OpenTelemetry community has codified these into SDKs and other resources to simplify implementation. -- **Trace Parent:** Packages unique identifiers describing a unit of work in one service and the associated trace to send to another service. -- **Trace State:** A simple list of key-value pairs that can carry additional information. +### HTTP Headers for Context Propagation -**Privacy Considerations:** It’s important to familiarize yourself with privacy considerations surrounding personally identifiable information (PII) and GDPR compliance. Following the W3C recommendations and using OpenTelemetry specifications is generally safe, but it's worth noting. +Part of the W3C recommendation is to use HTTP headers to send information between services. There are two headers recommended for this: **traceparent** and **tracestate**. -To summarize, distributed tracing helps us observe how a transaction or request traverses multiple microservices. Fortunately, there’s a widely adopted specification and tools available from OpenTelemetry to make this implementation straightforward. +- **Traceparent**: This header packages the unique identifiers that describe a unit of work happening in one service, along with the trace associated with it. +- **Tracestate**: This header is a simple list of key-value pairs that can carry additional contextual information. -Let me pause here because I know that was a lot of information. Does anyone have questions about these concepts before I dive into some code? +Using these headers allows services to communicate and understand what uniquely identifies a request. ---- +### Privacy Considerations + +It’s essential to familiarize yourself with privacy considerations regarding PII and GDPR when implementing distributed tracing. While following the W3C recommendations is generally safe, it’s crucial to be aware of potential data leakage. -Now, we're going to do some live coding! I've created a minimal example of a couple of microservices working together to demonstrate how to use the W3C recommendation and OpenTelemetry libraries. +### Implementation Example -In this scenario, we are building the Internet Band Database, which will allow us to create a source of truth for band names and reviews. We have an exposed endpoint that allows a caller to request a band by unique ID along with its reviews. +Now, I want to quickly show you a minimal example of a couple of microservices working together. Imagine we're building an Internet Band Database (similar to IMDb) that provides information about bands and their reviews. -**Live Coding Example:** +In this scenario, we expose an endpoint to call and retrieve a band by its unique ID and its associated reviews. -1. **Initial Setup:** - - I want to add tracing to our service to see what’s happening. I’ll bring in the OpenTelemetry SDK and use it to handle the trace context. +When I run the service, it takes a surprisingly long time. This is common; when someone reports performance issues, it often requires tedious debugging and log gathering. Adding distributed tracing can help. -2. **Adding Tracing to the Band Service:** - - I’ll allow the caller to send the trace context in the header. When I start a span for the get reviews function, I’ll inject the trace context from the caller. +### Adding Tracing -3. **Client Interaction:** - - When the client calls the API, they’ll send the trace parent header. We will observe the time it takes to get a response and see where any bottlenecks occur. +I will add some code to my service to enable distributed tracing. Using the OpenTelemetry SDK, I can extract the traceparent header and inject it into my service calls. This allows us to trace requests and see how our services perform. -4. **Using Jaeger for Visualization:** - - Jaeger is an open-source platform that helps visualize traces and spans. As we implement tracing, we can see which parts of our service are slow and where improvements can be made. +For instance, when calling the reviews service from the band service, I start a span and inject the trace context. This lets us observe what the band service is doing and how long it takes to get reviews. -5. **Debugging:** - - If we notice that the get reviews function is taking a long time, we can dig into that specific code and see what the issue might be. +### Observing with Jaeger -6. **Iterating and Improving:** - - We can continuously improve the service by observing the performance metrics we gather through tracing. +Now when we call the APIs, we can start digging into the performance. Using Jaeger, we can visualize the traces and spans. -Throughout this process, we will leverage the OpenTelemetry SDK to handle the heavy lifting and ensure our services are traceable. +When we examine the data, we see a majority of time is spent in the get reviews function, which helps us identify where the bottleneck is occurring. + +### Conclusion + +In conclusion, distributed tracing allows us to observe how a transaction traverses multiple microservices. The good news is that we have widely adopted specifications and tools from OpenTelemetry to make implementation straightforward. + +Let me pause here and see if anyone has questions before I jump into some code. --- -**Q&A Session:** +**Audience Questions** + +1. **Audience Member:** How did you find the documentation around context propagation? + + **Doug:** It was pretty good! The OpenTelemetry documentation has detailed information on things I glossed over today. Once you understand the concepts, building a reference implementation is quick. -After demonstrating the implementation, Doug encourages questions about context propagation and the overall implementation. +2. **Audience Member:** How do you integrate logging and metrics with observability? -**Audience Questions:** -- How did you find the documentation around context propagation? -- What’s your experience integrating logging and metrics with OpenTelemetry? + **Doug:** I recommend starting with logging because it’s familiar to most developers. By incorporating logging with OpenTelemetry, you can get Trace log correlation for free. Once you establish that connection, adding tracing becomes much easier. -Doug explains that he started with logging, as it is familiar to most developers, and recommended using OpenTelemetry libraries to include logging signals in their applications. He believes that this approach makes it easier to integrate tracing later on. +3. **Audience Member:** Could you do a follow-up presentation on logging? -Doug also expresses interest in collaborating with others on this reference implementation and suggests that it would be beneficial to have detailed comments in the code to clarify what each part does. + **Doug:** I would love to! I think there’s a lot of value in simple implementations that highlight these concepts. --- -**Conclusion:** +**Closing Remarks** -Thank you everyone for joining us today. We appreciate your engagement and Doug's insights on distributed tracing. Keep an eye out for our next Hotel in Practice session. +Thank you, everyone, for joining today, especially given our late start. Thanks, Doug, for adapting on the fly and for your insightful presentation. We will post updates about our next Hotel in Practice session on the various CNCF Slack channels. -If you're interested in further discussions, please reach out via the provided contact information. +If you're attending KubeCon EU, several of us will be there, and we’d love to connect. A blog post summarizing today's session will be available on the OpenTelemetry blog on Monday. -[Music] +Thank you again, everyone! ## Raw YouTube Transcript -[Music] thank you hi everyone thanks for joining us for hotel in practice and for being patient and sticking around um we've got Doug Ramirez from uplight joining us for a second time um this time in the hotel and practice capacity he has joined us previously for the hotel q a session and in case you missed that we do have a blog post summarizing that session in the hotel blogs in open Telemetry i o under the blog section so I guess without further Ado um here's Doug thank you hi everyone I am Doug Ramirez I'm a principal architect at uplight um we do cool things to try to save the planet um if uh just a Shameless plug uplight.com careers we're hiring we're a b Corp we do cool things uh but let's move on to open Telemetry and distributed tracing so I have a few things that I want to try to do today and uh that those are to introduce this concept of distributed tracing context propagation I want to share a very very minimal example of distributed tracing and action trying to peel away a lot of the uh extraneous stuff and noise around this so that you can just see in as few lines of code how this is implemented and then also share some resources around uh just more resources for people to go and explore distributed tracing so um so what does distributed tracing it's an observability concept that gives us a big picture of how our Myriad of services interact with each other if you think about the three main pillars of observability logs metrics tracing you know those have all been around for a while distributed tracing has been around for a while but as as more and more of um microservices architecture becomes a thing and becomes popular it really makes um it really kind of elevates the need for the ability to trace things that span uh process and service boundaries um there's a lot of things that monoliths give us and uh you know with with microservices it's it's really um like I really think that if you're going to move towards microservices you really need to make sure that you're you're treating distributed tracing as a first-class citizen as part of that implementation um and context propagation is the mechanism that gives us the ability to to observe things that happen in a service as a transactional request spans another service and another service and kind of your call chain of microservices context propagation is the thing that gives us the ability to watch and see what's happening um so a couple of Concepts that are important to know about when it when you're thinking about distributed tracing and context propagation or spans and traces um and in my experience oftentimes I think people flip the tune um but to be clear a span is is um it's basically a unit of operation um a unit of work you can think a database query is a great example of a span that's a unit of work that a service you know does there's a collection of these units of work that make up you know the the the the the transaction that may happen in the service and so a span is a span is nothing more than that unit of work and a span ID is the thing that uniquely identifies that you know unit of work a trace is just a collection of those spans and it tells the story of how a request you know traverses all of these different services that um that essentially enable a use case or a feature or a capability um and each Trace has a unique identifier um which is globally unique and random and I put some links in the readme here so that you can read more about um the uh the reasoning behind that but I think it's important for people to know and I know this has come up in conversations that I've had with people around what a trace ID is and it is important that it is globally unique and random for reasons that are explained better in the documentation from the w3c which I'll get into a moment until in a minute here um and a trace context is essentially kind of like the envelope or baggage that it's passed between services so that they can understand you know what uniquely identifies the request that's coming into them have a uniquely identify a transaction request that's going out of that service to another service and essentially um in order for all of this to work there has to be an agreement on what that format and what that specification is for a trace context and one of the things that we get as developers which is awesome is a recommendation from the w3c around what a trace contact should look like and we also get the benefit of a lot of the work that the open Telemetry Community has has done to codify uh those those recommendations and specifications into sdks and into other things like the open Telemetry collector that give us as developers a really easy way to implement distributed tracing context propagation there's an agreed upon format that is you know recommended by the w3c on how to do this there's a software in libraries and specifications that the open Telemetry Community gives us to make all of this really really easy and I'll show that in a moment um so with that specification that the w3c gives us um part of that recommendation is to use HTTP headers to allow a service to send information to another service so that that uh that Trace context can be propagated and there are two headers that the w3c recommends for doing this it's a trace parent and Trace State Trace parent is where we package up the unique identifiers that describe a unit of work that's happening in one service the trace that's associated with it to send to another service so that they essentially can unpack that tag that unit of work in the other service with information so that as that information is admitted to an observability platform that we can tie all these things together and we can see how this call chain of things that happen that span Services all work together and you know back in the day with monoliths it was pretty easy to observe and debug things that span different sub components of your monolithic architecture and we kind of lose that ability with microservices but we get it back by leveraging this Trace context concept and the um the the headers that are specified from that the w3c recommendation to pass that information between services so okay service a service calls another service it simply uses a header to package up some information it's an agreed upon format for other services to unpack and use to tag the spans and units of work that those other services are using um there's some details in here about like what the format of that is uh let me reduce my font here so you can see what a an example of a trace parent would look like so there's a version I'm not going to get into the version today nor will I get into the trace lags but the really important part of this is to understand that this ID here this unique identifier for the trace ID is the thing that will allow us to tie all of this work together so that we can observe how microservices are all working together to you know enable some feature satisfy some use case in addition to the the trace parent header the w3c recommendation also offers another additional header called tray state which is really nothing more than a simple list of key value pairs that you can add along with your the context that's being propagated between services so that you can attribute that to things that may be unique to whatever it is you're doing um vendors May uh use Trace State to provide observability into what they're doing but um you can just think of it almost just like a little bit of baggage that rides along with the trace context that moves between the services um so I have some links in here to the the w3c trace context specification and recommendation one of the things that I think is is worth calling out and I would encourage anybody who's going to go down this path is to make sure that you're familiarize yourself with the privacy considerations you know in this day and age with with pii and and gdpr and a lot of the other things that we really need to pay attention to as it relates to the the um potential leakage of personal information there are some considerations that are worth noting I will say that in general that following the w3c recommendation using the specifications in the libraries that we get from an open tem Elementary this is a very safe thing to do but just to kind of like be clear spend some time and just make sure you get yourself familiar with those privacy considerations so so I know that I ran through that quickly because I know we're kind of pressed for time but essentially you know this thing called distributed tracing again it's a way to you know observe how a transaction or a request traverses multiple microservices um again the good news here is that we have a specification that's widely adopted and it's widely known we have tools that we can use from the open Telemetry to employ this so for anyone who's thinking about like I'm building a bunch of microservices I need to observe how they're operating how do I do it the good news is is that that problem's been solved for you there's a de facto standard there's tools that you can use and uh as you'll see in a moment it's it's actually quite easy to do um let me pause there because I know that that was probably a fire hose of stuff but I want to see if anybody who's uh on the call right now has any questions about these Concepts before I jump into some code okay um we're gonna run with scissors now and we're gonna do some live coding so what I've done here is I have um again what I tried to do was to just create the most minimal example of a couple of microservices all working together um to demonstrate uh how you can use the w3c recommendation and use the open Telemetry libraries to do this stuff so in this scenario um we're building the um it's kind of like IMDb it's the instead of the Internet Movie Database it's the internet band database and so I want to I want to create like this source of Truth when it comes to bands names and uh and also um in this scenario we have we're leveraging a platform that just handles and reviews just a generic platform for reviews and so we're we've created it we're creating a band's API that our front-end developers are going to use to build web and mobile applications on top of and and and rather than building a reviews platform ourselves we're just going to leverage this fictitious reviews platform so in this case um we've exposed an endpoint that allows something to call and say hey give me this give me this band that has a unique ID and give me the reviews associated with it so I'm going to call the endpoint to get a band in its reviews and wow it took six seconds that's a long time let me run it again just to see what's happening here 12 seconds is a long time um so you can imagine we're all familiar with this right like you're building the service you've got people that are riding front ends to it and they're saying hey Doug uh I'm calling the endpoint for to get a band and it's taking a long time and that's not a lot of information to go on and debug you know I think we're all familiar with those scenarios where um you know somebody contacts you and says hey you know we've been noticing that this doesn't seem to be performing very well um we have to do that guessing game of like okay well when did you call this service like do I start grabbing a bunch of logs it becomes very tedious so one of the things that we that helps is to you know employ some tracing and some distributed tracing so what I'm going to do now is say all right hey uh from a developer um I'm going to add some stuff to my service and all I need you to do is to make sure you pass me some information when you call me so I know that you're calling me and then we can kind of debug this and see what's happening so the first thing I want to do is I want to go into my my my bands API and this is just a very very simple fast API application just a very simple model for a band I have an endpoint here that lets uh a caller get a ban by its ID I have a little helper function here that does some work to go actually access a data repository of reviews which happens to be this other review service that I mentioned before uh so let me bring up my my cheat sheet here so let's start by um bringing in some code that will provide some visibility with tracing so what I'm doing here is a couple things one is I'm going to bring in some um with the help of the open Telemetry SDK I'm going to bring in the library that's going to help me essentially tease out information from the header that I talked about a moment ago this Trace parent header and inject it into um essentially a hook that will call out to another service and omit that tracing signal and in this case I'm going to use Jaeger um uh all populated about open the open Telemetry collector but in this case I'm just going to bring in some code with the SDK give the band service the band's API the ability to unpack that header and inject that information to admit a trace so that we can see what's happening oh yeah so the get reviews so let's go to our get reviews so excuse me this is the function that does the call to the reviews service so what I want to do here is say okay for my when I call the get review service I want to start some kind of Trace so I can see what's happening and in this case I'm starting a span in my service and what I want to do is I want to actually I don't want to do this just yet sorry let's just do this and then back over here I'll explain what I'm doing here so what I'm doing now is um I'm adding a couple things I am um I'm creating the ability or what I'm doing is I'm bringing in a very very small Library of uh this is just a helper library that is essentially uh provisioning the uh the Tracer that we get as part of the open Telemetry SDK what this Tracer will do is this is the thing that will um build up the trays and send it to an observability platform in this case it's going to be Jaeger and uh in case you're wondering there's some default stuff that happens like the um the open Telemetry SDK knows to send to a certain Port that a service is listening to in this case it's four three one seven um and Jaeger's running locally so it will pick it up that's a lot of hand waving uh but in the interest of time just just know that the SDK is handling a lot of this stuff for you back in my band service what I want to do is say Okay anytime somebody calls the service um what I want to do is I want to allow the caller to send that Trace contacts in the header and so in my service what I want to do is I want to tease out that Trace parent header because I know that there's a standard and a specification for that I'm going to build up a carrier and I'm going to inject that or I'm going to extract that carrier from the payload and then I'm going to inject that Trace contacts from my caller into my contacts so that when I build up a span the open Telemetry SDK is going to do a bunch of magic and work for me and essentially emit that Trace to the observability platform so I have this now built up in my code so the bands API now can receive a request build up a Trace Loop in that Trace context from the caller and and when it calls its own internal function we're going to spin up another span which is a unit of work to make a query to the reviews platform so this will start to give us some of visibility into what the caller is doing and what the band service is doing so now let's go so what I want to say to the caller then the person who's building this application to say hey I'm adding some stuff to our service because I know you said it was slow and things don't seem right send this bit of information so that we can start to really observe what's happening here and kind of dig into what the problem may or may not be so if I go back to my client what I want to do is I want to essentially say hey help me help you by sending this header and when you send this header manufacturer a trace and a span on your end add that header to the call that you make to me and then we can kind of start to dig in to see what's happening here so now at this point we're kind of wired up now to like get a bit more observability into what these different services are doing so if I run my if the client now calls me I can see it took five seconds oh I forgot one thing sorry so I also just kind of want to print out to the console a little a nice little helper thing just to link me right to Jaeger where these traces are being sent to so I can start to see what's happening so now the client calls me again we should be able to start kind of digging into what's going on here wow 15 seconds is a long time so now I can kind of dig into this and I can see and I I don't I'm not going to spend too much time on Jaeger but Jaeger is an open source platform that's um that essentially helps visualize traces and spans so now that I have the caller following that w3c recommendation passing that Trace parent with some you know unique identifiers of the unit of work units of work it's doing sending it to the band service the band service is taking that information its context has been propagated to it it's leveraging that to emit information about what it's doing and now we can observe what's happening I can see that the uh the bands the get bands endpoint was called and I can see that the get reviews function unit of work was run and if I look at this um the spans table here I can see that the majority of time that was spent here was getting the reviews so that's a nice that's that's good helpful information to know right I can see like okay get reviews is taking a long time let's dig into that so let's go back to our code and I can say hey guys building folks building the client I see something going on in my service let me dig into it the get reviews is taking a long time hey what's this debugging thing uh somebody did something while they were debugging I just have a random timer here let me just get rid of this sorry about that folks right okay hey I just deployed a new version of my service call me again still taking a long time something doesn't seem right so now in this fictitious world I'm going to turn over and turn around to the folks that work on the reviews platform and say hey you know we're looking into an issue there was some random code that was sleeping in our end uh can you take a look and see what's going on in your platform because when we're calling you like it's taking a really long time to get reviews like our service seems to be working pretty fast so now let's go and do the same thing on the reviews end um very similar bit of refactoring let's go to the reviews especially let me bring yeah let's bring all this in so now the same uh the same service here gonna bring in this helper function to um start up the Tracer again leveraging the open Telemetry SDK to do that we'll bring in this Trace context propagator so that we can have this service do things they're interesting things with the trace context and let's see so here we want to do something that's a little bit different than the band service does right yeah I think that's right so now the review service can also receive that Trace parent that Trace context unpack it and Associate it with its span of work in addition to that we want to start a span and the uh at the internal um imagine this would be a piece of code that would more than likely read from a data repository a data store that's associated with the review service so now I can say okay folks that are building the front end of mobile app we've we've released a new bands API we've released a new reviews API that we we think we're going to help you know give us some visibility into what's going on so they're going to keep going doing their development they're going to start making calls to us four seconds isn't too bad let's try again it's still taking too long so now we got to figure out okay well what's going on so let's go back to our observability platform Jaeger and what is happening uh not skiing I'm not seeing the trace from the reviews out of this uh this is what you get when you try and run with scissors and do live coding um what am I missing here Chase parents so this is getting oh I know what we did all right folks there it is Okay the reason that the review service wasn't getting the context propagated because I didn't add that in the so in the band service is calling the review service we need to propagate that context um be quiet Siri here we go all right I know I'm jumping around quite a bit um so in the band service it's getting that Trace context from the caller it's attaching it to a span it's making this call to get reviews get reviews we want to see what's happening in that unit of work so we're starting up a new span but we also want to take the contacts from the caller and inject it into a header so that when the band service calls the review service that context gets propagated to it as well so now when the caller calls the van service it's spinning up this Trace ID this unique identifier sending it to the band service the band Service is like thank you very much I can attach this to my span this my units of work I'm going to send it Downstream to the thing that I rely on it's going to attach this information to its units of work and we should see all of that come together in Jaeger boom all right service is still running slow but at least now we have some visibility into what's happening oops and if I go to my uh Trace graph sorry my stands table actually where is it I can see here that the majority of the time seems to be sent be being sent being spent sorry and getting the reviews from the reviews database so when it's getting a review by getting the reviews for a band by the band ID it's taking a long time so let's go back and start digging into that code there to see what might be happening so in the review service if I look at this function that you know is grabbing data from some kind of data Repository again somebody put in some kind of sleep maybe they were debugging testing some timeouts who knows what but let's pull that out and now we can say hey folks that are building the mobile and web app we think we fixed it from real this time can you try again wow that's better let's go back and see what happened in Jaeger okay things are taking milliseconds that feels better I like this now I can observe what's happening across all of these different things yay team um so uh just to kind of walk through the code one last time the client is packaging up this header based on the w3c recommendation it's passing that header along to our bands API our bands API is leveraging the open Telemetry SDK and saying hey let me pull this stuff out and inject it into a carrier so that I can send it Downstream and by the way when I'm creating my spans um you know I'm gonna leverage that information when the review service gets a call into it it can tease out that Trace parent information and inject it into its fans by attaching that context and now we have distributed tracing with context propagation because we've got this awesome stuff from the open Telemetry community and we've got this awesome recommendation from the w3c let me stop there and ask if anybody has any questions about the the uh the kind of the basic minimal implementation of these Concepts in these handful of fast API services and maybe this is a good time just to kind of open up to general questions as well because I know we're coming up on times yoga I had a question for you how uh how did you find in terms of like finding the information the documentation around around the context propagation how how was that experience for you in terms of finding that information in the hotel Docs um it was pretty good actually uh so in the in the readme for this um repo which um oh which I hope that other people can help me uh test the documentation I would love for some folks to go and clone this repo and try and Spire all these Services up and make some calls and see if they can get it running locally um it was I it wasn't it was good um you know the the open Telemetry documentation has some some very good details around things that I glossed over today that I would I would really like to go into further detail around that I'm sure that anybody who watches this is probably going to start asking questions like well how does that actually work like there's a lot of magic under the hood but the open Telemetry documentation goes into a lot of detail around the things that I just waved my hands around like the tracers the trace exporters the context propagation the shape And format of spans um how these things all link together so I found I found it was pretty easy uh to find that and I also think that the the w3c documentation is also very good as well so I was able to kind of bring everything together pretty quickly like it didn't take me very long to build this reference implementation of these few applications um yeah open Telemetry also publishes a bunch of integration packages which lets you hook up to like node middleware or HTTP clients automatically so you can just have everything instrumented and doing the correct header insertion and things like that without really having to worry about doing it manually like like this demonstration noise yes that's a really good point so um uh one of the things that that I did as part of this uh this reference implementation was to do a very explicit implementation of these Concepts um we don't have time and I wanted to do that thing that you always see in a demo where somebody says do all this stuff and at the end of the demo and they say oh by the way you don't have to do anything you just do this other thing and it all gets handled for you there is a fast API instrument or um which will remove the need to do a lot of this explicit unpack um this explicit extraction and injection of the context um I didn't I didn't have time today to show that and I I kind of wanted to make it very clear to a developer who's learning these Concepts to walk through the code and see something very procedural that shows this for example there's a header and that header is very important and it is called a trace parent and it has a specification here's an example of teasing out that header in code the open Telemetry library has this thing where it can you know extract this context for you here's the line of code that does this when I make a call to another service I want to pass that along very explicitly so you know there was a lot of intention behind trying to make this application a bit more verbose even though it's very minimal to try to show somebody who's you know maybe new to this like what does this actually look like if you use the fast API instrumenter the good news is a lot of this gets handled for you um my recommendation would be for anyone who's getting started with microservices is to make sure that you include distributed tracing from the very very beginning and I would almost suggest that you spend a bit of time and implement this explicitly so that you understand what's happening so that other developers understand what's happening and once you solve the problem of observability then level up and start to leverage some of these other things that the open Telemetry gives us like Fast the fast API instrument or and these other things that that we get from the community um I feel I feel like that just kind of helps bake the concepts in for people and I I know that for me I'm a very tactile learner so like it helps me to see what's happening before I go and to start to do or start to let some libraries start to kind of handle that magic for me if that makes sense you know your mileage may vary but my recommendation would be to go ahead and do it explicitly it's a handful it's a few lines of code you can share it with people when you're doing code reviews you can explain to them what's happening it's not mysterious it's very explicit foreign have any questions for Doug thank you Doug for sharing that um one question from me is that um how does it look when you like so the obserability problem in production I suppose apart from this open Terminal tracing like part I suppose we also need to serve like logging and metrics and to find any difficulty like integrating like different part of obtibility um so the the last time I I spoke here um I I mentioned that the one of the the path that we took and had been taking that uplight has been to actually begin with the log signal and the reason is that um in terms of introducing these Concepts that these problems that open Telemetry solves these problems at the w3c helps us with solving is to begin with logging so that you because logging I think is very very familiar with most developers I think you know Junior intermediate developers are familiar with this concept of logging like a print statement and so what I've recommended and what we did at uplight when we started down this path was to start using the logging SDK even though it's newer than the the work for the signals of metrics and traces but to start with logging so that you can at least begin to practice including the packages and libraries into your code using open Telemetry to Omit that logging signal get that logging signal Landing in your your APM or whatever your observability platform is and once you make that connection of like I'm writing code I'm writing a statement that says logger.debug my message is landing in an APM it's structured it's it's it's being it's it's honoring the specification that the open Telemetry Community gives us then it's really easy to then um you know level up to add tracing and you get Trace law correlation for free and that's one of the things I would like to actually do with this this little reference implementation is to to add in some Trace log correlation so that people can see like how do I do the thing that I'm so familiar with doing like I like to send out a log I like to do a print statement like that's so we always do that all day every day and if you start with logging and you get the sdks into your repo and you have that information flow into your APM then adding the the layer of maturity around traces and the trace log correlation is almost free because you've already got everything packaged up into your application already so um you know that might not work for everybody to me that feels like a very natural way to evolve into using the specification using the sdks and and really kind of getting into observability Nirvana where you can walk up to your APM and say I have questions about the health and behavior of my services and I have a lot of them what's happening and you can get those answers if you do this Doug it sounds like you need to do a follow-up presentation on uh on logging you know I I would love to and I think that you know I would love some help from other people to take this this uh this repo and clone it or Fork it or just play with it and for if you know to help me test the documentation um some other things I would like to do is to go into the code itself and provide a bit more um kind of verbose commentary around what's happening again I think it's helpful to have very explicit implementation of these Concepts but for example I would I would really like to have you know some comments here that tell somebody hey when you call the extract method this is what it's doing for you because right now it you know it's magic it's just happening and that's awesome but I think for people to really understand what other Telemetry is doing and to understand what how all this works I think it would be helpful helpful to have in this reference implementation some pretty verbose comments and not the kind of thing you might want to put in your production code but for a reference implementation I think it would be helpful and I would love to do another one of these awesome um and you know I've been like looking through your repo as as you've been presenting and I really like this example it's it's uh it's simple but it's got enough complexity that it it illustrates what you're trying to um convey um so yeah I I think I I would love to at some point uh when I have some spare time to like poke around I I definitely encourage others to poke around as well maybe I I don't know uh as far as like I think it'd be cool to contribute this back to open Telemetry um although to be honest I I don't know if like other Hotel folks have uh have a better idea on the process around that because I I think I'm newer to the process but anyway um I think that yeah yeah and I you know I know that there are like there's a demo app and there's other reference implementations but one of one of the problems I was trying to solve here was how do I strip away every all the noise and give a developer with a modern operating system and docker the ability to get clone fire up these services and watch what's happening so they can really dig into these Concepts and understand how powerful they are in their Simplicity like it's pretty amazing like all we're doing is just you know sending a header and this these sdks are doing these wonderful things for us like it's it's pretty simple but it's unbelievably powerful and so the idea for this reference implementation was to try to just highlight that as best we could with as few lines of code as possible so that you know people can understand this and so that people can go and be successful with building this level of maturity in their observability in in their code yeah I totally agree and I think there's a lot of a lot of value in that like like you mentioned the open Telemetry demo app is is awesome because it shows like all of the things but there is definitely value in in simpler implementations like this one um so yeah um well on that note um I know we're a little bit over time um thank you everyone who was able to join us today um especially given that we uh started late and thanks Doug for uh uh you know working on the fly with our reduced time and for for lunch that's never uh never an easy task so definitely appreciate the time that you put into uh making this happen today um thanks everyone for joining us once again and um keep your eyes open for our next hotel in practice um I don't think we have one scheduled yet but we'll we'll post on on the various cncf slack channels for that um Rin do you have any additional upcoming announcements that uh folks should be aware of sure yeah so um first um there's an open Telemetry and practice Meetup if you came here through cncf um lightweight way to get notified whenever we have something and not have to look for the slack if you're not in the cncf slack and you want to talk further to um any of the hotel and practice folks Doug the audience um we're happy to add you um that information is in the chat and if you'll be at kubecon EU um several of us will be and we'd love to connect with you there will be a blog post on the open Telemetry blog on Monday with all of the activities and um yes I am happy to do a follow-up with Doug because of kubecon EU that will probably be in around the end of May and um we can figure out what format we want to do based on hopefully talking with you all in the slack thank you everybody [Music] +thank you hi everyone thanks for joining us for hotel in practice and for being patient and sticking around um we've got Doug Ramirez from uplight joining us for a second time um this time in the hotel and practice capacity he has joined us previously for the hotel q a session and in case you missed that we do have a blog post summarizing that session in the hotel blogs in open Telemetry i o under the blog section so I guess without further Ado um here's Doug thank you hi everyone I am Doug Ramirez I'm a principal architect at uplight um we do cool things to try to save the planet um if uh just a Shameless plug uplight.com careers we're hiring we're a b Corp we do cool things uh but let's move on to open Telemetry and distributed tracing so I have a few things that I want to try to do today and uh that those are to introduce this concept of distributed tracing context propagation I want to share a very very minimal example of distributed tracing and action trying to peel away a lot of the uh extraneous stuff and noise around this so that you can just see in as few lines of code how this is implemented and then also share some resources around uh just more resources for people to go and explore distributed tracing so um so what does distributed tracing it's an observability concept that gives us a big picture of how our Myriad of services interact with each other if you think about the three main pillars of observability logs metrics tracing you know those have all been around for a while distributed tracing has been around for a while but as as more and more of um microservices architecture becomes a thing and becomes popular it really makes um it really kind of elevates the need for the ability to trace things that span uh process and service boundaries um there's a lot of things that monoliths give us and uh you know with with microservices it's it's really um like I really think that if you're going to move towards microservices you really need to make sure that you're you're treating distributed tracing as a first-class citizen as part of that implementation um and context propagation is the mechanism that gives us the ability to to observe things that happen in a service as a transactional request spans another service and another service and kind of your call chain of microservices context propagation is the thing that gives us the ability to watch and see what's happening um so a couple of Concepts that are important to know about when it when you're thinking about distributed tracing and context propagation or spans and traces um and in my experience oftentimes I think people flip the tune um but to be clear a span is is um it's basically a unit of operation um a unit of work you can think a database query is a great example of a span that's a unit of work that a service you know does there's a collection of these units of work that make up you know the the the the the transaction that may happen in the service and so a span is a span is nothing more than that unit of work and a span ID is the thing that uniquely identifies that you know unit of work a trace is just a collection of those spans and it tells the story of how a request you know traverses all of these different services that um that essentially enable a use case or a feature or a capability um and each Trace has a unique identifier um which is globally unique and random and I put some links in the readme here so that you can read more about um the uh the reasoning behind that but I think it's important for people to know and I know this has come up in conversations that I've had with people around what a trace ID is and it is important that it is globally unique and random for reasons that are explained better in the documentation from the w3c which I'll get into a moment until in a minute here um and a trace context is essentially kind of like the envelope or baggage that it's passed between services so that they can understand you know what uniquely identifies the request that's coming into them have a uniquely identify a transaction request that's going out of that service to another service and essentially um in order for all of this to work there has to be an agreement on what that format and what that specification is for a trace context and one of the things that we get as developers which is awesome is a recommendation from the w3c around what a trace contact should look like and we also get the benefit of a lot of the work that the open Telemetry Community has has done to codify uh those those recommendations and specifications into sdks and into other things like the open Telemetry collector that give us as developers a really easy way to implement distributed tracing context propagation there's an agreed upon format that is you know recommended by the w3c on how to do this there's a software in libraries and specifications that the open Telemetry Community gives us to make all of this really really easy and I'll show that in a moment um so with that specification that the w3c gives us um part of that recommendation is to use HTTP headers to allow a service to send information to another service so that that uh that Trace context can be propagated and there are two headers that the w3c recommends for doing this it's a trace parent and Trace State Trace parent is where we package up the unique identifiers that describe a unit of work that's happening in one service the trace that's associated with it to send to another service so that they essentially can unpack that tag that unit of work in the other service with information so that as that information is admitted to an observability platform that we can tie all these things together and we can see how this call chain of things that happen that span Services all work together and you know back in the day with monoliths it was pretty easy to observe and debug things that span different sub components of your monolithic architecture and we kind of lose that ability with microservices but we get it back by leveraging this Trace context concept and the um the the headers that are specified from that the w3c recommendation to pass that information between services so okay service a service calls another service it simply uses a header to package up some information it's an agreed upon format for other services to unpack and use to tag the spans and units of work that those other services are using um there's some details in here about like what the format of that is uh let me reduce my font here so you can see what a an example of a trace parent would look like so there's a version I'm not going to get into the version today nor will I get into the trace lags but the really important part of this is to understand that this ID here this unique identifier for the trace ID is the thing that will allow us to tie all of this work together so that we can observe how microservices are all working together to you know enable some feature satisfy some use case in addition to the the trace parent header the w3c recommendation also offers another additional header called tray state which is really nothing more than a simple list of key value pairs that you can add along with your the context that's being propagated between services so that you can attribute that to things that may be unique to whatever it is you're doing um vendors May uh use Trace State to provide observability into what they're doing but um you can just think of it almost just like a little bit of baggage that rides along with the trace context that moves between the services um so I have some links in here to the the w3c trace context specification and recommendation one of the things that I think is is worth calling out and I would encourage anybody who's going to go down this path is to make sure that you're familiarize yourself with the privacy considerations you know in this day and age with with pii and and gdpr and a lot of the other things that we really need to pay attention to as it relates to the the um potential leakage of personal information there are some considerations that are worth noting I will say that in general that following the w3c recommendation using the specifications in the libraries that we get from an open tem Elementary this is a very safe thing to do but just to kind of like be clear spend some time and just make sure you get yourself familiar with those privacy considerations so so I know that I ran through that quickly because I know we're kind of pressed for time but essentially you know this thing called distributed tracing again it's a way to you know observe how a transaction or a request traverses multiple microservices um again the good news here is that we have a specification that's widely adopted and it's widely known we have tools that we can use from the open Telemetry to employ this so for anyone who's thinking about like I'm building a bunch of microservices I need to observe how they're operating how do I do it the good news is is that that problem's been solved for you there's a de facto standard there's tools that you can use and uh as you'll see in a moment it's it's actually quite easy to do um let me pause there because I know that that was probably a fire hose of stuff but I want to see if anybody who's uh on the call right now has any questions about these Concepts before I jump into some code okay um we're gonna run with scissors now and we're gonna do some live coding so what I've done here is I have um again what I tried to do was to just create the most minimal example of a couple of microservices all working together um to demonstrate uh how you can use the w3c recommendation and use the open Telemetry libraries to do this stuff so in this scenario um we're building the um it's kind of like IMDb it's the instead of the Internet Movie Database it's the internet band database and so I want to I want to create like this source of Truth when it comes to bands names and uh and also um in this scenario we have we're leveraging a platform that just handles and reviews just a generic platform for reviews and so we're we've created it we're creating a band's API that our front-end developers are going to use to build web and mobile applications on top of and and and rather than building a reviews platform ourselves we're just going to leverage this fictitious reviews platform so in this case um we've exposed an endpoint that allows something to call and say hey give me this give me this band that has a unique ID and give me the reviews associated with it so I'm going to call the endpoint to get a band in its reviews and wow it took six seconds that's a long time let me run it again just to see what's happening here 12 seconds is a long time um so you can imagine we're all familiar with this right like you're building the service you've got people that are riding front ends to it and they're saying hey Doug uh I'm calling the endpoint for to get a band and it's taking a long time and that's not a lot of information to go on and debug you know I think we're all familiar with those scenarios where um you know somebody contacts you and says hey you know we've been noticing that this doesn't seem to be performing very well um we have to do that guessing game of like okay well when did you call this service like do I start grabbing a bunch of logs it becomes very tedious so one of the things that we that helps is to you know employ some tracing and some distributed tracing so what I'm going to do now is say all right hey uh from a developer um I'm going to add some stuff to my service and all I need you to do is to make sure you pass me some information when you call me so I know that you're calling me and then we can kind of debug this and see what's happening so the first thing I want to do is I want to go into my my my bands API and this is just a very very simple fast API application just a very simple model for a band I have an endpoint here that lets uh a caller get a ban by its ID I have a little helper function here that does some work to go actually access a data repository of reviews which happens to be this other review service that I mentioned before uh so let me bring up my my cheat sheet here so let's start by um bringing in some code that will provide some visibility with tracing so what I'm doing here is a couple things one is I'm going to bring in some um with the help of the open Telemetry SDK I'm going to bring in the library that's going to help me essentially tease out information from the header that I talked about a moment ago this Trace parent header and inject it into um essentially a hook that will call out to another service and omit that tracing signal and in this case I'm going to use Jaeger um uh all populated about open the open Telemetry collector but in this case I'm just going to bring in some code with the SDK give the band service the band's API the ability to unpack that header and inject that information to admit a trace so that we can see what's happening oh yeah so the get reviews so let's go to our get reviews so excuse me this is the function that does the call to the reviews service so what I want to do here is say okay for my when I call the get review service I want to start some kind of Trace so I can see what's happening and in this case I'm starting a span in my service and what I want to do is I want to actually I don't want to do this just yet sorry let's just do this and then back over here I'll explain what I'm doing here so what I'm doing now is um I'm adding a couple things I am um I'm creating the ability or what I'm doing is I'm bringing in a very very small Library of uh this is just a helper library that is essentially uh provisioning the uh the Tracer that we get as part of the open Telemetry SDK what this Tracer will do is this is the thing that will um build up the trays and send it to an observability platform in this case it's going to be Jaeger and uh in case you're wondering there's some default stuff that happens like the um the open Telemetry SDK knows to send to a certain Port that a service is listening to in this case it's four three one seven um and Jaeger's running locally so it will pick it up that's a lot of hand waving uh but in the interest of time just just know that the SDK is handling a lot of this stuff for you back in my band service what I want to do is say Okay anytime somebody calls the service um what I want to do is I want to allow the caller to send that Trace contacts in the header and so in my service what I want to do is I want to tease out that Trace parent header because I know that there's a standard and a specification for that I'm going to build up a carrier and I'm going to inject that or I'm going to extract that carrier from the payload and then I'm going to inject that Trace contacts from my caller into my contacts so that when I build up a span the open Telemetry SDK is going to do a bunch of magic and work for me and essentially emit that Trace to the observability platform so I have this now built up in my code so the bands API now can receive a request build up a Trace Loop in that Trace context from the caller and and when it calls its own internal function we're going to spin up another span which is a unit of work to make a query to the reviews platform so this will start to give us some of visibility into what the caller is doing and what the band service is doing so now let's go so what I want to say to the caller then the person who's building this application to say hey I'm adding some stuff to our service because I know you said it was slow and things don't seem right send this bit of information so that we can start to really observe what's happening here and kind of dig into what the problem may or may not be so if I go back to my client what I want to do is I want to essentially say hey help me help you by sending this header and when you send this header manufacturer a trace and a span on your end add that header to the call that you make to me and then we can kind of start to dig in to see what's happening here so now at this point we're kind of wired up now to like get a bit more observability into what these different services are doing so if I run my if the client now calls me I can see it took five seconds oh I forgot one thing sorry so I also just kind of want to print out to the console a little a nice little helper thing just to link me right to Jaeger where these traces are being sent to so I can start to see what's happening so now the client calls me again we should be able to start kind of digging into what's going on here wow 15 seconds is a long time so now I can kind of dig into this and I can see and I I don't I'm not going to spend too much time on Jaeger but Jaeger is an open source platform that's um that essentially helps visualize traces and spans so now that I have the caller following that w3c recommendation passing that Trace parent with some you know unique identifiers of the unit of work units of work it's doing sending it to the band service the band service is taking that information its context has been propagated to it it's leveraging that to emit information about what it's doing and now we can observe what's happening I can see that the uh the bands the get bands endpoint was called and I can see that the get reviews function unit of work was run and if I look at this um the spans table here I can see that the majority of time that was spent here was getting the reviews so that's a nice that's that's good helpful information to know right I can see like okay get reviews is taking a long time let's dig into that so let's go back to our code and I can say hey guys building folks building the client I see something going on in my service let me dig into it the get reviews is taking a long time hey what's this debugging thing uh somebody did something while they were debugging I just have a random timer here let me just get rid of this sorry about that folks right okay hey I just deployed a new version of my service call me again still taking a long time something doesn't seem right so now in this fictitious world I'm going to turn over and turn around to the folks that work on the reviews platform and say hey you know we're looking into an issue there was some random code that was sleeping in our end uh can you take a look and see what's going on in your platform because when we're calling you like it's taking a really long time to get reviews like our service seems to be working pretty fast so now let's go and do the same thing on the reviews end um very similar bit of refactoring let's go to the reviews especially let me bring yeah let's bring all this in so now the same uh the same service here gonna bring in this helper function to um start up the Tracer again leveraging the open Telemetry SDK to do that we'll bring in this Trace context propagator so that we can have this service do things they're interesting things with the trace context and let's see so here we want to do something that's a little bit different than the band service does right yeah I think that's right so now the review service can also receive that Trace parent that Trace context unpack it and Associate it with its span of work in addition to that we want to start a span and the uh at the internal um imagine this would be a piece of code that would more than likely read from a data repository a data store that's associated with the review service so now I can say okay folks that are building the front end of mobile app we've we've released a new bands API we've released a new reviews API that we we think we're going to help you know give us some visibility into what's going on so they're going to keep going doing their development they're going to start making calls to us four seconds isn't too bad let's try again it's still taking too long so now we got to figure out okay well what's going on so let's go back to our observability platform Jaeger and what is happening uh not skiing I'm not seeing the trace from the reviews out of this uh this is what you get when you try and run with scissors and do live coding um what am I missing here Chase parents so this is getting oh I know what we did all right folks there it is Okay the reason that the review service wasn't getting the context propagated because I didn't add that in the so in the band service is calling the review service we need to propagate that context um be quiet Siri here we go all right I know I'm jumping around quite a bit um so in the band service it's getting that Trace context from the caller it's attaching it to a span it's making this call to get reviews get reviews we want to see what's happening in that unit of work so we're starting up a new span but we also want to take the contacts from the caller and inject it into a header so that when the band service calls the review service that context gets propagated to it as well so now when the caller calls the van service it's spinning up this Trace ID this unique identifier sending it to the band service the band Service is like thank you very much I can attach this to my span this my units of work I'm going to send it Downstream to the thing that I rely on it's going to attach this information to its units of work and we should see all of that come together in Jaeger boom all right service is still running slow but at least now we have some visibility into what's happening oops and if I go to my uh Trace graph sorry my stands table actually where is it I can see here that the majority of the time seems to be sent be being sent being spent sorry and getting the reviews from the reviews database so when it's getting a review by getting the reviews for a band by the band ID it's taking a long time so let's go back and start digging into that code there to see what might be happening so in the review service if I look at this function that you know is grabbing data from some kind of data Repository again somebody put in some kind of sleep maybe they were debugging testing some timeouts who knows what but let's pull that out and now we can say hey folks that are building the mobile and web app we think we fixed it from real this time can you try again wow that's better let's go back and see what happened in Jaeger okay things are taking milliseconds that feels better I like this now I can observe what's happening across all of these different things yay team um so uh just to kind of walk through the code one last time the client is packaging up this header based on the w3c recommendation it's passing that header along to our bands API our bands API is leveraging the open Telemetry SDK and saying hey let me pull this stuff out and inject it into a carrier so that I can send it Downstream and by the way when I'm creating my spans um you know I'm gonna leverage that information when the review service gets a call into it it can tease out that Trace parent information and inject it into its fans by attaching that context and now we have distributed tracing with context propagation because we've got this awesome stuff from the open Telemetry community and we've got this awesome recommendation from the w3c let me stop there and ask if anybody has any questions about the the uh the kind of the basic minimal implementation of these Concepts in these handful of fast API services and maybe this is a good time just to kind of open up to general questions as well because I know we're coming up on times yoga I had a question for you how uh how did you find in terms of like finding the information the documentation around around the context propagation how how was that experience for you in terms of finding that information in the hotel Docs um it was pretty good actually uh so in the in the readme for this um repo which um oh which I hope that other people can help me uh test the documentation I would love for some folks to go and clone this repo and try and Spire all these Services up and make some calls and see if they can get it running locally um it was I it wasn't it was good um you know the the open Telemetry documentation has some some very good details around things that I glossed over today that I would I would really like to go into further detail around that I'm sure that anybody who watches this is probably going to start asking questions like well how does that actually work like there's a lot of magic under the hood but the open Telemetry documentation goes into a lot of detail around the things that I just waved my hands around like the tracers the trace exporters the context propagation the shape And format of spans um how these things all link together so I found I found it was pretty easy uh to find that and I also think that the the w3c documentation is also very good as well so I was able to kind of bring everything together pretty quickly like it didn't take me very long to build this reference implementation of these few applications um yeah open Telemetry also publishes a bunch of integration packages which lets you hook up to like node middleware or HTTP clients automatically so you can just have everything instrumented and doing the correct header insertion and things like that without really having to worry about doing it manually like like this demonstration noise yes that's a really good point so um uh one of the things that that I did as part of this uh this reference implementation was to do a very explicit implementation of these Concepts um we don't have time and I wanted to do that thing that you always see in a demo where somebody says do all this stuff and at the end of the demo and they say oh by the way you don't have to do anything you just do this other thing and it all gets handled for you there is a fast API instrument or um which will remove the need to do a lot of this explicit unpack um this explicit extraction and injection of the context um I didn't I didn't have time today to show that and I I kind of wanted to make it very clear to a developer who's learning these Concepts to walk through the code and see something very procedural that shows this for example there's a header and that header is very important and it is called a trace parent and it has a specification here's an example of teasing out that header in code the open Telemetry library has this thing where it can you know extract this context for you here's the line of code that does this when I make a call to another service I want to pass that along very explicitly so you know there was a lot of intention behind trying to make this application a bit more verbose even though it's very minimal to try to show somebody who's you know maybe new to this like what does this actually look like if you use the fast API instrumenter the good news is a lot of this gets handled for you um my recommendation would be for anyone who's getting started with microservices is to make sure that you include distributed tracing from the very very beginning and I would almost suggest that you spend a bit of time and implement this explicitly so that you understand what's happening so that other developers understand what's happening and once you solve the problem of observability then level up and start to leverage some of these other things that the open Telemetry gives us like Fast the fast API instrument or and these other things that that we get from the community um I feel I feel like that just kind of helps bake the concepts in for people and I I know that for me I'm a very tactile learner so like it helps me to see what's happening before I go and to start to do or start to let some libraries start to kind of handle that magic for me if that makes sense you know your mileage may vary but my recommendation would be to go ahead and do it explicitly it's a handful it's a few lines of code you can share it with people when you're doing code reviews you can explain to them what's happening it's not mysterious it's very explicit foreign have any questions for Doug thank you Doug for sharing that um one question from me is that um how does it look when you like so the obserability problem in production I suppose apart from this open Terminal tracing like part I suppose we also need to serve like logging and metrics and to find any difficulty like integrating like different part of obtibility um so the the last time I I spoke here um I I mentioned that the one of the the path that we took and had been taking that uplight has been to actually begin with the log signal and the reason is that um in terms of introducing these Concepts that these problems that open Telemetry solves these problems at the w3c helps us with solving is to begin with logging so that you because logging I think is very very familiar with most developers I think you know Junior intermediate developers are familiar with this concept of logging like a print statement and so what I've recommended and what we did at uplight when we started down this path was to start using the logging SDK even though it's newer than the the work for the signals of metrics and traces but to start with logging so that you can at least begin to practice including the packages and libraries into your code using open Telemetry to Omit that logging signal get that logging signal Landing in your your APM or whatever your observability platform is and once you make that connection of like I'm writing code I'm writing a statement that says logger.debug my message is landing in an APM it's structured it's it's it's being it's it's honoring the specification that the open Telemetry Community gives us then it's really easy to then um you know level up to add tracing and you get Trace law correlation for free and that's one of the things I would like to actually do with this this little reference implementation is to to add in some Trace log correlation so that people can see like how do I do the thing that I'm so familiar with doing like I like to send out a log I like to do a print statement like that's so we always do that all day every day and if you start with logging and you get the sdks into your repo and you have that information flow into your APM then adding the the layer of maturity around traces and the trace log correlation is almost free because you've already got everything packaged up into your application already so um you know that might not work for everybody to me that feels like a very natural way to evolve into using the specification using the sdks and and really kind of getting into observability Nirvana where you can walk up to your APM and say I have questions about the health and behavior of my services and I have a lot of them what's happening and you can get those answers if you do this Doug it sounds like you need to do a follow-up presentation on uh on logging you know I I would love to and I think that you know I would love some help from other people to take this this uh this repo and clone it or Fork it or just play with it and for if you know to help me test the documentation um some other things I would like to do is to go into the code itself and provide a bit more um kind of verbose commentary around what's happening again I think it's helpful to have very explicit implementation of these Concepts but for example I would I would really like to have you know some comments here that tell somebody hey when you call the extract method this is what it's doing for you because right now it you know it's magic it's just happening and that's awesome but I think for people to really understand what other Telemetry is doing and to understand what how all this works I think it would be helpful helpful to have in this reference implementation some pretty verbose comments and not the kind of thing you might want to put in your production code but for a reference implementation I think it would be helpful and I would love to do another one of these awesome um and you know I've been like looking through your repo as as you've been presenting and I really like this example it's it's uh it's simple but it's got enough complexity that it it illustrates what you're trying to um convey um so yeah I I think I I would love to at some point uh when I have some spare time to like poke around I I definitely encourage others to poke around as well maybe I I don't know uh as far as like I think it'd be cool to contribute this back to open Telemetry um although to be honest I I don't know if like other Hotel folks have uh have a better idea on the process around that because I I think I'm newer to the process but anyway um I think that yeah yeah and I you know I know that there are like there's a demo app and there's other reference implementations but one of one of the problems I was trying to solve here was how do I strip away every all the noise and give a developer with a modern operating system and docker the ability to get clone fire up these services and watch what's happening so they can really dig into these Concepts and understand how powerful they are in their Simplicity like it's pretty amazing like all we're doing is just you know sending a header and this these sdks are doing these wonderful things for us like it's it's pretty simple but it's unbelievably powerful and so the idea for this reference implementation was to try to just highlight that as best we could with as few lines of code as possible so that you know people can understand this and so that people can go and be successful with building this level of maturity in their observability in in their code yeah I totally agree and I think there's a lot of a lot of value in that like like you mentioned the open Telemetry demo app is is awesome because it shows like all of the things but there is definitely value in in simpler implementations like this one um so yeah um well on that note um I know we're a little bit over time um thank you everyone who was able to join us today um especially given that we uh started late and thanks Doug for uh uh you know working on the fly with our reduced time and for for lunch that's never uh never an easy task so definitely appreciate the time that you put into uh making this happen today um thanks everyone for joining us once again and um keep your eyes open for our next hotel in practice um I don't think we have one scheduled yet but we'll we'll post on on the various cncf slack channels for that um Rin do you have any additional upcoming announcements that uh folks should be aware of sure yeah so um first um there's an open Telemetry and practice Meetup if you came here through cncf um lightweight way to get notified whenever we have something and not have to look for the slack if you're not in the cncf slack and you want to talk further to um any of the hotel and practice folks Doug the audience um we're happy to add you um that information is in the chat and if you'll be at kubecon EU um several of us will be and we'd love to connect with you there will be a blog post on the open Telemetry blog on Monday with all of the activities and um yes I am happy to do a follow-up with Doug because of kubecon EU that will probably be in around the end of May and um we can figure out what format we want to do based on hopefully talking with you all in the slack thank you everybody diff --git a/video-transcripts/transcripts/2023-09-25T04:00:25Z-otel-in-practice-fireside-chat-december-2022.md b/video-transcripts/transcripts/2023-09-25T04:00:25Z-otel-in-practice-fireside-chat-december-2022.md index b4c08ca..df9ae02 100644 --- a/video-transcripts/transcripts/2023-09-25T04:00:25Z-otel-in-practice-fireside-chat-december-2022.md +++ b/video-transcripts/transcripts/2023-09-25T04:00:25Z-otel-in-practice-fireside-chat-december-2022.md @@ -10,81 +10,88 @@ URL: https://www.youtube.com/watch?v=TYrIP_LkwgU ## Summary -The video is part of the OpenTelemetry and Practice series, featuring a panel discussion with speakers Adriana, Rhys, and Jess discussing various aspects of OpenTelemetry implementation and observability. Adriana, who works at Lightstep and has a background with HashiCorp, shares her experience managing a platform team and demonstrates how to run a hotel demo application using HashiCorp tools like Nomad and Console, illustrating the integration of OpenTelemetry. Jess presents a fun, interactive project called "Happy Ollie Days," which uses OpenTelemetry to create heat maps based on input images, showcasing its creative application in teaching observability concepts. Rhys later discusses how end users can get involved in the OpenTelemetry community, highlighting initiatives like user interviews, feedback sessions, and the importance of community engagement. The session emphasizes practical applications and community involvement in the OpenTelemetry ecosystem. +In this OpenTelemetry and Practice series video, the hosts, including Adriana, Rhys, and Jess, discuss various topics related to implementing OpenTelemetry across different platforms. Adriana shares her experiences with observability in HashiCorp products and demonstrates running a hotel demo app on Nomad, explaining how to translate Kubernetes manifests into Nomad job specifications. Rhys introduces the End User Working Group's efforts to engage users in the OpenTelemetry community and improve feedback loops between users and maintainers. Jess presents a fun project that visualizes spans in a heat map format using OpenTelemetry, showcasing how to create engaging visuals for educational purposes. The session emphasizes community involvement and encourages users to participate in discussions and feedback sessions. -# OpenTelemetry in Practice Series +## Chapters -Thank you for joining us! The OpenTelemetry in Practice series is typically a serious talk series where we dive deep into implementing OpenTelemetry in various programming languages. Every month, we usually speak with end users who are implementing it in production, and we also engage with maintainers of the project who can provide insights on best practices. +Sure! Here are the key moments from the livestream along with their timestamps: -Today, however, we have a different format. We will have three or four short talks that didn't fit well together. +00:00:00 Introductions to the OpenTelemetry and Practice Series +00:01:30 Overview of the format change for this session +00:03:00 Adriana discusses her background with HashiCorp products +00:05:30 Introduction to the Hotel Demo App and its features +00:07:00 Adriana shares her experience running the demo app on Nomad +00:10:00 Demonstration of setting up the observability tools locally +00:12:30 Explanation of Nomad job specs compared to Kubernetes deployments +00:15:00 Discussion on health checks and service dependencies in Nomad +00:20:00 Jess introduces her fun project for creating heat maps with OpenTelemetry +00:25:00 Rhys talks about the End User Working Group and how to get involved -## Speakers and Topics +Feel free to ask if you need more details on any specific section! -Adriana will kick things off by discussing **observability for HashiCorp products**. She currently works at Lightstep and previously worked at a Hashi shop. Later, Rhys will talk about how end users can get involved with the project, which I expect will be highly interactive. Finally, Jess "Jessatron" will present on using OpenTelemetry to create ASCII art heat maps, which is a fun and engaging way to learn about observability. +# OpenTelemetry and Practice Series -### Adriana's Talk +Thank you for joining today's session of the OpenTelemetry and Practice Series. This series is typically a serious talk where we dive deep into implementing OpenTelemetry in a specific programming language each month. We engage with an end user currently implementing it in production and usually include a maintainer of the project who can share the best practices for implementation. -Adriana begins by sharing her background: +Today, however, we have a different format. We will have three or four short talks that don't necessarily fit together. **Adriana** will kick things off with a discussion on observability for HashiCorp products. She currently works at Lightstep and previously worked at a Hashi shop. Later, **Rhys** will discuss how end users can get involved with the project, and **Jess** (Jessatron) will present on using OpenTelemetry to create ASCII art in heat maps. This is all about getting your team to learn about OpenTelemetry and engage with it effectively. -"I used to work at a Hashi shop, which gave me a Kubernetes background. I was thrust into the HashiCorp world where my previous employer ran their containerized workloads on Nomad. I quickly had to understand Nomad because I managed a platform team that supported Nomad, Console, and Vault for the entire enterprise. This experience led me to become a HashiCorp Ambassador as I started blogging and engaging with the community. +### Adriana's Talk on Observability -While at my former company, I also managed an observability team. I learned about observability around the same time I was getting to know Nomad, which naturally led me into OpenTelemetry. +Adriana began by sharing her background. -Now that I work at Lightstep, I’ve been able to contribute more to the OpenTelemetry community. I’ve worked on documentation, contributed to a demo app, and recently got the demo app running in a local Hashi environment on my machine. +> "As Rin mentioned, I used to work at a Hashi shop, which gave me a Kubernetes background. I was thrust into the HashiCorp world because my previous employer ran containerized workloads on Nomad. I had to quickly understand Nomad since I was managing a platform team that supported Nomad, Console, and Vault for the entire enterprise. This led me to become a HashiCorp Ambassador as I dug in, blogged about it, and continued to engage with the community." -There's a tool called HashiCube, similar to running Minikube or k3s, which provides a local Hashi environment. It replicates the setup you would have in a data center. Nomad by itself is somewhat limited, so you need the power of Console for service discovery and Vault for secrets management to create a robust system for managing containerized workloads. +Adriana then transitioned to her work at Lightstep, where she has contributed to the OpenTelemetry community. -I have HashiCube running locally, and I can show you the setup. Here’s the startup sequence for HashiCube, which uses Vagrant to provision a virtual machine or Docker image with all HashiCorp products. +> "I've contributed content to the documentation site, created a demo app, and was pleasantly surprised to see the demo app, which showcases what you can do with OpenTelemetry. I had the idea to get it running on Nomad, which has been on my wishlist for a while. I managed to get it up and running in a local Hashi environment on my machine." -Let’s take a look at Nomad and the services running on it. The hotel demo app is running locally, and it’s exciting to see all the services up and running. I had to convert Kubernetes manifests for each service into Nomad job specs, which was quite the process. +She demonstrated using a tool called HashiCube, a local HashiCorp stack environment, to run the demo app. -You can see the logs, services, and various components like Redis, Postgres, and Prometheus. I even used Traefik for load balancing to expose services to the outside world. +> "HashiCube essentially uses Vagrant to provision a virtual machine running all Hashi products in a single Docker image. I have it running on my local machine, which is exciting because Nomad alone is pretty useless without Console for service discovery and Vault for secrets management." -To access the hotel demo app's endpoints, I set up a local host endpoint, and I can share the Jaeger UI to visualize traces. It’s impressive to see the traces produced by the various services. +Adriana shared her screen to show the demo app running locally. -Let me show you what a Nomad job spec looks like. Nomad is an alternative to Kubernetes and can run various workloads, including VMs and containerized applications. In contrast to Kubernetes, where you have multiple YAML files for different objects, everything in Nomad is contained within a single job spec file written in HCL (HashiCorp Configuration Language). +> "This is Nomad, and you can see the services that make up the hotel demo app running on Nomad. By going through the Helm chart, I translated Kubernetes manifests for each service into the equivalent Nomad job specification. Everything is up and running, and we can check the Jaeger UI for our traces." -For example, I have the feature flag service defined here, and you can see the network definitions, ports, and environment variables. Nomad also allows dynamic references to services in Console for service discovery. +She also explained the Nomad job specifications and how they compare to Kubernetes deployments. -Additionally, we have health checks similar to Kubernetes liveness and readiness probes. This is essential to ensure that services are up and running and can communicate with each other efficiently. +> "In Nomad, everything is self-contained in one job spec file written in HCL (HashiCorp Configuration Language). For example, we define our network settings and service definitions here, which is quite different from the multiple YAML files required in Kubernetes." -That’s a brief overview of my setup and the hotel demo app running in Nomad. If you have any questions, feel free to ask!" +Adriana concluded her demo, showcasing how Nomad handles dependencies and restarts services if they fail, along with how health checks work in Nomad. -## Jessatron's Presentation +### Jess's Presentation on Heat Maps -Jessatron is up next with a unique and creative project. +Jess then presented an engaging project she created for Christmas. -"I created a fun project for the holidays called **Happy Ollie Days**. It’s a Node.js app that uses OpenTelemetry to create heat maps from images. You can clone the repository and use your Honeycomb API key to get started. +> "I made a fun app called 'Happy Holidays' that uses OpenTelemetry to generate heat maps. You can clone this repo and run it with your Honeycomb API key. It creates spans and traces for different spans and sends them to Honeycomb." -The application allows you to input a PNG image, and it converts it into a heat map based on the blue channel. You can customize the code for different systems, but it’s specific to Honeycomb’s UI. +Jess explained how the application processes images to create heat maps based on the PNG input. -In the app, I generate spans and traces, which allows you to visualize data in a fun way. You can see how different spans correspond to various attributes, and it illustrates how heat maps work. +> "It uses the blue channel in the PNG to convert it into a heat map, and you can interact with the data in Honeycomb to see how it visualizes the data." -This is not just a gimmick; it’s a fun way to teach observability concepts. You can explore how spans are created and used in a heat map context, which can help beginners understand the underlying principles of observability. +She demonstrated how to adjust settings in Honeycomb to view the generated heat maps and explore different attributes. -This project will be officially released on Monday, and I encourage you to check it out! You can find it on GitHub at `honeycombio/happy-ollie-days`. If you have any feedback or want to adapt it for another tool, I’d love to hear about it!" +> "You can see how different reindeer names are represented in the heat map, and you can customize it further if you want to use your images." -## Rhys's Discussion on End User Involvement +Jess encouraged attendees to check out her project and see how they can customize it for their needs. -Rhys wraps up the session by discussing ways for end users to get involved in the OpenTelemetry community. +### Rhys on End User Involvement -"A lot of people are not aware of how to navigate the community or that they even can get involved. We have two primary goals in our working group: fostering a vendor-agnostic community for end users and creating a feedback loop between end users and project maintainers. +Rhys discussed the importance of community involvement and how end users can engage with OpenTelemetry. -We have regular discussions, interviews, and feedback sessions where we gather insights from end users about their experiences and challenges with OpenTelemetry. If you’re interested in sharing your feedback or being part of these discussions, please reach out to me or any of the other group members. +> "We've found that many people are unaware of how to navigate the community or that they can get involved. Our goals in the end user working group are to foster a vendor-agnostic community and create a feedback loop between end users and project maintainers." -We also have a community survey for those who want to contribute without getting too involved. It’s a great way to share your thoughts on using OpenTelemetry. +He highlighted various activities and opportunities for involvement, including monthly discussion groups, end-user interviews, and community surveys. -Looking ahead, we plan to extend our user study function and possibly organize in-person meetups. We’re excited to bring the community together, especially after recent events where in-person interactions have become more common again. +> "If you're an end user interested in sharing feedback, please reach out to myself, Rin, or Adriana. We would love to get you on the schedule." -Finally, if you’re looking for help or want to contribute, there are multiple channels available, including the CNCF Slack and GitHub. We welcome any and all contributions, whether it’s code, blog posts, or documentation suggestions. +Rhys emphasized the importance of community contributions and the different ways individuals can engage with OpenTelemetry, whether through coding, documentation, or sharing experiences. -Thank you all for being here today, and I look forward to seeing how we can work together to improve OpenTelemetry!" +### Closing Remarks -## Conclusion - -Thank you for participating in today's session! If you have any questions or would like further information on any of the topics discussed, feel free to reach out. Happy holidays! +As we wrap up, thank you all for joining today. We appreciate your participation and encourage you to reach out with any questions or if you want to follow up on any of the topics discussed. Happy holidays! ## Raw YouTube Transcript -[Music] thank you so the open Telemetry and practice series is traditionally a pretty serious talk Series where we every every month we go deep into implementing open Telemetry in a particular language and we talk to an end user and uh um what in who's actually implementing it in production and we talked to um usually a maintainer of the project who can talk about like these are the best ways to implement um and this is a little bit different of a format because we decided to do three or four short talks that didn't fit well together um Adriana is going to lead off today with um talking about observability for hashicord products um she currently works at lightstep and um used to um work um fully in a Hashi shop yes and then a little bit later we'll have Rhys talking about um how end users can get involved with the project which I expect to be highly interactive and um Jess jessatron um is going to give us a talk about um using open Telemetry to make asksy ASCII are in heat Maps which is cool you are trying to get your team to learn about open Telemetry and pay attention and utterly useless um Adriana you ready to yeah I'll start so I'm ad Hocking this uh all right so um as Rin mentioned I used to work at a Hashi shop so I have a I actually have a kubernetes background was thrust into this hashicorp world where my previous employer um they ran their containerized workloads on on Nomads so I found myself in a situation where I had to quickly understand this Nomad thing because I was managing a platform team which supported Nomad console and Vault for the entire Enterprise um so that's how I began my dabblings in the Hashi world and and from that I I ended up becoming a hatchery Corp Ambassador because I I dug in blogged about it and continued to do so um and at the same time at this former company I was also managing an observability team and I came to learn about observability team at around the same time that I came to understand um Nomad so these were two things that I was doing at the same time learning about at the same time which led me into uh open Telemetry so um now that I work at lightstep I've had a chance to contribute more in the open Telemetry Community as I mentioned before um I've uh I've worked in the in in the comms area so I've I've contributed some content to the doc site I've worked I've made some contribution demo app and I was actually pleasantly surprised to learn about the demo app which I think launched It's relatively recent right that the demo app came out and and I like it because it showcases open to limit like what you can do with open Telemetry and I think the original version of the demo app that came out you could uh run everything in Docker compose locally and then more recently there's um uh the the team created um home charts for this one and she got me thinking hey if this thing runs Brunetti's wouldn't it be super cool to get this thing to run a nomad and so it's been part of my wish list ever since I I learned about like the hotel demo app Helm charts I'm like okay finally carved out some time to do this and I got it up and running um in a local Hashi environment on on my machine so there's um there's this really cool tool called Hashi Cube which is similar to you know like running mini Cube or k3s or whatever like one of these like local kubernetes Dev environments similar sort of thing for the hashy stack um which was created by a a third-party company called uh Serbian and and I love using this tool because it gives me like a full-fledged Hashi environment similar to what you would get I mean obviously smaller scale to what you would get in a data center but it replicates the the type of setup because in in real life the um Nomad by itself is is kind of useless so you need like the power of of console for service Discovery involved for for Secrets management along with Nomad to get all this stuff um to to really have like a a robust system to manage your your containerized workloads so I have uh I have this Hashi Cube running on on my machine locally which I can show you um I'll do like a little screen share well first I'll I'll show you my hashiq oh wait did uh am I sharing the right one hearing your hashes cool I have like so many windows the correct cashy tube I don't know over the time yeah this well okay so it says this is this is my Hatchi Cube startup sequence so hashikub what it what it is so it's it's basically using vagrants to provision either a um VM a virtualbox uh VM or um I'm I'm on an M1 Mac and virtualbox and M1 Max don't play nice I think there's like possibly a version of virtualbox that plays nice with M1 Max now but at the time it didn't so the uh the maintainers of hashic cube basically stand up all these Hashi products in a single Docker image um so this version of hashikub that I'm running where you can see like the tail end of the startup sequence here um it's running mad Vault console all in one all-in-one Docker image and so if I share I'll share you my Nomad console here so you can see um problem when I lose sight of all my windows um the oh here we go okay so this is this is Nomad and these are all of the services that make up um the hotel demo app running on Nomad locally on my machine which is pretty freaking exciting um I basically so I had to go through the um the home chart there's a for the for the hotel demo Helm chart um there's a there's a folder in the helm chart repo that shows like the rendered yaml so I basically started with the rendered yamls and went through the process of translating um these these um kubernetes manifests for each service into the equivalent Nomad job spec which is what you see here so for example like we're looking at the feedback service this is running on on Nomad um and if we go over here we can see um there's a lot of stuff here so we can see the logs of the true flag service nothing terrible is happening here we go it's running so we can see from from our our dashboard here that all of the services are up and running so it's made up of like all these uh different um different Services written in different languages plus we also have I had to convert um I had to nomadify um redis postgres we had the hotel collector um what else there was one more out there was Prometheus and grafana and I used trafic for load balancing so I can expose my services to the outside world and so what that looks like then um so to be able to access the hotel demo apps endpoints and hopefully my computer won't crash here um so I have IX endpoint called Hotel Dash demo dot localhost which is accessible this is just uh just something accessible off my local host and then the uh so Jaeger UI is is is exposed um as part of the my demo app configuration so we see Jaeger is up and running here we've got the demo apps UI going uh running here so we can buy stuff we can check stuff out and buy let's buy this telephone and super and then if we check Like Jaeger we can look to see sorry the screen sharing thing is like getting in my face um we can see our traces [Music] I think the hamsters are running really really hard here we can see it's produced some some traces here that we can see here in addition um when I know modified this there is also a some grafana dashboard so some cool dashboards that come pre-loaded pre-configured you can see there's some stuff going on here with the hotel collector and then there's also this demo dashboard which gets some stats from the recommendation service so that's um that's basically it I can show you also like what a job spec looks like in Nomad um I want to see more traces oh you want to see more traces yeah right I picked like the saddest Trace yeah that's what happens when you're like ad Hocking it okay let me uh let's let's pull up a more interesting Trace I think the recommendation service has interesting traces that we can see checkout's usually good check out cool let's find some traces oh yeah look at all those look at all those Services yeah there's some colors look at it go to all the different services yeah this is actually pretty cool you're right I picked the dinkiest to Showcase all this right um and all this is happening like on my machine in this like Docker image that's running like all the hashy things um which is super cool um I did want to show also like what a um um what a nomad job spec looks like um let me just share my screen for that so Nomad is like it's an alternative to kubernetes right exactly exactly yeah and it's it's interesting too because it doesn't just run containerized workloads like it can run like VM workloads it can even run like a jvm or like IIs um so it's kind of cool that way um here we go sorry I was looking for the one okay so let me pick um let's hear it a job spec is like a kubernetes deployment yeah exactly so here I'll uh I've got kubernetes deployment I'll put it side by side here all right cool okay so this is the feature flag service on the right we've got the feature flag service service definition this is our um our uh our Nomad job specs so unlike um kubernetes where you have like all these different yaml files that make up the the various um objects for that make up your manifest everything is self-contained in one um job spec file which is written in HCL which is a hash report configuration language so I liken it to like I mean if you've used if you've used terraform like this it looks familiar um I I find HCL is like if if Json was like slightly improved that's what it would look like I do find it's like easier to read than Json but this is basically what it looks like here so for example over here um in our Network definition you see that we Define two ports 8081 and uh five zero zero five three which lo and behold we defined those over here in our uh in our service yaml for the feature flag service um and then if we scroll over to here so to this task definition we're saying that we're using the docker driver so this is saying okay we're using containerized workload let's find the equivalent in the uh in the Manifest all right here so we go to our deployment and we can see where we Define our image I've defined an image pull time out because sometimes my network doesn't like to behave so I don't want Nomad to like crap out after five minutes and say sorry I can't pull your image sucker and then fail my deployment um over here we've got the ports configuration basically saying well just like here we're saying that this container requires these two ports well on the left side here in our in our HCL we see that our HTTP and grpc ports that we defined up here that's what they're pointing to um and then the other thing that I wanted to point out was over here in this end definition these are our environment variables which on the most part correspond to the ones that we Define in our deployment I did exclude some like you know these kubernetes Hotel kubernetes ones because obviously we're not running this kubernetes we're running them in Nomad but the non-kubernetes ones I translated over to Nomad land um and then I wanted to point out also that um over here in this uh what's called the temp stanza so be basically these these the groupings and the curly braces they call them the stanzas I'm defining two additional environment variables but they're defined in a slightly different way because they're referencing Services um uh from from other Nomad job specs so um here to be able to reference for example our database service um to get the information so the IP address and the port of the database service um we can't hard code that information right because that stuff can potentially change especially the IP so in order to get that information to Define this environment variable we have to use we basically have to look up this this service in in console which as I mentioned is for service Discovery and so um to find out what the information of that Services you refer to it by service name so if we open over here I'm going to open this FF postgres so this is the job spec for postgres I have a service named FF postgres service which if we look over here to do here we go over here so it basically says Hey console I want to pull up a service called FF postgres service and pretty please tell me the address and port number of the service so I can plug it in to this environment variable and so we use the template stanza to do this kind of dynamic definition and we have to tell it that the destination is going to be an environment uh variable because one of the things that you can do with the template stands as well is you can use it for configuration files so similarly very similar functionality to a config map in in kubernetes and then here is where we Define our resources this is in um megahertz or CPU and our our memory is in um megabytes um um and then the other thing that I wanted to point out is here I've got some rules on restarting the service so because there's no dependencies that you can Define in your services and Nomad like you know how you would do in Docker compose saying hey this service is dependent on that service you don't have that kind of dependency definition in Nomad so um basically the jobs start when they start which means that like for example example this feature flag search is actually relying on the database and the hotel collector to be up and running well what if the feature flag service starts before the Hotel collector and the database startup um if we don't put some restart rules in place then um it'll when it starts up and these two Services aren't up or one of these two Services isn't up it'll crap out and then that's it it's it's dead so what we want to do is put some rest over here basically saying within the span of two minutes we're going to try to restart this thing 10 times with a 15 minute sorry a 15 second interval between restarts and either after 10 attempts or two minutes whichever comes first if this fails to restart um we are basically going to say okay we're just gonna wait a little bit and reattempt that again the default mode for this this normally is fail so if if it didn't if it doesn't successfully restart in in 10 attempts within two minutes it would completely fail and then that means your whole your whole application deployment fails right because of all these dependent services so this basically gives you some protection saying okay well we'll keep trying we'll keep trying until things are up and running which ends up uh being very very be convenient um and then the final thing that I wanted to point out was that um we also have these checks here so these are uh these are basically health checks they're similar to uh the types of health checks that you would see in kubernetes like liveness liveness probes and Readiness probes and I wanted to show you actually what that looks like in console really briefly um let me just pull that up yeah and we're going to move on to Jess Citron after the console showing so if you have more Rihanna think them up please yes I'll show this very briefly but this basically shows all of our services and then it shows like if we look at the feature flag service we have two Services one for our Jr PCR and so this shows us that hey like we're we're able to to hit our uh our endpoints so um it sends a it sends a signal back to Nomad like all systems go hurray hurray so that's basically it in a nutshell so okay follow-up questions last parting thoughts about nice job yeah I agree nice job that was a super good demo and yeah good example of how to use the demo app um so just a lot of thinking is um actually going to show us something totally useless but super fun for training people or teaching um how to make um ask KR into a heat map using open telemetry right right so I made this thing for Christmas uh for Christmas gimmicks I guess I don't know um it's in a repo honeycombio slash happy Ollie days um which is cute and and you can you can clone this repo or run it and get pod uh and if you give it your honeycomb API key this is Honeycomb like the the UI is Honeycomb specific because it had to be specific to the the display um but it does use open telemetry so I'm going to give it a honeycomb API key oh it's still in my paste buffer great and then I'm going to run this little app and this app is in node um typescript so it has created a bunch of spans and uh in a Trace using where's my here we go uh start active so it does a start active span and node and then um for each it's figured out what spans it wants to send and then it does a start span for each of them and supplies a bunch of attributes and then ends each one they all go into one Trace uh it's given me a link to the data sets oh it's a good thing I'm following that link because apparently would you go to the link go to the link there it goes apparently my data set is called Fufu free but then in the readme it describes how to do this if you do a heat map on the height field and you get the last 10 minutes and you're starting to see something but then I need to pick graph no no no up here granularity five seconds oh that's so cute isn't it cute and and the cool part is that that you can use this program I mean I don't know what it's going to look like in any other system because uh because I I've customized the um the heights to the bucket sizes that honeycomb uses in heat Maps so you'd have to tweak the code to make it do something in some somebody else's heat Maps oh but it it actually pulls this out of um where is it images now there should be an images directory in here oh input input don't peak.png uh so if you give it a PNG that's between 25 and 50 pixels tall and you probably want it between 1 and 200 pixels wide then it can convert that into a heat map oh it's converting the heat map based on the blue channel in the PNG it uses the other one to get some attributes which does some fun things in honeycomb because you can pick bubble up and be like what is different about oh oh check this out okay my new favorite feature of honeycomb w ow cut on this stupid little do me anything is gone okay which is very useful sometimes I'm sure but not in here okay so now that I've gotten rid of that uh what is different about the spans in the second reindeer um and honeycomb does its little what is different analysis and it says the reindeer name that one has a reindeer name of dancer and then if you Group by the reindeer name you can go back and like oh oh wait hold on I've got to do the trick of uh wait wait give me give me the right 10 minutes um it wants to like the the Santa and his reindeer are like moving on I need to switch to Absolute time okay now let's group By Reindeer name okay now it's lost the granularity so let's fix that granularity five seconds okay but now uh results Tab and we have the different reindeer names so there's no reindeer name this there's dancer this one is Rudolph and this one is Prancer um and yeah so it's cute that way it's got different uh fields and those are based in case you want to use this yourself um and Supply your own image those are based on the red channel in the PNG and then there's a little key for uh how much red between 0 and 255 is in that color and what the fields are um so that's also encoded in a PNG and you can you can do it yourself and make your own um this this gets like officially released on Monday um and it'll be advertised and stuff but but this but I'm not and none of that stuff am I telling you how to fix it yourself so y'all are getting that um and there's one other trick in this data uh which is if you do a Max of Stack Heights and group by Stack group um then you get some nonsense but if you change the graph settings to use a stack graph then you start to get something and then if you get the order by right stack group descending come on do it a Christmas tree oh my God that's awesome and and that you can also generate yourself because that is based on house.png obviously um it used to be a house now it's Christmas tree uh and of course there's limitations you have to have like um all the colors have to be on top of one another and you can't have a color that's both under and over and you don't control what the colors actually show up as but um if you're clever you can make your own pngs uh and then oh yeah I've got some groups on that you could you you can Group by um oh it is great by Stack group that wasn't necessary but here's the names of them star and background and tree and stuff like that yes where do people go to find this app it is at Honeycutt miles slash happy Ollie days and so yeah you can download this and if you have a free honeycomb account um or if if you get it to work in another tool that would be awesome I would love to hear about it oh thanks Johnny um yeah so that's that's that we'll find stop share maybe not Daniel's like I'm running not walking to try this out it's fun it's true um and uh I have a question which is how do you think this can be useful to people as a tool to teach about heat Maps observability yeah yeah because you can I mean you can ask like how how do I draw it and the answer is that um I send a different number of spans which of course is it this one which of course I also stick all the intermediate data on the spans because the owls what I debug it um no that's the clearly the wrong field uh show me the trace I haven't made the trace cute yet someday I'll make this have a drawing in it too um span you know count amount how many oh well uh I send multiple spans per uh pixel if I want a dark color because honeycomb says darker is more and lighter is fewer so there's just one Span in in these um oh you know what I can find that field if I do Bubble Up what's different about these that only have one um spans at once there it is okay so this spans at once these only have one span at a time and uh some of the the darkest ones have 10 spans at a time um so if you think if you can use it to be like how how does a heat map um mostly it's just entertaining and it does really illustrate like bubble up um letter all of these most of these are letters and some of them have oh they have different letters yeah uh so Bubble Up is pretty well illustrated by this I think um I want to say what Bubble Up is for folks who aren't honeycomb users Bubble Up is Honeycombs what is different so you can draw the Box and um and um it does the statistical analysis on What fields are different it's just cute that was super good [Music] there's not question threes do you want to go ahead and talk about um end user working group work I mean yes but also how do I follow up we're doing is systematic and important and slowly marching through things um Jessica to do the fun creative yeah yes yeah this is um I mean I have some like one or two fun Graphics but it's not gonna be anything as cool as Jesse Jesse trance um so you'll have to make two um yeah I just wanted to just talk real quick on something that we've surfaced um just from talking to a wide variety of end users is um a lot of people are not really aware of how to navigate the community on how they can get involved um or that they even can get involved um so I'm not sure um I know there's um a couple contributors on the call um not sure everyone else's end user um but hopefully this will be of some utility utility for you so a couple things we'll go over um this time of the community at large some of the different parts of it and then we'll talk about how you can get involved if you're interested in some of the different ways that you can get involved so we've mentioned a few times the end user working group that we a lot of us are a part of um what is the andeserving group we have two primary goals one is to Foster sense of vendor agnostic Community for end users and two is to create a feedback loop between the end users and project maintainers but the overarching goal of improving the um project software so what are some of the working group activities that we have implemented so first of all this is one of them um this is rin's child um but we are very excited um a hotel in practice is what it is so every month keep an eye out for more fun talks um presentations and you know casual conversations um we have the monthly discussion groups now with a maintainer per session and in all regions so America emea which is which I just found out stands for Europe Middle East and Africa and APAC which I'm actually not sure where those things were but it's Asia and the Pacific region imagine where AC stands for um and those are all on the open symmetry public calendars we also have end user interview and feedback sessions where we will talk to um an end user about their adoption implementation and challenges that they face and get the feedback uh shared back to the open telemetry maintainers so if you're interested if you're an end user and you're interested in Sharing feedback in one of these sessions please reach out to myself Rin or Adriana we would be happy to get you on the schedule and we are also new um for uh end user interviews moving forward we're going to turn them into profiles for the open slim shoe blog to kind of make the um implementation and adoption um that other end users have done and their organizations more discoverable and we also have a community survey so that's another way that you can contribute if you don't really want to get too involved but you still want to have a way to share your thoughts and opinions on how using a potomacy is for you some of the stuff that's upcoming for the working group um we want to extend our user study function and also the governance committee is going to work to implement a project management function for the specifications Sig to one streamline the feedback loop um from like all the feedback that we're gathering at these sessions and activities and drive prioritization for work based on user feedback and in person meetups this might be a thing coming to a town near you maybe we'll see um I think this would be a really fun thing to do especially as like I think you know we a lot of us are just at kubecon and it seems like people are pretty into meeting in person again so we'll see um and then six probably most of you are maybe fully familiar with these but um I'll just go over the go over them um special interest groups the goal is to improve the workflow and manage the project more efficiently each Sig needs regularly um you can access the meeting notes and recordings um through the public calendar and that's pretty much a thing or working group for pretty much all components um languages of open telemetry there's also a governance committee and a technical committee which I won't get too deep into they can kind of see the roles of each committee here show me for a second and I can also share the slide deck too because there are some links to the resources that I mentioned in here um documentation yes we have a lot of questions about documentation if you do have questions you can always talk to the comsig at Hotel Dash coms Channel some languages we are aware um are a little bit more comprehensive than others there is standardization and improvement work in progress at the moment if you would like to help contribute feel free to jump into the channel attend one of their meetings or open an issue directly in the repo so I personally find it this kind of interesting to see what um what oteps are have been proposed um they are open selling to enhancement proposals it's a process for proposing changes to the spec and they have to be cross-cutting changes that introduce new Behavior changes our Behavior or otherwise modify requirements and it's kind of fun personally I think to go and look at what people are proposing or wanting to do there's some interesting stuff in there okay and also getting involved um the first one I want to cover is how do I get help using a photometry there are multiple ways CNC of slack you have to sign up for an account with cncf slack instance but once you get in there um there's pretty much an ozone channel for um whatever it is you're looking for languages components collector Etc there's also vendor specific um channels or you can go to the general Hotel vendor Channel if you're having problems with a specific vendor that you're using and of course GitHub um you can open issues jump in with comments on anything that's open um and we also have the end user discussion groups which I mentioned earlier if you want to join and ask questions or share how you're using open Telemetry or help other people who are using open filament organizations that's another great way to get involved and for as far as contributions we welcome any and all code in oncoming contributions um if there's a specific language you're interested or a specific component go check out the Sig um Sig notes see what they're talking about or you can hop into any of the meetings and just kind of check it out blog posts um could be something that you've you know it could be something like fun and so-called useless but really you know if it brings so much joy is it really useless no so something you know fun that just did could be anything open until related basically we would love to see it um as I mentioned documentation you are welcome to join the end user working group and you know give suggestions on things you'd like us to help with or do um as for you as an end user and of course you can also just share feedback and there's different ways to do that if you don't really want to get engaged um with a interview um you are welcome to take part in our survey which I thought I linked somewhere I have linked the survey in here if you would like to share feedback that way that is great if you would like to participate in the Indies working group um or uh by way of the discussion group or end user interview we would be very happy to have you and that's it that's all I have thank you so much I should have added more Jingle Bells and Christmas stuff I feel not very festive but I promise I am Fair Justice thank you Reese and saying that for those not following the chat Johnny's saying um they participate in some conferences related to devops and they like to share about the power of observability and how open Telemetry can help us make more easy our lives do folks have questions um since we've got 10 home minutes we're happy to entertain if you have random questions about open Telemetry we may or may not be able to answer them but lots of knowledge in this room yeah and if anyone is having specifically a question about racist presentation absolutely well actually I was going to say about open telemetry.net we have a maintainer on the line is that Alan yeah yes there are two Mike Blanche Blanchard hello Mike very important spot that's why I'm like it's okay if I call him out I will say that I tried to uh pull down the um honeycomb um The Happy Holidays happy hour oh it is it it's gonna take a little bit of work but I was I was gonna try to see how easy it would be to get that reporting into a different tool than to like be Relic and see if that uh see what it would look like I'm sure it would look probably amazing [Music] it will look mangled initially I'm like if if you want to work at that and um I'm happy to work with you on like I know where in the code it's choosing the height that works and I also know like how I use the browser Tools in honeycomb to figure out what height it was needing um uh which uh some of those tricks will also work in new relics so happy to work with you on that if you want because I would love to see what it looks like somewhere else I don't know if they're still doing them but last year Grace and I were doing a lot of Legos for New Relic I've got some of them here and I wonder if they're like amazing art oh that's true Grace's New Relic social media person and a huge Lego fan um but yeah I think Legos tie in very well with asciar and you know lots of tech people are Lego fans so true true that works other other questions that folks have we can also go ahead and end with call Early yeah yeah Ellen I'm jessatron at honeycomb if you want to get in touch yes and I'll post the link to um Jessica trance calendar sounds good yeah nice to meet y'all you too well thanks everyone it was great to see you all yeah great to see you have a good fun to have a a chatty group thank you all for joining and a couple of you for putting yourself on video that's always great for us yeah I'll Zoom to like see faces cool thank you hey yeah feel free to reach out to any of us with questions or if there's anything you want to follow up on happy holidays [Music] +thank you so the open Telemetry and practice series is traditionally a pretty serious talk Series where we every every month we go deep into implementing open Telemetry in a particular language and we talk to an end user and uh um what in who's actually implementing it in production and we talked to um usually a maintainer of the project who can talk about like these are the best ways to implement um and this is a little bit different of a format because we decided to do three or four short talks that didn't fit well together um Adriana is going to lead off today with um talking about observability for hashicord products um she currently works at lightstep and um used to um work um fully in a Hashi shop yes and then a little bit later we'll have Rhys talking about um how end users can get involved with the project which I expect to be highly interactive and um Jess jessatron um is going to give us a talk about um using open Telemetry to make asksy ASCII are in heat Maps which is cool you are trying to get your team to learn about open Telemetry and pay attention and utterly useless um Adriana you ready to yeah I'll start so I'm ad Hocking this uh all right so um as Rin mentioned I used to work at a Hashi shop so I have a I actually have a kubernetes background was thrust into this hashicorp world where my previous employer um they ran their containerized workloads on on Nomads so I found myself in a situation where I had to quickly understand this Nomad thing because I was managing a platform team which supported Nomad console and Vault for the entire Enterprise um so that's how I began my dabblings in the Hashi world and and from that I I ended up becoming a hatchery Corp Ambassador because I I dug in blogged about it and continued to do so um and at the same time at this former company I was also managing an observability team and I came to learn about observability team at around the same time that I came to understand um Nomad so these were two things that I was doing at the same time learning about at the same time which led me into uh open Telemetry so um now that I work at lightstep I've had a chance to contribute more in the open Telemetry Community as I mentioned before um I've uh I've worked in the in in the comms area so I've I've contributed some content to the doc site I've worked I've made some contribution demo app and I was actually pleasantly surprised to learn about the demo app which I think launched It's relatively recent right that the demo app came out and and I like it because it showcases open to limit like what you can do with open Telemetry and I think the original version of the demo app that came out you could uh run everything in Docker compose locally and then more recently there's um uh the the team created um home charts for this one and she got me thinking hey if this thing runs Brunetti's wouldn't it be super cool to get this thing to run a nomad and so it's been part of my wish list ever since I I learned about like the hotel demo app Helm charts I'm like okay finally carved out some time to do this and I got it up and running um in a local Hashi environment on on my machine so there's um there's this really cool tool called Hashi Cube which is similar to you know like running mini Cube or k3s or whatever like one of these like local kubernetes Dev environments similar sort of thing for the hashy stack um which was created by a a third-party company called uh Serbian and and I love using this tool because it gives me like a full-fledged Hashi environment similar to what you would get I mean obviously smaller scale to what you would get in a data center but it replicates the the type of setup because in in real life the um Nomad by itself is is kind of useless so you need like the power of of console for service Discovery involved for for Secrets management along with Nomad to get all this stuff um to to really have like a a robust system to manage your your containerized workloads so I have uh I have this Hashi Cube running on on my machine locally which I can show you um I'll do like a little screen share well first I'll I'll show you my hashiq oh wait did uh am I sharing the right one hearing your hashes cool I have like so many windows the correct cashy tube I don't know over the time yeah this well okay so it says this is this is my Hatchi Cube startup sequence so hashikub what it what it is so it's it's basically using vagrants to provision either a um VM a virtualbox uh VM or um I'm I'm on an M1 Mac and virtualbox and M1 Max don't play nice I think there's like possibly a version of virtualbox that plays nice with M1 Max now but at the time it didn't so the uh the maintainers of hashic cube basically stand up all these Hashi products in a single Docker image um so this version of hashikub that I'm running where you can see like the tail end of the startup sequence here um it's running mad Vault console all in one all-in-one Docker image and so if I share I'll share you my Nomad console here so you can see um problem when I lose sight of all my windows um the oh here we go okay so this is this is Nomad and these are all of the services that make up um the hotel demo app running on Nomad locally on my machine which is pretty freaking exciting um I basically so I had to go through the um the home chart there's a for the for the hotel demo Helm chart um there's a there's a folder in the helm chart repo that shows like the rendered yaml so I basically started with the rendered yamls and went through the process of translating um these these um kubernetes manifests for each service into the equivalent Nomad job spec which is what you see here so for example like we're looking at the feedback service this is running on on Nomad um and if we go over here we can see um there's a lot of stuff here so we can see the logs of the true flag service nothing terrible is happening here we go it's running so we can see from from our our dashboard here that all of the services are up and running so it's made up of like all these uh different um different Services written in different languages plus we also have I had to convert um I had to nomadify um redis postgres we had the hotel collector um what else there was one more out there was Prometheus and grafana and I used trafic for load balancing so I can expose my services to the outside world and so what that looks like then um so to be able to access the hotel demo apps endpoints and hopefully my computer won't crash here um so I have IX endpoint called Hotel Dash demo dot localhost which is accessible this is just uh just something accessible off my local host and then the uh so Jaeger UI is is is exposed um as part of the my demo app configuration so we see Jaeger is up and running here we've got the demo apps UI going uh running here so we can buy stuff we can check stuff out and buy let's buy this telephone and super and then if we check Like Jaeger we can look to see sorry the screen sharing thing is like getting in my face um we can see our traces I think the hamsters are running really really hard here we can see it's produced some some traces here that we can see here in addition um when I know modified this there is also a some grafana dashboard so some cool dashboards that come pre-loaded pre-configured you can see there's some stuff going on here with the hotel collector and then there's also this demo dashboard which gets some stats from the recommendation service so that's um that's basically it I can show you also like what a job spec looks like in Nomad um I want to see more traces oh you want to see more traces yeah right I picked like the saddest Trace yeah that's what happens when you're like ad Hocking it okay let me uh let's let's pull up a more interesting Trace I think the recommendation service has interesting traces that we can see checkout's usually good check out cool let's find some traces oh yeah look at all those look at all those Services yeah there's some colors look at it go to all the different services yeah this is actually pretty cool you're right I picked the dinkiest to Showcase all this right um and all this is happening like on my machine in this like Docker image that's running like all the hashy things um which is super cool um I did want to show also like what a um um what a nomad job spec looks like um let me just share my screen for that so Nomad is like it's an alternative to kubernetes right exactly exactly yeah and it's it's interesting too because it doesn't just run containerized workloads like it can run like VM workloads it can even run like a jvm or like IIs um so it's kind of cool that way um here we go sorry I was looking for the one okay so let me pick um let's hear it a job spec is like a kubernetes deployment yeah exactly so here I'll uh I've got kubernetes deployment I'll put it side by side here all right cool okay so this is the feature flag service on the right we've got the feature flag service service definition this is our um our uh our Nomad job specs so unlike um kubernetes where you have like all these different yaml files that make up the the various um objects for that make up your manifest everything is self-contained in one um job spec file which is written in HCL which is a hash report configuration language so I liken it to like I mean if you've used if you've used terraform like this it looks familiar um I I find HCL is like if if Json was like slightly improved that's what it would look like I do find it's like easier to read than Json but this is basically what it looks like here so for example over here um in our Network definition you see that we Define two ports 8081 and uh five zero zero five three which lo and behold we defined those over here in our uh in our service yaml for the feature flag service um and then if we scroll over to here so to this task definition we're saying that we're using the docker driver so this is saying okay we're using containerized workload let's find the equivalent in the uh in the Manifest all right here so we go to our deployment and we can see where we Define our image I've defined an image pull time out because sometimes my network doesn't like to behave so I don't want Nomad to like crap out after five minutes and say sorry I can't pull your image sucker and then fail my deployment um over here we've got the ports configuration basically saying well just like here we're saying that this container requires these two ports well on the left side here in our in our HCL we see that our HTTP and grpc ports that we defined up here that's what they're pointing to um and then the other thing that I wanted to point out was over here in this end definition these are our environment variables which on the most part correspond to the ones that we Define in our deployment I did exclude some like you know these kubernetes Hotel kubernetes ones because obviously we're not running this kubernetes we're running them in Nomad but the non-kubernetes ones I translated over to Nomad land um and then I wanted to point out also that um over here in this uh what's called the temp stanza so be basically these these the groupings and the curly braces they call them the stanzas I'm defining two additional environment variables but they're defined in a slightly different way because they're referencing Services um uh from from other Nomad job specs so um here to be able to reference for example our database service um to get the information so the IP address and the port of the database service um we can't hard code that information right because that stuff can potentially change especially the IP so in order to get that information to Define this environment variable we have to use we basically have to look up this this service in in console which as I mentioned is for service Discovery and so um to find out what the information of that Services you refer to it by service name so if we open over here I'm going to open this FF postgres so this is the job spec for postgres I have a service named FF postgres service which if we look over here to do here we go over here so it basically says Hey console I want to pull up a service called FF postgres service and pretty please tell me the address and port number of the service so I can plug it in to this environment variable and so we use the template stanza to do this kind of dynamic definition and we have to tell it that the destination is going to be an environment uh variable because one of the things that you can do with the template stands as well is you can use it for configuration files so similarly very similar functionality to a config map in in kubernetes and then here is where we Define our resources this is in um megahertz or CPU and our our memory is in um megabytes um um and then the other thing that I wanted to point out is here I've got some rules on restarting the service so because there's no dependencies that you can Define in your services and Nomad like you know how you would do in Docker compose saying hey this service is dependent on that service you don't have that kind of dependency definition in Nomad so um basically the jobs start when they start which means that like for example example this feature flag search is actually relying on the database and the hotel collector to be up and running well what if the feature flag service starts before the Hotel collector and the database startup um if we don't put some restart rules in place then um it'll when it starts up and these two Services aren't up or one of these two Services isn't up it'll crap out and then that's it it's it's dead so what we want to do is put some rest over here basically saying within the span of two minutes we're going to try to restart this thing 10 times with a 15 minute sorry a 15 second interval between restarts and either after 10 attempts or two minutes whichever comes first if this fails to restart um we are basically going to say okay we're just gonna wait a little bit and reattempt that again the default mode for this this normally is fail so if if it didn't if it doesn't successfully restart in in 10 attempts within two minutes it would completely fail and then that means your whole your whole application deployment fails right because of all these dependent services so this basically gives you some protection saying okay well we'll keep trying we'll keep trying until things are up and running which ends up uh being very very be convenient um and then the final thing that I wanted to point out was that um we also have these checks here so these are uh these are basically health checks they're similar to uh the types of health checks that you would see in kubernetes like liveness liveness probes and Readiness probes and I wanted to show you actually what that looks like in console really briefly um let me just pull that up yeah and we're going to move on to Jess Citron after the console showing so if you have more Rihanna think them up please yes I'll show this very briefly but this basically shows all of our services and then it shows like if we look at the feature flag service we have two Services one for our Jr PCR and so this shows us that hey like we're we're able to to hit our uh our endpoints so um it sends a it sends a signal back to Nomad like all systems go hurray hurray so that's basically it in a nutshell so okay follow-up questions last parting thoughts about nice job yeah I agree nice job that was a super good demo and yeah good example of how to use the demo app um so just a lot of thinking is um actually going to show us something totally useless but super fun for training people or teaching um how to make um ask KR into a heat map using open telemetry right right so I made this thing for Christmas uh for Christmas gimmicks I guess I don't know um it's in a repo honeycombio slash happy Ollie days um which is cute and and you can you can clone this repo or run it and get pod uh and if you give it your honeycomb API key this is Honeycomb like the the UI is Honeycomb specific because it had to be specific to the the display um but it does use open telemetry so I'm going to give it a honeycomb API key oh it's still in my paste buffer great and then I'm going to run this little app and this app is in node um typescript so it has created a bunch of spans and uh in a Trace using where's my here we go uh start active so it does a start active span and node and then um for each it's figured out what spans it wants to send and then it does a start span for each of them and supplies a bunch of attributes and then ends each one they all go into one Trace uh it's given me a link to the data sets oh it's a good thing I'm following that link because apparently would you go to the link go to the link there it goes apparently my data set is called Fufu free but then in the readme it describes how to do this if you do a heat map on the height field and you get the last 10 minutes and you're starting to see something but then I need to pick graph no no no up here granularity five seconds oh that's so cute isn't it cute and and the cool part is that that you can use this program I mean I don't know what it's going to look like in any other system because uh because I I've customized the um the heights to the bucket sizes that honeycomb uses in heat Maps so you'd have to tweak the code to make it do something in some somebody else's heat Maps oh but it it actually pulls this out of um where is it images now there should be an images directory in here oh input input don't peak.png uh so if you give it a PNG that's between 25 and 50 pixels tall and you probably want it between 1 and 200 pixels wide then it can convert that into a heat map oh it's converting the heat map based on the blue channel in the PNG it uses the other one to get some attributes which does some fun things in honeycomb because you can pick bubble up and be like what is different about oh oh check this out okay my new favorite feature of honeycomb w ow cut on this stupid little do me anything is gone okay which is very useful sometimes I'm sure but not in here okay so now that I've gotten rid of that uh what is different about the spans in the second reindeer um and honeycomb does its little what is different analysis and it says the reindeer name that one has a reindeer name of dancer and then if you Group by the reindeer name you can go back and like oh oh wait hold on I've got to do the trick of uh wait wait give me give me the right 10 minutes um it wants to like the the Santa and his reindeer are like moving on I need to switch to Absolute time okay now let's group By Reindeer name okay now it's lost the granularity so let's fix that granularity five seconds okay but now uh results Tab and we have the different reindeer names so there's no reindeer name this there's dancer this one is Rudolph and this one is Prancer um and yeah so it's cute that way it's got different uh fields and those are based in case you want to use this yourself um and Supply your own image those are based on the red channel in the PNG and then there's a little key for uh how much red between 0 and 255 is in that color and what the fields are um so that's also encoded in a PNG and you can you can do it yourself and make your own um this this gets like officially released on Monday um and it'll be advertised and stuff but but this but I'm not and none of that stuff am I telling you how to fix it yourself so y'all are getting that um and there's one other trick in this data uh which is if you do a Max of Stack Heights and group by Stack group um then you get some nonsense but if you change the graph settings to use a stack graph then you start to get something and then if you get the order by right stack group descending come on do it a Christmas tree oh my God that's awesome and and that you can also generate yourself because that is based on house.png obviously um it used to be a house now it's Christmas tree uh and of course there's limitations you have to have like um all the colors have to be on top of one another and you can't have a color that's both under and over and you don't control what the colors actually show up as but um if you're clever you can make your own pngs uh and then oh yeah I've got some groups on that you could you you can Group by um oh it is great by Stack group that wasn't necessary but here's the names of them star and background and tree and stuff like that yes where do people go to find this app it is at Honeycutt miles slash happy Ollie days and so yeah you can download this and if you have a free honeycomb account um or if if you get it to work in another tool that would be awesome I would love to hear about it oh thanks Johnny um yeah so that's that's that we'll find stop share maybe not Daniel's like I'm running not walking to try this out it's fun it's true um and uh I have a question which is how do you think this can be useful to people as a tool to teach about heat Maps observability yeah yeah because you can I mean you can ask like how how do I draw it and the answer is that um I send a different number of spans which of course is it this one which of course I also stick all the intermediate data on the spans because the owls what I debug it um no that's the clearly the wrong field uh show me the trace I haven't made the trace cute yet someday I'll make this have a drawing in it too um span you know count amount how many oh well uh I send multiple spans per uh pixel if I want a dark color because honeycomb says darker is more and lighter is fewer so there's just one Span in in these um oh you know what I can find that field if I do Bubble Up what's different about these that only have one um spans at once there it is okay so this spans at once these only have one span at a time and uh some of the the darkest ones have 10 spans at a time um so if you think if you can use it to be like how how does a heat map um mostly it's just entertaining and it does really illustrate like bubble up um letter all of these most of these are letters and some of them have oh they have different letters yeah uh so Bubble Up is pretty well illustrated by this I think um I want to say what Bubble Up is for folks who aren't honeycomb users Bubble Up is Honeycombs what is different so you can draw the Box and um and um it does the statistical analysis on What fields are different it's just cute that was super good there's not question threes do you want to go ahead and talk about um end user working group work I mean yes but also how do I follow up we're doing is systematic and important and slowly marching through things um Jessica to do the fun creative yeah yes yeah this is um I mean I have some like one or two fun Graphics but it's not gonna be anything as cool as Jesse Jesse trance um so you'll have to make two um yeah I just wanted to just talk real quick on something that we've surfaced um just from talking to a wide variety of end users is um a lot of people are not really aware of how to navigate the community on how they can get involved um or that they even can get involved um so I'm not sure um I know there's um a couple contributors on the call um not sure everyone else's end user um but hopefully this will be of some utility utility for you so a couple things we'll go over um this time of the community at large some of the different parts of it and then we'll talk about how you can get involved if you're interested in some of the different ways that you can get involved so we've mentioned a few times the end user working group that we a lot of us are a part of um what is the andeserving group we have two primary goals one is to Foster sense of vendor agnostic Community for end users and two is to create a feedback loop between the end users and project maintainers but the overarching goal of improving the um project software so what are some of the working group activities that we have implemented so first of all this is one of them um this is rin's child um but we are very excited um a hotel in practice is what it is so every month keep an eye out for more fun talks um presentations and you know casual conversations um we have the monthly discussion groups now with a maintainer per session and in all regions so America emea which is which I just found out stands for Europe Middle East and Africa and APAC which I'm actually not sure where those things were but it's Asia and the Pacific region imagine where AC stands for um and those are all on the open symmetry public calendars we also have end user interview and feedback sessions where we will talk to um an end user about their adoption implementation and challenges that they face and get the feedback uh shared back to the open telemetry maintainers so if you're interested if you're an end user and you're interested in Sharing feedback in one of these sessions please reach out to myself Rin or Adriana we would be happy to get you on the schedule and we are also new um for uh end user interviews moving forward we're going to turn them into profiles for the open slim shoe blog to kind of make the um implementation and adoption um that other end users have done and their organizations more discoverable and we also have a community survey so that's another way that you can contribute if you don't really want to get too involved but you still want to have a way to share your thoughts and opinions on how using a potomacy is for you some of the stuff that's upcoming for the working group um we want to extend our user study function and also the governance committee is going to work to implement a project management function for the specifications Sig to one streamline the feedback loop um from like all the feedback that we're gathering at these sessions and activities and drive prioritization for work based on user feedback and in person meetups this might be a thing coming to a town near you maybe we'll see um I think this would be a really fun thing to do especially as like I think you know we a lot of us are just at kubecon and it seems like people are pretty into meeting in person again so we'll see um and then six probably most of you are maybe fully familiar with these but um I'll just go over the go over them um special interest groups the goal is to improve the workflow and manage the project more efficiently each Sig needs regularly um you can access the meeting notes and recordings um through the public calendar and that's pretty much a thing or working group for pretty much all components um languages of open telemetry there's also a governance committee and a technical committee which I won't get too deep into they can kind of see the roles of each committee here show me for a second and I can also share the slide deck too because there are some links to the resources that I mentioned in here um documentation yes we have a lot of questions about documentation if you do have questions you can always talk to the comsig at Hotel Dash coms Channel some languages we are aware um are a little bit more comprehensive than others there is standardization and improvement work in progress at the moment if you would like to help contribute feel free to jump into the channel attend one of their meetings or open an issue directly in the repo so I personally find it this kind of interesting to see what um what oteps are have been proposed um they are open selling to enhancement proposals it's a process for proposing changes to the spec and they have to be cross-cutting changes that introduce new Behavior changes our Behavior or otherwise modify requirements and it's kind of fun personally I think to go and look at what people are proposing or wanting to do there's some interesting stuff in there okay and also getting involved um the first one I want to cover is how do I get help using a photometry there are multiple ways CNC of slack you have to sign up for an account with cncf slack instance but once you get in there um there's pretty much an ozone channel for um whatever it is you're looking for languages components collector Etc there's also vendor specific um channels or you can go to the general Hotel vendor Channel if you're having problems with a specific vendor that you're using and of course GitHub um you can open issues jump in with comments on anything that's open um and we also have the end user discussion groups which I mentioned earlier if you want to join and ask questions or share how you're using open Telemetry or help other people who are using open filament organizations that's another great way to get involved and for as far as contributions we welcome any and all code in oncoming contributions um if there's a specific language you're interested or a specific component go check out the Sig um Sig notes see what they're talking about or you can hop into any of the meetings and just kind of check it out blog posts um could be something that you've you know it could be something like fun and so-called useless but really you know if it brings so much joy is it really useless no so something you know fun that just did could be anything open until related basically we would love to see it um as I mentioned documentation you are welcome to join the end user working group and you know give suggestions on things you'd like us to help with or do um as for you as an end user and of course you can also just share feedback and there's different ways to do that if you don't really want to get engaged um with a interview um you are welcome to take part in our survey which I thought I linked somewhere I have linked the survey in here if you would like to share feedback that way that is great if you would like to participate in the Indies working group um or uh by way of the discussion group or end user interview we would be very happy to have you and that's it that's all I have thank you so much I should have added more Jingle Bells and Christmas stuff I feel not very festive but I promise I am Fair Justice thank you Reese and saying that for those not following the chat Johnny's saying um they participate in some conferences related to devops and they like to share about the power of observability and how open Telemetry can help us make more easy our lives do folks have questions um since we've got 10 home minutes we're happy to entertain if you have random questions about open Telemetry we may or may not be able to answer them but lots of knowledge in this room yeah and if anyone is having specifically a question about racist presentation absolutely well actually I was going to say about open telemetry.net we have a maintainer on the line is that Alan yeah yes there are two Mike Blanche Blanchard hello Mike very important spot that's why I'm like it's okay if I call him out I will say that I tried to uh pull down the um honeycomb um The Happy Holidays happy hour oh it is it it's gonna take a little bit of work but I was I was gonna try to see how easy it would be to get that reporting into a different tool than to like be Relic and see if that uh see what it would look like I'm sure it would look probably amazing it will look mangled initially I'm like if if you want to work at that and um I'm happy to work with you on like I know where in the code it's choosing the height that works and I also know like how I use the browser Tools in honeycomb to figure out what height it was needing um uh which uh some of those tricks will also work in new relics so happy to work with you on that if you want because I would love to see what it looks like somewhere else I don't know if they're still doing them but last year Grace and I were doing a lot of Legos for New Relic I've got some of them here and I wonder if they're like amazing art oh that's true Grace's New Relic social media person and a huge Lego fan um but yeah I think Legos tie in very well with asciar and you know lots of tech people are Lego fans so true true that works other other questions that folks have we can also go ahead and end with call Early yeah yeah Ellen I'm jessatron at honeycomb if you want to get in touch yes and I'll post the link to um Jessica trance calendar sounds good yeah nice to meet y'all you too well thanks everyone it was great to see you all yeah great to see you have a good fun to have a a chatty group thank you all for joining and a couple of you for putting yourself on video that's always great for us yeah I'll Zoom to like see faces cool thank you hey yeah feel free to reach out to any of us with questions or if there's anything you want to follow up on happy holidays diff --git a/video-transcripts/transcripts/2023-09-28T04:00:36Z-otel-end-user-discussions-amer-january-2023.md b/video-transcripts/transcripts/2023-09-28T04:00:36Z-otel-end-user-discussions-amer-january-2023.md index bd96d97..10adf42 100644 --- a/video-transcripts/transcripts/2023-09-28T04:00:36Z-otel-end-user-discussions-amer-january-2023.md +++ b/video-transcripts/transcripts/2023-09-28T04:00:36Z-otel-end-user-discussions-amer-january-2023.md @@ -10,185 +10,140 @@ URL: https://www.youtube.com/watch?v=qn4x0DgG5SI ## Summary -In this YouTube video, Reese, who works at New Relic and is involved with OpenTelemetry, hosts a discussion with participants including Derek, Dan, and others, focusing on the role of "enablers" in technology teams, particularly in adopting OpenTelemetry across various programming languages. They explore challenges faced when dealing with languages outside one's expertise, the importance of creating champions within teams, and strategies for fostering understanding and adoption of observability practices. The conversation shifts to technical issues such as clock drift and pipeline configuration for data processing, discussing best practices for scaling collectors and handling telemetry data efficiently. Participants share insights and experiences on creating effective observability cultures within their organizations, fostering knowledge sharing, and the importance of community support. The session concludes with an informal moment featuring a kitten named Taco. +In this YouTube video, Reese, a member of the OpenTelemetry working group and an employee at New Relic, leads a discussion about enabling observability across various programming languages within organizations. Participants, including Derek, Dan, and others, share their experiences with OpenTelemetry, discussing challenges like dealing with clock drift, defining the role of an enabler, and strategies for championing observability in teams that primarily work in languages where they lack expertise. The conversation emphasizes the importance of creating community practices, the need for documentation on successful implementations, and the idea of establishing a service catalog to improve observability standards. They also explore practical solutions like implementing connectors and routing processors to optimize data flow in observability pipelines. The session concludes with a light-hearted moment featuring Reese's kitten, Taco. -# YouTube Transcript Cleanup - -[Music] - -**Reese**: Of course, I was looking forward to the sessions. My name is Reese. My day job is with New Relic, and I also do a lot of work with the OpenTelemetry user working group, including hosting these sessions. - -Alright, let's see... Okay, let's start with this one. - -**Question**: Does your role in Compass involve being an enabler? How do you deal with helping in languages you're not an expert in? - -Usually, I run these sessions based on whoever put the topic or question in. If you can unmute and maybe give us a little bit more detail... It looks like Derek has a question on how to define "enabler." - -**Derek**: Yeah, this one's mine. Happy to expand on this. Basically, the role I have in my company is kind of twofold. The first is to maintain OpenTelemetry collector deployments and be responsible for the data pipeline—receiving data and sending that to various backends. But it's also what I call the enabler, which involves being a champion for OpenTelemetry, driving adoption within the company, troubleshooting, and helping teams troubleshoot issues they have in their respective services. - -In my company, for instance, we're probably about 70% .NET, 10% Java, 10% Go, and 5% Python. We're primarily focused in one area, so we build examples in .NET. I personally am more of a .NET developer, and my team is as well. - -A great aspect of OpenTelemetry, in my opinion, is that the concepts are extremely similar across languages. However, I don't really know how to write Java. It's not too far away from .NET, but sometimes it's troubling for me to provide specific feedback or look at a PR and really understand everything happening. - -I'm curious if teams have this problem at the companies they work in. Do you try to create champions in those languages you're not familiar with, or communities of practice around that? If anyone's experiencing this, how are you solving for it? - -**Response**: Sure! As a former enabler, I can share that my strategy was to lead the way by showing how to enable observability within my domain of expertise, which at the time was Go. As I was demonstrating the benefits of using OpenTelemetry—or at that time, OpenTracing—people would come in and ask me questions. Many of these individuals worked in different teams and languages. They were interested in starting out but weren't familiar with the concepts. - -My role was to pair with those people who were experts in whatever languages they were trying to become enablers in. I worked alongside them until they felt comfortable with the concepts of the observability platform I was using. This strategy worked out pretty well, and we managed to deploy things to three or four different teams within the organization. - -**Derek**: Okay, cool! That's kind of what I mean by champions—basically leveling up individuals or groups within those smaller subsets and then letting them run with it. Do you find that when you did this, those people sought information on their own? Did they jump into the OpenTelemetry Slack channels and ask questions, or were they routing things through you? - -**Response**: It was a mixed bag. Some people felt more comfortable talking to the person they knew, so to a certain extent, I became a router for questions. However, I did see a couple of people who were really into talking to the community directly. They would just go into the Slack channels or reach out to people directly on GitHub. Over time, as I directed people to specific issues in GitHub or specific spec changes, they tended to get more comfortable with the idea that they could go there themselves. - -**Derek**: That's great! One last follow-up, if you don't mind. How long would you say it took for those individuals you worked with to get up to speed? Was it a matter of weeks, or was it a longer period? How easy was it for them to sort of understand the domain when it wasn't their primary focus? +## Chapters -**Response**: Sorry to give you a mixed answer, but it really depended on the individuals. Some people who were keen on learning new ways of doing things got up to speed a lot faster. Others who were more accustomed to doing things a certain way took a bit longer. To be honest, some people I tried to get distributed tracing adopted just never really understood the benefits. It really is a mixed bag, depending on where people are in your organization and how resistant they are to change. +Here are 10 key moments from the livestream along with their timestamps: -**Derek**: Thank you! I appreciate that because I feel like we struggle with that sometimes. They don't really understand the purpose or how it may have value outside of their individual team, especially when we start connecting services or something. Just trying to gain insights into improving the culture, so I appreciate the response. +00:00:00 Introductions and overview of the session +00:02:30 Discussion on the role of an enabler in OpenTelemetry +00:05:00 Defining the concept of "champions" in different programming languages +00:12:45 Sharing strategies for helping teams adopt OpenTelemetry +00:20:00 The importance of auto instrumentation in OpenTelemetry +00:28:15 Challenges of dealing with clock drift in telemetry data +00:35:00 Best practices for bifurcating data in a telemetry pipeline +00:40:30 Introduction of the concept of connectors in OpenTelemetry +00:45:00 Discussion on scaling collector deployments and configuration +00:50:00 Closing thoughts and kitten reveal at the end of the session -**Response**: It's really hard to take someone from their day-to-day job, where most people are already strained with their tasks, and ask them to dive into what appears to be an unnecessary aspect of their jobs—like implementing OpenTelemetry. It’s not until they have those "aha" moments of understanding how much better their lives will be in the future that they can really wrap their heads around the benefits. - -One thing I've tried to help people understand is how to get started as quickly as possible. One of the main areas of OpenTelemetry that I'm really excited about is all the auto-instrumentation work. I think this allows people to gain some benefit without a tremendous amount of upfront investment. - -Even if it only gets them 60 or 70% of the way, it tends to give people enough visibility into what their systems are doing through auto-instrumentation that they can then get excited about investing more time in this work. - -**Derek**: Makes sense. - -**Response**: I sympathize with you. As someone who codes less than what you described in your role, it feels weird to lead the way as an enabler for these things in your work without being the person who can just say, "Alright, let me show you some examples." - -In my previous organization, when we were trying to bring OpenTelemetry into the development teams, we faced interesting challenges. The teams recognized the importance of OpenTelemetry but were constantly fighting fires. They didn't have time to instrument their code, which was ironic because instrumenting their code would probably help them fight those fires more effectively. - -While my team was the enabler, we defined best practices and taught teams how to instrument their code, they kept trying to use my team as, "Oh, you know this stuff, so why don't you instrument our code for us?" We had to push back really hard on that because we didn't know their code. They had to instrument their code because they knew it best. - -**Derek**: I feel like we're all in the same boat. It's super interesting because I've experienced what you guys have said. What Gerald mentioned is something I'm also experiencing. I sometimes try to help a few open-source projects add instrumentation to their code base to have good references to point other people to. - -The problem is that some projects want good instrumentation, but they aren’t ready for it. I’m experiencing that within my company and other companies as well. We can try to get a network to instrument things for them, but if they aren't ready for it, it's just not going to work. - -**Response**: I agree. First, we need to sell the idea that if teams recognize the benefit earlier, it helps in all these avenues. It helps them understand the purpose and, therefore, maybe want to contribute more to solving the problem. Maybe we should hone in on showing examples where doing this solves some problems, especially if they're experiencing similar issues. - -**Derek**: Is my microphone better now? Am I closer? - -**Response**: Yes, that's much better! - -**Derek**: Good! I had a conversation with someone who mentioned something similar. They have an observability group that serves as the reference group for observability. They can monitor what the whole company is doing in terms of observability. There was one team using a specific programming language that found a specific metric helped them recover very quickly from an outage. - -So, the observability group shared that information with the entire company, saying, "Teams using this programming language should include this metric, or else..." I see the enabler's role as spreading information—getting information from one team and sharing it with the whole company. +# YouTube Transcript Cleanup -**Response**: Would it be beneficial to have documentation around real-world cases like the one Gerald mentioned? Do you think that might help if you could share similar stories with your team? +**Reese:** Of course, I was looking for near to the sessions. My name is Reese. My day job is with New Relic, and I also do a lot of work with the OpenTelemetry User Working Group, including hosting these sessions. -**Derek**: I think we try to do that already. I think it's a combination of everything mentioned here. I don't want to say what we're doing isn't working, but progress is slower than I would prefer. Having a collection of good news stories could help. +Let’s start with this question: Does your role in Compass as an enabler help you deal with languages you're not an expert in? Usually, I run these sessions based on whoever put the topic or question in. If you can unmute, maybe give us a bit more detail. It looks like Derek has a question about how to define "enabler." -For example, "Hey, I work at company X, and we were able to solve this problem and reduce mean time to resolution." If people are looking to get into this space, I think those examples are great. As someone in the observability group, I read blogs, but I don't think everyone at my company does that. +**Derek:** Yeah, this one's mine. I'm happy to expand on this. Basically, the role I have in my company is twofold. The first is to maintain OpenTelemetry collector deployments and be responsible for the data pipeline—receiving data and sending it to various backends. The second part is what I call the enabler. This involves being a champion for OpenTelemetry, driving adoption within the company, troubleshooting, and helping teams with issues in their respective services. -**Response**: Maybe we should consider having periodic forums where organizations looking to adopt OpenTelemetry practices have a chance to interact with folks in the community for Q&A. Just to ease their concerns. Would that feel like too much overlap with what this group already does? +In my company, for instance, we're probably like 70% .NET, 10% Java, 10% Go, and 5% Python. We primarily focus on one area, so we build examples in .NET. I personally am more of a .NET developer, and my team is as well. A great thing about OpenTelemetry, in my opinion, is that the concepts are extremely similar across languages. However, I don’t really know how to write Java, which makes it difficult for me to provide specific feedback or look at a PR and fully understand everything happening. -**Derek**: My first thought is that it does have some overlap with this group. I think I would need to think about that a bit more. +I'm curious if teams have this problem in the companies they work in. Do you try to create champions in those languages you're not familiar with or establish communities of practice around that? If anyone’s experiencing this, I’d love to hear how you’re solving for it. -One thing Reese mentioned is that she's recording this meeting, which is good. People can go back and gain insights from past discussions. Some topics brought up here aren't necessarily easy to find answers for, as they are more gray. +**Another Participant:** I can tell you that my strategy, as a former enabler, was to lead the way by showing how to enable observability within my domain of expertise, which at the time was Go. As I demonstrated the benefits of using OpenTelemetry, people from different teams and languages would come in and ask me questions. They were interested in starting out but weren't familiar with the concepts. -**Response**: It’s a valid concern, and I’d like to think about this more as well. +My role was to pair with those who were experts in whatever languages they were using, working alongside them until they felt comfortable with the observability platform I was using. This strategy worked out pretty well; we managed to deploy things to three or four different teams within the organization. -I just realized I haven’t been time-boxing this, so I apologize. It sounds like we might be good on this topic. Dan, was there anything else you wanted to add before we move to the next one? +**Derek:** That’s kind of what I mean by champions—leveling up individuals or groups within those smaller subsets and letting them run with it. Did you find that those people, once they got up to speed, sought information on their own or routed questions through you? -**Dan**: No, I'm good. +**Another Participant:** It’s a mixed bag. Some people feel more comfortable talking to the person they know, so I became a router for questions. However, I did see a few people who were really into talking to the community directly, so they would go into Slack channels or reach out on GitHub. Over time, as I directed them to specific issues or spec changes, they became more comfortable going there themselves. -**Response**: Can I hop in real quick just to share one last thought? +**Derek:** How long would you say it took for those individuals to get up to speed? -We're trying to drive change across various departments through the idea of a service catalog and scorecard for services. Various subject matter experts can define a set of standards in a particular domain that should apply to services and provide a rubric for those standards. +**Another Participant:** It really depended on the individual. Some people who were eager to learn got up to speed much faster. Others, who were used to doing things a certain way, took longer. Honestly, some people never really understood the benefits of distributed tracing. It varies based on where people are in your organization and how resistant they are to change. -This way, service owners can consult that rubric to grade their own services according to the criteria. If a service is graded a C, they can refer to the documentation provided by subject matter experts to improve their service to an A. +**Derek:** That resonates because we sometimes struggle with that; they don’t really understand the purpose or how it benefits them outside of their team. -This is something new at our organization, and we’re trying this to drive change across different areas, like SRE practices, Kubernetes, code quality, test coverage, observability, etc. +**Another Participant:** It's tough to take someone from their day-to-day job and ask them to invest time in something that might seem unnecessary. It’s only when they have those “aha” moments that they start to see the benefits. I've helped people understand how to get started quickly, and the work on auto-instrumentation has been great for that. It allows people to see benefits without a tremendous upfront investment, which can spark excitement about investing more time later. -**Derek**: I see. +**Derek:** That makes sense. Sometimes, I feel weird leading the way as an enabler without being able to just show examples from start to finish. -**Response**: Since the next two topics have equal votes, we’ll go with the one about dealing with clock/time drift. +**Another Participant:** In my previous organization, we faced challenges where teams recognized the importance of OpenTelemetry but were constantly fighting fires and didn’t have time to instrument their code. Ironically, instrumenting their code would help them fight those fires more effectively. My team defined best practices and taught teams how to instrument their code, but they often wanted us to instrument their code for them, which we had to push back on because we didn’t know their code. -**Derek**: These are actually all mine for today. I noticed our backend doesn't accept data that's in the future—specifically, data points that are more than 10 minutes ahead. +**Derek:** I feel like we're all in the same boat. It’s super interesting because I’ve experienced all of what you guys have said. -I'm curious if people are experiencing this problem. The naive side of me wants to say they should just fix their servers, but if you are experiencing this, do you notify services or implement something in the collector to understand that it's happening? +**Another Participant:** Yes, I’ve tried to help open-source projects by adding instrumentation to their codebase as references for others. The problem is that some projects wanted good instrumentation but weren’t ready for it. -**Response**: I can talk a bit about how Jaeger deals with that. Clock skew is going to happen, and you just have to account for it. It's important to realize that there's no way to synchronize clocks, especially in a microservices architecture. +**Derek:** It sounds like we need to sell the idea of observability more effectively. If teams recognize the benefits early on, it helps them understand the purpose and maybe contributes more to solving the problem. -One way to deal with that is for Jaeger to assume that you don't have asynchronous processing, which is very synchronous. The parent span or the first span is adjusted to be as long as the longest span that it has. However, this created confusion for users who had asynchronous processing, so we ended up doing a flag to disable this behavior. +**Another Participant:** One interesting thing I heard recently is about an observability group that acts as a reference group. They watch what others in the company are doing in terms of observability. They found that one specific metric helped a team recover quickly from an outage and spread that information to the whole company. -The owner of the system generating the telemetry data is in a better position to understand clock drift than any general-purpose tool like the collector. +**Derek:** Would documentation around real-world cases be helpful? -**Derek**: Thank you! I think there are two use cases here—like a low mild clock skew and the really bad ones that are 10 minutes off. I was thinking of creating a processor that detects incoming data points that are over a certain threshold—like five minutes. +**Another Participant:** I think it would help. We already try to do that, but progress is often slower than I would prefer. Having good news stories can motivate people to get into the space, especially if they see how others solved similar problems. -Creating a data point that captures the service name could help identify that this data is out of sync. I don't know if this would be useful, but right now, my backend is rejecting data, so more visibility into the issue would be a first step. Does that sound like a weird use case? +**Derek:** Would it be beneficial to have periodic forums for organizations looking to adopt OpenTelemetry practices? -**Response**: It's an interesting use case. You could have a processor that detects things that are out in the future. The hard part is that you wouldn't catch all of your clock drifts; you would only catch the very obvious ones. +**Another Participant:** It may overlap with what this group already does, but putting folks who are unsure in touch with others who have experience could ease their concerns. -If your backend is rejecting data, it would be good to know that. If you had a 10-minute threshold and data was only three minutes delayed, you wouldn't detect that, but it could still cause issues with data interpretation. It might be better to shore up data loss and have service owners interpret the drift since they are the experts. +**Another Participant:** Recording these meetings is good because people can go back and gain insights. There are good topics discussed that aren’t necessarily easy to answer. -**Derek**: Are we talking specifically about metrics or any telemetry data type? +**Reese:** I haven’t been time-boxing this, so I apologize. It sounds like we might be good on this topic. Dan, is there anything else you wanted to add before we move to the next one? -**Response**: I would need to double-check, but I think it was for spans. In my opinion, it would be useful for any type. +**Dan:** No, I’m good. -**Derek**: Okay, cool! +**Reese:** Can I share one last thought? -**Response**: For metrics, some metric stores will block new metrics from the past and future, so you would have a warning in your logs. For traces, it might not be too hard to detect clock drift by looking at the spans of a trace. Logs are tricky because you generate so many logs on a single server, and they generally have the same timestamps. +**Dan:** Of course! -**Derek**: I see. +**Reese:** We're trying to drive change across various areas through a service catalog and scorecard for services. Subject matter experts define standards and provide a rubric that allows service owners to grade their services. This separates the implementation from leading the way in improving something. -**Response**: The simplest thing that would work is to flag log timestamps that are still in the future. However, this would only catch clock drift in the future, not in the past. +**Derek:** Sounds like a good approach! -**Derek**: Makes sense. Thanks for the input, everyone! +**Reese:** Let's move on to the next topic: How are you dealing with clock/time drift? -**Response**: Alright, moving on to best practices for bifurcating data in a pipeline. For example, having a filter processor with positive and negative condition clarity around fan-in. +**Dan:** These are all mine! I've noticed that our backend doesn’t accept data that is in the future—specifically, 10 minutes in the future. People are creating data points from the future because their server times are incorrect. Are others experiencing this problem? -This is also mine, and I don't know how to phrase it simply. I have data flowing into two pipelines. For one pipeline, I want histograms to go to backend A, and for the second pipeline, I want non-histograms to go to backend B. Is configuring two pipelines like that the right thing to do? +**Another Participant:** Clock skew is going to happen, especially in microservices architecture. One way of dealing with that is to assume that you don’t have asynchronous processing. -**Response**: There's a new type of component for the collector coming up called connectors. The collector currently has four types of components—extensions and pipeline-specific components like receivers, processors, and exporters. The connectors can act as both receivers and exporters. +**Dan:** I was thinking of creating a processor that detects incoming data points that are beyond a certain threshold—like five minutes out of sync—just to identify it. -You can have one pipeline that receives OTLP, does some processing, and ends with an exporter. The connector then connects with another pipeline as a receiver. In your case, you could have one receiver and two connectors—one for histogram-specific and one for non-histogram-specific data. Each connector can filter what should be passed through. +**Another Participant:** That’s an interesting use case. It would capture obvious issues, but it may not catch all clock drifts. -**Derek**: That sounds like a good feature! I'll look into it. +**Dan:** I just want more visibility into the issue, especially since our backend is rejecting data. -**Response**: It is! We have a routing processor that routes data points based on their characteristics to specific exporters. However, it makes another network connection, which can be inefficient. +**Another Participant:** For metrics, some metric stores block new metrics from the future but not from the past. For traces, it might be easier to find clock drift by looking at the spans of a trace. -**Derek**: I'm currently using a filter processor that works. I just felt like having two receivers receiving the same data was doubling up on memory. +**Dan:** That makes sense. Thanks for the input! -**Response**: Profiling might be helpful. Check out the connectors, as they could help reduce memory usage. +**Reese:** Let’s move to the next topic: Best practices for bifurcating data in a pipeline. -**Derek**: Sounds good! +**Dan:** I have data flowing into two different pipelines. I want one to have histograms go to backend A and the other to have non-histograms go to backend B. Is configuring two pipelines the right thing to do? -**Response**: Does anyone else have anything to add or ask? +**Another Participant:** There’s a new type of component coming up called connectors, which will allow you to connect pipelines efficiently. Currently, the routing processor routes data based on their characteristics but does that by making another network connection. -**Derek**: I do! I asked this a couple of sessions ago. I’m trying to understand how to scale my collector deployment correctly. I opened a GitHub issue that was large in scope, so I haven’t had responses. +**Dan:** I feel like I’m doubling the memory by having two receivers receiving the same data. Should I be profiling that? -I’m curious about criteria for when to horizontally scale my pods versus modifying the configuration of an individual collector. Does that make sense? +**Another Participant:** Yes, and you might want to look into connectors as they would only keep one copy of the data point in memory. -**Response**: If you only have stateless components in your collector, you can just increase the number of replicas. Your workload will dictate how you scale—not just for demand but also for high availability. Is it better to have one per namespace? Should you have one per tenant? +**Dan:** Sounds good! I’ll definitely follow that. -If you have one collector that all traffic goes through, it can be a critical failure point. +**Reese:** Does anyone else have anything to add or ask before we wrap up? -**Derek**: I understand. +**Dan:** I have a question regarding scaling my collector deployment. -**Response**: There’s no easy answer, but you could chart your pipeline based on the type of processing. You might want to split by metrics, logs, and traces because they have different workloads. +**Reese:** Please go ahead! -**Derek**: Okay, cool! +**Dan:** I’m trying to determine when to horizontally scale my pods versus modifying the configuration of an individual collector. What criteria should I use to decide between the two? -**Response**: Regarding the "num consumers" setting, it relates to the signing queue. The number of consumers is the number of workers picking from the queue. It’s complicated because the optimal value depends on your workload and the backend you're sending data to. +**Another Participant:** If you only have stateless components in your collector, scaling up by adding more replicas would be the solution. -**Derek**: So there's no one-size-fits-all answer? +**Dan:** I have an HPA configured, but I still need to manage spikes in traffic, especially with sticky sessions. -**Response**: Exactly. You’ll need to do specific testing around your expected data volume to determine the optimal settings. +**Another Participant:** It’s important to have your deployment done correctly. If one collector fails, having a headless service allows clients to connect to a list of known backends. -**Derek**: Awesome! Thank you for the insight. +**Dan:** That makes sense. Thanks for clarifying! -**Response**: Thank you all! I want to be respectful of everyone’s time. Dan, thank you for your questions. Alex, Gerald, it was great to have you on. If anyone has questions about these sessions or anything else, feel free to reach out to the end-user working group on Slack. +**Reese:** Thank you all for your insights. If anyone has questions about these sessions or anything else, feel free to reach out. -Oh, and before I go, I just want to show you my kitten, Taco. +**Dan:** Thank you all! -**Derek**: Hi Taco! +**Reese:** I just realized I haven’t introduced my kitten. This is Taco! -**Response**: Thank you all so much! +**Dan:** Hi Taco! -[Music] +**Reese:** Thank you all so much! ## Raw YouTube Transcript -[Music] of course I was looking for near to the sessions um my name is Reese I my day job is uh with New Relic and I also do a lot of work with the open Telemetry and user working group including hosting these sessions all righty let's see okay let's start with this one does your role in compass being an enabler how do you deal with helping in languages you're not an expert in um usually I run these is whoever put the topic or question in if you can unmute and um maybe give us a little bit more detail it looks like Derek has a question on how do you define enabler yeah this one's mine uh happy to expand on this um so like basically the role I have in my company is kind of twofold the first is like you know maintain uh open Telemetry collector deployments and and kind of be responsible for the data pipeline uh receiving data and sending that to the various backends but it's also like what I call the enabler which is like uh being a champion for open Telemetry driving adoption within the company uh you know troubleshooting helping teams troubleshoot issues that they have in their respective Services uh you know things like that um so like in my company for instance we're probably like 70. Net 10 Java 10 go five percent python you know like we're primarily focused in one area so like we build examples in.net we I personally am more of like a.net developer my team is as well uh you know a great thing about open Telemetry my opinion is that like the concepts are extremely similar across languages but like you know I don't really know how to write Java you know it's not too far away from.net but for instance um so sometimes it's like troubling for me to provide specific feedback or like look at a PR and like really understand the scope of everything that's happening and I'm curious like if teams have this problem at the companies that they're working in do you try to like you know I don't know create champions in those languages you're not familiar in or you know communities of practice around that um just you know if anyone's experiencing this and maybe how you're you're you're solving for it I hope that makes sense there totally yeah as a as a former enabler um I I can tell you that my my strategy was um really to kind of first lead the way by showing how to enable observability or you know within my within my domain of expertise which at the time was uh go and you know as as I was demonstrating the the benefits of using open Telemetry or I guess at the time it was open tracing that if that gives you an idea of how long ago this was um but you know the one of the benefits as I was demoing the benefits people would come in ask me questions and a lot of the times these people work in different teams and different languages um and you know they would be interested in in starting out but they wouldn't be familiar with the concepts of open tracing open Telemetry and so you know my my role as I thought was to kind of pair with those people who were experts in whatever languages that they were trying to be um becoming enabler in and you know really work alongside with them until they felt comfortable with the concepts of the observability platform I was using so I think that's that's kind of my strategy it worked out pretty well I think we managed to deploy things to you know like three or four different teams within the organization and it worked pretty well yeah okay cool that's kind of I guess what I mean by like Champions so like basically leveling up you know maybe individuals or groups within those like smaller subsets and then letting them kind of run with it um do you find when you did that that those people you know as like I wasn't familiar with open tracing but like you know as the spec evolves and as new features get added do they do they come back to you do they seek information under their own do they jump in the hotel slack channels and ask questions or uh you know were they routing things through you and your experience um it's kind of a mixed bag I think I think some people feel more comfortable with just talking to the person they know and so you know you to a certain extent I became a bit of a bit of a router for questions um but you know I did see a couple of people who were really into um you know talking to the community directly and so they they just went into the slide channels or reached out to people directly in GitHub or whatever so it it's a bit of a mixed bag um you know I I think at some point as you direct people to specific issues in GitHub or specific spec changes people tend to get more comfortable with the idea that they can go there themselves but um yeah cool and then maybe one more follow-up if you don't mind like how long would you say like it took for those individuals you worked with uh to like I don't know get up to speed whatever that means in your uh opinion you know is that like a matter of weeks was that like a longer period you know how how would you engage like how easy it was for them to sort of like understand the domain when it is maybe not their like primary focus yeah um again sorry to give you maybe uh uh a mixed answer but again it was it depended on individuals I think some people who are really keen on trying to learn new ways of doing things we're really excited about the prospect of of trying out um you know something new and then you know those people kind of got up to speed a lot faster the people that were you know um you know maybe more uh used to doing things a certain way took a little bit longer um and then you know if I'm honest I think some people you know I was trying to get distributed tracing adopted some people just never really understood uh what the benefits were and I think that you know it really it really is a mixed bag so it really depends on where people are in your organizations and how resistant to change they are I think yeah cool no thank you I appreciate that because I I feel like we struggle with sometimes exactly that like they don't really understand the purpose or they don't understand how it may be has value outside of their individual team you know like when we start connecting services or something um so just trying to you know come up with any insight I can into like improving like the culture and that sort of thing so appreciate the response yeah I think you know you know I think the it's really hard to get to take someone from their day-to-day job which you know most people are already kind of strained on doing whatever it is that they're doing for the business um and asking them to do more by you know heading off into this completely what a kind of appears unnecessary aspect of their jobs right to implement open Telemetry um and it's not really until people have those aha moments of you know understanding how much better their lives will be in the future um to that they can really wrap their head around the benefits I think one of the things that I've really helped try to help people understand is you know how do you get started as quickly as possible and I think that's one of the one of the main areas of open Telemetry that I'm really really excited about is all of the auto instrumentation uh work um because I think that allows people to get some amount of benefit without a tremendous amount of investment up front and a lot of the time I find even you know even if it only gets you 60 or 70 of the way they're um it tends to give people enough of visibility into what their systems are doing through Auto instrumentation that they can then like get excited about the prospect of investing more time in doing this work yeah makes sense awesome Dan I don't really have an answer all I can say is I sympathize with you uh as someone who like codes even probably a lot less than you know what you described in your role it feels weird to kind of want to lead the way as an enabler for these things in your work without necessarily being the person who's able to kind of just all right let me let me do a couple examples for you let me kind of show you from start to finish how this looks so I don't know I'm trying to figure it out too probably further behind you actually so in my uh previous organization um like when we were trying to bring open Telemetry into uh the development teams we Face some interesting challenges where the teams recognize the importance of open Telemetry but they were constantly fighting fires so they didn't have time to instrument their code which was kind of ironic because instrumenting their code would probably help them fight those fires more effectively and so like while my my team was uh their job was to be that enabler so we you know Define the best practices and taught teams how to instrument their code they kept trying to use my team as like oh well you guys know this stuff so why don't you instrument our code for us which we had to push back really hard on because like we don't know your code so you have to instrument your code because you know it best right you know what the things are that you need to look for yeah I feel like everything like we're all talking about like we all have like somehow are in the same boat and I've just it's like super interesting because I feel like I've experienced all all of what you guys have said yeah what what a general mentioned is something that I'm experiencing as well um and even if it was wider than just one company it goes even throughout the community so sometimes I um I try to help a few open source projects to instrument uh to add these orientation to their code base because I wanted to use to have good references of what other people can can do you know so I could then point other people to that code base and say you know this is a very well instrumented application you can use it as reference for instrumenting your own application um the problem with that was some projects were they wanted to have a good instrumentation but they didn't they didn't actually um they weren't ready for it right and this is something that I'm experiencing within the company uh within you know other companies as well is that we we can try to get a network and instrument things for them but if they are not ready to have their code instrumented it's just not gonna work I don't know yep um I I could by the way you sound a little distant I I was able to hear you um but just that way um I I think that's true like I feel like first somehow we need to like sell the like if teams maybe recognize the benefit earlier like it helps in all these avenues like it helps with them understanding the purpose and therefore like maybe wanting to contribute more how to solve the problem so maybe I don't know I'm just thinking out loud here and maybe like pushing to show more like here are some examples where doing this solves some problems like if you're experiencing similar problems and perhaps like this would be a really good idea for you um maybe like you know I don't know honing in on that kind of an attack might might be beneficial yeah so is my microphone better now am I closer yeah that's much better yeah okay that's good uh yeah so um I I just came with a performing conversation with another person that um that person was mentioning something exactly like that you know um and one thing that that they're doing and I found very interesting is um they they have like an observability group and that group is uh the reference group in terms of observability and they they can watch what other people what the whole company is doing in terms of observability and what what he mentioned was um there was one team using this specific programming language and they found that one specific metric helped them recover very fast from an outage so what the observability group did then was to spread that information to the whole company and saying so teams were using that programming language you should you should you now have to include this metric here um otherwise you know because that metric helped that team to recruit were very fast from an outage so um I see the role of the neighbor here like the observation neighbor as also to spread information to get information from one title and break this item and bring to the whole company what should the whole Community for that matter um deal would it be you know as far as something like the community could do with this um would maybe some documentation around like real world cases like the one gerasi mentioned um do you think that might be helpful if you could share like similar stories with your team it it might I mean we we try to do that already I I think it's like a combination of everything that's mentioned here maybe like I don't want to say like what we're doing isn't working I think it is working it's just like progress is slower than I maybe would prefer you know um I mean yeah I think like you know having a collection of like good news stories that you know or whatever you want to call it like uh you know like hey I work at company X and we were able to solve this problem and you know reduce mean time resolution blah blah blah blah like yeah I do think like that that you know you know if people are like looking to get into the space I think those are great and I think like having those examples of like specifics um you know like as my role here as like in the observability group for my company like I go and I read blogs and stuff but I don't think like everybody at my company would do that like maybe they are reading blogs specific to like the.net runtime or something you know I don't know um so yeah I think it would it would help um I just I don't know what the best I guess format would be I'm just wondering would would it be beneficial to have like periodic like some sort of forum um where like every month or whatever or every quarter where folks from organizations that are looking to adopt open Telemetry practices open Telemetry in general um have a chance to like interact with folks in the community for for like a pointed q a um to like just to sort of you know ease their concerns like or or would you all feel like this is too much of an overlap of what this group already does I'm just wondering um because I I do feel like just putting putting folks who are kind of like not really sure about this um in touch with with other folks who have gone through this or you know have some expertise in in the area I can like put them at ease I don't know my first thought is like it does have some overlap with this group um I mean I think I don't know maybe I would need to think about that a bit more well one thing like at Rhys mentioned like she's recording this uh meeting now which I think is good like people can go back and like get insight into the past which before I think was like a there's some good topics here that are brought up that that like aren't necessarily easy to get answers because there aren't really answers like they're they're more gray if you will um I don't know I would have to think about that yeah um no I think it's it's a very valid um observator concern um and yeah I would like to Noodle on this more as well um and that said I just realized well I realized a few minutes ago that I haven't been time boxing this so I apologize um but it sounds like um we might be good on this topic Dan was there anything else you wanted to add before we move to the next one no I'm good okay can I hop in real quick just share one last thought of course um this is kind of just a more General approach for trying out our organization but we're kind of trying to drive change across various laterals through the idea of like a service catalog and scorecard for services where various like subject matter experts can Define like a set of standards in a particular domain that should apply for services and kind of provide a rubric for those things so what does like you know sufficient or like get give a service of grading in a particular set of criteria right c a A plus right and you don't have to be responsible for implementing what is like an a look like in your service but you're allowed to you know you can define those standards and then the actual service owners that are responsible for that service can consult that rubric that scorecard kind of grade their own Services according and then kind of refer to whatever standardized documentation you provide as a subject matter expert for figuring out okay how do I take my service that is maybe a a great C and observability to like a a grade A right and so in that kind of a situation you're setting forth the standards you're providing a clear rubric that allows service owners to kind of grade themselves and you're providing more information if they're looking to kind of level up uh their service so this is kind of something that is very new at our organization we're going to try to drive change in a bunch of different places things like SRA practices how you tune your services and kubernetes code quality test coverage observability stuff so we're just gonna try that out and see if that makes it kind of easier to gamify making strides um and just improving services and things like that so I don't know if that's helpful at all but it kind of separates implementation from like leading the way and improving something all right Derek I don't know if you can see this but Dan has put a thumbs up oh cool yep alrighty so since the next two have equals how more votes we'll just go with the uh in order how are you dealing with clock slash time drift yeah these are actually all mine but today um yeah um yeah so like I uh I don't know I noticed well I noticed because our backend doesn't accept data that's like in the future I think it's 10 minutes in the future um obviously people are creating data points from the future because there are times on their servers are not correct um one I guess are people experiencing this problem like the naive side of me wants to just say well like they should just fix their servers which I do agree with um if you are experiencing this do you do you notify services do you implement something in the collector to to understand that it's happening uh uh yeah that so um I can probably talk a little bit about how jiggers deals with that um and the way that eager does or used to do is um it tries to detect well so I guess the first realization is that uh clock skew is going to happen um and you just have to account for it there is no way for clocks with synchrons or synchronized especially on a microservices architecture um and one way of of dealing with that is younger chance is um it first assumes that um you don't have asynchronous processing so one Trace is uh very synchronous so the the parent span or the very first span um and uh at most when the the the the is the last span finishes you know so and if that's not the case then Jaeger by default will try to adjust the parent spans of that span uh to be as big or as as long as the longest span that it has um it did generate a lot of confusion among users especially for users who do have a synchrance processing so much that we ended up doing a flag to disable this behavior on the eager side on the eager collector side and I think we at some point thought about not having that that behavior by default so only users who would know what they're doing would then enable the clock skew I can't remember what is the feature name but uh they wouldn't have this uh clock um fixing a feature enabled um on The Collector side we're not doing anything that I know of perhaps Alex Cannon can share uh if he knows whether we are doing something like that there but uh and I guess the short answer is um the owner of the of the the system that generates the Telemetry data is in a way better position to understand the clock drift then uh any any general purpose tool like The Collector so the collector cannot in a it's not in a very good position to detect and and fix this issue for you um so yeah great thank you um so like I don't know for to me at least and maybe this is not true but um like there's sort of like two use cases here there's like like a lower mild kind of clock skew where it's like I don't know a couple seconds or something and yeah sure maybe it makes the the tracing Vision like look a little awkward or something and then there's like like the really bad you know ones that are like in this case like I I know they were like 10 minutes off um so I I was thinking of like creating a processor that detects some you know incoming data point that is like some some like distance I don't know what the right word is some you know if it's more than five minutes or some threshold uh like just I don't know creating create a data point that like captures you know the service name or something and and like you know just just at least like identifies it like hey this is a this is something that's like way out of sync with what we consider real time I don't know if like something like that would be useful um right now our back end there's no good way to like correlate like the air that our back end throws with like the data that's coming through so I don't even know like what data is missing unless a service owner were to reach out to me um it could be specific to my back end maybe my use case um I know so I was just like maybe just like having more visibility into the issue would like be a first step does that sound like a weird use case like I don't know creating a processor that would kind of detect that I think it's definitely an interesting use case if you know you could have a processor that just detects things that are out in the future um I I would guess that the the hard part is you know you still wouldn't catch all of your clock drifts I guess you would catch a very obvious ones that you kind of know about um that you've seen in the past but yeah so I I guess then yeah that's like I guess because in my case I'm like dropping data because the back end's rejecting it it would be good to know that but if there was like say it was a 10 minute threshold and it was I don't know three minutes delayed and I didn't want to detect that or I couldn't detect that now it still cause issues with the interpretation of the data uh but it would be like a secondary concern like maybe I should just Shore up like the data loss if you will and then I don't know just be better at you know having having that like like drawsi is that how you pronounce your name having like the service owners sort of interpret the drift or the skew because they're that the experts like like you said are you um are you talking specifically about metrics or are you talking about any Telemetry data type um I would have to double check I want to say the data that I was looking at in this case was spans [Music] um but in my opinion it would be for for any type yeah so for metrics there's some metrics um um stores like you know every every single permittee is based storage uh will block new metrics that are from the past so not from the future I suppose but from the past um so you would have a a warning on some logs already because of that before using Primitives um for for traces uh it might be relatively easy to to find it out if the clock drift can be detected by looking at a trace right so by looking at the spans of a Trace and I guess it would not be too hard to to make a a processor that detects that now for logs that um for logs I don't know I I wouldn't even know if we would want to have something like that for logs um because you generate such a huge amount of logs on one specific server and supposedly all of the logs on that server are having the same timestamps for things are a lot at the same time um and uh so perhaps just yeah I don't know perhaps this data it's hard to think about logs in this case because it would be having a bunch of logs at once uh and then you have to compare those logs with a batch of logs from another server right coming from another server on The Collector so your your comparison base would be huge yeah I guess my my take there would be just do the simplest thing that would work which in this case if the log time stamps are still in the future you could you could flag it you know you could create a metric that would at least capture that information but again this would only capture clock drift that's in the future nothing nothing clear would happen if it was like drifting in the past yeah makes sense okay cool thanks for the input guys I'm good with this one recent other people don't have comments excellence alrighty best practices for bifurcating data in a pipeline for example have a filter processor with positive and negative condition Clarity around Fan in yeah so this is also mine and I don't really know how to phrase this in a in a simple way um maybe I'll tell you a really weird case use case that I have and [Music] yeah anyway so like I have data flowing um let's say I have an otlp I have a collector I have an otlp receiver I have some processors configured and then I have uh a back end let's call it back end a and then I have another pipeline also otlp receiver well let's just say we're using metrics here um some processors in backend B so I want to like have um so the same set of data is flowing into both of these pipelines and then for like pipeline one I want say just to have histograms go to backend a and for pipeline two I wanted to have non-histograms go to backend B is configuring two Pipelines um like that the right thing to do um I was trying to like it works like I have this configured it works I was thinking about like is this the most CPU or memory efficient in terms of like uh I'm not I'm not I guess like the fan and fan out of how like the receivers and and stuff work in the in the in the exporters work I was trying to like you know understand if I'm doing it in like the optimal way does that question make sense at all is that a use case anyone else has um I hope that made sense there's a new type of component for The Collector that is coming up and that's called the connector so it's it's uh so the collector right now has uh four types of components right extensions and then pipeline specific components like receivers processors and exporters and there's going to be a fourth one or a fifth uh which is connectors so connectors they can act as receivers and exporters so you have one pipeline that receives otlp for instance then there is some processing and um you end up with a with an exporter and exporter for that pipeline is going to be a connector now the connector then connects with another pipeline um and as a receiver right so it's an exporter in one Pipeline and a receiver on the next pipeline um so what you can do is you can you can translate signals so you can translate its bands to metrics for instance but in your case here it would work uh in a way that you have like one receiver and then uh two exporters or two connectors as part of the same pipeline one that is going to connect this data or this this Pipeline with one that is histogram specific and one that is going to connect with a non-histogram specific with the other one and then each one of those connectors would then be able to filter what is interesting to be passed through this disconnector and block everything else so it ended up with uh three pipelines in there and so one that receives the raw data one that deals with a histogram data and one that deals with other data so this is what we have planned for the future so the basic building blocks for that are are merged already so um from what I remember so Alex can correct me if I'm wrong um but uh the basic building blocks are there and it's it's we're all looking forward to seeing connectors being ready to be used because we have so many use cases in mind to implement now what what we have today for that specific case is the routing processor so we have a routing processor and it it doesn't really work the way that you that you that you want here but it it works in a very similar way so what we do is we route the data points based on their characteristics and we rush them to specific exporters but we do that by making another network connection so it's very inefficient but it works so those are the two possible solutions for that with the router I'll obviously look into this thank you I didn't know what it is maybe this is specific use case but what would be the advantage of like the router solution versus like having a filter processor that like you know having the two pipelines and having the filter processor just like get rid of a you know part of like the part A of the set or Part B of the set would there be a advantage to you either one of those I guess that would be a third solution yeah I guess um that's what I'm doing right now um and it's working uh there were some bugs in the filter processor that have been resolved so now it seems to be working perfectly as far as I know it just I I just felt like I was because I had like two receivers receiving the same data like I was doubling up on the memory that you know my collector was using uh I haven't done any profiling but maybe that's something I should do um but I'll definitely look at the at the routing processor um actually take a look at the connectors because I I've made a very similar question to the to the author of The connectors on the pr that they introduced and I think I think indeed the the answer is that the connector is using connectors you wouldn't only have one receiver and only only on the costume only one copy of the of the data point in memory it only keeps one copy of the data point to memory let me try to find a PR and you can get more information from there okay awesome that would be great thank you that sounds like a great feature I'll definitely follow that thank you yeah that's neat I haven't heard about the connectors so I'm interested to learn more as well well jurassi is getting us that PR does anyone else have anything they'd like to add or ask well we still have 10 minutes on the clock I I do if no one else does that would if someone else obviously go first since I've asked enough I think you're in the clarity and go ahead okay um I asked this a couple of these sessions ago um I was trying to I'm trying to find I had opened a GitHub issue um it was very large in scope so I don't think I have any responses on it um it was it was it was about like understanding more about um um I'll just pop this in the zoom chat uh if you're curious um it was more about just like figuring out like how to scale my collector deployment correctly in dressy I saw that you had like a post yesterday that you posted on one of the channels that talks a little bit about that um there were some other things I had put in here like um understanding like when the num consumers setting uh for instance should be modified um I'm curious if you guys have any like insight into like when I should be I guess horizontally scaling my pod versus when I should be modifying the configuration of uh an individual pod if that or an individual collector if that makes sense like how do I what wonder what criteria do I use to determine one verse versus the other does that make any sense it's probably a loaded question as well so are you asking like at what point should the configuration be relegated to the Pod versus to The Collector uh so like I have it like an HPA configured so like as my traffic increases you know presumably I spin up new pods um but why are there settings like the number of consumers like should I be is that is that purely for non you know deployments that can't automatically scale um more so like when when do I when do I create more collectors or when do I change collector specific configuration um I'm sorry I I missed the very beginning of the question because I was looking for for the issue I found the issue and I linked here and I linked directly to the comment that I've made that is close to your question um you can read then a dance and search that um but coming coming back to this question here I'm and forgive me if I'm if I'm missing soft some context from the very beginning of the question but um the way that I that I would I would say this the way that I would tackle is um if you only have stateless connect or or components in your collector you can then just dot use a specific metric and scale up or scale adding more more replicas would be the solution um the other answer to that is you know your workload um and you would probably have to think about that not only as in the scaling to attend the demands uh like like the the load that I have but also scaling to improve my high availability so if a node goes down what am I going to what what are the effects across my observability pipeline so how would I want to isolate failures on my observability pipeline so is it better to have one per namespace is that and is that good enough or um or should I have a a one per tenant or perhaps a a even within tenants in my cluster perhaps I should have different layers of collectors um because you know failing one specific branch of your collection of collectors isn't not going to be so critical as something that is going to fail only you know if you have everything going through one collector then it's really going to be a an epic failure at one point um I guess there's no easy answer to that um yeah I I know there's no like one size fits all I guess I was just trying to like okay like so obviously like I have some like bait like some bass throughput like you know that I want to kind of I want to serve like you know I get a certain amount of data points per second or whatever uh base and then like I get spikes right so I have to like have a minimum deployment that can handle those spikes yeah but that but then there are like some connections that are I think like like have like sticky sessions so I need to be careful about like if one of those things like burst real high then like a given you know a given pod or given collector if you will will will like I don't know have to be able to to handle that all right um so in most of the cases you're going to use grpc for the connection between collectors or between your workload and a collector um and the good thing about Char PC is when a connection fails uh it will attempt to connect to another um to another backhand automatically right so if you have your deployment done correctly and correctly here means on kubernetes you wouldn't be having a a headless surveys and your clients would be then connecting to the Headless service so that the the client has a list of known backends up front so whenever one of them fails it just fails over to the next one and uh so so that's one way of of um dealing with the sticky problem they sticky session problems so most other connections they are long-lived when we talk about the observability pipeline so there are grpc connections and grpc connections they are long-lived by Design which also means that if you have like only three collectors at the very beginning of your life cycle of your cluster for instance and then you keep adding more collectors to that but you don't increase the number of clients then you're not going to see any effects until the clients reconnect uh due to some failure right so you're not going to see any advantage uh the moments that you scale up you're only going to see advantages the moment things start to fail and then you start seeing the other collectors receive some traffic or when you have new clients so new clients are going to be load balanced through the to the new nodes so then you start seeing that um yeah so I guess that's there's that uh one other thing you may consider is um charting your pipeline based on the type of processing that it's doing all right so one pattern that we've seen before and uh that I've recorded at some point is having one pipeline per data point so you have one one Matrix pipeline you have one logs Pipeline and you have one traces pipeline because the types of workloads or you know the workload for metrics is way different than the workload for for a traces um the way that they work is really different so you might want to split by that and you also might want to split by the type of processing that you do on on the Telemetry data that has been generated by your workload so if you have more pii information being generated by nodes or by pods on one specific namespace then it makes sense to have one collector on that namespace with a specific um processor to remove this pii and then goes to the next layer of collectors that deals with more generic data yeah so you're only affecting like you're only slowing down for like that subset right exactly yeah okay yeah so yeah okay cool awesome and then I know we're like almost that time so just like harping on one specific setting that num consumers thing like is there when when would I ever want to change that do you know I'm sorry um consumers as in yeah there's on the otlp exporter there's a maybe this is two technical maybe I should just open an issue just for this uh there's a property called num consumers I think it's on otlp exporter um it seemed like I got better at the root book when I increased it um and I was just wondering what like why wouldn't I increase that like what trade-off am I sacrificing okay um sorry I see now so this is about it's about the signing queue uh this is a blocking queue and um the way that it works is um whenever things are being sent from from one place to another from from an exporter and into a back end um it's placed on a queue and then things are are picked from the queue like from like workers and uh this is you know number of consumers is basically the number of workers that are picking things up from the queue now um I'm not the author of this component here of this Helper but um I we had a similar component on Jager and I can tell you by experience that we don't actually know what is the optimal value for that you have to find that out by yourself um a and it's complicated because um in in Java for instance uh the number of workers would be typically uh closely related to the number of processors in a specific like CPU processors in a machine ACP using a machine now with Goal it's not like that so one one worker thread is not a thread it's a good routine which is not related to an OS threat a Linux thread so there's no relation between or no very explicit relation between a number of consumers with the number of of uh processors on a specific laptop or a machine server bare metal um so you have to play with that number I I think I kind of played with this information on on an article recently um and the idea is that the more um if you have backhands that are are taking very long to answer to you having more consumers here means that you have more HTTP connections with the backend that you're sending data to and it might be a problem with the back end that you're you're I mean if you're the backend is having trouble with a specific amount of connections and you're adding more connections you're adding more load to a server that is already overloaded in that case you want to decrease the number of consumers but what it means is it is processing less data um um simultaneously concurrently so what is effectively means is it it increases the concurrency of data being sent to the remote server so in theory it is related to the CPU number of CPUs that you have uh but at the same time I think it is more important to the back end that we are talking to okay yeah so awesome that's like super helpful I I think the main takeaway for me is like I should probably be doing some more specific testing around like my expected you know like how much data I'm receiving and expecting to send to my back ends and just you know trying to optimize for my specific use case rather than like having a generic uh this type of situation you know you should set it to X Y or Z absolutely okay awesome really really appreciate the Insight thank you Brad thank you thank you all all or a few minutes a few minutes or so I want to be respect to respect everyone's time um Dan thank you so much for all your questions um Alex gerasi it was great to have you on and um if anyone has any questions about these sessions or anything else they are curious myself Adriana as well as Rin are all on the end user working group so feel free to reach out to any of us on cnco slack um and for I I just have to drop real quick but um Alex and C wanted to see the kitten I'm just gonna do a quick kitten kitten show this is Taco thank you hi taco and all right thank you all so much [Music] +of course I was looking for near to the sessions um my name is Reese I my day job is uh with New Relic and I also do a lot of work with the open Telemetry and user working group including hosting these sessions all righty let's see okay let's start with this one does your role in compass being an enabler how do you deal with helping in languages you're not an expert in um usually I run these is whoever put the topic or question in if you can unmute and um maybe give us a little bit more detail it looks like Derek has a question on how do you define enabler yeah this one's mine uh happy to expand on this um so like basically the role I have in my company is kind of twofold the first is like you know maintain uh open Telemetry collector deployments and and kind of be responsible for the data pipeline uh receiving data and sending that to the various backends but it's also like what I call the enabler which is like uh being a champion for open Telemetry driving adoption within the company uh you know troubleshooting helping teams troubleshoot issues that they have in their respective Services uh you know things like that um so like in my company for instance we're probably like 70. Net 10 Java 10 go five percent python you know like we're primarily focused in one area so like we build examples in.net we I personally am more of like a.net developer my team is as well uh you know a great thing about open Telemetry my opinion is that like the concepts are extremely similar across languages but like you know I don't really know how to write Java you know it's not too far away from.net but for instance um so sometimes it's like troubling for me to provide specific feedback or like look at a PR and like really understand the scope of everything that's happening and I'm curious like if teams have this problem at the companies that they're working in do you try to like you know I don't know create champions in those languages you're not familiar in or you know communities of practice around that um just you know if anyone's experiencing this and maybe how you're you're you're solving for it I hope that makes sense there totally yeah as a as a former enabler um I I can tell you that my my strategy was um really to kind of first lead the way by showing how to enable observability or you know within my within my domain of expertise which at the time was uh go and you know as as I was demonstrating the the benefits of using open Telemetry or I guess at the time it was open tracing that if that gives you an idea of how long ago this was um but you know the one of the benefits as I was demoing the benefits people would come in ask me questions and a lot of the times these people work in different teams and different languages um and you know they would be interested in in starting out but they wouldn't be familiar with the concepts of open tracing open Telemetry and so you know my my role as I thought was to kind of pair with those people who were experts in whatever languages that they were trying to be um becoming enabler in and you know really work alongside with them until they felt comfortable with the concepts of the observability platform I was using so I think that's that's kind of my strategy it worked out pretty well I think we managed to deploy things to you know like three or four different teams within the organization and it worked pretty well yeah okay cool that's kind of I guess what I mean by like Champions so like basically leveling up you know maybe individuals or groups within those like smaller subsets and then letting them kind of run with it um do you find when you did that that those people you know as like I wasn't familiar with open tracing but like you know as the spec evolves and as new features get added do they do they come back to you do they seek information under their own do they jump in the hotel slack channels and ask questions or uh you know were they routing things through you and your experience um it's kind of a mixed bag I think I think some people feel more comfortable with just talking to the person they know and so you know you to a certain extent I became a bit of a bit of a router for questions um but you know I did see a couple of people who were really into um you know talking to the community directly and so they they just went into the slide channels or reached out to people directly in GitHub or whatever so it it's a bit of a mixed bag um you know I I think at some point as you direct people to specific issues in GitHub or specific spec changes people tend to get more comfortable with the idea that they can go there themselves but um yeah cool and then maybe one more follow-up if you don't mind like how long would you say like it took for those individuals you worked with uh to like I don't know get up to speed whatever that means in your uh opinion you know is that like a matter of weeks was that like a longer period you know how how would you engage like how easy it was for them to sort of like understand the domain when it is maybe not their like primary focus yeah um again sorry to give you maybe uh uh a mixed answer but again it was it depended on individuals I think some people who are really keen on trying to learn new ways of doing things we're really excited about the prospect of of trying out um you know something new and then you know those people kind of got up to speed a lot faster the people that were you know um you know maybe more uh used to doing things a certain way took a little bit longer um and then you know if I'm honest I think some people you know I was trying to get distributed tracing adopted some people just never really understood uh what the benefits were and I think that you know it really it really is a mixed bag so it really depends on where people are in your organizations and how resistant to change they are I think yeah cool no thank you I appreciate that because I I feel like we struggle with sometimes exactly that like they don't really understand the purpose or they don't understand how it may be has value outside of their individual team you know like when we start connecting services or something um so just trying to you know come up with any insight I can into like improving like the culture and that sort of thing so appreciate the response yeah I think you know you know I think the it's really hard to get to take someone from their day-to-day job which you know most people are already kind of strained on doing whatever it is that they're doing for the business um and asking them to do more by you know heading off into this completely what a kind of appears unnecessary aspect of their jobs right to implement open Telemetry um and it's not really until people have those aha moments of you know understanding how much better their lives will be in the future um to that they can really wrap their head around the benefits I think one of the things that I've really helped try to help people understand is you know how do you get started as quickly as possible and I think that's one of the one of the main areas of open Telemetry that I'm really really excited about is all of the auto instrumentation uh work um because I think that allows people to get some amount of benefit without a tremendous amount of investment up front and a lot of the time I find even you know even if it only gets you 60 or 70 of the way they're um it tends to give people enough of visibility into what their systems are doing through Auto instrumentation that they can then like get excited about the prospect of investing more time in doing this work yeah makes sense awesome Dan I don't really have an answer all I can say is I sympathize with you uh as someone who like codes even probably a lot less than you know what you described in your role it feels weird to kind of want to lead the way as an enabler for these things in your work without necessarily being the person who's able to kind of just all right let me let me do a couple examples for you let me kind of show you from start to finish how this looks so I don't know I'm trying to figure it out too probably further behind you actually so in my uh previous organization um like when we were trying to bring open Telemetry into uh the development teams we Face some interesting challenges where the teams recognize the importance of open Telemetry but they were constantly fighting fires so they didn't have time to instrument their code which was kind of ironic because instrumenting their code would probably help them fight those fires more effectively and so like while my my team was uh their job was to be that enabler so we you know Define the best practices and taught teams how to instrument their code they kept trying to use my team as like oh well you guys know this stuff so why don't you instrument our code for us which we had to push back really hard on because like we don't know your code so you have to instrument your code because you know it best right you know what the things are that you need to look for yeah I feel like everything like we're all talking about like we all have like somehow are in the same boat and I've just it's like super interesting because I feel like I've experienced all all of what you guys have said yeah what what a general mentioned is something that I'm experiencing as well um and even if it was wider than just one company it goes even throughout the community so sometimes I um I try to help a few open source projects to instrument uh to add these orientation to their code base because I wanted to use to have good references of what other people can can do you know so I could then point other people to that code base and say you know this is a very well instrumented application you can use it as reference for instrumenting your own application um the problem with that was some projects were they wanted to have a good instrumentation but they didn't they didn't actually um they weren't ready for it right and this is something that I'm experiencing within the company uh within you know other companies as well is that we we can try to get a network and instrument things for them but if they are not ready to have their code instrumented it's just not gonna work I don't know yep um I I could by the way you sound a little distant I I was able to hear you um but just that way um I I think that's true like I feel like first somehow we need to like sell the like if teams maybe recognize the benefit earlier like it helps in all these avenues like it helps with them understanding the purpose and therefore like maybe wanting to contribute more how to solve the problem so maybe I don't know I'm just thinking out loud here and maybe like pushing to show more like here are some examples where doing this solves some problems like if you're experiencing similar problems and perhaps like this would be a really good idea for you um maybe like you know I don't know honing in on that kind of an attack might might be beneficial yeah so is my microphone better now am I closer yeah that's much better yeah okay that's good uh yeah so um I I just came with a performing conversation with another person that um that person was mentioning something exactly like that you know um and one thing that that they're doing and I found very interesting is um they they have like an observability group and that group is uh the reference group in terms of observability and they they can watch what other people what the whole company is doing in terms of observability and what what he mentioned was um there was one team using this specific programming language and they found that one specific metric helped them recover very fast from an outage so what the observability group did then was to spread that information to the whole company and saying so teams were using that programming language you should you should you now have to include this metric here um otherwise you know because that metric helped that team to recruit were very fast from an outage so um I see the role of the neighbor here like the observation neighbor as also to spread information to get information from one title and break this item and bring to the whole company what should the whole Community for that matter um deal would it be you know as far as something like the community could do with this um would maybe some documentation around like real world cases like the one gerasi mentioned um do you think that might be helpful if you could share like similar stories with your team it it might I mean we we try to do that already I I think it's like a combination of everything that's mentioned here maybe like I don't want to say like what we're doing isn't working I think it is working it's just like progress is slower than I maybe would prefer you know um I mean yeah I think like you know having a collection of like good news stories that you know or whatever you want to call it like uh you know like hey I work at company X and we were able to solve this problem and you know reduce mean time resolution blah blah blah blah like yeah I do think like that that you know you know if people are like looking to get into the space I think those are great and I think like having those examples of like specifics um you know like as my role here as like in the observability group for my company like I go and I read blogs and stuff but I don't think like everybody at my company would do that like maybe they are reading blogs specific to like the.net runtime or something you know I don't know um so yeah I think it would it would help um I just I don't know what the best I guess format would be I'm just wondering would would it be beneficial to have like periodic like some sort of forum um where like every month or whatever or every quarter where folks from organizations that are looking to adopt open Telemetry practices open Telemetry in general um have a chance to like interact with folks in the community for for like a pointed q a um to like just to sort of you know ease their concerns like or or would you all feel like this is too much of an overlap of what this group already does I'm just wondering um because I I do feel like just putting putting folks who are kind of like not really sure about this um in touch with with other folks who have gone through this or you know have some expertise in in the area I can like put them at ease I don't know my first thought is like it does have some overlap with this group um I mean I think I don't know maybe I would need to think about that a bit more well one thing like at Rhys mentioned like she's recording this uh meeting now which I think is good like people can go back and like get insight into the past which before I think was like a there's some good topics here that are brought up that that like aren't necessarily easy to get answers because there aren't really answers like they're they're more gray if you will um I don't know I would have to think about that yeah um no I think it's it's a very valid um observator concern um and yeah I would like to Noodle on this more as well um and that said I just realized well I realized a few minutes ago that I haven't been time boxing this so I apologize um but it sounds like um we might be good on this topic Dan was there anything else you wanted to add before we move to the next one no I'm good okay can I hop in real quick just share one last thought of course um this is kind of just a more General approach for trying out our organization but we're kind of trying to drive change across various laterals through the idea of like a service catalog and scorecard for services where various like subject matter experts can Define like a set of standards in a particular domain that should apply for services and kind of provide a rubric for those things so what does like you know sufficient or like get give a service of grading in a particular set of criteria right c a A plus right and you don't have to be responsible for implementing what is like an a look like in your service but you're allowed to you know you can define those standards and then the actual service owners that are responsible for that service can consult that rubric that scorecard kind of grade their own Services according and then kind of refer to whatever standardized documentation you provide as a subject matter expert for figuring out okay how do I take my service that is maybe a a great C and observability to like a a grade A right and so in that kind of a situation you're setting forth the standards you're providing a clear rubric that allows service owners to kind of grade themselves and you're providing more information if they're looking to kind of level up uh their service so this is kind of something that is very new at our organization we're going to try to drive change in a bunch of different places things like SRA practices how you tune your services and kubernetes code quality test coverage observability stuff so we're just gonna try that out and see if that makes it kind of easier to gamify making strides um and just improving services and things like that so I don't know if that's helpful at all but it kind of separates implementation from like leading the way and improving something all right Derek I don't know if you can see this but Dan has put a thumbs up oh cool yep alrighty so since the next two have equals how more votes we'll just go with the uh in order how are you dealing with clock slash time drift yeah these are actually all mine but today um yeah um yeah so like I uh I don't know I noticed well I noticed because our backend doesn't accept data that's like in the future I think it's 10 minutes in the future um obviously people are creating data points from the future because there are times on their servers are not correct um one I guess are people experiencing this problem like the naive side of me wants to just say well like they should just fix their servers which I do agree with um if you are experiencing this do you do you notify services do you implement something in the collector to to understand that it's happening uh uh yeah that so um I can probably talk a little bit about how jiggers deals with that um and the way that eager does or used to do is um it tries to detect well so I guess the first realization is that uh clock skew is going to happen um and you just have to account for it there is no way for clocks with synchrons or synchronized especially on a microservices architecture um and one way of of dealing with that is younger chance is um it first assumes that um you don't have asynchronous processing so one Trace is uh very synchronous so the the parent span or the very first span um and uh at most when the the the the is the last span finishes you know so and if that's not the case then Jaeger by default will try to adjust the parent spans of that span uh to be as big or as as long as the longest span that it has um it did generate a lot of confusion among users especially for users who do have a synchrance processing so much that we ended up doing a flag to disable this behavior on the eager side on the eager collector side and I think we at some point thought about not having that that behavior by default so only users who would know what they're doing would then enable the clock skew I can't remember what is the feature name but uh they wouldn't have this uh clock um fixing a feature enabled um on The Collector side we're not doing anything that I know of perhaps Alex Cannon can share uh if he knows whether we are doing something like that there but uh and I guess the short answer is um the owner of the of the the system that generates the Telemetry data is in a way better position to understand the clock drift then uh any any general purpose tool like The Collector so the collector cannot in a it's not in a very good position to detect and and fix this issue for you um so yeah great thank you um so like I don't know for to me at least and maybe this is not true but um like there's sort of like two use cases here there's like like a lower mild kind of clock skew where it's like I don't know a couple seconds or something and yeah sure maybe it makes the the tracing Vision like look a little awkward or something and then there's like like the really bad you know ones that are like in this case like I I know they were like 10 minutes off um so I I was thinking of like creating a processor that detects some you know incoming data point that is like some some like distance I don't know what the right word is some you know if it's more than five minutes or some threshold uh like just I don't know creating create a data point that like captures you know the service name or something and and like you know just just at least like identifies it like hey this is a this is something that's like way out of sync with what we consider real time I don't know if like something like that would be useful um right now our back end there's no good way to like correlate like the air that our back end throws with like the data that's coming through so I don't even know like what data is missing unless a service owner were to reach out to me um it could be specific to my back end maybe my use case um I know so I was just like maybe just like having more visibility into the issue would like be a first step does that sound like a weird use case like I don't know creating a processor that would kind of detect that I think it's definitely an interesting use case if you know you could have a processor that just detects things that are out in the future um I I would guess that the the hard part is you know you still wouldn't catch all of your clock drifts I guess you would catch a very obvious ones that you kind of know about um that you've seen in the past but yeah so I I guess then yeah that's like I guess because in my case I'm like dropping data because the back end's rejecting it it would be good to know that but if there was like say it was a 10 minute threshold and it was I don't know three minutes delayed and I didn't want to detect that or I couldn't detect that now it still cause issues with the interpretation of the data uh but it would be like a secondary concern like maybe I should just Shore up like the data loss if you will and then I don't know just be better at you know having having that like like drawsi is that how you pronounce your name having like the service owners sort of interpret the drift or the skew because they're that the experts like like you said are you um are you talking specifically about metrics or are you talking about any Telemetry data type um I would have to double check I want to say the data that I was looking at in this case was spans um but in my opinion it would be for for any type yeah so for metrics there's some metrics um um stores like you know every every single permittee is based storage uh will block new metrics that are from the past so not from the future I suppose but from the past um so you would have a a warning on some logs already because of that before using Primitives um for for traces uh it might be relatively easy to to find it out if the clock drift can be detected by looking at a trace right so by looking at the spans of a Trace and I guess it would not be too hard to to make a a processor that detects that now for logs that um for logs I don't know I I wouldn't even know if we would want to have something like that for logs um because you generate such a huge amount of logs on one specific server and supposedly all of the logs on that server are having the same timestamps for things are a lot at the same time um and uh so perhaps just yeah I don't know perhaps this data it's hard to think about logs in this case because it would be having a bunch of logs at once uh and then you have to compare those logs with a batch of logs from another server right coming from another server on The Collector so your your comparison base would be huge yeah I guess my my take there would be just do the simplest thing that would work which in this case if the log time stamps are still in the future you could you could flag it you know you could create a metric that would at least capture that information but again this would only capture clock drift that's in the future nothing nothing clear would happen if it was like drifting in the past yeah makes sense okay cool thanks for the input guys I'm good with this one recent other people don't have comments excellence alrighty best practices for bifurcating data in a pipeline for example have a filter processor with positive and negative condition Clarity around Fan in yeah so this is also mine and I don't really know how to phrase this in a in a simple way um maybe I'll tell you a really weird case use case that I have and yeah anyway so like I have data flowing um let's say I have an otlp I have a collector I have an otlp receiver I have some processors configured and then I have uh a back end let's call it back end a and then I have another pipeline also otlp receiver well let's just say we're using metrics here um some processors in backend B so I want to like have um so the same set of data is flowing into both of these pipelines and then for like pipeline one I want say just to have histograms go to backend a and for pipeline two I wanted to have non-histograms go to backend B is configuring two Pipelines um like that the right thing to do um I was trying to like it works like I have this configured it works I was thinking about like is this the most CPU or memory efficient in terms of like uh I'm not I'm not I guess like the fan and fan out of how like the receivers and and stuff work in the in the in the exporters work I was trying to like you know understand if I'm doing it in like the optimal way does that question make sense at all is that a use case anyone else has um I hope that made sense there's a new type of component for The Collector that is coming up and that's called the connector so it's it's uh so the collector right now has uh four types of components right extensions and then pipeline specific components like receivers processors and exporters and there's going to be a fourth one or a fifth uh which is connectors so connectors they can act as receivers and exporters so you have one pipeline that receives otlp for instance then there is some processing and um you end up with a with an exporter and exporter for that pipeline is going to be a connector now the connector then connects with another pipeline um and as a receiver right so it's an exporter in one Pipeline and a receiver on the next pipeline um so what you can do is you can you can translate signals so you can translate its bands to metrics for instance but in your case here it would work uh in a way that you have like one receiver and then uh two exporters or two connectors as part of the same pipeline one that is going to connect this data or this this Pipeline with one that is histogram specific and one that is going to connect with a non-histogram specific with the other one and then each one of those connectors would then be able to filter what is interesting to be passed through this disconnector and block everything else so it ended up with uh three pipelines in there and so one that receives the raw data one that deals with a histogram data and one that deals with other data so this is what we have planned for the future so the basic building blocks for that are are merged already so um from what I remember so Alex can correct me if I'm wrong um but uh the basic building blocks are there and it's it's we're all looking forward to seeing connectors being ready to be used because we have so many use cases in mind to implement now what what we have today for that specific case is the routing processor so we have a routing processor and it it doesn't really work the way that you that you that you want here but it it works in a very similar way so what we do is we route the data points based on their characteristics and we rush them to specific exporters but we do that by making another network connection so it's very inefficient but it works so those are the two possible solutions for that with the router I'll obviously look into this thank you I didn't know what it is maybe this is specific use case but what would be the advantage of like the router solution versus like having a filter processor that like you know having the two pipelines and having the filter processor just like get rid of a you know part of like the part A of the set or Part B of the set would there be a advantage to you either one of those I guess that would be a third solution yeah I guess um that's what I'm doing right now um and it's working uh there were some bugs in the filter processor that have been resolved so now it seems to be working perfectly as far as I know it just I I just felt like I was because I had like two receivers receiving the same data like I was doubling up on the memory that you know my collector was using uh I haven't done any profiling but maybe that's something I should do um but I'll definitely look at the at the routing processor um actually take a look at the connectors because I I've made a very similar question to the to the author of The connectors on the pr that they introduced and I think I think indeed the the answer is that the connector is using connectors you wouldn't only have one receiver and only only on the costume only one copy of the of the data point in memory it only keeps one copy of the data point to memory let me try to find a PR and you can get more information from there okay awesome that would be great thank you that sounds like a great feature I'll definitely follow that thank you yeah that's neat I haven't heard about the connectors so I'm interested to learn more as well well jurassi is getting us that PR does anyone else have anything they'd like to add or ask well we still have 10 minutes on the clock I I do if no one else does that would if someone else obviously go first since I've asked enough I think you're in the clarity and go ahead okay um I asked this a couple of these sessions ago um I was trying to I'm trying to find I had opened a GitHub issue um it was very large in scope so I don't think I have any responses on it um it was it was it was about like understanding more about um um I'll just pop this in the zoom chat uh if you're curious um it was more about just like figuring out like how to scale my collector deployment correctly in dressy I saw that you had like a post yesterday that you posted on one of the channels that talks a little bit about that um there were some other things I had put in here like um understanding like when the num consumers setting uh for instance should be modified um I'm curious if you guys have any like insight into like when I should be I guess horizontally scaling my pod versus when I should be modifying the configuration of uh an individual pod if that or an individual collector if that makes sense like how do I what wonder what criteria do I use to determine one verse versus the other does that make any sense it's probably a loaded question as well so are you asking like at what point should the configuration be relegated to the Pod versus to The Collector uh so like I have it like an HPA configured so like as my traffic increases you know presumably I spin up new pods um but why are there settings like the number of consumers like should I be is that is that purely for non you know deployments that can't automatically scale um more so like when when do I when do I create more collectors or when do I change collector specific configuration um I'm sorry I I missed the very beginning of the question because I was looking for for the issue I found the issue and I linked here and I linked directly to the comment that I've made that is close to your question um you can read then a dance and search that um but coming coming back to this question here I'm and forgive me if I'm if I'm missing soft some context from the very beginning of the question but um the way that I that I would I would say this the way that I would tackle is um if you only have stateless connect or or components in your collector you can then just dot use a specific metric and scale up or scale adding more more replicas would be the solution um the other answer to that is you know your workload um and you would probably have to think about that not only as in the scaling to attend the demands uh like like the the load that I have but also scaling to improve my high availability so if a node goes down what am I going to what what are the effects across my observability pipeline so how would I want to isolate failures on my observability pipeline so is it better to have one per namespace is that and is that good enough or um or should I have a a one per tenant or perhaps a a even within tenants in my cluster perhaps I should have different layers of collectors um because you know failing one specific branch of your collection of collectors isn't not going to be so critical as something that is going to fail only you know if you have everything going through one collector then it's really going to be a an epic failure at one point um I guess there's no easy answer to that um yeah I I know there's no like one size fits all I guess I was just trying to like okay like so obviously like I have some like bait like some bass throughput like you know that I want to kind of I want to serve like you know I get a certain amount of data points per second or whatever uh base and then like I get spikes right so I have to like have a minimum deployment that can handle those spikes yeah but that but then there are like some connections that are I think like like have like sticky sessions so I need to be careful about like if one of those things like burst real high then like a given you know a given pod or given collector if you will will will like I don't know have to be able to to handle that all right um so in most of the cases you're going to use grpc for the connection between collectors or between your workload and a collector um and the good thing about Char PC is when a connection fails uh it will attempt to connect to another um to another backhand automatically right so if you have your deployment done correctly and correctly here means on kubernetes you wouldn't be having a a headless surveys and your clients would be then connecting to the Headless service so that the the client has a list of known backends up front so whenever one of them fails it just fails over to the next one and uh so so that's one way of of um dealing with the sticky problem they sticky session problems so most other connections they are long-lived when we talk about the observability pipeline so there are grpc connections and grpc connections they are long-lived by Design which also means that if you have like only three collectors at the very beginning of your life cycle of your cluster for instance and then you keep adding more collectors to that but you don't increase the number of clients then you're not going to see any effects until the clients reconnect uh due to some failure right so you're not going to see any advantage uh the moments that you scale up you're only going to see advantages the moment things start to fail and then you start seeing the other collectors receive some traffic or when you have new clients so new clients are going to be load balanced through the to the new nodes so then you start seeing that um yeah so I guess that's there's that uh one other thing you may consider is um charting your pipeline based on the type of processing that it's doing all right so one pattern that we've seen before and uh that I've recorded at some point is having one pipeline per data point so you have one one Matrix pipeline you have one logs Pipeline and you have one traces pipeline because the types of workloads or you know the workload for metrics is way different than the workload for for a traces um the way that they work is really different so you might want to split by that and you also might want to split by the type of processing that you do on on the Telemetry data that has been generated by your workload so if you have more pii information being generated by nodes or by pods on one specific namespace then it makes sense to have one collector on that namespace with a specific um processor to remove this pii and then goes to the next layer of collectors that deals with more generic data yeah so you're only affecting like you're only slowing down for like that subset right exactly yeah okay yeah so yeah okay cool awesome and then I know we're like almost that time so just like harping on one specific setting that num consumers thing like is there when when would I ever want to change that do you know I'm sorry um consumers as in yeah there's on the otlp exporter there's a maybe this is two technical maybe I should just open an issue just for this uh there's a property called num consumers I think it's on otlp exporter um it seemed like I got better at the root book when I increased it um and I was just wondering what like why wouldn't I increase that like what trade-off am I sacrificing okay um sorry I see now so this is about it's about the signing queue uh this is a blocking queue and um the way that it works is um whenever things are being sent from from one place to another from from an exporter and into a back end um it's placed on a queue and then things are are picked from the queue like from like workers and uh this is you know number of consumers is basically the number of workers that are picking things up from the queue now um I'm not the author of this component here of this Helper but um I we had a similar component on Jager and I can tell you by experience that we don't actually know what is the optimal value for that you have to find that out by yourself um a and it's complicated because um in in Java for instance uh the number of workers would be typically uh closely related to the number of processors in a specific like CPU processors in a machine ACP using a machine now with Goal it's not like that so one one worker thread is not a thread it's a good routine which is not related to an OS threat a Linux thread so there's no relation between or no very explicit relation between a number of consumers with the number of of uh processors on a specific laptop or a machine server bare metal um so you have to play with that number I I think I kind of played with this information on on an article recently um and the idea is that the more um if you have backhands that are are taking very long to answer to you having more consumers here means that you have more HTTP connections with the backend that you're sending data to and it might be a problem with the back end that you're you're I mean if you're the backend is having trouble with a specific amount of connections and you're adding more connections you're adding more load to a server that is already overloaded in that case you want to decrease the number of consumers but what it means is it is processing less data um um simultaneously concurrently so what is effectively means is it it increases the concurrency of data being sent to the remote server so in theory it is related to the CPU number of CPUs that you have uh but at the same time I think it is more important to the back end that we are talking to okay yeah so awesome that's like super helpful I I think the main takeaway for me is like I should probably be doing some more specific testing around like my expected you know like how much data I'm receiving and expecting to send to my back ends and just you know trying to optimize for my specific use case rather than like having a generic uh this type of situation you know you should set it to X Y or Z absolutely okay awesome really really appreciate the Insight thank you Brad thank you thank you all all or a few minutes a few minutes or so I want to be respect to respect everyone's time um Dan thank you so much for all your questions um Alex gerasi it was great to have you on and um if anyone has any questions about these sessions or anything else they are curious myself Adriana as well as Rin are all on the end user working group so feel free to reach out to any of us on cnco slack um and for I I just have to drop real quick but um Alex and C wanted to see the kitten I'm just gonna do a quick kitten kitten show this is Taco thank you hi taco and all right thank you all so much diff --git a/video-transcripts/transcripts/2023-10-02T15:08:33Z-otel-in-practice-presents-a-hands-on-otel-migration-story-with-jacob-aronoff.md b/video-transcripts/transcripts/2023-10-02T15:08:33Z-otel-in-practice-presents-a-hands-on-otel-migration-story-with-jacob-aronoff.md new file mode 100644 index 0000000..52adf9b --- /dev/null +++ b/video-transcripts/transcripts/2023-10-02T15:08:33Z-otel-in-practice-presents-a-hands-on-otel-migration-story-with-jacob-aronoff.md @@ -0,0 +1,125 @@ +# OTEL in Practice Presents: A Hands-On OTel Migration Story with Jacob Aronoff + +Published on 2023-10-02T15:08:33Z + +## Description + +What happens when you're an Observability vendor migrating to OpenTelemetry? Jacob Aronoff knows exactly what that's like, ... + +URL: https://www.youtube.com/watch?v=pHHINe9D94w + +## Summary + +In this YouTube video, Jacob Arof, a staff software engineer at Lightstep, discusses the migration from StatsD and OpenTracing to OpenTelemetry, focusing on the challenges and strategies involved in the process. Jacob outlines various migration paths, such as the "All or Nothing" approach, the "Slow Tail" method, and using a bridge for gradual migration. He emphasizes the importance of gaining organizational buy-in, the need for thorough testing, and the benefits of OpenTelemetry's API support across multiple programming languages. Jacob also shares personal insights from their migration journey, including performance issues encountered and solutions implemented. The session encourages audience participation, with attendees asking questions about practical challenges and seeking advice on transitioning to OpenTelemetry effectively. The video concludes by inviting viewers to engage with the OpenTelemetry community and participate in future discussions. + +## Chapters + +Here's a summary of the key moments from the livestream: + +00:00:00 Welcome and Introductions +00:01:30 Jacob's Background and Role +00:02:45 Overview of the Migration from StatsD to OpenTelemetry +00:05:00 Challenges in Migration and Gaining Organizational Buy-in +00:08:15 Different Migration Paths Explained +00:12:30 Discussion on the All or Nothing Migration Approach +00:16:45 Exploring the Slow Tail Migration Method +00:20:00 Advantages and Disadvantages of the Bridge Approach +00:25:00 Audience Questions on Migration Experiences +00:30:00 Jacob Shares His Team's Migration Process and Insights +00:35:00 Discussion on OpenTelemetry Protocols and Collectors +00:40:00 Managing Access Tokens and Security in OpenTelemetry +00:45:00 Future Plans for Logging Migration and Infrastructure Updates +00:50:00 Closing Remarks and Encouragement for Further Discussion + +Feel free to ask if you need more details or have other questions! + +# OpenTelemetry in Practice + +Welcome everyone to OpenTelemetry in Practice! I'm really excited about such a great turnout; I think this is one of our best turnouts yet. For anyone who couldn't make it today, there will be a recording available. If you have friends who missed this, please let them know we will be posting the recording soon and polishing it up once it's ready. + +Jacob presented last month for the OpenTelemetry Q&A. We worked together at Lightstep, which is now part of ServiceNow. Without further ado, I'll let Jacob take it away. + +--- + +## Presentation by Jacob Arof + +Thank you! My name is Jacob Arof, and I am a Staff Software Engineer working on our Telemetry Pipeline team. Our team is responsible for our internal and external open-source telemetry efforts, including OpenTelemetry SDKs in Go, the Collector, and the Operator. Essentially, we handle everything you would use to collect and send data to your vendor. + +Today, I’m going to talk about a migration we led last year, transitioning from StatsD to OpenTelemetry and from OpenTracing to OpenTelemetry. I’ll cover various migration paths, what went well, what went wrong during our migration, and I'm open to questions throughout, so please feel free to ask. My slides are pretty light, and I'm interested to hear about the challenges you're facing and the features you're interested in. + +Let's dive into the migration story. + +### The Problem + +I’ve been in the industry for many years and have led several migrations where hand-rolled metrics, tracing, or logging libraries just aren’t cutting it anymore. Some common issues include performance not being sufficient, high maintenance costs, or features that everyone wants but can't be implemented. Depending on the number of services your company runs and the organization's willingness to undergo migration, these can be daunting challenges. + +This theme resonates with many engineers, especially SREs or DevOps folks who are passionate about making things work but struggle to get the necessary buy-in. The solution, of course, is to migrate to something new, like OpenTelemetry's metric API, trace API, or logging API, which are accepted by most major vendors out of the box. + +### Migration Challenges + +When considering migration, you might face several questions: + +- How do you break up the work? +- How do you ensure all your telemetry works the same as before? +- What are the risks of different migration models? +- How do you convince leadership that this is worth doing? + +Trying to brute-force your way into a migration can lead to burnout. Hence, the first step is making a convincing case for migration. Sometimes, a proof of concept helps, but the key is getting a group of people who agree with you and can help communicate the value to stakeholders. + +In my case, we were a main OpenTelemetry contributor. The OpenTelemetry metrics API and SDK were looking for quick feedback from a team with high traffic, making it easy to sell the migration. In past roles, the selling point often revolved around maintenance costs and performance improvements, especially when we counted the number of tickets and hours spent on features and maintenance. + +### Migration Paths + +Let's discuss the migration paths we considered: + +1. **All or Nothing**: + - This method involves completely ripping out existing instrumentation in favor of OpenTelemetry. + - Pros: You reduce the time for split-brain scenarios, where some services are on one system, and others on another. It makes issues very visible since any problems will be apparent immediately. + - Cons: This requires thorough testing in a robust development environment. If staging and production environments differ significantly, you may face production bugs that you couldn't catch in staging. + +2. **Slow Tail**: + - This method allows application developers to migrate themselves by toggling an environment variable. + - Pros: Less upfront work and the ability to confirm functionality for one service at a time. It also gives you more time to develop dashboards and alerts. + - Cons: This can be slow, especially in larger environments. Bugs may not reveal themselves uniformly if services are not instrumented the same way. + +3. **Bridge**: + - OpenTelemetry provides a bridge for some migrations, allowing you to transition from one instrumentation to another without significant code changes. + - Pros: Minimal code changes required. + - Cons: You may experience performance drawbacks compared to using the new method directly. + +### Transition to OpenTelemetry + +In our case, we began migrating from StatsD to OpenTelemetry metrics using the Slow Tail approach. We had several motivating factors, including wanting to utilize new metrics features that StatsD didn’t support, like asynchronous instruments and exponential histograms. + +While we started our tracing migration, we faced some performance issues due to the wrapper library we were using, which required conversion to OpenTelemetry tags. This led us to begin our tracing migration using the All or Nothing approach, as tracing had been stable for some time. + +We wrote the code for the migration and pushed an image to our staging and production environments, confirming that each version looked the same before and after the migration. This was a crucial step, as it allowed us to catch integration issues early on. + +### Next Steps + +Currently, we are focusing on migrating our infrastructure metrics to use OpenTelemetry's first receivers and plan to migrate to the new logging cases soon. + +--- + +### Questions and Discussion + +Do we have any questions? + +**Question from Rahul**: I'm currently experiencing challenges in shifting mindsets from our previous vendor to OpenTelemetry. Any tips on winning over various developer teams? + +**Jacob's Answer**: It's crucial to sell on features. For instance, highlight cost savings—if you’re moving from a service like DataDog to Prometheus, the cost difference can be significant. Additionally, emphasize the local development experience for metrics and the ease of using Grafana for dashboards. Building local tooling around metrics can also help developers see the benefits firsthand. + +**Question from Jay**: What is the best protocol to use within OpenTelemetry for performance and security? + +**Jacob's Answer**: We typically use gRPC for our setups, but it can depend on your company's security requirements. I recommend using the OpenTelemetry Collector where possible, as it offers numerous options for exporting data to your desired backend. + +--- + +As we wrap up, I want to thank everyone for their participation today. If you have any further questions or would like to share your OpenTelemetry use cases, feel free to reach out! + +Thank you, Jacob, for sharing your insights and experiences with us. We appreciate your time and expertise. Have a great day, everyone! + +## Raw YouTube Transcript + +welcome everyone to otel in practice um really stoked for such a good turnout I think this is one of our best turnouts yay um and for anyone who uh could make it today there will be a recording so if you have friends who are like damn it I missed this you can let them know there that we will be posting the the recording um and prettying it up once it's available so uh Jacob presented um I want to say last month for otel Q&A we worked together at lightstep from service now um and uh I'll let's Jacob take it away yeah thank you um so yeah my name is Jacob arof I am a staff software engineer um I work on our Telemetry pipeline team uh our team is sort of in charge of our internal and external open source Telemetry efforts whether it's like OTL sdks and go or The Collector or the operator um sort of anything that you would be using to collect and send data to your vendor is what we do so today I'm going to be talking about a migration that we LED last year and a little bit year and a half ago uh for when we migrated from uh stasy to open Telemetry and also from open tracing to open Telemetry going to talk about the you know various migration paths that are out there some of the things that went well and went wrong or our migration uh but I also would like this to be you know feel free to ask as many questions as you have um my slides are pretty light so I'm really interested to here you know where people struggle currently uh what type of challenges you're running into what features you're interested in um so please you know raise a hand or you know comment in the chat uh and I'd be happy to answer any questions and go down any interesting avenues that you might have so with that I will share my screen uh can we all see my slides yes great thank you very much so yeah today I'll be talking about that oel migration story so first let's talk about a problem uh I've been in the industry for you know many years many years um and I've had to lead a few of these migrations where your handrolled metrics or tracing or logging Library just isn't cutting it anymore something where it's not performant enough maybe the maintenance is really expensive maybe um there's a feature that everybody wants that you just can't Implement um you know whatever it might be you want to do a migration um it can be very challenging and daunting depending on the amount of services that your company runs depending on what the organization's uh you know willingness to like undergo this migration might be uh so all of these are problems that I've faced um in various jobs not just where I am now but you know a few past roles as well uh it can be really hard to make this happen especially when you know you as maybe an SRE or a devops person or maybe just a you know engineer just want to make this work uh when you feel very passionate about making it happen and you just can't seem to get the buying that you need um so that is sort of the theme for today um and there is a solution right it's like migrating to this new thing um otels metric API I'm going be talking about metrics traces but you know this is really true about a lot of stuff um for otel the metric API the traces API and even the logging API is accepted by most major vendors out of the box now um the API is support for you know all these instrument types um in all of your favorite languages probably they're very few that don't um and the performance and compatibility is always top of mind for us um throughout the stack of otel um components everybody is always thinking about performance um so you know with all this in mind you're like yeah this sounds great uh I want to start using this um but there's a problem right like using these new tools migrating these new tools is hard how do you break up the work how do you know all of your Telemetry is working the same as before uh what are the risks of these different migration models what even are the different migration models um and probably the most important one here is how do you convinced leadership that this is worth doing um you can try and you know brute force your way into doing a migration but that to me is a recipe for Burnout um doing a migration in general where you know there's a new architecture that you think is really going to improve uh quality of life for your team for your company uh without getting organizational buyin is how you spend months in a project that may never see the light of day and that is really demoralizing and very difficult to work around so the first thing that you have to do when you want to migrate to a new tool is make the case um it's you know not even um you know show a proof of concept it's just can you convince people that this is worth doing sometimes a proof concept helps but that's not you know the key to success the key to success is really getting a group of people who agree with you that you're able to sell one-on-one who can help you really make it clear to the stakeholders uh you know in your leadership that this is worth doing um for me was pretty straightforward we're you know a main open tary contributor uh the oot metrics API and SDK wanted to go uh into their you know 1.0 stable um and they really wanted some quick feedback from someone who has a lot of traffic uh to test it out and see where the edges are so so in that case it was very easy to sell the migration in past jobs the selling point is usually you know you look at maintenance cost uh and performance overall um I worked with a handrolled uh solution uh in few jobs ago and when it came time to do a migration the thing that really sold people was just counting the amount of tickets and hours that we have to spend on you know new features and maintenance on our internal Library um as well as the amount of times where you know we've had an incident because something related to our instrumentation is just incorrect which you know does happen a lot with your own handrolled stuff so that is maybe a good basis to go off of uh does anyone have any questions sort of before I go into more specifics about this section and uh I'm going to do like a five count something to learned from a teacher of mine uh I'm just going to we're going to do 5 Seconds of Silence um until someone raises their hand or I'll just keep going so no cool oh is there something in the chat oh that's just me moving it a little cool no worries uh thank you um so let's talk about the migration paths um the first one is is what I call the All or Nothing um this method has you entirely rip out your existing instrumentation in favor of otel this would be you know a lot of people talk about you know replacing you know the engine of a of a running plane or running car that a lot of people will say that um this would be like you know just getting a new plane and have every and you have everyone hop to the new one while it's flying so it's difficult but you know maybe it has its benefits one of the pros is that you know once you've pushed your code and you've confirmed things are working and you have a good enough cicd system your work is really done right it just rolls out and everybody's happy this reduces the time for split brain split brain is what happens when you're on two systems that may not be compatible together you could imagine you know if you're on stats D or if you're on Prometheus even um if most of your metrics come from Prometheus they don't have periods in them they have you know pretty Str requirements about um their shape overall um you can't do things like um up down counters like that's not a type that they have um they used to not have a proper histogram they now do uh they now have an exponential histogram support which is experimental but you know there are things that otel has that um Prometheus or statsd just does not have and doesn't have the ability to do um so being in a split brain where some Services have it and some Services don't and they're emitting different metrics and different shapes different variables um that can get pretty unwieldy pretty fast and so if you're not very deliberate about planning how you're going to migrate your dashboards and alerts then you're going to be stuck with both of them for some time which if you're on call and if you've been on call during a migration for this type of stuff that gets really painful if you get paged at 3:00 a. you have to wake up and go oh you know which uh which dashboard should I check for this is the service on otel or is the service on statsd um that's really frustrating uh that that's the type of thing that you don't want to have an on call engineer uh think about so the other benefit of doing this All or Nothing approach is that the issues are incredibly visible if a dashboard breaks if an alert Pages or a service crashes um it's pretty obvious right hopefully you're looking at a dashboard or you know the person you take the pager when you're doing this might ation so that you experience that pain and hopefully you also page on no data it's very important um and then service crashes you should have an alert on that ideally um so all of these things though make it really clear that as soon as you do this All or Nothing thing um you know if all of your services are rolling out within an hour with this new change and everything is looking good that's a very good sign and that that gives a lot of confidence um this does though you know the Clauses here this requires a lot more thorough testing in a really good development environment if you're unable if you have a lot of uh environment drift where your staging environment is entirely different than your public environment this might be really challenging because this means you might have uh production bugs that you're just unable to catch in staging um and if you do if the blast radius is all of your services that can be really dangerous um also you know you do have those compatibility problems I mentioned and so you have to be very deliberate about um observing which dashboards and alerts break and fixing them proactively um or even in advance if you know what the metrics are going to be um the example I have here is is definitely a common one where you could imagine a metric type changes but a metric name doesn't which most vendors will just reject um or the dashboard that you're looking at will look very strange um this also does mean that there's more upfront effort to migrate your services it's going to take you know probably a group of people to help you monitor this depending on how many services you have to Mig if you're you know a company with maybe five to 10 Services that's not too bad to do with one person but if you're a company with hundreds of services you're going to have you're going to want a team of people to uh monitor this with you so on to the next one we have the slow tail um this method gives your application developers the ability to migrate themselves by usually flipping an environment variable on and off for whichever method they want um benefit here is that there's less upfront work uh you can confirm that it works for one service and push it out for that service only and you can do that in all of your environments so if you have that um you know if you don't have that confidence that I mentioned earlier about your saging environment versus your production environment um this would be really helpful because you could actually just push a single service without the fear of you know all these other services rolling out to verify that your change is worked as expected and finally this also allows you to have more time to develop dashboards and alerts to handle those compatibility issues problem is is that this can be very slow um if you have more than 50 services in you know at least two environments and it takes an hour to migrate a single service because you're trying to be extra careful then you're looking at a multiple weeks or months long process um if you leave a migration to app developers without a strong why also they'll never do it so it's usually going to fall on your team to make that happen um bugs Also may not reveal themselves if your services are not uniform imagine you have something like a q worker that works off of kofka uh which whose topology looks entirely different than something like you know a classic API server um if you're only you know instrumenting your API servers and then you go to instrument one of those kofka servers if there's something that's significantly different in how you did your instrumentation that might take some time to figure out uh what's going wrong um you know ideally otel um has figured out a lot of this but doing any of these migrations you always need to check um you know we just cannot and honestly should not know how you instrument your services um you know you don't want to have to explain your whole um observability backend to us that doesn't really make much sense so ultimately it's important that you do some work to check that the migration is working as you expected uh one thing that I did just think of is that you could do a combination of this slow tail all-in-one where you do a migration for you know say some good sample of your services and then once you're pretty confident with those you could move to just enabling for everyone at once um that's another good option uh it I'm trying to think about the drawbacks of that not sure there are any I think that's probably the way to go unless you're really nervous about um some of these compatibility problems or some of these like unknown unknowns that might be the only time that that would be a little uh scary so the last one is the brdge um open Telemetry for some migrations actually provides a bridge where you can go from you know instrumentation a to instrumentation B um without having to do any real code changes other than implementing the bridge there are a few issues here um well already you can see the pros I mean it's pretty obvious right you don't have to make many code changes problem is is that you're going to have some worse performance in comparison to writing using the new method um it's going to be you know a fair you know there's a conversion cost to anything that you're doing in your application um if another option is you could just send from you know instrumentation a and then convert it to instumentation B with otel collector so if you wanted to go from stats d The otel Collector has a stats D receiver and an otel exporter so that's also fine but both of these have the same drawback which is that if you wanted to take advantage of some of the capabilities of the otel sdks you know then this Migra you're not really doing a migration um it means that you can you know try and migrate stuff um Peace meal and like for dashboards and alerts which is great but it does have this drawback of like you're not actually doing the migration you're still you know just putting off the hard work later um and so this can also be confusing to your app developers where someone says well my code says open tracing but this trace says Hotel why can't I you know use x feature um and that that can be a little confusing I think that's like not the end of the world though it's it's you know as long as you're communicating well that should be all right um but yeah so this is really I think using the collector is another really good path forward overall though um doing the thing where you just send some traffic to The Collector from one of your services uh you migrate your apps and dashboards and then you have everything sent to the collector and then you can migrate your apps to otel with that allinone approach um and then your dashboards and alerts should just you know already be changed and you should be all set before I move to the next portion do I have any questions any thoughts I'm going do a five C one question Jacob I'm actually living through this right now um one thing that I'm running into quite a bit is like getting the various developer teams to shift their mindset from sort of our previous vendor that we were with into our newer one and it's you know there's underlying things we're doing like the statsd into like premier Prometheus style type thing with it and yeah that's led to a couple people who are like oh these things aren't apples to apples here anymore and that's causing a bunch of stuff um any tips and suggestions on like winning the hearts and Minds here yeah so really like the thing to do is sell on features when you can so if you could say you know this improves our well the easiest one is cost right if you're using staty you're probably coming from data dog you're going to say you know our previous spend with data dog was X thousands maybe millions of dollars and with Prometheus it's like you know 100 100 a month or something right that's the easiest one um but the more valuable one is you know you sell in the ecosystem where um for me the real benefit of Prometheus is that um the local development experience for metrics is actually much simpler um you don't have to run you know a data dog agent you don't have to do anything else you can just hit your metrics endpoint and verify that your metrics are doing what you expect them to um and that that's a really good developer flow um for testing that stuff um the other thing that is I I mean people from data dog I used to work for data dog um and they their dashboard product is great um but grafana if you're using grafana also has a really strong dashboards product showing something like you know maybe you use redus in your Cube cluster um you install a redus uh service Monitor and then you install the off-the-shelf grafana dashboard for it right like that's a pretty great experience and that's all done at you know zero which is pretty incredible you know you know Prometheus metric cost which is very small in cents on the dollar um so the last thing you can do is like trainings I I have a storied history with trainings I think that they never really achieve the thing that you want um which is for more people to be excited um really what they do is can they can cause more confusion if you're not careful uh with like your language um I think maybe a strategy that I've always wanted to do is like build some local tooling around the stuff to developers um whether it's you know writing maybe an endtoend test or a little UI around their applications uh metrics so they can do something with them locally to be like oh yes like this thing is working as expected um I mean I think that the local Dev for Prometheus is just pretty fantastic um and that's probably what I would sell on um but again you know it's so it's very company specific and in many ways right if people are really bought into the datadog model of things which is you know high cost very low thinking um Prometheus and grafana is not really that it's it's low cost but much more thinking and ultimately like you don't want your developers to have to think too much about their instrumentation um the thing that uh I tend to do is have a uh rapper library before doing a migration um so that people are used to the people are using the same signatures it's just doing a different function under the hood it also makes your migration easier um I didn't mention it here because most companies already have some type of wrapper because they have some needs that are like specific um it's usually like a thin wrapper but doing as much as you can to not change their workflow and make it very simple is always going to be good um the thing with Prometheus you can do is check what metrics are available and then you can autogenerate dashboards from those metrics and uh you can add into your um Helm charts you know these are the metrics I care about and then you could just generate alerts automatically from that as well um so there's a lot of that quality of life stuff that again like it's easy with the data dog UI but is automatic with um you know infrastructure as code so that's like another trade-off I would say because not a lot of people I don't think use terraform at data dog um so I don't know maybe they do and I just don't know that oh thanks for that check it yeah problem uh any more questions on this part cool okay so now I'll talk about what I did what the team did um to forour migration from statsd to otel or metrics obviously um so so we began migrating using that slow tail path um the reasons were uh I'd say good motivating factors as I mentioned earlier the metrix API andd weren't declared stable at the time which meant we would have had to deal with some signature changes um and that would have been potentially a lot of signatures uh we did have a wrapper library but still doing those types of changes can be pretty frustrating um we also wanted to use some new metrics features that statsd didn't support this is like asynchronous instruments um the biggest one was exponential histograms which otel sort of pioneered um and then the last one is that we would you know because we are the group that helps write these libraries uh we wanted to understand the performance and quality of life features to make it as easy as possible and as you know um as quick a decision it's just you know you don't have to worry about performance you don't have to worry about all this other stuff how do we make this simple um so in migrating to otel for metrics we did find a few performance issues um the first one was you know because we were using that rapper Library I mentioned um our implementation was working off of Open tracing tags which at the time was our tracing instrumentation which we then would need to convert into otel tags anytime you wanted to make a metric which if you think about the amount of times that you call you know metric. record um that's a lot of time so that gets pretty expensive um and so while we waited for improved performance in the metric libraries uh we just began our tracing migration because the otel team needed time to investigate some of the stuff that we brought to them um and it also meant that we could fix you know the very um our company specific problem of this uh conversion and so then we began our tracing migration um going from open tracing an open sensus to otel and so we went for this one with All or Nothing approach because otel for tracing had been stable for some time weren't really concerned about compatibility um and ultimately at the time there weren't a lot of guides written um but the process was relatively straightforward and the compiler is really doing most of the work for you um the structure for it we already supported oel traces in our product so there's actually like no in theory there were no real fun changes either uh for our own dashboards and alerts that we were going off of so what I did for this was write some write the code for the migration um and then push up build and push an image to our staging and production environments for a service that doesn't get much traffic does get some it's important that it gets some um and then it confirmed that each version in our product looked the same before and after we have a view uh of sort of you know these red metrics per version and it it shows you the difference in those versions and if any of those looked different if the rate for example uh dropped off um then that's a sign that we did something totally incorrect um doing that for one surface is good but then the reason that the you know the All or Nothing approach is really useful is that you get that integration test so when we when I migrated all of our services to this new method it was really clear that service to service Trace propagation wasn't working you know you just did a spot check of a few different um traces and it's really that's like such an obvious thing um so that was a pretty easy fix I actually had to just go into the oel community um and Implement something that uh had a Todo around it um another issue that we had right before you know sort of in the the last bit of this migration was that our sampler wasn't config configured correctly um otel provides a lot of new features for sampling and I was under sampling in one environment and then oversampling in another environment uh because I misconfigured it so I had to roll that back really quickly fix it and then push it out a day or two later but overall this whole process took maybe a month of work um to migrate I think it's like a hundred or so services in all of our environments um which is if you've ever led a migration that that's a pretty good time I was pretty happy with that one it also meant that we could get back to our metrics migration uh because we had sort of achieved that goal with open tracing tags um we were able to use those use the fact that we didn't need do this conversion to continue our metrix migration it also we came back to the otel team having shipped some real performance and quality of life improvements um that really let us continue with this without the fear of performance problems um these changes also let us use this feature called metrics views uh which let you create a new metric series to provide seamless compatibility with statsd statsd emits their histogram is what's really what's called a Prometheus summary um but we wanted to Emmit exponential histograms but that's the really the only change that was happening here um all of our other metrics were were able to say about the same so uh we just needed to dual write the old summary which otel didn't really support at the time and I think doesn't and shouldn't it's a bad metric type um and we also wanted to dualite the exponential histogram so that we could migrate all of our dashboards and alerts from this old approach to this new approach um and then finally you know as I said we put this Library behind a fature flag and then we could just flip that on and off whenever we wanted um and then we would roll it out pretty slowly over a two or three month period um to all of our environments this also let us like test the change really effectively as well um I think what's next let me go back so any questions about this migration process do five count again Jacob this is Rahul um hi I have one question around the OTP protocol so I'm guessing you must have used the OTP receivers and exporters vastly during your metrics and Trace migration so what is the go-to protocol within OTP it supports stdp and grpc but from security and performance point of view which is the go-to protocol for traces and metrics uh I'm not sure I follow we we actually just um emitted OTP directly from our sdks to our SAS so we didn't go through a collector for this one um we could have gone through a collector but uh we didn't want the operational overhead at the time it was enough migrations to to handle it once but that didn't really answer your question can you rest your question yeah I mean I wanted to know um what is the best protocol to use under OTP is it stdp or grpc um in terms of metrics and traces yeah in terms of performance and security yeah so I'm not the best with for security recommendations um I would say we use grpc or everything um just because it's I don't know our sort of deao internal standard um HTTP is more accepted for some like depending on your company security requirements um I'm not sure of the real differences like the like to compare and contrast them I don't think I'd be um the right person to speak to those maybe some of the other oel folks have more of an opinion or more information on that though yeah don't okay cool were there any learnings around managing access tokens um and did you use multiple access tokens uh does it portal you know um if you're sending a lot of metrics or traces or a single access token yeah we uh we just used um the same access to token approach that we did um where I I don't know if I can speak to that just because it's it's you know internal security stuff um yeah sorry about that yeah no way I'd say that you know the best thing you can do for security in like a cloud mative environment um is use some sort of Secrets provider um the kubernetes external Secrets operator um is great uh you can hook it up to something like gcps uh KMS um and um decrypts inversion your secrets um to then load access tokens from though um you can do the same thing with AWS or vault or any of these other providers as well um if you're particularly like security inclined it's also important to use things like mtls as well um something like ISO can help you with some of that um the otel operator actually provides some mtls features in open shift um so you know there there are a lot of security features out there to be used um hopefully that that seeds some interesting um investigation for you yep yep thanks no problem um so moving on uh you might be wondering what's next are we going to do a logs migration um and right now we're under a different migration which is changing our infrastructure metrics to use the otel first receivers like the CP cluster receiver the CET receiver um and so forth um because we really want to start using some of these things the community is writing um after that we should be able to begin migrating to use the new oo loging case which um should be looking pretty good in a few they're looking good now but um I'm not sure what the state of it is uh for go just yet given that go just released like a new um standard logging Library um yeah any more questions hi this is uh Jay speaking um I have one general question um uh regarding open telary so um I I'm pretty new to this new to open tet Tre and uh the the reason why I was investigating into open Telemetry was uh for its ability to be backend agnostic for generating uh metric uh data uh the question is however uh I was interested in exploring po based uh metrics uh uh exporters uh but uh uh so far uh I don't if I'm right or wrong but the promethus export exporter within the open Telemetry SDK is the only one that is supporting uh the pull based metrics approach is that correct so um it sounds like you're going so there are a few different types of exporters within um uh within Hotel so there's an SDK exporter um which yeah there's a prus exporter I think there might even be a D exporter um but one what might be a better fit is to use the otel collector and use their exporters which are numerous and and pretty much every backend um has some type of exporter um so I think I would try and and say you should if you're GNA use otel sdks you should export an OTP to an Noel collector and then export to you know your protocol of choice um yeah but when using this uhel exporter this would rather be a push based mechanism right uh to send data to The oel Collector yeah that's correct okay okay uh you can also if your instrumentation is in Prometheus right now though um you can have the otel collector scrape the Prometheus metrics and then export them as OTP or export them as statd or you know whatever you want good thank you any more question do you use any solution to manage or you know update the AML file um of the fleet of agent Fleet um on the go and is there any built-in solution that you guys are using to manage the collectors yeah so um the I am a maintainer for the oel operator um and internally and externally I recommend using the oel operator with the otel collector crds um they're pretty uh easy to use easy to set up easy to manage um and we're always developing and thinking about new features as well um and so that that'll be the place to get those uh now and in the future and I'd continue recommending that okay thanks Jacob could ask a question following on from the last question um would you recommend using something like opamp in terms of managing your Fleet of collectors or would you say it's preferable to do it in an operator basis you know kind the C pattern yeah so if you're in kubernetes um OPM by the way is still pretty early Alpha right now well the protocol itself stable but the actual implementations are in Alpha um so I'm going to answer this question with the assumption that the implementations are done if that's all right sure yeah um so if you're in kubernetes I would recommend using the opamp bridge that we're developing um the bridge is a component that can connect to your vendor um and will be able to manage uh pools of collectors rather than just having an extension that works on or a supervisor and an extension that works on a single collector pod um the reason for that is usually you are running a collector um and what you you're running a collector in a pool not as a monolith and so whereas opamp would be opamp on a using a supervisor would be very useful for um you know a a VM or just running it as a binary um doing it in kubernetes if you're running it as a pool it's not a great pattern in kubernetes to have a supervisor update a single pods configuration to make it not uniform with the other pods in its replica set so what that means is you know if you're going to run it in kubernetes if you're going to run a replica set of pods in kubernetes you want those pods to be the same configuration and if you're only running a supervisor um if you're running a supervisor on each of those pods but you're only making the change to a single one um that's an anti-pattern and can get you into some trouble um so the bridge however can manage pools of col ctors um and is definitely what I would recommend to use again with this stuff being completed that's what I would recommend for kubernetes okay thank you no problem I've got a few questions um related to the work so I work at a large US Bank and I'm trying to bring in hotel um and essentially have that as our main strategy um to try and move away from B on vendors specifically um the question that we running into just now are the challenges like vanilla versus vendor and what I mean by that is um do we just pull down if I take the collector for example do we just pull down the collector configure The Collector the way that we want it with receivers and exporters um or with the specific the Strategic vendors that we work with uh do we look to bring their vendor wrapped collector and deploy that within our Enterprise there's obviously pros and cons in doing both it doesn't sound like you know from your side Jacob obviously you're working from the vendor side but I have spoken to other vendors who have given me an interesting range of opinions on what what area or which of those to look at um it'd be quite interesting to see what your thoughts are yeah this this is a great uh really great question um it totally depends on your deployment model I would say so one option especially you know if you're running thousands of PODS and you know hundreds of clusters um the model that I would recommend you use is the Gateway model where you know let's say you have a collector per group of apps and then all of those collectors forward to a centralized pool of collectors that then forward to your vendor um this means that your uh those pools for your applications are vendor neutral um because all they're really doing is gathering in forwarding stuff um for your application teams and then a centralized team would manage the Gateway uh the Gateway collectors and then that's the place where you make the decision about am I going to use my vendor wrapped collector or am I going to use my uh you know vendor neutral one the Choice becomes really easy to you know if for some there's some feature that your vendor provides um that's only available in their WS collector um you could choose to use the rapid collector there and then in the future if you wanted to change vendors you still have OTL data sending to that vender collector and so you can just change that one out very very easy the reason that this is good is because you wouldn't have to go to your application teams um or any of these other you know Orcs and say hey you know you have to reconfigure your whole um setup because we're we're changing our underlying vendor here whereas you as the centralized team could be the one to just change a single pool to make it all um consistent um that's probably the what I that that's like a pretty uh future proofed approach um the conf you know the configuration for what what you're going to do no matter what is going to be complicated um but that one is probably going to do you best if you really want to use the vendor collector if not still doing the Gateway approach is a pretty good one um you can centralize things like officiation sampling rates um or even like you know requirements for Telemetry things like attribute requirements um can be centralized before they can egress um it also means you can have you know set points of egress as well um um which you know if you're in a pretty lock down kubernetes cluster as you know and is is really important to have um you know only a certain amount of applications that can erress from the cluster okay I think um I think I definitely follow in terms of the the deployment patterns and layouts and having the multiple level of collector I think I think that's where we are thinking and going I think for me being in the Enterprise what what we are worried about is again moving stuff like this to production so if we have a theoretical issue in in a uh H sorry a vanilla collector you know again just theoretical example if we've got RTO and we've got to fix issues within a one or two hours for example um that's probably going to be the Tipping Point for us on the vendor versus vanilla question and because we we we would obviously be looking for some kind of support uh from a vender and typically we we would have that as as most folks would have through a vendor but then that then it certainly in my mind gives us another problem where instead of going from agent Poli proliferation where you are just now it's almost like going to Hotel proliferation you know it's almost like you're solving one problem and creating another so um I think like the others on the call you know we we're relatively early in in our journey and we're just trying to to go through the not necessarily the technical questions but the hardening questions what what what would reality look like when we're in production with with you know very high volumes of traffic coming through yeah I mean that that's it sounds like you're on the right path here overall um as far as you're thinking going um there definitely is that worry of like collector proliferation um you can avoid that you know multiple pools to gateways if you'd like um if you're going from I mean usually you're doing this if you're going for like a legacy protocol something like statsd which doesn't like to go over the Internet because it's UDP um or something like preus where you have all of these targets and you don't want to worry about managing the scrapes for them right um doing this at scale is going to be really environment and volume dependent um I think if your vendor provides their own collector they should be able to give you some support for um you know the vanilla collectors that you run I you know with the people that I work with do give support for you know whatever collectors they run and I mean we don't have a vendor specific collector like like like uh our company just doesn't give out a vendor specific one um but I you know I provide support for for any collectors that are customer runs um so if that's the fear I would check with your vendor to see they also will give you that type of support um I also found that like the steady state of these things is pretty once you tune it with uh resourcing and autoscaling um it's pretty hands-off I found um I actually was just working yesterday in a cluster that I touch every six months or so to do um some like Helm chart testing and it's been running for six months without issue um and with like a huge varying scale of traffic um because of autoscaling and and sort of just because of how simple we keep I I keep those collectors um this is for both metrics and traces too um for infrastructure and and application so I mean it's a much smaller example than than what you're talking about for sure um but the point remains where it's like once you reach a good steady state um especially with like your the Bal cycle um if that's what you have um if you're Autos scaling the setup correctly and if your configuration is is pretty uh you know nailed down you should be it should be pretty hands off knock on wood but that's definitely the hope yeah yeah okay thanks Jacob appreciate that yeah no problem thank you so folks we are coming up on time so we've got about five more minutes in case anyone has any more burning questions for Jacob Al righty I will take that as a no but uh thank you Jacob so much for um for joining today and sharing your migration story I think uh I think this has resonated with a lot of folks so we definitely appreciate you um coming on and and and sharing this experience with everyone um like I said this recording will be made available on the otel um YouTube channel also for anyone who has missed um Jacob's um otel Q&A uh session that we had last month there is a video up on the otel channel and reys um who works with Ren and me on the otel and user working group did a wonderful write up of the uh the Q&A in case video isn't your jam so definitely be sure to check that out okay so if you go to this link here you should be able to find our um our our various slack channels and we love um we encourage everyone to uh to just ask questions share use cases we love hearing all that stuff and also if you or anyone you know has a really cool um Hotel use case you're just getting started or you're a more advanced user does not matter we would love love to hear from you we're always looking for folks for otel in practice otel Q&A and we also have monthly uh otel end user discussions which we run those for three different time zones so we have them for emia APAC and Americas so be sure to join any one of those because always there's always uh really amazing and thoughtful discussions coming out of of these so uh yeah everyone thank you very much and once again Jacob thank you so much for taking the time to uh to chat with us twice yeah thanks so much for having me I appreciate thank you by have a good rest of your day bye yeah + diff --git a/video-transcripts/transcripts/2023-10-11T20:52:28Z-the-evolution-of-observability-practices.md b/video-transcripts/transcripts/2023-10-11T20:52:28Z-the-evolution-of-observability-practices.md index 673804c..3ec98ee 100644 --- a/video-transcripts/transcripts/2023-10-11T20:52:28Z-the-evolution-of-observability-practices.md +++ b/video-transcripts/transcripts/2023-10-11T20:52:28Z-the-evolution-of-observability-practices.md @@ -10,154 +10,100 @@ URL: https://www.youtube.com/watch?v=zSeKL2-_sVg ## Summary -In this YouTube video, a panel discussion hosted by the OpenTelemetry User Working Group explores the evolution of observability practices. The panelists include David W, Iris, Vijay Samuel, Austin Parker, and Noika Melera, who share their experiences with implementing OpenTelemetry in various organizations. The conversation covers the challenges faced in observability before adopting OpenTelemetry, such as the lack of standardization and difficulties in developer engagement. They discuss how OpenTelemetry has facilitated a more unified and standardized approach to observability, allowing developers to emit telemetry data more effectively across different programming languages and platforms. The panelists also address the ongoing improvements needed in OpenTelemetry, including better logging support and dynamic configuration reloading. Overall, the discussion emphasizes the transformative impact of OpenTelemetry on the observability landscape and the future potential for further integration and innovation in this space. +In this YouTube panel discussion hosted by the OpenTelemetry User Working Group, panelists David Win, Iris, VJ Samuel, Austin Parker, and Noika Melera explored the evolution and current state of observability practices, particularly focusing on OpenTelemetry (OTel). David initiated the conversation by sharing his background in observability and the challenges faced when transitioning to cloud-native architectures, emphasizing the need for better tools for developers to understand complex systems. The discussion highlighted how OTel provides a standardized approach to telemetry data, enhancing developer productivity and enabling smoother integrations. Panelists shared personal experiences with implementing OTel in their organizations, addressing both successful strategies and lingering challenges, such as the need for improved logging capabilities and dynamic configuration management. The conversation concluded with a forward-looking perspective, expressing excitement about future developments in OTel and its potential to revolutionize observability practices across the industry. -# Observability Panel Discussion Transcript - -[Music] - -All right, I think we can go ahead and get started. Thank you all so much for being here. The panelists we have today are **David W.**, **Austin Parker**, **VJ Samuel**, **Iris Dear**, **Mishi**, and **Noika Melera**. I hope I got all those correct. We'll do a quick round of introductions. - -This discussion is hosted by the OpenTelemetry User Working Group, of which I am a part, as is Adriana. I see her there. Today, we're going to just have a casual conversation, so feel free to get as opinionated as possible about the evolution of observability practices. - -**David**, since this was inspired by you, I would love for you to do a quick introduction, and then we'll go through the rest of the panelists. Let's hear a little bit more from David, whose brainchild this was. - ---- - -**David:** -Hello everyone, my name is David Win. I am the Principal of People and Machine something at Edge Delta, which is in the observability pipeline space. The thing about being a startup is that you just sort of do whatever needs to be done, so I flex the title appropriately. Ree and I were chatting about different ideas that might be fun to discuss, and one of them that seemed very appropriate to the group is the evolution of observability and where things are going. How are people tackling the challenges of shifting to OpenTelemetry, and what are some of the interesting lessons learned along the way? - -We're looking at this from both a high-level view and a ground-level view, where it can be challenging at times to get feedback. - -To continue with introductions, I'll go ahead and do it popcorn style. **Iris**, why don’t you introduce yourself next? - ---- - -**Iris:** -Hello everyone, my name is Iris—depending on the country I’m in, it's Iris or Irish. I'm currently based in Portugal. I’m a Platform Engineer and Observability Engineer at Farfetch. My day-to-day involves building, maintaining, and modernizing our observability platform and offering this kind of service to the engineers in my company. That’s about it! +## Chapters ---- +Here are the key moments from the livestream, along with their timestamps: -**VJ:** -Hi everyone, I’m VJ Samuel. I work at eBay, where my day job predominantly revolves around doing architecture for our internal observability platform. This includes everything from logs and metrics to events and tracing, helping all our developers with alerting, visualization, and anomaly detection—the whole shebang. +00:00:00 Introductions of panelists and overview of the discussion +00:02:30 David's introduction and the concept of evolving observability practices +00:05:00 Iris shares her experience as an observability engineer +00:06:30 Vijay discusses his role in observability architecture at eBay +00:08:15 Austin introduces his background with OpenTelemetry +00:09:45 Noika talks about her experience working with observability and OpenTelemetry +00:13:00 Discussion on the state of observability before OpenTelemetry adoption +00:19:30 Challenges faced with observability tools and getting developers on board +00:27:00 The importance of standardization in observability practices +00:32:15 Insights on the role of open source in observability evolution ---- - -**Austin:** -Hi everybody! I’m Austin Parker, Community Maintainer for OpenTelemetry. I was formerly at LightStep and part of ServiceNow. Currently, it’s a surprise, and you’ll find out very soon what I'm doing. I’ve been part of OpenTelemetry since it was created, having been an OpenTracing maintainer. I’ve been working in observability for over five years now and have a lot of thoughts from witnessing its growth and evolution. - ---- - -**Noika:** -Hi everyone, I’m Noika. I’m at the open-source startup Signos and have been working with observability since we called it Real's Performance Management. I’m mainly focused on OpenTelemetry and Kubernetes. - ---- - -Thank you all so much again for the introductions! We have a list of questions intended to help guide the conversation, but once everyone gets going, I expect it to become a lot more dynamic. So, to get us going, we want to know about the state of the world before you all undertook your OpenTelemetry journey. - -What was working pretty well, what did not work, or what sucked? What moment prompted you to change? Feel free to raise your hand if you want to chime in! - ---- - -**David:** -I’ll start because I think I have what might not be a very unique story but is an interesting one. Before I got into observability, I started out in software doing QA. I was a Software Developer in Test around 2013-2014. The cloud was a concept, but "cloud-native" wasn’t a term yet. I saw the company I was at go through various transformations, including a DevOps transformation. We wanted to tighten up the feedback loop and reduce the deployment time from 24-48 hours to minutes. - -However, we soon realized that the infrastructure issues weren’t just due to bad servers; we were encountering problems we didn’t know existed. The question arose: “How do we know what’s going on?” We had various logs in different places, and keeping track of service interactions was complex. We tried tools like New Relic and DataDog, but ironically, it didn’t matter what tools we brought in; the developers weren’t interested because they didn’t find the data relatable or useful. - -That’s what got me into observability—trying to answer fundamental questions about service interactions and performance. +# Observability Panel Discussion Transcript ---- +**Moderator:** All right, I think we can go ahead and get started. Thank you all so much for being here. The panelists we have today are David W., Austin Parker, VJ Samuel, Iris Dear, Mishi, and Noika Melera. I'm hoping I got all those correct. -**VJ:** -That resonates with me. Before OpenTelemetry, we had a centralized application logging platform at eBay for over 20 years, but developers had to learn proprietary clients. When cloud-native architecture came along with Prometheus, we still faced issues with standardization across SDKs. OpenTelemetry promised standardization, which was a game-changer for developer productivity. +This discussion is hosted by the OpenTelemetry User Working Group, which I am part of, as is Adriana. Today, we're just going to have a casual conversation. Feel free to get as opinionated as possible about the evolution of observability practices. ---- +**David**, since you inspired this, I would love for you to do a quick introduction, and then we can go through the rest of the panelists. -**Iris:** -My experience mirrors some of that. I started in environments with zero observability and then jumped into a company with a robust observability culture. We faced challenges with using multiple APM vendors and gathering information from various places. We went the OpenTelemetry route to centralize everything, moving from vendor to vendor seamlessly while standardizing our practices. +### Introductions ---- +**David:** Hello everyone, my name is David W. I am a principal at Edge Delta, which is in the observability pipeline space. The thing about being a startup is you just sort of do whatever it needs to be doing, so I flex the title appropriately. Ree and I were chatting about different ideas that might be fun to discuss, and one of them that seemed very appropriate to the group would be the evolution of observability. We can discuss how people are tackling the challenge of shifting to OpenTelemetry and what some of the interesting lessons learned are along the way. -**Austin:** -What’s transformative about OpenTelemetry is that it offers a universal method for developers to emit telemetry information. It’s becoming a core part of being a competent developer. We're still in the early days of seeing how OpenTelemetry impacts the industry. +So, to continue with introductions, I'll go ahead and do it popcorn style. **Iris**, why don't you introduce yourself next? ---- +**Iris:** Hello everyone! My name is Iris, or Iris Dear, depending on the country. I'm currently based in Portugal. I'm a platform engineer, specifically an observability engineer at Farfetch. My day-to-day involves building, maintaining, and modernizing our observability platform and offering this kind of service to the engineers in my company. That's about it. Go ahead and call someone else out, **VJ**. -**Iris:** -I want to add that OpenTelemetry allows you to implement it without breaking your entire system. It’s compatible with most technologies, especially open source ones, which minimizes downtime and developer blind spots. +**VJ:** Hi everyone, I'm VJ Samuel. I work at eBay. My day job predominantly revolves around doing architecture for our observability platform internally. This includes everything like logs, metrics, events, and tracing—helping all our developers with alerting, visualization, anomaly detection, the whole shebang. ---- +**Austin:** Hi everybody, I'm Austin Parker, a community maintainer for OpenTelemetry. Formerly, I was at LightStep, then part of ServiceNow, and currently, I'm involved in something you'll find out about very soon. I've been part of OpenTelemetry since it was created and have worked in observability for over five years now. I have a lot of thoughts from seeing it grow and evolve. -**David:** -There are two fundamental shifts. First, we adopted architectures that are harder to diagnose using traditional methods. Second, when discussing funding, VCs inquire about your observability setup, which changes the conversation from internal to external. +**Noika:** Hi everyone, I'm Noika, and I'm at the open-source startup Signos. I've been working with observability since back when we called it Real Performance Management. Now, I'm mainly working with OpenTelemetry and Kubernetes stuff. ---- +**Moderator:** Thank you all so much again! We have a list of questions intended to help guide the conversation, but I expect it to become a lot more dynamic as we go. -**VJ:** -I think it's essential to have a standardization on the consumer side of things as well. With OpenTelemetry, there’s a promise of standardization, which will help developers and organizations. +### State of Observability Before OpenTelemetry ---- +To get us going, we want to know about the state of the world before you all undertook your OpenTelemetry journey. What was working well, what didn't work, or what sucked? What was the moment that prompted you to change? Feel free to raise your hand, and we'll go from there. -**Austin:** -I agree. The OpenTelemetry framework allows developers to focus on emitting telemetry data without worrying about the underlying infrastructure. It’s about making the telemetry process a fundamental part of software development. +**David:** I’ll start. I think I have what is probably not a very unique story, but an interesting one. Before I got into observability, I started out in software doing QA. I was a software developer in test around 2013-2014. The cloud was a thing, but "cloud-native" wasn't quite a term yet. I saw my company go through various transformations, one of which was a DevOps transformation. ---- +We wanted to tighten the feedback loop and move from 24-48 hours before changes got into tests to minutes. A big part of this was getting on-demand infrastructure rather than static infrastructure. We found that moving to the cloud didn’t actually fix many of the problems we had. -**David:** -It's interesting to see the evolving relationship between observability and development practices. As we continue to explore these themes, I’d like to know how you first heard about OpenTelemetry and how you managed to get leadership buy-in. +We started seeing issues we didn’t know existed before, and the question became, “How do we know what’s breaking?” We had hundreds of nodes with logs in various places, which made it difficult to understand. An engineer created a topology service that looked like a nail art project with strings everywhere, and nobody could keep it in their head. ---- +We tried tools like New Relic and DataDog, but developers weren't interested in using them because the data wasn't at their level. This got me into observability—trying to answer that fundamental question. -**Iris:** -I became obsessed with tracing early on and discovered OpenTelemetry while searching online. At Farfetch, we needed something new and better, and our leadership immediately supported the idea after we presented the benefits. +**VJ:** I resonate with that, David. Our problem space was a bit different. Pre-OpenTelemetry, we had a centralized logging platform for over 20 years. Developers had to learn proprietary clients, which limited our observability efforts. ---- +With the cloud-native era and large-scale Kubernetes adoption, we needed standardized SDKs across the board. OpenTelemetry came in with the promise of standardization, allowing us to offer our developers a community-managed solution they could use across the board. -**VJ:** -I first learned about OpenTelemetry when it was announced as the standard, and we were passively trying it out. Once we saw its stability, we moved forward with it, presenting a detailed memo to leadership about the benefits of standardization. +**Iris:** My experience is similar to VJ's. I started with companies that had zero observability. When I joined my current company, I encountered a robust observability culture. Our challenge was that we were using multiple open-source and APM vendors, which meant we had to gather information from ten different places. ---- +We decided to go the OpenTelemetry route because it centralized everything, allowing us to move from vendor to vendor easily. -**Austin:** -For me, the excitement surrounding OpenTelemetry was palpable in the developer community. The promise of a unified standard was compelling, and it garnered significant attention. +**Austin:** To both of your points, I think what’s transformative about OpenTelemetry is that it offers a universal idea of how developers should emit telemetry information. Many companies have had some form of transactional tracing for years, but the difference is that OpenTelemetry makes it a core part of being a developer—an essential tool in your toolbox. ---- +### Surprising Aspects of the OpenTelemetry Journey -**David:** -That’s a great point, Austin. The foundational aspect of OpenTelemetry is crucial for its adoption. As we wrap up, how has the focus of your observability practice changed after implementing OpenTelemetry, and what are you looking forward to next? +**Moderator:** As we move along, what was most surprising as your OpenTelemetry journey got underway? What were the trickiest stakeholders you encountered? ---- +**David:** I often say in observability, there are no technical problems, only people problems. So, what were some of the surprising things you found either that were sticky and tricky to navigate or that were surprisingly smooth? -**VJ:** -One surprising challenge was the lack of dynamic configuration reloads in the collector, which we had to build ourselves. We'd also like more reference implementations and case studies around OpenTelemetry. +**Iris:** For us, there hasn’t been a challenge we couldn’t find a solution for. The challenges we’re working on now involve the OpenTelemetry operator and the collector. We have a love-hate relationship there. The presets often differ from the normal collector, which creates challenges. ---- +Logging isn’t part of OpenTelemetry at all in our case, so I would like to see more stability in that area. -**Iris:** -For us, the challenges have been manageable, but I’d like to see improvements in logging, as it’s not being fully addressed by OpenTelemetry at the moment. +**VJ:** I agree, Iris. One of the features we were shocked didn’t exist was dynamic reloading of configurations. This was crucial for us while scraping Prometheus endpoints in Kubernetes. If that existed in the collector, it would simplify a lot for us. ---- +**Austin:** I think everyone will be excited about the announcements we have coming up at KubeCon this November, which will address many of these concerns. -**Austin:** -I think everyone will be excited about the announcements at KubeCon this year. We’re making significant strides with OpenTelemetry, and I’m looking forward to what’s coming next. +### Future Focus of Observability Practices ---- +**Moderator:** How has the focus of your observability practice changed, or not changed, after implementing OpenTelemetry? What are you looking forward to next, either from the project or as a result of your observability efforts? -**David:** -There’s a strong momentum behind OpenTelemetry. As new developers recognize the importance of implementing observability from the start, I believe we’ll see it become a standard practice in the industry. +**David:** We’re seeing enterprises realize that the amount of control they have with OpenTelemetry compensates for some of the grit in meeting their needs. New developers are also realizing that implementing observability is part of being a competent developer. ---- +**Iris:** I would like to see improvements in logging and overall stability in the OpenTelemetry operator. -**Austin:** -I agree with David. I’m excited about the future and looking forward to seeing OpenTelemetry win as a standard across the board. +**VJ:** I agree. More reference implementations for tracing and metrics would be beneficial. Case studies on how to use OpenTelemetry effectively would also help. ---- +**Austin:** I think OpenTelemetry is foundational for the future of observability. We’re just beginning to see its impact, and there’s still plenty of room for growth. -[Music] +**Moderator:** Thank you all for your insights today. This has been a fruitful discussion, and I look forward to seeing how OpenTelemetry evolves in the future. If anyone has to go, feel free to jump off. -Thank you all for a fantastic discussion! Feel free to reach out if you'd like to continue this conversation. Have a great day everyone! +**Everyone:** Thank you! ## Raw YouTube Transcript -[Music] all right I think we can go ahead and get started thank you all so much for being here um the panelists we have today are David W Austin Parker VJ Samuel Iris dear Mishi and noika melera I'm hoping I got all those correct and we'll do a quick run of introductions um this discussion is hosted by the open tet tree and user working group which I am part of as um as is Adriana um I see her there and yeah today we're gonna just have a casual conversation um feel free to get as opinionated as possible um about basically the OB uh evolution of observability um practices and actually David since you kind since you the that inspired this um I would love for you to do a quick introduction and um after that we'll just kind of go through the rest of the panalysts but um yeah let's hear a little bit more from David who whose brainchild this was hello everyone my name is David win I am principal people machine something uh at Edge Delta which is in obser the observability pipeline space the thing about being a startup is you just sort of do whatever it needs to be doing so I flexx the title appropriately as such uh but yes we uh Ree and I were chatting about different ideas that might be fun to discuss and one of them that seemed very appropo to the group would be sort of the evolution of observability and where things are going how people are tackling the challenge of Shifting to otel and and what are some of the interesting Lessons Learned along the way not only from the the 10,000 foot view of like we see where the mountains will go but also at the 10 foot view of boy this grass is tall sometimes and trying to get a little bit of a feedback on all different directions of it um yeah so to continue with introductions why don't I'll go ahead and do it popcorn style Iris why don't you introduce yourself next hello everyone uh my name is Iris Iris Irish depending on the country where I am currently I'm based in Portugal uh I'm a platform engineer observability engineer at farfetch uh so my day-to-day is building an observability platform maintaining it modernizing it and offering this kind of service to the engineers in my company I don't know that's that's all about it go ahead and call someone else out Vijay hi everyone uh I'm Vijay Samuel uh I work at eBay uh my day job predominantly revolves around doing architecture for observability platform internally so everything logs metric events tracing helping all our developers uh do uh alerting visualization anomaly detection the whole shebang uh with regards to observability that's that's pretty much what I do Austin hi everybody so I'm Austin Parker uh Community maintainer for open Telemetry um formerly uh light step a part of service now and currently uh it's a surprise and you'll find out very soon what I'm currently doing so yeah I've been a part of open Telemetry since it was created I was a open tracing maintainer I've been working in observability um for over over five years now now and got a lot of thoughts from seeing it kind of grow and evolve from you know what it was to what it is and am I the last person NAA or did NAA that's NAA no I didn't go hi everybody I'm noika I'm at the um open source startup signos and uh have been working with observability stuff from back when we called it Real's Performance Management uh so uh yeah um mainly now working with uh open Telemetry and kubernetes stuff excellent thank you all so much again um and yeah I we have a list of questions that are kind of like intended to help guide the conversation but once everyone gets going I expect it to become a lot more Dynamic so you know I am think we're all totally happy to see where this takes us um so so I guess to get us going um we want to know about the state of the world before you all undertook your open solum to Journey what was working pretty well what did not work or slash what sucked um and kind of what was the moment that prompted you to change and feel free to raise your hand um I think yeah all the panels at least are on camera so if you need visual cues us to when you can step and hopefully that helps but feel free to raise your hand too and then we'll do that way as long as Austin doesn't thumbs up we'll be good I turned off the uh the reaction thing I'll actually start um because I think I have what is probably not a very hope maybe not a very unique story but an interesting one so before I got into observability as a field I um I started out in software doing QA uh I was a soft estet software developer and test and this was you know uh 20 2013 2014 I guess is when I started really getting into into technology as a career or software as a career I should say and the cloud was you know a thing but Cloud native wasn't quite a word yet right like we we didn't have this concept of like oh we're just building all these things with all these cool apis and this idea of like infrastructure on demand or whatever and so I saw you know the company I was that go through these various Transformations and one of them and what I helped kind of lead there was a devops transformation where we went from okay um when you build your code and you deploy your C you know you you write you pull a ticket you write some code works on your machine great you push it and then that night someone else gets to to deploy it and see if it actually worked and one of the things we wanted to do was really tighten up the feedback loop here right we wanted to get from 24 48 hours before changes got into test to minutes or you know hours and minutes and a big part of that was getting on demand infrastructure rather than sort of static infrastructure so we're going through this and we're building all this out and we're we're getting stuff into the cloud and it's great and what we started to see though was it wasn't actually fixing a lot of the problems we had like there was kind of this you know ground truth that everyone had agreed on beforehand it's like well the problem is is that we have you know the infrastructure is bad right like these servers are not properly cared for uh we're just doing we're wiping stuff and recreating it rather than you know actually getting fresh um images every time so it must be some just config thing it's probably not the code and then you know something we go into production it' make it into a patch and uh then the customer come back and say like hey this is actually broken and we missed it because we thought this is because of the you know our testing infrastructure so when we start going into the cloud we have fresh images we have all this stuff and we're finding all these problems that we really didn't even know about before and the question came back it's like well what you know we don't how do we know what's going on how do we know what's breaking our product was a um platform as a service right so we had you know hundreds or you know hundreds and hundreds of nodes um various logs in all sorts of different places it was Windows servers it was Linux servers you know we had all these different databases and it was a very diffic you know it was kind of big it was hard to keep your your head around and someone one of the engineers actually came back and said okay I made a topology service topology and it looked like um you know if you seen one of those like nail art things where someone will make a picture by putting a bunch of nails and a piece of wood and then tying string together it was like that right where you have just like lines everywhere and things connecting each other nobody can keep this in their head nobody could understand how Services actually talk to one another you could look at like a very small section and you could say like okay I I get this but looking at it holistically was impossible and I brought in um at the time you know we tried New Relic we tried data dog we tried a couple things and what I found was ironically enough that it didn't matter what tools I brought in the developers weren't interested in using them just because it wasn't data it wasn't information that was kind of like at their level right like they didn't understand you know how do these things correlate how do these you know how what does it mean when SQL Server spikes and memory usage increases but there's no way to really tell like what was happening and that's got me into observability right is is trying to answer that fundamental question uh someone was piggybacking yeah that so so right off of that I remember working in New Relic uh TR thean of my time there and there was this really fundamental thing of like oh this shows like how this request hit all these these spots and it shows it as like a a trace right with a bunch of time spans including like individual fun calls on all these different Services pretty cool but we we get these questions back that were like well just show me where the request went show me what services were hit and also like in an interpretable version and it was an example where there's a lot of focus on getting a certain piece of information back right but the developers wanted information that was at a different level this is was exactly it is like hey I just want to know where the request is going what services are involved and how the failure on the SQL Server might come back up and affect the front end in these ways and that that was very very hard to tease out whatever this was seven years ago but it's a similar theme right it's it's really about making sense of a pile of stuff again trying to zero in on that moment before we get into these things there's there's the point where you have everything and then you realize that have you thought that having everything was the problem and then it's not and then you're like no now I have everything and I still don't understand everything well maybe I need to actually draw a map and then you draw a map and you're like wow this computer doesn't know how to draw maps and somhow this just looks like a bowl of spaghetti um and then you're like cool how do I Zone this back down again in and out and in and out so it seems like sense making is at least one of the unifying Concepts we see there BJ IRS whoever wants to take it is that a similar feeling that you guys got when you were hitting this this inflection point uh for us the problem space was a little bit different in the in the sense that um pre-open Telemetry we have a pre- cloud native uh era and during Cloud native as well so if you take the pre- cloud native like we we have had something called the centralized application logging platform inside of eBay for more than 20 years and it had the concept of transactional logging where you have a root transaction nested transaction ction events very similar to what we have in the tracing world today uh but the problem with the with the pre- cloud night era is always that developers come into eBay they have to learn proprietary clients uh we had clients only for a few languages so if they are not using that or writing their own code you cannot observe things inside uh inside the company or you're on your own to figure out uh spin up your own elk stack or uh anything that you can do to to monitor the system and when Cloud native H uh and we had the large scale kuus adoption we had the Prometheus end points scraping from Lock files a little more flexible but you do not have standardized sdks across the board even for metrics it used to be that a few people used to use the official Prometheus client some used micrometer and for nodejs you didn't even have an official community supported SDK so I think that's where like when open telary came in with the promise of standardization it was like okay now we can just offer all our developers one standard um uh and it's Community managed they can hop from any company into eBay and they should be able to uh be able to use the observ platform as long as we are open Elementary compliant so I think ours was our story is a lot more developer uh productivity focused at least in the beginning yeah I would say that um my my viewpoint my story is kind of a bit like VJ uh my observability experience first started with some companies that had zero observability in place and that's how I I I got to know it I was like okay what is happening so that was my my job and then where I'm currently working I jumped on a completely different world that had a very nice observability platform very nice observability culture which was a big shock and our job was actually not to come up with waste to monitor but to just keep improving and of course we came to that uh bottleneck that we were using a lot of Open Source we were using APM vendors and to get the information you have to go in 10 different places which is not not great and I would say that it is in developer productivity Viewpoint as well uh why we went the open Telemetry uh route because everything is centralized and we can move from vendor to vendor if we need we are collecting everything standardizing everything so I would say we have have kind of the the same case here I think to both of your points like that to me is what is really transformative about open Telemetry is that it it offers a you know for the first time I think um This truly Universal idea of like how should I as a developer emit this Telemetry information right like I think VJ your story you know is not unique like most especially large distributed um most companies that have like large distributed systems have had some sort of transactional tracing you know some sort of structured lws with transaction IDs that they can you know look at for a decade or more like you know when we think about an open Telemetry Trace you know the actual data model is very similar to what um Google produced in Dapper you know almost 25 years ago now like there there's nothing new Under the Sun what's been missing is the idea that this is actually like a core part of being a developer like a core um tool in the toolbox a core part of your trade is learning how to emit good Telemetry about what your system is doing and open Telemetry provides a standards-based way to do this that also can be natively integrated and available through your MOS service framework or your RPC library or whatever else and that to me is you know we're we're we're at the beginning of the beginning almost of seeing how that actually impacts uh the industry n oh yeah see Iris also has her hand up as well let let Iris go first I've talk too much there is a problem I just want to add a little bit of what Austin was saying as well it's absolutely I mean nothing to add on top of that but uh the other good thing about of telemetry is that you don't have to break your whole system to implement it as well that is one of the the great things because it's compatible with almost all technologies that we currently have especially the open source ones so you don't have to cause downtime have your developers blind for hours and days you can just put it there flawlessly so that's the best the best of all for for me yeah I think there's really two fundamental shifts right there's one is we started adopting architectures that became much harder to diagnose with the stuff that you taught yourself when you taught yourself to code right that like hey adding in some locking lines and such is not going to work on a large microservice Arch Ure the other change and this is bit it's been a while but but it's still there is that when you have conversations about funding when you have conversation with VC they're going to ask you about your observability setup right so those two changes combined have really changed what where the conversation is with observability where it went from an internal discussion about maybe developer velocity and roll back times and these other pieces to like yeah we we we have to have this and it has to be applicable across this this stack I think uh there is some Fair bit of standardization that gets promoted uh on the on the consumer side of things as well uh given that there is a standard in how you instrument Tech metrics for example gauges counters exponential histograms U whatever like earlier it used to be that if you picked one time series database you would get certain capabilities defined in a certain way if you have to switch then the likelihood of of that capability existing may not be um always possible but now since there is a standardization in the inest eventually there will be a good amount of standardization on what technology that the providers either open source projects or vendors provide um so I think uh the net benefits are definitely there uh across the board not just on the on the instrumentation side so I think that's actually you bring up a really interesting point right like historically um everything about Telemetry has really been a a pretty binding abstraction to the data store so if you were use you know statsd metrics worked in the way they did because of the way that stats the you know store and let you query that Prometheus metrics work the way they do because of how you store and query them um logging databases tend to you know if you're using elastic search or something like the format mattered and that influenced how the client libraries were designed and so on and so on and so on and so forth with open Telemetry you know we kind of break that right like you have to open symetry tells you like hey this is what a histogram looks like this is what a structured me event of any kind looks like and I think it's interesting that you also see um this rise in popularity of column stores for storing observability data like metrics logs traces session whatever um you throw a rock and you'll hit a new you know column store based or heck new click house based um Telemetry thing like and it's great and I think open Telemetry has actually been a huge factor in this because it used to be if you wanted to go and build like an observability tool or build some sort of analysis tool you would have to like come up with all this stuff or no one would use it you'd have to come up with your API you'd have to come up with your s you'd have to build Integrations for 40 billion things and open Telemetry means like actually that's just that's all a commodity now you just get that for free with otel and you can build really interesting um workflows on top of that data and I think that's very friendly for developers right like going forward we haven't even really started to see like how do these tools become integral in you know devel vment workflows right when do we get to you know IDE level integration and all that so it's I know it's very easy to feel like open Telemetry is done or or like that it's kind of like hitting this Plateau but I think we're really really still in early days um with what people can actually do with it I I have to agree on this being early days for otel um so a little bit about my background is I was in observability before I hopped over to CL for a bunch of years and now I'm back in the observability space and the conversations that I'm having with people today are they remind me a lot about conversations about get like 15 years ago or something like that when it was sort of a perennial joke that everyone's like oh yeah get looks really interesting we're thinking about moving to get I mean we're still on you know we're still on insert you know uh insert system here but uh it feels almost every conversation I have and again we're in the p blind space so it's about um it's about routing and moving and transforming all the data to the right spot but everything that comes out of us is otel so if they just need to bolt on something and they want to do that they can do that but the first thing they talk to us about is something with some sort of data not being in the right place and the second thing is oh also we're we're starting to look at otel is that good and being the second question I think is a good sign of like climbing the curve uh but I think it's I think there's still plenty of room to grow and I guess speaking of that um you know how did you first hear about the projects um I think it's kind of clear like some of the things have drawn you all to it um something I think too that people might be interested in is how did you get leadership Buy in um I can say something about this because for me it's it's easy so how I heard about the project I have um when I start started working with observability I became obsessed with tracing I'm still fascinated by tracing so I was like oh tracing is so amazing and I started just searching online about then I came across open Telemetry I started playing with it uh for for a long time and not actually implementing anything so when I joined farfetch uh we were like okay we need something new something better okay about open Telemetry and it was very surprising because there was an immediate support uh I was I like to say and brag that we have very good observability culture but I maybe everyone had heard about it we just made a small presentation and our leadership was on board of course it has a lot of benefits and we had to present what we're lacking uh the issues that we're currently facing it what how and what MRI can help us with and it was immediate support so every time I tell the story some people don't believe me but that's that's exactly how it happened it was it was very interesting very very fast support in my case um I can maybe go next um at least in um our case U uh I think uh I first came to know about it uh as part of the uh open telary will will basically be the standard uh and uh we would eventually retire open tracing and open census I think that uh we were passively uh trying out open tracing around that time um and this announcement came and it was like Hey interesting this is something that we need to watch out for and uh we were doing passive experiments with the the open telary SDK and uh The Collector uh for a few months and once the the hit stability that's when we made the informed call that okay we'll we'll start tracing with the open Telemetry as an offering inside the company so now we had the open Telemetry SDK for traces and open uh Telemetry collector for the for for the accepting of all the spans but on the other hand we had uh metric beat and file beat for uh collecting uh logs events and metrics so uh at that point we were at a cross roads of if we should support multiple agents or just standardize across open Telemetry collector for everything and I wrote a very big memo internally on what it would take to migrate out of uh uh beats for uh metrix logs and events um presented it to leadership and I think we felt that uh being on a vendor neutral industry standard would be the better thing for us in the long run and it's best to do it now than than later on so that's that's how we did the migration but and on on our end the metric migration was a massive one because uh we we at that time if I'm not wrong we were already at 32 million samples per second in just across more than 100 kubernetes clusters um million plus Prometheus and points and whatnot so it was a very very involved migration that we did but the but the but the benefits are definitely there now BJ were there any other when you're coming from a culture that has a strong kind of build Vibe uh I'm what I'm trying to get is do you guys run your own collector drro or are you taking something like how how do you managing that in terms of the particular flavor that you decided to land on uh we do run our own um drro internally but I think uh uh the number of custom processors that we run um are very less uh it's mostly that we don't need all the exporters for all the various vendors that are there being packaged in so we create only the ones that we absolutely need uh but for now like we export using either OTP or Prometheus remote right um protocol which are pretty much open standards right now uh so it's about being lean actually I are you all using the collector Builder to make your images uh right now no uh we just craft our own uh go do mod and then just just do it ourself yall thinking about using the collector Builder I think we can like we haven't we haven't really explored it uh uh but yeah we should yeah I'm just trying to get more peopleware The Collector Builder because I think it's really cool aw is it could you say what the collector Builder is so the collector Builder or OCB is a tool that we provide from The Collector repo and what you can do is you can give it a manifest of go modules and it will create a custom image or a custom build of the open Telemetry collector for you so you like eBay can you know get something that has just the receivers processors exporters connectors and extensions that you want it's also a really great way to extend the collector because in this way instead of having to fork contrib or Fork whichever you know raw uh base you want and then bringing in the code you can simply have a go module published in wherever GitHub and pull that in through the collector Builder um and bring in your custom extensions and and things that way so it's a really great Tool uh it's also very friendly for your security teams because it gives them a manifest that they can kind of look at and say like hey this is exactly what's in there and that way I know if you're you know at a larger or um Security review can be extremely taxing and if you're trying to use the contrib image there's an awful lot of dependencies so something to kind of cut that down to size is very friendly to your friends in security wow there's a new user now for the Builder I actually had missed this so there's actually some there is so check out the website open open Telemetry doio does have some has a little tutorial on doing building with the collector um I actually recently used it for a personal project where I was making some custom processors to talk to an API um and and do some stuff some data so it was really really fun to use and it makes it really easy to you know it makes it a lot easier I think to kind of use the collector maybe the way it's sort of intended um I don't know necessarily know if Our intention was for people to just be pulling this giant contrib image constantly um as long as we oh go on oh I just gonna point out that um VJ and Seth very helpfully uh shared a couple links about the custom collector Builder ah perfect so um the other sorry go ahead go on first you were talking sure uh yeah I'm just going to mention we' found it very useful we um have forked just a exporter and are working on trying to get a whole request approved for it but in the meantime we continue to need to persist those features and newer versions of The Collector so this helps us to kind of drag our our single exporter along the way and keep rebuilding it with new uh releases so it's been really helpful yeah that that's another really great use case is if you have custom stuff and you or if you have just Mo modifications I know that PRS can um sit for a bit sometimes so yeah um but the other cool use case the other and also my current one of my current hobby horses is if you are using you know uh large language models to assist you in development so things like co-pilot or chat GP then you know it's not perfect because of the data cut offs on that but over time like as um especially as things like P data and other um structures in The Collector repo stabilize then you know it's actually pretty easy to ask chat you can ask chat GP chat GPT like hey scaffold me a collector extension uh it'll use the wrong apis but they're actually but um it'll do 80% of the for you and then you can ask it like hey I need to transform this data or do what you know do whatever and it's actually a pretty good use case for uh AI assistants I think in in programming so yeah especially that once you get to the point of like just wanting the Rex to like transform it in a certain way it's a good good moment for the co-pilot yeah um I don't know if it understands OTL open tary transform language yet probably not but you know the models will get better and um I think the biggest thing is just St you know once we get to the point where things are changing a little less frequently and it can get into the model so stay tuned maybe maybe we'll have something to say about that as a project in the future um okay so moving the conversation along since um we've coming up on time almost um what was most surprising as your opary journey got under way so like what were the trickiest stakeholders in your case um I think David you like to say in observability there's no technical problems only people problems yeah yeah I should stop saying that even though I'm probably gonna keep saying that but there's you know particularly in this space the the technical problems mostly are straightforward it's all the people problem so what to the whole panel what were some of the surprising things that you found either that were sticky and a little bit tricky and had to navigate or that were surprisingly smooth both of those are valid surprises because um we've we've gone over some of the key selling points here that I think a lot of people would recognize as being uh useful with going in an Hotel Direction but what's what's the part we didn't say that was either good or that was almost good yeah I'm currently running a poll on LinkedIn where I ask people think if they think Opa telemeter is easy to use um I won't disclose the results of that poll quite yet I think there's a yeah I'd be interested to hear from people that have gone through implementation Journeys I can say we're in a a pilot for integration with applications and U the the net library is not quite having the uh log exporter stable until just very recently has been a large deterrent for a very long time um so about two years ago I started trying to convince the application teams to all integrate with open Telemetry and there's been kind of a an appetite for some of the more Cutting Edge teams in our company to try out something that still doesn't have a stable label but uh that that was a big Det current but now that with I think it's the 16 release now that that's that's stable on the net uh instrumentation Library there's a better story there um and there's more Acceptance Now um outside of that the The Collector itself just getting lots of new features for free has been amazing so that's part of the reason we know about the Builder just because we have these custom features that we really want and we keep seeing better newer versions of The Collector come out it's like oh wow we better upgrade because we don't want to sit on this old stuff and all these other great people have been contributing a lot of good enhancements so that's been an easy cell to say here's why part of the why of helping teams move to this instrumentation challenge that will take some time for them to work through um actually I have a can we play a fun game can I get everyone to open up your Zoom chat and tell me um what is what what language do you use otel in and do you think it's good do you think the SDK is good in that language this data may be used by the project see what you don't know is that the part of this request that's hard for me is that the zoom client in on my machine is very temperamental in terms of opening the chat window so that's the hard part okay well I can see the chat node yes it's good I think node's got I think JS has gotten a lot better I think most of the problems in JS right now have a lot more to do with like esm and CJs and various um JavaScript them things ecosystem things rather than any problem with the language s De uh go lot of changes for metrics yeah go needs some love I feel like go was written by um several dear friends of mine um who know go in out left right and indifferent and uh it's great and it's actually pretty idiomatic go I think but it's kind of hard to use because it's idiomatic go personal opinion uh exemplars not getting implemented in what which l language all of them some of them most of them I think the the only one where it's actually implemented is is Java and Java inet and then from what I've asked around like not really anywhere else like I've seen a bunch of open PRS around that but no actual implementations I think we need to get better about closing PRS um net python Java pretty good yeah I'd say Java Java cotland multiplat form nonexistent um have you looked at the uh to Derek there's uh I don't know if it actually is in the repos yet but I know Splunk donated their Android uh client SDK thing that I think targets cotlin I know they want they either either it has been donated or is being donated um I didn't check yeah I've been watching that donation process um okay yeah for us we were looking for an SDK that could be used by cotland multiplatform to go both on IOS and Android clients um I I know some people are using the Java SDK you know for purely for Android but uh we have to just Implement that separately yeah um that is interesting uh will you be a come by any chance I will uh look me up when you're there we'll talk about it cool uh go terrible yeah go go is right now a little bit of a hobby horse for me I'm I'm I think it's is high on my list of languages that feels like it needs some developer experience love I'm sorry for taking over your panel Reese good I kind of like the in it's always you know know I feel like it's hard to get a big group of people online to participate so down API yeah kades is also an interesting one um when I went to when we went to kubernetes years ago you if you're not aware U kubernetes uh has started to feature gate an alpha and beta um Native oel traces from things like cuet and a few other comp and the API server so there is more support for otel coming in the biggest thing is uh kubernetes metrics are all super Prometheus out and and that is probably not going to change anytime soon um but there is some interesting stuff with Prometheus talking about like how you know should Prometheus just use like Hotel libraries and stuff like that so we might see some changes there um and hopefully that can lead to a more unified sort of met Telemetry story for kubernetes a your functions as a service um I'm sure the majority of people are using Lambda and that seems to be the main focus of the community at this point yeah I think we really just need people to I actually I mean I will also plead ignorance I don't know if they have an equivalent for uh Lambda layers or extension layers do you know i' I've looked across the project and while at the top of the just the Faz offering of the open ometry if you look at the documentation it says for example Azure functions AWS Lambda there's really not much integration if any with Azure itself so my my experience on the pipeline side with Azure functions I believe is the official name spending a lot on that naming convention uh is there's a dual writer that you can enable but then you essentially have to set up something that collects all of that information so you would we recommend like throwing up a small Cas cluster depending on how much traffic you want to throw at it um but you could be running anything in there and it's basically a pretty good dual writing mechanism whereas the lamb layer stuff tends to have start up and cool down times that I think I think actually Azure functions is probably a more scalable approach particularly if you get really is it is it function to app insights and then app insights split out into otel it's not it doesn't bounce off of app insights it's from the function itself I believe yeah a writer SDK that Microsoft provides to right out to wherever um part of the challenge was like well is there a way from Azure Monitor to Monitor the actual use of it rather than incurring extra Cycles within the function itself toit its own symetry but yeah okay I think the only tricky part is if you definitely want to use an alternative Source you have to essentially restrict the IM permissions of the function to not write to app insites and that's the only way to actually cut it off I think okay also I'm just gonna step in real quick to to bring around wor sorry very this is a very interesting discussion uh feel free to ping me on the cncf slack if you'd like to continue it but I think it's going to require [Music] um several moving Parts yeah okay thanks we can definitely um Host this another conversation around this and another time um but to bring it back to to this panel um I would like to know how has the focus of your observability practice changed or not changed after implementing open Telemetry and what are you looking forward to next either from the project or as a result of your observability efforts and David is not very emphatically SU I'm gonna go with David no no I was mostly nodding because I definitely want to hear this answer because there's a I alluded to this earlier but there's there's sort of this moment where we think if we have everything that definitely is valuable but it feels like it's not enough and then if we feel like we could put it all I'll I'll use the phrase single paint of glass if we single paint of glass something then something will be better and and then it's not and so then we got to figure out the next next thing so I very interested to hear from the rest of the panel VJ I'll pick on you okay uh so um um uh starting from the previous question right like uh when we did the migration there was one feature that we were shocked that did not exist and I don't think it still exists which is dynamic reloading of configurations uh that was very crucial for us especially while scraping Prometheus and points in kubernetes so that was something that we had to build on our own as a custom receiver uh but but that is a feature that I would I would really like to see on the collector um uh because if that exists then we don't have to have a custom receiver that can like watch on the APA server look for annotations create configurations when the Pod comes up and then tear it down when the Pod goes down um you should definitely pay attention to cubec con Hotel announcements then okay okay yeah I'd be I'd be definitely interested in that uh that that that was one uh but uh the other one is Asian management at scale I know we have opamp but we don't have like uh the an offthe shelf implementation within the open Elementary community that people can uh spin up and then do agent management uh we we have we'd probably have to uh do it ourselves um for larger companies it might might be still uh easy to do but uh I think for the general public I think it would be easy to just have something that they can take up and run it run with it uh with tracing one of the challenges that we had was not enough uh reference implementations uh especially around the collector uh saying okay where does enrich uh K enrichment sit in the pipeline where does red metric uh or span metric connector sit and what are the parameters that we could potentially dun uh not enough people have talked about it uh at least for metrics we did a case study ourself to say that these are the challenges that we might people might run into but for tracing it would be really good if there were more case studies on how people are using exemplars how they're setting up the um collectors um anything that people need to be aware of with regards to grpc I've heard many folks uh complain about uh grpc pinning the connection to a certain collector and then killing it um but like we haven't seen good enough good blog post that actually solve it um so I think like yeah a lot lot more case studies would definitely be helpful um for our part like we are doing two talks in cucon uh to share about what we are we have learned specifically with tracing in metrics around open Telemetry but uh it would be great if we hear a lot more Aris well from our side I think there hasn't been a challenge so far that we haven't been able to find the solution and that goes back to also the poll that Austin was mentioning if you were to choose how is it is open Tel easy I would say in our case has been pretty easy um the challenges that we're working on and that of course I'd like to see improved in the future uh we're mostly using the op Telemetry operator and I have a LoveHate relationship with with a collector that is being deployed by The Collector there's a lot of things missing there especially the presets uh you test something on the normal collector and then you go to implement it on the out of the box and it's not there so that's something that has been a challenge of course uh we can can do it in with custom configurations U and I would say that the other challenge is logging uh we're doing pretty well with and traces and generally we get the good information that we need if we're missing something here on or there we find the work around R it but logging is not part of uh is not being touched by op Telemetry at all for us so that's something that I would like to see it a bit more stable uh and hopefully have it uh full full unified observability with open Telemetry not just the metrics and traces which of course have been the the center of of the project right now so just waiting for more changes on the logging part Austin did you have anything to add to that um I think that uh I think everyone will be really excited to hear about the stuff that we will be announcing and talking about at cubec con this year coming up in November live from Chicago if you are attending in person there will be a ton of great open Telemetry announcements at activities and various other things um and we're really excited about you know not only the progress that the project has made this year but also what is coming next so stay tuned W activities too ni or David did you have anything to add as far as like the focus of your observably practice um and or what are you looking forward to next from otel or your Ali efforts I think there's I think there's two things we're seeing one of course we're seeing Enterprises really see that the amount of control they have with running open Telemetry sort of compensates for some of the some of the grit that exists in the gearbox as they you know try to make it meet their needs um and then on the other side we're seeing new developers realize that like part of being a competent new developer and I think the get comparison is very apt like you know like yeah of course you know Source control if you're going to write code of any kind for production right and so the same thing's happen with absorbability where previously maybe hey you were just entering a few log lines and then you were ready to go it's like no we need to know how to actually uh you know Implement one of these tools from the start and Implement tracing from the start so sort of from two directions I think open Telemetry is getting a lot a lot of momentum in the next year yeah I I am looking forward to the day when you have to have a reason to be off hotel instead of to be on Hotel um because and and to see how that how that affects the vendor landscape so I'm I'm here on behalf of a vendor but because we're in pipelines like if I could get rid of all of our Integrations that weren't otel it would just be so much easier um instead of having to do something really bespoke for everything that's out there but there was a great article not that long ago that you wrote right NOA isn't that right about how otel at the moment is being treated in various uh tools differently need to talk about that I've got some great stuff coming out next week that is not quite so mean to my good friends attended I can do Relic but I I think we all we all need a little bit of a kick in the pants sometimes I think it's allowed yeah one of the really fundamental things like when I was working at New Relic on the integration with x-ray data right is that it's like hey yes we can get we can shove these things together and we can we can make it kind of work but until we all adopt one open standard this is just too complex a data type to just really fluidly chart in one View and see from one View and do things like what is the average amount of time consumed by this single function by the single DV call right that is going to be very tough to do until you just say look we all had to be on these Open Standards some really weird llm training task yes okay it's going to use some some super strange system for for this doesn't really look like tracing or time spans at all cool but everything else right yeah we need to we need to adopt an open CER to really be able to have otel data be a first class citizen so I'm looking forward to otel winning that's all same I think it's already won in many ways right like just not to toot my own horn or to toot the Project's horn but in you know Everyone likes to refer to the x kcd like there's 14 competing standards I'm going to make the unifying one now there are 15 competing standards like I think whatever you know I'm I'm under [Music] no delusion that like this is the Apex of innovation in the space like there will be you know future developments and other things but I think in terms of you know consolidating the last two three decades worth of thinking about application and INF Str Telemetry data and how it can be you know used together like I I think open Telemetry is going to be foundational for a generation of you know observability practitioners for engineers right like there will be something in the future but I think in terms of where we are now this is kind of the this is the thing um and whatever comes next is going to be influenced by things that we don't even see like you I've been doing a lot of research on AI and large language models and stuff and if you look at what you know what are you actually doing when you make a request to a large language model you're making a trace it's literally a trace it is a Dapper style Trace where you have requests and responses you know even the most advanced thing we can think about in terms of software architectures right now and and whatever looks like stuff that we have a way to model in otel today so until that changes um I I feel like open Telemetry is you know long term going to be what at least at least until I am retired so you know give it 30 years but like it's going to be the thing so Justin in the chat asks any thought on micrometer approach of blending the signals into a single observation API from which traces logs and metrics are all derived I think that's fine like I think it's I do think that like the thing you actually like the thing we actually need is we need as a project for people to commit to OTL as being an output format and we need people to respect you know hey this is Trace data and it should be interpreted differently the metric data or whatever even if you are going to like splay those out into a generic sort of event um data structure there are use cases for keeping those things separate in terms of observability pipelining [Music] um yeah it's there's maybe not a cut and dry answer to this um I would like to point out that it is the top of the hour so if you have to go um obviously feel free to jump see you have a good day everyone take everyone care everybody so much everyone byebye +all right I think we can go ahead and get started thank you all so much for being here um the panelists we have today are David W Austin Parker VJ Samuel Iris dear Mishi and noika melera I'm hoping I got all those correct and we'll do a quick run of introductions um this discussion is hosted by the open tet tree and user working group which I am part of as um as is Adriana um I see her there and yeah today we're gonna just have a casual conversation um feel free to get as opinionated as possible um about basically the OB uh evolution of observability um practices and actually David since you kind since you the that inspired this um I would love for you to do a quick introduction and um after that we'll just kind of go through the rest of the panalysts but um yeah let's hear a little bit more from David who whose brainchild this was hello everyone my name is David win I am principal people machine something uh at Edge Delta which is in obser the observability pipeline space the thing about being a startup is you just sort of do whatever it needs to be doing so I flexx the title appropriately as such uh but yes we uh Ree and I were chatting about different ideas that might be fun to discuss and one of them that seemed very appropo to the group would be sort of the evolution of observability and where things are going how people are tackling the challenge of Shifting to otel and and what are some of the interesting Lessons Learned along the way not only from the the 10,000 foot view of like we see where the mountains will go but also at the 10 foot view of boy this grass is tall sometimes and trying to get a little bit of a feedback on all different directions of it um yeah so to continue with introductions why don't I'll go ahead and do it popcorn style Iris why don't you introduce yourself next hello everyone uh my name is Iris Iris Irish depending on the country where I am currently I'm based in Portugal uh I'm a platform engineer observability engineer at farfetch uh so my day-to-day is building an observability platform maintaining it modernizing it and offering this kind of service to the engineers in my company I don't know that's that's all about it go ahead and call someone else out Vijay hi everyone uh I'm Vijay Samuel uh I work at eBay uh my day job predominantly revolves around doing architecture for observability platform internally so everything logs metric events tracing helping all our developers uh do uh alerting visualization anomaly detection the whole shebang uh with regards to observability that's that's pretty much what I do Austin hi everybody so I'm Austin Parker uh Community maintainer for open Telemetry um formerly uh light step a part of service now and currently uh it's a surprise and you'll find out very soon what I'm currently doing so yeah I've been a part of open Telemetry since it was created I was a open tracing maintainer I've been working in observability um for over over five years now now and got a lot of thoughts from seeing it kind of grow and evolve from you know what it was to what it is and am I the last person NAA or did NAA that's NAA no I didn't go hi everybody I'm noika I'm at the um open source startup signos and uh have been working with observability stuff from back when we called it Real's Performance Management uh so uh yeah um mainly now working with uh open Telemetry and kubernetes stuff excellent thank you all so much again um and yeah I we have a list of questions that are kind of like intended to help guide the conversation but once everyone gets going I expect it to become a lot more Dynamic so you know I am think we're all totally happy to see where this takes us um so so I guess to get us going um we want to know about the state of the world before you all undertook your open solum to Journey what was working pretty well what did not work or slash what sucked um and kind of what was the moment that prompted you to change and feel free to raise your hand um I think yeah all the panels at least are on camera so if you need visual cues us to when you can step and hopefully that helps but feel free to raise your hand too and then we'll do that way as long as Austin doesn't thumbs up we'll be good I turned off the uh the reaction thing I'll actually start um because I think I have what is probably not a very hope maybe not a very unique story but an interesting one so before I got into observability as a field I um I started out in software doing QA uh I was a soft estet software developer and test and this was you know uh 20 2013 2014 I guess is when I started really getting into into technology as a career or software as a career I should say and the cloud was you know a thing but Cloud native wasn't quite a word yet right like we we didn't have this concept of like oh we're just building all these things with all these cool apis and this idea of like infrastructure on demand or whatever and so I saw you know the company I was that go through these various Transformations and one of them and what I helped kind of lead there was a devops transformation where we went from okay um when you build your code and you deploy your C you know you you write you pull a ticket you write some code works on your machine great you push it and then that night someone else gets to to deploy it and see if it actually worked and one of the things we wanted to do was really tighten up the feedback loop here right we wanted to get from 24 48 hours before changes got into test to minutes or you know hours and minutes and a big part of that was getting on demand infrastructure rather than sort of static infrastructure so we're going through this and we're building all this out and we're we're getting stuff into the cloud and it's great and what we started to see though was it wasn't actually fixing a lot of the problems we had like there was kind of this you know ground truth that everyone had agreed on beforehand it's like well the problem is is that we have you know the infrastructure is bad right like these servers are not properly cared for uh we're just doing we're wiping stuff and recreating it rather than you know actually getting fresh um images every time so it must be some just config thing it's probably not the code and then you know something we go into production it' make it into a patch and uh then the customer come back and say like hey this is actually broken and we missed it because we thought this is because of the you know our testing infrastructure so when we start going into the cloud we have fresh images we have all this stuff and we're finding all these problems that we really didn't even know about before and the question came back it's like well what you know we don't how do we know what's going on how do we know what's breaking our product was a um platform as a service right so we had you know hundreds or you know hundreds and hundreds of nodes um various logs in all sorts of different places it was Windows servers it was Linux servers you know we had all these different databases and it was a very diffic you know it was kind of big it was hard to keep your your head around and someone one of the engineers actually came back and said okay I made a topology service topology and it looked like um you know if you seen one of those like nail art things where someone will make a picture by putting a bunch of nails and a piece of wood and then tying string together it was like that right where you have just like lines everywhere and things connecting each other nobody can keep this in their head nobody could understand how Services actually talk to one another you could look at like a very small section and you could say like okay I I get this but looking at it holistically was impossible and I brought in um at the time you know we tried New Relic we tried data dog we tried a couple things and what I found was ironically enough that it didn't matter what tools I brought in the developers weren't interested in using them just because it wasn't data it wasn't information that was kind of like at their level right like they didn't understand you know how do these things correlate how do these you know how what does it mean when SQL Server spikes and memory usage increases but there's no way to really tell like what was happening and that's got me into observability right is is trying to answer that fundamental question uh someone was piggybacking yeah that so so right off of that I remember working in New Relic uh TR thean of my time there and there was this really fundamental thing of like oh this shows like how this request hit all these these spots and it shows it as like a a trace right with a bunch of time spans including like individual fun calls on all these different Services pretty cool but we we get these questions back that were like well just show me where the request went show me what services were hit and also like in an interpretable version and it was an example where there's a lot of focus on getting a certain piece of information back right but the developers wanted information that was at a different level this is was exactly it is like hey I just want to know where the request is going what services are involved and how the failure on the SQL Server might come back up and affect the front end in these ways and that that was very very hard to tease out whatever this was seven years ago but it's a similar theme right it's it's really about making sense of a pile of stuff again trying to zero in on that moment before we get into these things there's there's the point where you have everything and then you realize that have you thought that having everything was the problem and then it's not and then you're like no now I have everything and I still don't understand everything well maybe I need to actually draw a map and then you draw a map and you're like wow this computer doesn't know how to draw maps and somhow this just looks like a bowl of spaghetti um and then you're like cool how do I Zone this back down again in and out and in and out so it seems like sense making is at least one of the unifying Concepts we see there BJ IRS whoever wants to take it is that a similar feeling that you guys got when you were hitting this this inflection point uh for us the problem space was a little bit different in the in the sense that um pre-open Telemetry we have a pre- cloud native uh era and during Cloud native as well so if you take the pre- cloud native like we we have had something called the centralized application logging platform inside of eBay for more than 20 years and it had the concept of transactional logging where you have a root transaction nested transaction ction events very similar to what we have in the tracing world today uh but the problem with the with the pre- cloud night era is always that developers come into eBay they have to learn proprietary clients uh we had clients only for a few languages so if they are not using that or writing their own code you cannot observe things inside uh inside the company or you're on your own to figure out uh spin up your own elk stack or uh anything that you can do to to monitor the system and when Cloud native H uh and we had the large scale kuus adoption we had the Prometheus end points scraping from Lock files a little more flexible but you do not have standardized sdks across the board even for metrics it used to be that a few people used to use the official Prometheus client some used micrometer and for nodejs you didn't even have an official community supported SDK so I think that's where like when open telary came in with the promise of standardization it was like okay now we can just offer all our developers one standard um uh and it's Community managed they can hop from any company into eBay and they should be able to uh be able to use the observ platform as long as we are open Elementary compliant so I think ours was our story is a lot more developer uh productivity focused at least in the beginning yeah I would say that um my my viewpoint my story is kind of a bit like VJ uh my observability experience first started with some companies that had zero observability in place and that's how I I I got to know it I was like okay what is happening so that was my my job and then where I'm currently working I jumped on a completely different world that had a very nice observability platform very nice observability culture which was a big shock and our job was actually not to come up with waste to monitor but to just keep improving and of course we came to that uh bottleneck that we were using a lot of Open Source we were using APM vendors and to get the information you have to go in 10 different places which is not not great and I would say that it is in developer productivity Viewpoint as well uh why we went the open Telemetry uh route because everything is centralized and we can move from vendor to vendor if we need we are collecting everything standardizing everything so I would say we have have kind of the the same case here I think to both of your points like that to me is what is really transformative about open Telemetry is that it it offers a you know for the first time I think um This truly Universal idea of like how should I as a developer emit this Telemetry information right like I think VJ your story you know is not unique like most especially large distributed um most companies that have like large distributed systems have had some sort of transactional tracing you know some sort of structured lws with transaction IDs that they can you know look at for a decade or more like you know when we think about an open Telemetry Trace you know the actual data model is very similar to what um Google produced in Dapper you know almost 25 years ago now like there there's nothing new Under the Sun what's been missing is the idea that this is actually like a core part of being a developer like a core um tool in the toolbox a core part of your trade is learning how to emit good Telemetry about what your system is doing and open Telemetry provides a standards-based way to do this that also can be natively integrated and available through your MOS service framework or your RPC library or whatever else and that to me is you know we're we're we're at the beginning of the beginning almost of seeing how that actually impacts uh the industry n oh yeah see Iris also has her hand up as well let let Iris go first I've talk too much there is a problem I just want to add a little bit of what Austin was saying as well it's absolutely I mean nothing to add on top of that but uh the other good thing about of telemetry is that you don't have to break your whole system to implement it as well that is one of the the great things because it's compatible with almost all technologies that we currently have especially the open source ones so you don't have to cause downtime have your developers blind for hours and days you can just put it there flawlessly so that's the best the best of all for for me yeah I think there's really two fundamental shifts right there's one is we started adopting architectures that became much harder to diagnose with the stuff that you taught yourself when you taught yourself to code right that like hey adding in some locking lines and such is not going to work on a large microservice Arch Ure the other change and this is bit it's been a while but but it's still there is that when you have conversations about funding when you have conversation with VC they're going to ask you about your observability setup right so those two changes combined have really changed what where the conversation is with observability where it went from an internal discussion about maybe developer velocity and roll back times and these other pieces to like yeah we we we have to have this and it has to be applicable across this this stack I think uh there is some Fair bit of standardization that gets promoted uh on the on the consumer side of things as well uh given that there is a standard in how you instrument Tech metrics for example gauges counters exponential histograms U whatever like earlier it used to be that if you picked one time series database you would get certain capabilities defined in a certain way if you have to switch then the likelihood of of that capability existing may not be um always possible but now since there is a standardization in the inest eventually there will be a good amount of standardization on what technology that the providers either open source projects or vendors provide um so I think uh the net benefits are definitely there uh across the board not just on the on the instrumentation side so I think that's actually you bring up a really interesting point right like historically um everything about Telemetry has really been a a pretty binding abstraction to the data store so if you were use you know statsd metrics worked in the way they did because of the way that stats the you know store and let you query that Prometheus metrics work the way they do because of how you store and query them um logging databases tend to you know if you're using elastic search or something like the format mattered and that influenced how the client libraries were designed and so on and so on and so on and so forth with open Telemetry you know we kind of break that right like you have to open symetry tells you like hey this is what a histogram looks like this is what a structured me event of any kind looks like and I think it's interesting that you also see um this rise in popularity of column stores for storing observability data like metrics logs traces session whatever um you throw a rock and you'll hit a new you know column store based or heck new click house based um Telemetry thing like and it's great and I think open Telemetry has actually been a huge factor in this because it used to be if you wanted to go and build like an observability tool or build some sort of analysis tool you would have to like come up with all this stuff or no one would use it you'd have to come up with your API you'd have to come up with your s you'd have to build Integrations for 40 billion things and open Telemetry means like actually that's just that's all a commodity now you just get that for free with otel and you can build really interesting um workflows on top of that data and I think that's very friendly for developers right like going forward we haven't even really started to see like how do these tools become integral in you know devel vment workflows right when do we get to you know IDE level integration and all that so it's I know it's very easy to feel like open Telemetry is done or or like that it's kind of like hitting this Plateau but I think we're really really still in early days um with what people can actually do with it I I have to agree on this being early days for otel um so a little bit about my background is I was in observability before I hopped over to CL for a bunch of years and now I'm back in the observability space and the conversations that I'm having with people today are they remind me a lot about conversations about get like 15 years ago or something like that when it was sort of a perennial joke that everyone's like oh yeah get looks really interesting we're thinking about moving to get I mean we're still on you know we're still on insert you know uh insert system here but uh it feels almost every conversation I have and again we're in the p blind space so it's about um it's about routing and moving and transforming all the data to the right spot but everything that comes out of us is otel so if they just need to bolt on something and they want to do that they can do that but the first thing they talk to us about is something with some sort of data not being in the right place and the second thing is oh also we're we're starting to look at otel is that good and being the second question I think is a good sign of like climbing the curve uh but I think it's I think there's still plenty of room to grow and I guess speaking of that um you know how did you first hear about the projects um I think it's kind of clear like some of the things have drawn you all to it um something I think too that people might be interested in is how did you get leadership Buy in um I can say something about this because for me it's it's easy so how I heard about the project I have um when I start started working with observability I became obsessed with tracing I'm still fascinated by tracing so I was like oh tracing is so amazing and I started just searching online about then I came across open Telemetry I started playing with it uh for for a long time and not actually implementing anything so when I joined farfetch uh we were like okay we need something new something better okay about open Telemetry and it was very surprising because there was an immediate support uh I was I like to say and brag that we have very good observability culture but I maybe everyone had heard about it we just made a small presentation and our leadership was on board of course it has a lot of benefits and we had to present what we're lacking uh the issues that we're currently facing it what how and what MRI can help us with and it was immediate support so every time I tell the story some people don't believe me but that's that's exactly how it happened it was it was very interesting very very fast support in my case um I can maybe go next um at least in um our case U uh I think uh I first came to know about it uh as part of the uh open telary will will basically be the standard uh and uh we would eventually retire open tracing and open census I think that uh we were passively uh trying out open tracing around that time um and this announcement came and it was like Hey interesting this is something that we need to watch out for and uh we were doing passive experiments with the the open telary SDK and uh The Collector uh for a few months and once the the hit stability that's when we made the informed call that okay we'll we'll start tracing with the open Telemetry as an offering inside the company so now we had the open Telemetry SDK for traces and open uh Telemetry collector for the for for the accepting of all the spans but on the other hand we had uh metric beat and file beat for uh collecting uh logs events and metrics so uh at that point we were at a cross roads of if we should support multiple agents or just standardize across open Telemetry collector for everything and I wrote a very big memo internally on what it would take to migrate out of uh uh beats for uh metrix logs and events um presented it to leadership and I think we felt that uh being on a vendor neutral industry standard would be the better thing for us in the long run and it's best to do it now than than later on so that's that's how we did the migration but and on on our end the metric migration was a massive one because uh we we at that time if I'm not wrong we were already at 32 million samples per second in just across more than 100 kubernetes clusters um million plus Prometheus and points and whatnot so it was a very very involved migration that we did but the but the but the benefits are definitely there now BJ were there any other when you're coming from a culture that has a strong kind of build Vibe uh I'm what I'm trying to get is do you guys run your own collector drro or are you taking something like how how do you managing that in terms of the particular flavor that you decided to land on uh we do run our own um drro internally but I think uh uh the number of custom processors that we run um are very less uh it's mostly that we don't need all the exporters for all the various vendors that are there being packaged in so we create only the ones that we absolutely need uh but for now like we export using either OTP or Prometheus remote right um protocol which are pretty much open standards right now uh so it's about being lean actually I are you all using the collector Builder to make your images uh right now no uh we just craft our own uh go do mod and then just just do it ourself yall thinking about using the collector Builder I think we can like we haven't we haven't really explored it uh uh but yeah we should yeah I'm just trying to get more peopleware The Collector Builder because I think it's really cool aw is it could you say what the collector Builder is so the collector Builder or OCB is a tool that we provide from The Collector repo and what you can do is you can give it a manifest of go modules and it will create a custom image or a custom build of the open Telemetry collector for you so you like eBay can you know get something that has just the receivers processors exporters connectors and extensions that you want it's also a really great way to extend the collector because in this way instead of having to fork contrib or Fork whichever you know raw uh base you want and then bringing in the code you can simply have a go module published in wherever GitHub and pull that in through the collector Builder um and bring in your custom extensions and and things that way so it's a really great Tool uh it's also very friendly for your security teams because it gives them a manifest that they can kind of look at and say like hey this is exactly what's in there and that way I know if you're you know at a larger or um Security review can be extremely taxing and if you're trying to use the contrib image there's an awful lot of dependencies so something to kind of cut that down to size is very friendly to your friends in security wow there's a new user now for the Builder I actually had missed this so there's actually some there is so check out the website open open Telemetry doio does have some has a little tutorial on doing building with the collector um I actually recently used it for a personal project where I was making some custom processors to talk to an API um and and do some stuff some data so it was really really fun to use and it makes it really easy to you know it makes it a lot easier I think to kind of use the collector maybe the way it's sort of intended um I don't know necessarily know if Our intention was for people to just be pulling this giant contrib image constantly um as long as we oh go on oh I just gonna point out that um VJ and Seth very helpfully uh shared a couple links about the custom collector Builder ah perfect so um the other sorry go ahead go on first you were talking sure uh yeah I'm just going to mention we' found it very useful we um have forked just a exporter and are working on trying to get a whole request approved for it but in the meantime we continue to need to persist those features and newer versions of The Collector so this helps us to kind of drag our our single exporter along the way and keep rebuilding it with new uh releases so it's been really helpful yeah that that's another really great use case is if you have custom stuff and you or if you have just Mo modifications I know that PRS can um sit for a bit sometimes so yeah um but the other cool use case the other and also my current one of my current hobby horses is if you are using you know uh large language models to assist you in development so things like co-pilot or chat GP then you know it's not perfect because of the data cut offs on that but over time like as um especially as things like P data and other um structures in The Collector repo stabilize then you know it's actually pretty easy to ask chat you can ask chat GP chat GPT like hey scaffold me a collector extension uh it'll use the wrong apis but they're actually but um it'll do 80% of the for you and then you can ask it like hey I need to transform this data or do what you know do whatever and it's actually a pretty good use case for uh AI assistants I think in in programming so yeah especially that once you get to the point of like just wanting the Rex to like transform it in a certain way it's a good good moment for the co-pilot yeah um I don't know if it understands OTL open tary transform language yet probably not but you know the models will get better and um I think the biggest thing is just St you know once we get to the point where things are changing a little less frequently and it can get into the model so stay tuned maybe maybe we'll have something to say about that as a project in the future um okay so moving the conversation along since um we've coming up on time almost um what was most surprising as your opary journey got under way so like what were the trickiest stakeholders in your case um I think David you like to say in observability there's no technical problems only people problems yeah yeah I should stop saying that even though I'm probably gonna keep saying that but there's you know particularly in this space the the technical problems mostly are straightforward it's all the people problem so what to the whole panel what were some of the surprising things that you found either that were sticky and a little bit tricky and had to navigate or that were surprisingly smooth both of those are valid surprises because um we've we've gone over some of the key selling points here that I think a lot of people would recognize as being uh useful with going in an Hotel Direction but what's what's the part we didn't say that was either good or that was almost good yeah I'm currently running a poll on LinkedIn where I ask people think if they think Opa telemeter is easy to use um I won't disclose the results of that poll quite yet I think there's a yeah I'd be interested to hear from people that have gone through implementation Journeys I can say we're in a a pilot for integration with applications and U the the net library is not quite having the uh log exporter stable until just very recently has been a large deterrent for a very long time um so about two years ago I started trying to convince the application teams to all integrate with open Telemetry and there's been kind of a an appetite for some of the more Cutting Edge teams in our company to try out something that still doesn't have a stable label but uh that that was a big Det current but now that with I think it's the 16 release now that that's that's stable on the net uh instrumentation Library there's a better story there um and there's more Acceptance Now um outside of that the The Collector itself just getting lots of new features for free has been amazing so that's part of the reason we know about the Builder just because we have these custom features that we really want and we keep seeing better newer versions of The Collector come out it's like oh wow we better upgrade because we don't want to sit on this old stuff and all these other great people have been contributing a lot of good enhancements so that's been an easy cell to say here's why part of the why of helping teams move to this instrumentation challenge that will take some time for them to work through um actually I have a can we play a fun game can I get everyone to open up your Zoom chat and tell me um what is what what language do you use otel in and do you think it's good do you think the SDK is good in that language this data may be used by the project see what you don't know is that the part of this request that's hard for me is that the zoom client in on my machine is very temperamental in terms of opening the chat window so that's the hard part okay well I can see the chat node yes it's good I think node's got I think JS has gotten a lot better I think most of the problems in JS right now have a lot more to do with like esm and CJs and various um JavaScript them things ecosystem things rather than any problem with the language s De uh go lot of changes for metrics yeah go needs some love I feel like go was written by um several dear friends of mine um who know go in out left right and indifferent and uh it's great and it's actually pretty idiomatic go I think but it's kind of hard to use because it's idiomatic go personal opinion uh exemplars not getting implemented in what which l language all of them some of them most of them I think the the only one where it's actually implemented is is Java and Java inet and then from what I've asked around like not really anywhere else like I've seen a bunch of open PRS around that but no actual implementations I think we need to get better about closing PRS um net python Java pretty good yeah I'd say Java Java cotland multiplat form nonexistent um have you looked at the uh to Derek there's uh I don't know if it actually is in the repos yet but I know Splunk donated their Android uh client SDK thing that I think targets cotlin I know they want they either either it has been donated or is being donated um I didn't check yeah I've been watching that donation process um okay yeah for us we were looking for an SDK that could be used by cotland multiplatform to go both on IOS and Android clients um I I know some people are using the Java SDK you know for purely for Android but uh we have to just Implement that separately yeah um that is interesting uh will you be a come by any chance I will uh look me up when you're there we'll talk about it cool uh go terrible yeah go go is right now a little bit of a hobby horse for me I'm I'm I think it's is high on my list of languages that feels like it needs some developer experience love I'm sorry for taking over your panel Reese good I kind of like the in it's always you know know I feel like it's hard to get a big group of people online to participate so down API yeah kades is also an interesting one um when I went to when we went to kubernetes years ago you if you're not aware U kubernetes uh has started to feature gate an alpha and beta um Native oel traces from things like cuet and a few other comp and the API server so there is more support for otel coming in the biggest thing is uh kubernetes metrics are all super Prometheus out and and that is probably not going to change anytime soon um but there is some interesting stuff with Prometheus talking about like how you know should Prometheus just use like Hotel libraries and stuff like that so we might see some changes there um and hopefully that can lead to a more unified sort of met Telemetry story for kubernetes a your functions as a service um I'm sure the majority of people are using Lambda and that seems to be the main focus of the community at this point yeah I think we really just need people to I actually I mean I will also plead ignorance I don't know if they have an equivalent for uh Lambda layers or extension layers do you know i' I've looked across the project and while at the top of the just the Faz offering of the open ometry if you look at the documentation it says for example Azure functions AWS Lambda there's really not much integration if any with Azure itself so my my experience on the pipeline side with Azure functions I believe is the official name spending a lot on that naming convention uh is there's a dual writer that you can enable but then you essentially have to set up something that collects all of that information so you would we recommend like throwing up a small Cas cluster depending on how much traffic you want to throw at it um but you could be running anything in there and it's basically a pretty good dual writing mechanism whereas the lamb layer stuff tends to have start up and cool down times that I think I think actually Azure functions is probably a more scalable approach particularly if you get really is it is it function to app insights and then app insights split out into otel it's not it doesn't bounce off of app insights it's from the function itself I believe yeah a writer SDK that Microsoft provides to right out to wherever um part of the challenge was like well is there a way from Azure Monitor to Monitor the actual use of it rather than incurring extra Cycles within the function itself toit its own symetry but yeah okay I think the only tricky part is if you definitely want to use an alternative Source you have to essentially restrict the IM permissions of the function to not write to app insites and that's the only way to actually cut it off I think okay also I'm just gonna step in real quick to to bring around wor sorry very this is a very interesting discussion uh feel free to ping me on the cncf slack if you'd like to continue it but I think it's going to require um several moving Parts yeah okay thanks we can definitely um Host this another conversation around this and another time um but to bring it back to to this panel um I would like to know how has the focus of your observability practice changed or not changed after implementing open Telemetry and what are you looking forward to next either from the project or as a result of your observability efforts and David is not very emphatically SU I'm gonna go with David no no I was mostly nodding because I definitely want to hear this answer because there's a I alluded to this earlier but there's there's sort of this moment where we think if we have everything that definitely is valuable but it feels like it's not enough and then if we feel like we could put it all I'll I'll use the phrase single paint of glass if we single paint of glass something then something will be better and and then it's not and so then we got to figure out the next next thing so I very interested to hear from the rest of the panel VJ I'll pick on you okay uh so um um uh starting from the previous question right like uh when we did the migration there was one feature that we were shocked that did not exist and I don't think it still exists which is dynamic reloading of configurations uh that was very crucial for us especially while scraping Prometheus and points in kubernetes so that was something that we had to build on our own as a custom receiver uh but but that is a feature that I would I would really like to see on the collector um uh because if that exists then we don't have to have a custom receiver that can like watch on the APA server look for annotations create configurations when the Pod comes up and then tear it down when the Pod goes down um you should definitely pay attention to cubec con Hotel announcements then okay okay yeah I'd be I'd be definitely interested in that uh that that that was one uh but uh the other one is Asian management at scale I know we have opamp but we don't have like uh the an offthe shelf implementation within the open Elementary community that people can uh spin up and then do agent management uh we we have we'd probably have to uh do it ourselves um for larger companies it might might be still uh easy to do but uh I think for the general public I think it would be easy to just have something that they can take up and run it run with it uh with tracing one of the challenges that we had was not enough uh reference implementations uh especially around the collector uh saying okay where does enrich uh K enrichment sit in the pipeline where does red metric uh or span metric connector sit and what are the parameters that we could potentially dun uh not enough people have talked about it uh at least for metrics we did a case study ourself to say that these are the challenges that we might people might run into but for tracing it would be really good if there were more case studies on how people are using exemplars how they're setting up the um collectors um anything that people need to be aware of with regards to grpc I've heard many folks uh complain about uh grpc pinning the connection to a certain collector and then killing it um but like we haven't seen good enough good blog post that actually solve it um so I think like yeah a lot lot more case studies would definitely be helpful um for our part like we are doing two talks in cucon uh to share about what we are we have learned specifically with tracing in metrics around open Telemetry but uh it would be great if we hear a lot more Aris well from our side I think there hasn't been a challenge so far that we haven't been able to find the solution and that goes back to also the poll that Austin was mentioning if you were to choose how is it is open Tel easy I would say in our case has been pretty easy um the challenges that we're working on and that of course I'd like to see improved in the future uh we're mostly using the op Telemetry operator and I have a LoveHate relationship with with a collector that is being deployed by The Collector there's a lot of things missing there especially the presets uh you test something on the normal collector and then you go to implement it on the out of the box and it's not there so that's something that has been a challenge of course uh we can can do it in with custom configurations U and I would say that the other challenge is logging uh we're doing pretty well with and traces and generally we get the good information that we need if we're missing something here on or there we find the work around R it but logging is not part of uh is not being touched by op Telemetry at all for us so that's something that I would like to see it a bit more stable uh and hopefully have it uh full full unified observability with open Telemetry not just the metrics and traces which of course have been the the center of of the project right now so just waiting for more changes on the logging part Austin did you have anything to add to that um I think that uh I think everyone will be really excited to hear about the stuff that we will be announcing and talking about at cubec con this year coming up in November live from Chicago if you are attending in person there will be a ton of great open Telemetry announcements at activities and various other things um and we're really excited about you know not only the progress that the project has made this year but also what is coming next so stay tuned W activities too ni or David did you have anything to add as far as like the focus of your observably practice um and or what are you looking forward to next from otel or your Ali efforts I think there's I think there's two things we're seeing one of course we're seeing Enterprises really see that the amount of control they have with running open Telemetry sort of compensates for some of the some of the grit that exists in the gearbox as they you know try to make it meet their needs um and then on the other side we're seeing new developers realize that like part of being a competent new developer and I think the get comparison is very apt like you know like yeah of course you know Source control if you're going to write code of any kind for production right and so the same thing's happen with absorbability where previously maybe hey you were just entering a few log lines and then you were ready to go it's like no we need to know how to actually uh you know Implement one of these tools from the start and Implement tracing from the start so sort of from two directions I think open Telemetry is getting a lot a lot of momentum in the next year yeah I I am looking forward to the day when you have to have a reason to be off hotel instead of to be on Hotel um because and and to see how that how that affects the vendor landscape so I'm I'm here on behalf of a vendor but because we're in pipelines like if I could get rid of all of our Integrations that weren't otel it would just be so much easier um instead of having to do something really bespoke for everything that's out there but there was a great article not that long ago that you wrote right NOA isn't that right about how otel at the moment is being treated in various uh tools differently need to talk about that I've got some great stuff coming out next week that is not quite so mean to my good friends attended I can do Relic but I I think we all we all need a little bit of a kick in the pants sometimes I think it's allowed yeah one of the really fundamental things like when I was working at New Relic on the integration with x-ray data right is that it's like hey yes we can get we can shove these things together and we can we can make it kind of work but until we all adopt one open standard this is just too complex a data type to just really fluidly chart in one View and see from one View and do things like what is the average amount of time consumed by this single function by the single DV call right that is going to be very tough to do until you just say look we all had to be on these Open Standards some really weird llm training task yes okay it's going to use some some super strange system for for this doesn't really look like tracing or time spans at all cool but everything else right yeah we need to we need to adopt an open CER to really be able to have otel data be a first class citizen so I'm looking forward to otel winning that's all same I think it's already won in many ways right like just not to toot my own horn or to toot the Project's horn but in you know Everyone likes to refer to the x kcd like there's 14 competing standards I'm going to make the unifying one now there are 15 competing standards like I think whatever you know I'm I'm under no delusion that like this is the Apex of innovation in the space like there will be you know future developments and other things but I think in terms of you know consolidating the last two three decades worth of thinking about application and INF Str Telemetry data and how it can be you know used together like I I think open Telemetry is going to be foundational for a generation of you know observability practitioners for engineers right like there will be something in the future but I think in terms of where we are now this is kind of the this is the thing um and whatever comes next is going to be influenced by things that we don't even see like you I've been doing a lot of research on AI and large language models and stuff and if you look at what you know what are you actually doing when you make a request to a large language model you're making a trace it's literally a trace it is a Dapper style Trace where you have requests and responses you know even the most advanced thing we can think about in terms of software architectures right now and and whatever looks like stuff that we have a way to model in otel today so until that changes um I I feel like open Telemetry is you know long term going to be what at least at least until I am retired so you know give it 30 years but like it's going to be the thing so Justin in the chat asks any thought on micrometer approach of blending the signals into a single observation API from which traces logs and metrics are all derived I think that's fine like I think it's I do think that like the thing you actually like the thing we actually need is we need as a project for people to commit to OTL as being an output format and we need people to respect you know hey this is Trace data and it should be interpreted differently the metric data or whatever even if you are going to like splay those out into a generic sort of event um data structure there are use cases for keeping those things separate in terms of observability pipelining um yeah it's there's maybe not a cut and dry answer to this um I would like to point out that it is the top of the hour so if you have to go um obviously feel free to jump see you have a good day everyone take everyone care everybody so much everyone byebye diff --git a/video-transcripts/transcripts/2023-11-21T17:59:37Z-otel-in-practice-filtering-parsing-data-with-the-otel-collector.md b/video-transcripts/transcripts/2023-11-21T17:59:37Z-otel-in-practice-filtering-parsing-data-with-the-otel-collector.md index 61e6964..3346048 100644 --- a/video-transcripts/transcripts/2023-11-21T17:59:37Z-otel-in-practice-filtering-parsing-data-with-the-otel-collector.md +++ b/video-transcripts/transcripts/2023-11-21T17:59:37Z-otel-in-practice-filtering-parsing-data-with-the-otel-collector.md @@ -10,78 +10,91 @@ URL: https://www.youtube.com/watch?v=y3AAerwIFfg ## Summary -In the latest episode of "Filtering the Noise," the presenter, NA, discusses the importance of filtering telemetry data at the OpenTelemetry (OTel) collector level, particularly for traces. The session emphasizes the need for operational tools to manage data without requiring code changes, especially when dealing with sensitive or excessive data. NA illustrates how to filter out unwanted spans using configuration settings within the OTel collector and highlights the limitations and capabilities of various processors, such as the filter and transform processors. The discussion also covers best practices for testing configurations and the significance of maintaining clean observability data. The session concludes with a Q&A, addressing viewer inquiries and encouraging further exploration of OTel functionalities. +In this YouTube video, the speaker discusses filtering techniques for telemetry data using OpenTelemetry (OTel) collectors, focusing on traces rather than logging. They emphasize the necessity of filtering to handle noisy, sensitive, or excessive data generated by code-level instrumentation, which often cannot be modified directly by operational teams. The session covers various methods for data filtering, including removing spans based on attributes, transformation processors, and the importance of maintaining clean observability practices without altering the underlying codebase. The speaker provides practical demonstrations of configuring filters in the OTel collector and discusses the limitations of these processes, such as the inability to change span relationships or drop entire traces. Notable attendees included Adrian and Ren, who contributed questions and insights throughout the presentation. The video also highlights future sessions and resources available through the OpenTelemetry community. -# Filtering the Noise in OpenTelemetry +## Chapters -[Music] +Here are 10 key moments from the livestream transcript, along with their timestamps: -Welcome to "Filtering the Noise." I initially promoted this as better filtering for logging, but today, I will mainly demonstrate filtering for traces. The principles for both are essentially the same. +00:00:00 Introduction to the topic of filtering in OpenTelemetry +00:02:15 Importance of filtering at the OPA Telemetry collector level +00:06:40 Discussion on the challenges of noisy and sensitive data +00:10:30 Example of excessive data being collected in spans +00:15:00 Demonstration of filtering data at the collector level +00:20:45 Live demo: adding filter configurations in the OpenTelemetry collector +00:25:10 Explanation of filtering by attributes in spans +00:30:00 Overview of using regex for filtering data +00:35:15 Discussion on head vs. tail-based sampling +00:40:00 Final thoughts and audience questions on configuration testing and deployment -Let's start by discussing some considerations regarding why we would want filtering at the OpenTelemetry (OPA) Telemetry collector level. +Feel free to let me know if you need any further details! -## Level Setting and Considerations +# Filtering the Noise: A Deep Dive into OPA Telemetry Collector -OpenTelemetry instrumentation, like any code-level instrumentation, can generate a lot of data, but some of that data can be noisy, sensitive, or simply excessive. After we go through the effort of instrumentation—especially at the code level—we ideally want to address data collection issues without requiring changes to our codebase. +Welcome, everyone! Today, we're going to explore filtering in the context of OPA Telemetry Collector. While we may have initially framed it around logging, the principles we'll discuss apply broadly to tracing as well. Let's dive right in. -For many operational teams, accessing the codebase to make changes isn't feasible. They might not know where the hooks are added or may not want to approach the product team with issues like, "Hey, we’re accidentally sending Social Security numbers to our observability backend. Can you fix this tomorrow?" This workflow doesn’t make sense for many people. +## Introduction -While we can discuss the importance of having access to the codebase and better connections between teams, let's focus on the practical aspects. For instance, if we find ourselves sending too much data—like recording every step of a long loop as a separate span—do we really want to bother the product team to edit the code? It would be ideal if we had operational tools to manage this. +First, let’s establish why filtering at the OPA Telemetry Collector level is crucial. OpenTelemetry instrumentation, like any code-level instrumentation, can generate a lot of data—some of which is noisy, sensitive, or simply excessive. Ideally, we want to address these issues without requiring changes to the codebase, especially since many operational teams lack the ability to edit the code directly. -## Example Scenario +### Considerations for Filtering -Let's consider a scenario where a developer was asked to add custom code points and ended up sending more data than necessary—like user IDs and database keys. This is not a critical security issue, but it is excessive. We may not want to send the full database key across the internet. +1. **Sensitive Data**: We might inadvertently send sensitive information, such as personally identifiable information (PII), to our observability backend. +2. **Excessive Data**: Sometimes, we may be recording too much data. For instance, if every step of a 500-step loop is recorded as a separate span, it clutters our traces. +3. **Operational Independence**: It’s often impractical for operational teams to reach out to product teams for code edits to address these issues. -Editing this at the service level would require multiple changes across the codebase, which can be cumbersome. Instead, we often utilize a collector-to-collector architecture to handle this at the collector level. Alternatively, we could implement filtering at the data store level, which would involve editing how we handle spans and traces. +### Example Scenario -## Why Filtering Matters +Let’s imagine a scenario where a developer adds multiple custom code points while trying to enhance observability. They might include user IDs, role counts, and even database keys in the spans. While this may seem beneficial, it can lead to unnecessary data exposure, such as sending full database keys over the internet. -Filtering is particularly important for sensitive data or when dealing with excessive data. Sometimes, when looking at a trace, we might notice something that isn't helpful. For example, if a request is asynchronous but is rolled into database time, it could mislead us into thinking the database is slow when that’s not the case. +This highlights the need for operational tools that allow us to filter or scrub sensitive or excessive data without modifying code across potentially numerous services. -So, how do we filter? Inside the collector, we have receivers and exporters for transport, and processors in the middle that handle data scrubbing, normalization, and sampling. +## Filtering Approaches -## Filtering by Attributes +Now, let’s discuss some filtering approaches we can implement at the collector level: -We can filter spans based on specific attributes. For example, if we want to avoid recording spans for certain internal test users, we can add a processor section that filters out those spans. +### 1. **Filtering by Attributes** -Here’s how you can implement this in your OpenTelemetry collector configuration: +We can filter spans based on specific attributes. For instance, if we identify certain internal test users whose spans clutter our production environment, we can configure the collector to drop spans for these users. -1. Define your filter with a descriptive name. -2. Add it to the pipeline, ensuring the correct order of operations (filters should precede transforms). +Here’s how to set it up: -After saving the configuration and restarting the collector, any spans with unwanted attributes will be dropped, while valid spans will continue to be processed. +- **Configuration**: We add a processor section starting with a filter, naming it descriptively (e.g., `Dev users`), and include it in the pipeline configuration. +- **Live Demonstration**: After saving the config and restarting the collector, we can test it by sending requests from both allowed and disallowed users. The spans for disallowed users should be dropped, while those for allowed users should still appear in our observability backend. -## Key Takeaways +### 2. **Transforming Data** -1. **Filter Processor Behavior**: Once a span matches a filtering condition, it won’t execute later conditions, ensuring efficiency. -2. **Event Handling**: If you drop span events, the span itself remains; however, if all data points from a metric are dropped, the metric will not be reported. -3. **Filter Configurations**: Ensure to add your filters to the pipeline correctly to avoid issues in execution. +Sometimes we don't just want to drop attributes; we may want to transform them. For instance, if we have a sensitive database key in our spans, we might want to replace it with a generic identifier while preserving other useful information, like user roles. -## Transforming Data +This can be done through a transform processor that captures specific patterns in the data and replaces them accordingly. -We can also transform data using regex patterns. When we want to drop specific attributes but keep useful information, we can utilize a transform processor. This allows us to replace unwanted values while preserving necessary information. +## Limitations of Collector Processors -### Cautions with Transformations +While collector processors are powerful, they do have limitations: -- **Metric Transformations**: Be careful when transforming metrics, as improper conversions can create meaningless data and obscure important signals. -- **Naming Conflicts**: Changing labels can lead to orphan values in your data store, making it difficult to retrieve useful insights. +- **No Changing Relationships**: You cannot redefine the parent-child relationships of spans. +- **Cannot Drop Entire Traces**: While we can drop individual spans, dropping an entire trace is more complex and generally not supported in the current filtering architecture. +- **Head vs. Tail Sampling**: We need to consider the differences between head-based and tail-based sampling. Head-based sampling occurs before a trace starts, while tail-based sampling evaluates traces after they are collected. -## Conclusion +## Testing Config Changes -This presentation covered the importance of filtering and transforming data at the collector level, especially for managing excessive or sensitive data in OpenTelemetry. +When implementing changes to the collector configuration, it's critical to test them before deploying to production. Here are some recommended strategies: -Do we have any questions before we wrap up? +1. **Test Harness**: Use a local version of the collector for testing. +2. **Canary Deployments**: Deploy changes to a subset of your collector architecture to evaluate their impact. +3. **Resource Values**: Leverage resource values in your conditions to limit the scope of your changes. -### Q&A Session +## Final Thoughts -- **Testing Config Changes**: To test configuration changes, consider using a test version of your observability setup or a canary deployment to limit the impact of changes. -- **Collector Restart for Config Changes**: Generally, a collector restart is required for new configurations to take effect. +As we conclude, it's important to remember that while we have the power to filter and transform data, we should exercise caution to avoid creating meaningless metrics or data that could lead to confusion later on. -Thank you for joining this session, and I hope you found it informative. If you have further questions or need clarification on any topics, feel free to reach out! +Thank you for joining this session! If you have any further questions or need clarification on any points, feel free to ask. Also, for those interested, recordings of this session and future discussions can be found on the OpenTelemetry YouTube channel. Don’t forget to check out the upcoming Q&A sessions as well! -[Music] +--- + +Let’s open the floor for questions! ## Raw YouTube Transcript -[Music] this is filtering the noise uh wish I I think I promoted as like better filtering for logging but I'm mainly going to demonstrate filtering for traces the principles are the same I promise we'll get there we'll get there folks so let's talk about some level setting for considerations with let me just turn off my ageback here sorry considerations with why we would want filtering at the OPA Telemetry collector level so just review these like first principles everybody seeing the slideshow okay it's got the little green line around that's why I assume you're seeing this great I'm gonna assume that's great give me a negative Emoji react if if if you're not seeing it but otherwise uh we're good um so open lemetry instrumentation like any code level instrumentation unless you send a lot of data rather easily some of that data is noisy some of it is sensitive and some of it is good but needs tweaking and once we've done the work to do instrumentation especially instrumentation at the code level when we're faced with a data collection problem we would ideally want to fix that without requiring changes to our code base so for most people who are working operationally the the thing is like no I can't cannot edit the code base right I don't I don't know where these these hooks are added I don't want to message the product team and say hey uh turns out we're sending you know people Social Security numbers to the to the our observability back end please go do this code commit tomorrow that's just not that's not the workflow that makes sense for a lot of people and we could talk all day and all night about how well you should have access to the code base you should have a better connection that's real DeVos blah blah blah great points but let's talk about like you know okay it's one thing when oh my God we're sending people's a ton of people's pii into observability when we shouldn't what about the situation where it's like hey we're sending a little too much hey the traces they like you know every step of a 500 step four Loop they're recording is a separate span and they shouldn't is that really something where you want to bother the so the the the the product people about it and ask them to like edit the code like you know for sure that's like' be really nice if we had operational tools to do that so there are a let's take a look at this code where uh we can sort of Imagine where we might have gone a little far here so we had our developer we said hey would you add some custom code points maybe they got excited about it we did our little brown bag they added a bunch of points they added hey let's send the user ID as a value for this um for this span let's send like how many uh rolles are we doing we'll use that uh or right I think we decorate the child spam with that yeah and then um let's also send their database key up as part of what we're sending and that probably was going a little far right we're like oh no we're shipping this data like across the internet and maybe we don't want to send their full database Key by the way key is a somewhat overloaded term here but let's just say like this is not critical security uh issue but it is like hey that's more than we really want to be sending around and showing to everybody in the the the back end so there are a ton of this actually a diagram I drew for something else but works F here there are a ton of points where we could say hey I don't want that database key uh um completely uh uh a as a complete string to be stored and so going in and editing these code points right would be editing it at the service level which might be in a whole lot of places and then we might be able to we very often have these collector to collector architectures so so what we're going to be talking about today is is hitting it in one of these collector levels and then the last and and relatively common is doing it at the data store level so this would be like hey maybe we're hitting we're sending way too many spans on a single trace uh maybe that example of like Hey we're sending 50 identical spans actually this example I don't think I I don't have the code here I'm not going to hop over to my IDE for this this thing but it is like it's saying hey roll if you roll seven dice it's sending a span for every single dice roll like be like I don't know I don't want all those those those sub spans right that might be an example where instead of trying to filter The Collector level a a pretty normal step would be like oh let's just go and edit the data store um and say hey maybe older than this amount we want to compress our traces in this way that's a perfectly reasonable way to do it depending on whether or not you have access to do that I know with uh chronosphere generally you do um certainly if you like have a a you know self-hosted data store for this um that that should be very doable if you're using like Loki or signos to do that but um we're going to talk about filtering at The Collector level any questions about like why we're doing all this why this this matters or is a tool that we would want to have yes why uh yeah so in general it's going to be sensitive data or it's be over over like overfit data um are sort of the two main reasons but there's also just going to be this General sort of vibe which is hey I'm looking at this Trace here and I don't like what I see for some reason so see change the share here I'm saying hey I'm looking at this trace and this isn't helpful for a reason I will explain right like like and and that really could be anything but you know when we work on the op operation side work on the SRE side like here's an example that where it's like hey you roll the dice three times I don't need these little sub spans and once the request gets complex it really starts to clog up the view and it's not so much about like having an elegant view that should be something that you can control on on the UI side but it's like yeah something's not looking right and it's leading to distracting data a perfect example is um hey maybe you have a large request that is in fact asynchronous but it's getting rolled into your database time and so it's like hey that's very deceptive and it's it's causing everybody to say hey the database is running really slowly but that's not what's going on and so that pretty good pretty good uh time when we would say hey we we don't want to go and actually edit application code or custom code calls we want to do something at the collector level so just the reminder of like what can happen inside the collector right is like we have receivers and exporters right which do our our uh transport standard um communication can sell stuff out to Stats d can take stuff in from a million places um and then we have our processors in the middle and we're going to talk about data scrubbing data normalization and a little bit about sampling batching you know it's batching that that is that is what it is but um that is the concepts that we care about okay so let's talk about filtering by attributes so we have some attributes on this um uh a span that we say hey if we get these attributes we really don't want to save this so let's go ahead and demo that live let's do a new share and here so here's my open Telemetry collector configuration and just stepping through what was necessary to uh add this um to add this hey if if a span has these attributes I do not want to send it um across so in our application right we take a request attribute um and we add it to that span right down here we say hey the user for this is this request attribute user ID that we received obviously with a real application we should have the user ID from a million other places but um we picked it up here and it turns out that a couple of our users are like our internal test users right and so in fact that's so consistent because it's like a automated testing Suite so we know they're always going to be named this we say hey don't don't record any spans for these people because maybe they're testing stuff maybe those spans get crazy long they introduce a bunch of fuzzy unnecessary data to our to our production environment so we can add a processor section which starts with filter and then has whatever name we want it to have um so this is slash we say hey we can call this Dev users because that's that's descriptive and then critically and I think I have a slide about this too we also add it to the pipeline down here which is where I usually mess up I usually go ahead and create craft a very careful filter in The Collector config and let me give like one stke of Zoom here um and then forget to add it to this thing and and this uh pipeline is red left to right so uh you know if you're doing things like trans transform filters transform processors which we'll talk about you know you want to have just this in the correct order so that your Transformations will have effect when you filter and you generally want batch at the end so if we save this config and go and restart our collector come on bud cool cool cool cool and then we send in a couple of requests so that was from David and if I check my config David's like an allowed user but if I send one from Bob so here's the service right it's this it's the same dice roller that you've seen in the open Telemetry examples right it returns an array of dice rolls uh you know Bob still gets his data because it's not like we're you know this is all observability side so the the application works exactly the same but we should see that this span is is dropped um we can hop over to our UI and I have gone so far as to so this guy is just coming in as normal and I believe has a user attached to it yeah this is David he's allowed and this is like the Juliet child chopped onions because no I did not wait for this to show up in my observability back in it would be there probably by by now but just so just so I didn't hit refresh and have it not show up uh but then if we did send it in with a um with this guy who we didn't want um it will go ahead and drop that span now in this case that span did have ch have child spans so it still shows up as hey there was there was time spent here but we did drop the span so we know it existed but we're we're we're we're dropping all data from it which is in this case is what we wanted what we wanted was we wanted to say hey um just this span is what doesn't matter this is like we're doing like span level metrics for execution time and so then we care about dropping that single span okay I think I have like screenshots of that back in the presentation we'll find out uh new share okay so um we want to go ahead and drop out the span yep there's our nice little thing and then and then our our regular one rolls on just as we'd expect okay so uh some stuff to remember about the filter processor first of all once it hits an attribute that'll filter it's not going to hit the later conditions so if there's an error on a later condition for some reason um it's not actually going to execute um and then uh you know any matching condition will work so so you know the the config that you just saw where it's like hey user equal to Bob user equal to David user equal to Alice filter right that is going to work it doesn't have to meet all those conditions just needs to meet one and then um if we drop out events this is a confusion for some people depending on the model that they're using for for monitoring uh the span is going to stay there so if we say hey the type is a span event and we want to drop every single span event we're still G to have that span and then um a metric is works not the same way if we drop data points from a metric until there are no data points and the metric itself will not be reported which makes sense you wouldn't generally report right like hey this data point but but but nil metric or sorry this metric but nil data points was not something you would normally report so yeah this is my this is my slide to reminder myself and everyone else that you want to actually add these to the pipeline okay so now let's talk about this problematic data here where this is the key that we're getting through which we really didn't want so in this case obviously we could write something that just said hey Buy name drop this attribute um but we don't want to drop the attribute because we we have this guest down in here that's useful to us so we want to know if it's guest admin whatever else whatever other role that's useful information to us we just want to drop the the the full key and so for that this this got me for a minute so um you can have a um filter that says hey it does it by match and we'll take do star say accept any um attribute but then it will not accept a full REX and then I read this portion of the documentation which is hey here's some alternative config which could soon be deprecated and that includes using Rex's as your match type and so I was like wait can I not am I not supposed to be able to use rexes here am I supposed to just use this like relatively simple like direct string matching so you can and uh my thing is write down this link because the something that I I've had on my to-dos for a few weeks is to get this um this doc a little better Linked In to the other documentation because a complete list of the open Telemetry transform language functions is like not super well linked into the like filter and transform processor um don't actually write this down I will share this uh I'll drop it into the chat you don't have to write down a whole link but uh yeah open Telemetry transform language those functions include uh a pattern-based replace which we'll demonstrate in a minute here so some notes on using this Rex in a collector config uh you want to double your escapes like this uh which you can set as a style thing on Rex 101 or wherever other Rex test tool that you're using and then you can use capture groups they do work but it will seem like they don't because you have to use this double dollar sign thing to do it so in this case it's like hey whatever the actual in this case this person wants to change Cube to k8s and U they want to change it to KES Dot and but they want the ID they want to they want to preserve the ID so they're saying hey match this key which is q but then some string of letters and numbers and then go ahead and replace it with K8 but that include what you matched in the in the uh uh parenthesis apologies that's r review for you all but uh Rex is my my my home and my joy so I always like to talk about it but yeah so uh you need to escape the dollar sign for this capture group with a double dollar sign and you have to use triple dollar sign to use a single dollar sign so sorry uh this is just one of those things you have to note because the beautiful thing about redx is that it always works except when it doesn't and so uh yeah these are just things you want to remember about your syntax so yeah so in this case what we'd want is we'd want a transform processor because we want to change our value right to just be underscore guest or just be some kind of escape and underscore guest and so um in this case right we say okay go and find it and I've been a little bit uh greedy here where I've said hey look for this thing it's a whole bunch of numbers uh more than one number for this like don't just take anything that is a user DB key and then go ahead and overwrite the ID with replacement ID and then and then your uh whatever group you first matched so then you'll get something like this replacement idore guest I have a version of this Trace in the signal dashboard but take my word on it this works so transfer processors really effective way you can um grab some other values if you need to you can chain them together but the big thing is just to to do uh like really smart collapse of cardinality and collapse of pii um really really nice tool to do that so what can't you do collector processors you know we've seen this beautifully impressive uh uh uh demo here of the stuff we can do um a lot there's a lot that you cannot do with collector processors um so here I would say is probably the big one big ones is you you cannot change the relationships that spans have uh with the processor you cannot just say hey you go be part of this Trace now nor can you create a parent child relationship like oh you actually were kicked off and we're inside this other one and I'm going to know that from like your the N your naming convention and then I'm going to stick you on to this other one so you can do filtering and transformation and you can touch Trace ID I haven't experimented with this extensively but but apparently it is not possible to like push something into a trace after the fact this makes some sense because of the way that like sampling and other kind of uh working with uh a traces happen there's like some computer science reasons why this is like not uh a trivial thing at all would not be a trivial thing at all to implement but you want to be aware of it and I I learned this for the first time it's sort of you know it's hard to notice an absence sometimes you feel like oh yeah I'm fully in control this I can do whatever I want but then uh Hazel's great talk for this same User Group clued me into like hey this is a limitation here and um I haven't like tried to hack it to every degree but it's definitely yeah there's no like there's no like built-in call say hey like ADD child relationship not at all you also can't drop a whole Trace so you notice that I had this example where it's like hey don't look that the you know the span is gone no worries but you would sort of like to say hey I can go in and say hey um this span looks bad let me go ahead and just lose this Trace um and that is quite a new kettle of fish called tail based sampl right where you're looking at all of the traces that you receive and um you're saying hey I only want these ones to come through based on some information about the the trace so this is the the the dichotomy between head and tail based sampling head based sampling is right before Trace even starts you decide if you're going to run it or not maybe based on like the tiniest amount of data like what route it's in or uh whatever like request information you have but otherwise basically it's just you know um uh uh I want to say the word I know it's the wrong word is skoric skoric has a completely different re meaning but you're doing probabilistic sampling in that case with tailbase sampling uh this thing that we all want whether or not we can have it is you're saying hey now that I'm looking at the trace I decide that you're intered you want to send you on tailbase sampling has a bunch of promise for the efficiency that it's going to add to your system and only sending interesting traces traces is relatively expensive to transmit store process so it's very promising but it has its own caveats to make work um so this is Mo we did most of the presentation I did not pause for questions a bunch of times but do do we have questions about that about head and tail Bas sampling and why that is not supported in the simple uh filter transform system love it I have to go give this same talk not the same talk but a version of this talk um to cgl next week I think and so thank you so much for joining me on this journey um because there's not a ton out there about these p i mean midly like the transform processor is in Alpha so I totally get it why they're wouldn't uh be a ton out there about it but uh I um yeah I'm glad we're getting to talk about it now so to do feel free to make comments or drop in the chat if you have other questions there are 29 things in the chat and I have not been looking at the chat at all sorry most of it is our but Paige is asking how do you recommend testing config changes before shipping them to prod or monitoring to know if you messed up a filtering rule yeah really good question so um I'm gonna hit back a million times sorry about that uh I I I do think that a really good way to do kind of experimental stuff where you're saying Hey I want to do a kind of a multile level change to what we're doing I want to do a complex transform uh the two things are one is to just have a test version have some piece of your observability inside your test harness um and so have your local version of a collector or somewhere else to to test it um the other is to this um collector to collector architecture is extremely extremely useful here so you can do like a canary deploy to say I'm just going to send it out to this one sub piece instead of just our our uh Master collector architecture and then the third is to do some further limitations because you can you can chain together an and here so you can and and then you can use um uh resource values so you can say hey only do this to this service name so that's super useful for for doing this I didn't I didn't show that piece I didn't I didn't do that here but you can say hey I want this to have X service name and also check for this span value and then drop the span so you can started out as being more limited before it spread out to everybody else really good question um is it collector restart always required to pick up new config great question from a long time ago I would get kicked off of twitch so fast because I'm not looking at chat H embarrassing um I used to know the answer to that page and now I do not I I I seem to recall that there was some uh there was some issue that I ran about about the the feasibility of some kind of hot reloading of config but essentially yes you have to do a collector restart certainly in your like little test test bed environment um yeah there's also not like um The Collector has relatively limited like remote abilities to pick up config so yeah um but yeah that's uh something maybe one of the honeycom people will know something about has as has fiddle with a bit and I can check with the rest of sign on Steam as well um Okay so we dropped this replacement ID we did that talk about what we can and can't do um and yeah there is uh a tail sampling processor out there um it has its limitations but you want to you want to explore that separately as its own processor and its own its own service for dropping entire um for dropping entire traces okay so words of caution with metric Transformations so uh conversion between data types isn't supported by the metric model but you can can do it with a Transformer but don't do it but you can do it but don't do it um it's exceedingly easy to CL create meaningless metrics right to like doing things like creating averages from Max values right which you shouldn't do um yeah I I would say very much like you're in the realm of you know creating statistical Transformations just with like Excel equations at that point so you are in danger certainly of messing up what you're doing and uh getting stuff that looks okay but it's in fact extremely bad signal um what have I seen I've seen like uh somebody processing in creating a new average with a uh single new value so they have an average the last 10 values and they get a single new value and they say okay cool I can just add that one to the average which was not quite right uh um and so yeah things like that are are are worth a concern um so yeah a bunch of the Transformations between um uh uh measurement types are one directional and so you should not use the transform to try to go back um and or to try to move between these Lanes uh because you will end up with bad data that is still data that still looks like data and will chart like data so so that's that's a concern very similar to going in and doing like uh just queries directly into your data Store to edit your data right like um you want to be exceedingly careful with that because of course there is always the possibility of destroying the information that you have uh slide two of cautions about metric Transformations so you can change the labels on metric and traces so uh you can cause yourself data store Problems by giving things the same uh names uh as as two metrics two time series you give the same name attributes and scope and um I tried to break signas the like click house data store doing this because I was very excited to see something break but uh I couldn't quite do it but but presumably you can get to the point where you're having your know dashboards not load or other queries not working because you have um double results when there should be unique values in a in a column so more common outcome of that will be orphan values will be spans that are not part of any trace and traces that are not connected to any service um oh yeah Ren I'll put it we we'll put it in the the uh cncf collector Channel I'll ask it in The cncf Collector Channel and the first thing I'm going to do is actually search The cncf Collector Channel because it's interesting the question was um do you have to do a collector restart always to get to pick up new collector config I think the answer is yes but um I yes yeah and I I thought there was an issue about it there was a talk about exploring some some uh support for that but uh I I'm certain that by default right now the answer is no that was the end of the slideshow folks which really just crept right up on me um uh do we want we we saw live demo stuff oh yeah let's go do let's go do one last live demo thing that's right that's what I wanted to do um new share new share share okay so um man there was totally some oh right uh we have the situation where we are getting way too many components this thing like um roll it it you know it sends you back an array of roll dice um and we had a situation where when we sended a request to say like hey roll um you know 41 dice for me we get back this nice array very very quickly but uh the problem is that when we go and look at the uh traces for these guys they just have a ton a ton of these individual single dice rolls which you know in this version right is not really a problem right you know you're you're Trac dashboard should be able to just hide those a whole bunch of n rules but for whatever reason in our example we're saying hey this is this is an issue we have hundreds of these we have thousands of these we know they're all identical they always take the same amount of time we we want to get these out of here this is an older Trace that didn't have 40 we just had six but you get the point and so what we want to do is we want to say hey I I want to drop these spans these roll one spans but some helpful and also uh sabotage focused developer has named each of spans uniquely so we can't just say hey go ahead and drop any span that has this name because it includes like this is the counter on that uh role so um that is something that we want to handle we don't need a reject for this we just need the match tool so if we go back into our IDE let me oh I started typing the command but I won't actually show this so if we go back into our IDE we go back into our config we can say hey span if you have a match on your name that is roll ones followed by anything let's go ahead and drop that span and if we save this and then we restart our collector um our app's been running this whole time so now asking for 41 dice rolls let's ask for 40 uh who likes using a prime number I certainly don't um while I politely wait for a second for these traces to hit my local collector get to the batch moment which is like every I think every five minutes on my thing and then get sent off to the back end do we have other questions about this stuff about um oh uh I didn't mention it but you can uh you have the same filter functions available on metrics and logs so uh we demoed this all with spans but it is totally doable with metrics and logs by the same principles but yeah other questions feel free to drop them in chat or come off mute I promise I am watching the chat now no worries okay so yeah now let me go ahead and start sharing this this is just our last five minutes of uh requests and so we look at we we just looking at a previous one where it was like hey we have way too many of these now if we go ahead and sort this and look at our most recent traces we should sure enough be able to see oh I I also gave it the name that's uh that's like hey do drop drop this name so we're dropping two things here but our actual R dice is happening it it drops the routing span because again it has this name it doesn't like and then it also drops the sub like individual Ro once spans um okay folks that's been my stuff man 38 minutes wow usually when I'm doing something for like the first or second time I like time it out talk it all through I'm like 45 minutes and then when I actually get in front of people 11 minutes uh just zipping zipping but this one I guess there was there's a lot to say about this um final stuff uh the um functions are all out Are all uh fully supported the transform processor is still in Alpha and you're going to need your own build of The Collector to make sure you include it uh which I mean you need to do for for other you know collector contrib stuff so that's that's that's pretty standard but just be aware of it um you wouldn't want to base like your whole DIY observability stuff on transform processors like um do do some work ahead of time to to like get your stuff labeled right and don't don't be like completely relying on it because it is and outfit is going to shift a little bit and there are some standards conversations happening but it's definitely worth doing for stuff like this for like hey you have this filtering or or transformation task and you don't want to bother your product team okay folks that's been my time I don't know Adrian if you want to give like final stuff oh um while you're filtering out these spans not to be shipped to signas can you route them to something like S3 oh that's an interesting question so not with this set of tooling you cannot use like a filter processor to say now go to a different exporter Lane but you certainly can do that with the with um you know uh uh with with your import tools right so so with your um components where you're taking in data you can obviously say on your receivers oops don't do that you can say on your receivers like hey I want to uh pick up this these values and send them to this other endpoint yeah and then go go into a separate pipeline um I don't yeah filter filter is like Drop filter is like drop or remove data it's not route this data to another endpoint great question pretty sure you can do that routing by span name but also also uh should you do that should do that's a good question um that's a really good I should write something about that uh butare that's that's really good I'm gonna I'm gonna like a look at a future blog post about that that's a really good question any other questions for na Ren says in chat yeah there are a lot of tools out there that help with managing your pipeline uh including like especially you know uh we got three I think observability no four observability team people here who can talk a lot about how like there are times when I I think all of us want to be like hey we're your One-Stop shop to look at your data but there's going to be reasons that you're that you're sending it to multiple places uh examples of course be like uh sending stuff to cloudwatch but also sending it to a New Relic dashboard sending stuff to S3 as like hey this is we're going to Cold Storage this but we want to have it uh obviously classically logs right sending them to multiple places to one place where the retention is 10 days and another place where the retention is 10 years so yeah that's that's going to make a ton of sense so yeah there's there's something written about that Ren maybe I'll maybe I'll hit you up for like a we'll do like a a partner post or something on it because it's an interesting question and and not one where there's a ton written about it sure yeah that sounds good um most of the pipeline stuff I've seen is from pipeline companies it'd be interesting to break it down from a perspective of folks who are more neutral towards which pipeline you choose yeah yeah yeah really good page is like is like um uh Paige says in the last webinar I did they asked us how many absorbability tools they had in a significant portion of the groups at five plus I think if we were being honest everyone's gonna say at least two right every well no at least three they say hey I have an observability tool which is the the thing called observability that says observability on its um SEO filing then I have some kind of logging to and then I have a debugging tool that I use right I have my click Ops tool that I use to go in and look at what the heck's going on and so that's at least three right um if not then okay well now what's what do you use as like a pinger or other synthetic metrics right that can can so yeah now we're at five right with just with just what I would say like oh like you you list all five you're like yeah that that's normal that's normal I need all this it's like me looking in my purse I'm like uh I need to get some stuff out of here they're like no I need all of this I need two batteries for my phone uh okay rid of these oh man great chat thank you so much uh and and again big shout out to R you missed it right at the start when I was like really thanking you for uh doing the last minute promo uh I will be giving a a similar but not identical version of this talk because we can all admit there were parts that the dragg um I'll be a similar version of this talk at cgl next week um and then also we'll be doing it at another conference who Name Escapes Me in uh in January uh will will to follow uh coach smash Cod smash which is in January or something I'm gonna be giving the same talk um and also be a cubec con and if you want to come say hi cubec con uh come say hi and also tric signas and also thank you thank you so much na for uh for joining us today um and thank you everyone who was able to make it this was actually a really good turnout for uh for otel and practice so thank you for taking the time um also for um if you know anyone who um wanted to attend but couldn't make it let them know that we will be posting a recording of this on the open Telemetry YouTube channel um the handle is otel Das official if you don't haven't subscribed yet um we've got a bunch of uh previous videos from otel and practice otel Q&A and even some of it are in user discussion groups so definitely check it out out um please check out the Hazel weekly one from six weeks ago now was really really key stuff so that's that's really worth so if you want a reason to go check it out that's that's a really good one yeah it's it's next level it's very very good content and we've got an otel Q&A coming up at on November the 16th with uh Jennifer Moore where she talks about uh her experiences with uh with otel implementation at one of her previous companies so that will be also a really great session to look forward to it'll be in this time slot same time slot um yeah um does anyone else have anything else they want to share um reys Ren nope just a huge thank you to NAA yeah this was awesome +this is filtering the noise uh wish I I think I promoted as like better filtering for logging but I'm mainly going to demonstrate filtering for traces the principles are the same I promise we'll get there we'll get there folks so let's talk about some level setting for considerations with let me just turn off my ageback here sorry considerations with why we would want filtering at the OPA Telemetry collector level so just review these like first principles everybody seeing the slideshow okay it's got the little green line around that's why I assume you're seeing this great I'm gonna assume that's great give me a negative Emoji react if if if you're not seeing it but otherwise uh we're good um so open lemetry instrumentation like any code level instrumentation unless you send a lot of data rather easily some of that data is noisy some of it is sensitive and some of it is good but needs tweaking and once we've done the work to do instrumentation especially instrumentation at the code level when we're faced with a data collection problem we would ideally want to fix that without requiring changes to our code base so for most people who are working operationally the the thing is like no I can't cannot edit the code base right I don't I don't know where these these hooks are added I don't want to message the product team and say hey uh turns out we're sending you know people Social Security numbers to the to the our observability back end please go do this code commit tomorrow that's just not that's not the workflow that makes sense for a lot of people and we could talk all day and all night about how well you should have access to the code base you should have a better connection that's real DeVos blah blah blah great points but let's talk about like you know okay it's one thing when oh my God we're sending people's a ton of people's pii into observability when we shouldn't what about the situation where it's like hey we're sending a little too much hey the traces they like you know every step of a 500 step four Loop they're recording is a separate span and they shouldn't is that really something where you want to bother the so the the the the product people about it and ask them to like edit the code like you know for sure that's like' be really nice if we had operational tools to do that so there are a let's take a look at this code where uh we can sort of Imagine where we might have gone a little far here so we had our developer we said hey would you add some custom code points maybe they got excited about it we did our little brown bag they added a bunch of points they added hey let's send the user ID as a value for this um for this span let's send like how many uh rolles are we doing we'll use that uh or right I think we decorate the child spam with that yeah and then um let's also send their database key up as part of what we're sending and that probably was going a little far right we're like oh no we're shipping this data like across the internet and maybe we don't want to send their full database Key by the way key is a somewhat overloaded term here but let's just say like this is not critical security uh issue but it is like hey that's more than we really want to be sending around and showing to everybody in the the the back end so there are a ton of this actually a diagram I drew for something else but works F here there are a ton of points where we could say hey I don't want that database key uh um completely uh uh a as a complete string to be stored and so going in and editing these code points right would be editing it at the service level which might be in a whole lot of places and then we might be able to we very often have these collector to collector architectures so so what we're going to be talking about today is is hitting it in one of these collector levels and then the last and and relatively common is doing it at the data store level so this would be like hey maybe we're hitting we're sending way too many spans on a single trace uh maybe that example of like Hey we're sending 50 identical spans actually this example I don't think I I don't have the code here I'm not going to hop over to my IDE for this this thing but it is like it's saying hey roll if you roll seven dice it's sending a span for every single dice roll like be like I don't know I don't want all those those those sub spans right that might be an example where instead of trying to filter The Collector level a a pretty normal step would be like oh let's just go and edit the data store um and say hey maybe older than this amount we want to compress our traces in this way that's a perfectly reasonable way to do it depending on whether or not you have access to do that I know with uh chronosphere generally you do um certainly if you like have a a you know self-hosted data store for this um that that should be very doable if you're using like Loki or signos to do that but um we're going to talk about filtering at The Collector level any questions about like why we're doing all this why this this matters or is a tool that we would want to have yes why uh yeah so in general it's going to be sensitive data or it's be over over like overfit data um are sort of the two main reasons but there's also just going to be this General sort of vibe which is hey I'm looking at this Trace here and I don't like what I see for some reason so see change the share here I'm saying hey I'm looking at this trace and this isn't helpful for a reason I will explain right like like and and that really could be anything but you know when we work on the op operation side work on the SRE side like here's an example that where it's like hey you roll the dice three times I don't need these little sub spans and once the request gets complex it really starts to clog up the view and it's not so much about like having an elegant view that should be something that you can control on on the UI side but it's like yeah something's not looking right and it's leading to distracting data a perfect example is um hey maybe you have a large request that is in fact asynchronous but it's getting rolled into your database time and so it's like hey that's very deceptive and it's it's causing everybody to say hey the database is running really slowly but that's not what's going on and so that pretty good pretty good uh time when we would say hey we we don't want to go and actually edit application code or custom code calls we want to do something at the collector level so just the reminder of like what can happen inside the collector right is like we have receivers and exporters right which do our our uh transport standard um communication can sell stuff out to Stats d can take stuff in from a million places um and then we have our processors in the middle and we're going to talk about data scrubbing data normalization and a little bit about sampling batching you know it's batching that that is that is what it is but um that is the concepts that we care about okay so let's talk about filtering by attributes so we have some attributes on this um uh a span that we say hey if we get these attributes we really don't want to save this so let's go ahead and demo that live let's do a new share and here so here's my open Telemetry collector configuration and just stepping through what was necessary to uh add this um to add this hey if if a span has these attributes I do not want to send it um across so in our application right we take a request attribute um and we add it to that span right down here we say hey the user for this is this request attribute user ID that we received obviously with a real application we should have the user ID from a million other places but um we picked it up here and it turns out that a couple of our users are like our internal test users right and so in fact that's so consistent because it's like a automated testing Suite so we know they're always going to be named this we say hey don't don't record any spans for these people because maybe they're testing stuff maybe those spans get crazy long they introduce a bunch of fuzzy unnecessary data to our to our production environment so we can add a processor section which starts with filter and then has whatever name we want it to have um so this is slash we say hey we can call this Dev users because that's that's descriptive and then critically and I think I have a slide about this too we also add it to the pipeline down here which is where I usually mess up I usually go ahead and create craft a very careful filter in The Collector config and let me give like one stke of Zoom here um and then forget to add it to this thing and and this uh pipeline is red left to right so uh you know if you're doing things like trans transform filters transform processors which we'll talk about you know you want to have just this in the correct order so that your Transformations will have effect when you filter and you generally want batch at the end so if we save this config and go and restart our collector come on bud cool cool cool cool and then we send in a couple of requests so that was from David and if I check my config David's like an allowed user but if I send one from Bob so here's the service right it's this it's the same dice roller that you've seen in the open Telemetry examples right it returns an array of dice rolls uh you know Bob still gets his data because it's not like we're you know this is all observability side so the the application works exactly the same but we should see that this span is is dropped um we can hop over to our UI and I have gone so far as to so this guy is just coming in as normal and I believe has a user attached to it yeah this is David he's allowed and this is like the Juliet child chopped onions because no I did not wait for this to show up in my observability back in it would be there probably by by now but just so just so I didn't hit refresh and have it not show up uh but then if we did send it in with a um with this guy who we didn't want um it will go ahead and drop that span now in this case that span did have ch have child spans so it still shows up as hey there was there was time spent here but we did drop the span so we know it existed but we're we're we're we're dropping all data from it which is in this case is what we wanted what we wanted was we wanted to say hey um just this span is what doesn't matter this is like we're doing like span level metrics for execution time and so then we care about dropping that single span okay I think I have like screenshots of that back in the presentation we'll find out uh new share okay so um we want to go ahead and drop out the span yep there's our nice little thing and then and then our our regular one rolls on just as we'd expect okay so uh some stuff to remember about the filter processor first of all once it hits an attribute that'll filter it's not going to hit the later conditions so if there's an error on a later condition for some reason um it's not actually going to execute um and then uh you know any matching condition will work so so you know the the config that you just saw where it's like hey user equal to Bob user equal to David user equal to Alice filter right that is going to work it doesn't have to meet all those conditions just needs to meet one and then um if we drop out events this is a confusion for some people depending on the model that they're using for for monitoring uh the span is going to stay there so if we say hey the type is a span event and we want to drop every single span event we're still G to have that span and then um a metric is works not the same way if we drop data points from a metric until there are no data points and the metric itself will not be reported which makes sense you wouldn't generally report right like hey this data point but but but nil metric or sorry this metric but nil data points was not something you would normally report so yeah this is my this is my slide to reminder myself and everyone else that you want to actually add these to the pipeline okay so now let's talk about this problematic data here where this is the key that we're getting through which we really didn't want so in this case obviously we could write something that just said hey Buy name drop this attribute um but we don't want to drop the attribute because we we have this guest down in here that's useful to us so we want to know if it's guest admin whatever else whatever other role that's useful information to us we just want to drop the the the full key and so for that this this got me for a minute so um you can have a um filter that says hey it does it by match and we'll take do star say accept any um attribute but then it will not accept a full REX and then I read this portion of the documentation which is hey here's some alternative config which could soon be deprecated and that includes using Rex's as your match type and so I was like wait can I not am I not supposed to be able to use rexes here am I supposed to just use this like relatively simple like direct string matching so you can and uh my thing is write down this link because the something that I I've had on my to-dos for a few weeks is to get this um this doc a little better Linked In to the other documentation because a complete list of the open Telemetry transform language functions is like not super well linked into the like filter and transform processor um don't actually write this down I will share this uh I'll drop it into the chat you don't have to write down a whole link but uh yeah open Telemetry transform language those functions include uh a pattern-based replace which we'll demonstrate in a minute here so some notes on using this Rex in a collector config uh you want to double your escapes like this uh which you can set as a style thing on Rex 101 or wherever other Rex test tool that you're using and then you can use capture groups they do work but it will seem like they don't because you have to use this double dollar sign thing to do it so in this case it's like hey whatever the actual in this case this person wants to change Cube to k8s and U they want to change it to KES Dot and but they want the ID they want to they want to preserve the ID so they're saying hey match this key which is q but then some string of letters and numbers and then go ahead and replace it with K8 but that include what you matched in the in the uh uh parenthesis apologies that's r review for you all but uh Rex is my my my home and my joy so I always like to talk about it but yeah so uh you need to escape the dollar sign for this capture group with a double dollar sign and you have to use triple dollar sign to use a single dollar sign so sorry uh this is just one of those things you have to note because the beautiful thing about redx is that it always works except when it doesn't and so uh yeah these are just things you want to remember about your syntax so yeah so in this case what we'd want is we'd want a transform processor because we want to change our value right to just be underscore guest or just be some kind of escape and underscore guest and so um in this case right we say okay go and find it and I've been a little bit uh greedy here where I've said hey look for this thing it's a whole bunch of numbers uh more than one number for this like don't just take anything that is a user DB key and then go ahead and overwrite the ID with replacement ID and then and then your uh whatever group you first matched so then you'll get something like this replacement idore guest I have a version of this Trace in the signal dashboard but take my word on it this works so transfer processors really effective way you can um grab some other values if you need to you can chain them together but the big thing is just to to do uh like really smart collapse of cardinality and collapse of pii um really really nice tool to do that so what can't you do collector processors you know we've seen this beautifully impressive uh uh uh demo here of the stuff we can do um a lot there's a lot that you cannot do with collector processors um so here I would say is probably the big one big ones is you you cannot change the relationships that spans have uh with the processor you cannot just say hey you go be part of this Trace now nor can you create a parent child relationship like oh you actually were kicked off and we're inside this other one and I'm going to know that from like your the N your naming convention and then I'm going to stick you on to this other one so you can do filtering and transformation and you can touch Trace ID I haven't experimented with this extensively but but apparently it is not possible to like push something into a trace after the fact this makes some sense because of the way that like sampling and other kind of uh working with uh a traces happen there's like some computer science reasons why this is like not uh a trivial thing at all would not be a trivial thing at all to implement but you want to be aware of it and I I learned this for the first time it's sort of you know it's hard to notice an absence sometimes you feel like oh yeah I'm fully in control this I can do whatever I want but then uh Hazel's great talk for this same User Group clued me into like hey this is a limitation here and um I haven't like tried to hack it to every degree but it's definitely yeah there's no like there's no like built-in call say hey like ADD child relationship not at all you also can't drop a whole Trace so you notice that I had this example where it's like hey don't look that the you know the span is gone no worries but you would sort of like to say hey I can go in and say hey um this span looks bad let me go ahead and just lose this Trace um and that is quite a new kettle of fish called tail based sampl right where you're looking at all of the traces that you receive and um you're saying hey I only want these ones to come through based on some information about the the trace so this is the the the dichotomy between head and tail based sampling head based sampling is right before Trace even starts you decide if you're going to run it or not maybe based on like the tiniest amount of data like what route it's in or uh whatever like request information you have but otherwise basically it's just you know um uh uh I want to say the word I know it's the wrong word is skoric skoric has a completely different re meaning but you're doing probabilistic sampling in that case with tailbase sampling uh this thing that we all want whether or not we can have it is you're saying hey now that I'm looking at the trace I decide that you're intered you want to send you on tailbase sampling has a bunch of promise for the efficiency that it's going to add to your system and only sending interesting traces traces is relatively expensive to transmit store process so it's very promising but it has its own caveats to make work um so this is Mo we did most of the presentation I did not pause for questions a bunch of times but do do we have questions about that about head and tail Bas sampling and why that is not supported in the simple uh filter transform system love it I have to go give this same talk not the same talk but a version of this talk um to cgl next week I think and so thank you so much for joining me on this journey um because there's not a ton out there about these p i mean midly like the transform processor is in Alpha so I totally get it why they're wouldn't uh be a ton out there about it but uh I um yeah I'm glad we're getting to talk about it now so to do feel free to make comments or drop in the chat if you have other questions there are 29 things in the chat and I have not been looking at the chat at all sorry most of it is our but Paige is asking how do you recommend testing config changes before shipping them to prod or monitoring to know if you messed up a filtering rule yeah really good question so um I'm gonna hit back a million times sorry about that uh I I I do think that a really good way to do kind of experimental stuff where you're saying Hey I want to do a kind of a multile level change to what we're doing I want to do a complex transform uh the two things are one is to just have a test version have some piece of your observability inside your test harness um and so have your local version of a collector or somewhere else to to test it um the other is to this um collector to collector architecture is extremely extremely useful here so you can do like a canary deploy to say I'm just going to send it out to this one sub piece instead of just our our uh Master collector architecture and then the third is to do some further limitations because you can you can chain together an and here so you can and and then you can use um uh resource values so you can say hey only do this to this service name so that's super useful for for doing this I didn't I didn't show that piece I didn't I didn't do that here but you can say hey I want this to have X service name and also check for this span value and then drop the span so you can started out as being more limited before it spread out to everybody else really good question um is it collector restart always required to pick up new config great question from a long time ago I would get kicked off of twitch so fast because I'm not looking at chat H embarrassing um I used to know the answer to that page and now I do not I I I seem to recall that there was some uh there was some issue that I ran about about the the feasibility of some kind of hot reloading of config but essentially yes you have to do a collector restart certainly in your like little test test bed environment um yeah there's also not like um The Collector has relatively limited like remote abilities to pick up config so yeah um but yeah that's uh something maybe one of the honeycom people will know something about has as has fiddle with a bit and I can check with the rest of sign on Steam as well um Okay so we dropped this replacement ID we did that talk about what we can and can't do um and yeah there is uh a tail sampling processor out there um it has its limitations but you want to you want to explore that separately as its own processor and its own its own service for dropping entire um for dropping entire traces okay so words of caution with metric Transformations so uh conversion between data types isn't supported by the metric model but you can can do it with a Transformer but don't do it but you can do it but don't do it um it's exceedingly easy to CL create meaningless metrics right to like doing things like creating averages from Max values right which you shouldn't do um yeah I I would say very much like you're in the realm of you know creating statistical Transformations just with like Excel equations at that point so you are in danger certainly of messing up what you're doing and uh getting stuff that looks okay but it's in fact extremely bad signal um what have I seen I've seen like uh somebody processing in creating a new average with a uh single new value so they have an average the last 10 values and they get a single new value and they say okay cool I can just add that one to the average which was not quite right uh um and so yeah things like that are are are worth a concern um so yeah a bunch of the Transformations between um uh uh measurement types are one directional and so you should not use the transform to try to go back um and or to try to move between these Lanes uh because you will end up with bad data that is still data that still looks like data and will chart like data so so that's that's a concern very similar to going in and doing like uh just queries directly into your data Store to edit your data right like um you want to be exceedingly careful with that because of course there is always the possibility of destroying the information that you have uh slide two of cautions about metric Transformations so you can change the labels on metric and traces so uh you can cause yourself data store Problems by giving things the same uh names uh as as two metrics two time series you give the same name attributes and scope and um I tried to break signas the like click house data store doing this because I was very excited to see something break but uh I couldn't quite do it but but presumably you can get to the point where you're having your know dashboards not load or other queries not working because you have um double results when there should be unique values in a in a column so more common outcome of that will be orphan values will be spans that are not part of any trace and traces that are not connected to any service um oh yeah Ren I'll put it we we'll put it in the the uh cncf collector Channel I'll ask it in The cncf Collector Channel and the first thing I'm going to do is actually search The cncf Collector Channel because it's interesting the question was um do you have to do a collector restart always to get to pick up new collector config I think the answer is yes but um I yes yeah and I I thought there was an issue about it there was a talk about exploring some some uh support for that but uh I I'm certain that by default right now the answer is no that was the end of the slideshow folks which really just crept right up on me um uh do we want we we saw live demo stuff oh yeah let's go do let's go do one last live demo thing that's right that's what I wanted to do um new share new share share okay so um man there was totally some oh right uh we have the situation where we are getting way too many components this thing like um roll it it you know it sends you back an array of roll dice um and we had a situation where when we sended a request to say like hey roll um you know 41 dice for me we get back this nice array very very quickly but uh the problem is that when we go and look at the uh traces for these guys they just have a ton a ton of these individual single dice rolls which you know in this version right is not really a problem right you know you're you're Trac dashboard should be able to just hide those a whole bunch of n rules but for whatever reason in our example we're saying hey this is this is an issue we have hundreds of these we have thousands of these we know they're all identical they always take the same amount of time we we want to get these out of here this is an older Trace that didn't have 40 we just had six but you get the point and so what we want to do is we want to say hey I I want to drop these spans these roll one spans but some helpful and also uh sabotage focused developer has named each of spans uniquely so we can't just say hey go ahead and drop any span that has this name because it includes like this is the counter on that uh role so um that is something that we want to handle we don't need a reject for this we just need the match tool so if we go back into our IDE let me oh I started typing the command but I won't actually show this so if we go back into our IDE we go back into our config we can say hey span if you have a match on your name that is roll ones followed by anything let's go ahead and drop that span and if we save this and then we restart our collector um our app's been running this whole time so now asking for 41 dice rolls let's ask for 40 uh who likes using a prime number I certainly don't um while I politely wait for a second for these traces to hit my local collector get to the batch moment which is like every I think every five minutes on my thing and then get sent off to the back end do we have other questions about this stuff about um oh uh I didn't mention it but you can uh you have the same filter functions available on metrics and logs so uh we demoed this all with spans but it is totally doable with metrics and logs by the same principles but yeah other questions feel free to drop them in chat or come off mute I promise I am watching the chat now no worries okay so yeah now let me go ahead and start sharing this this is just our last five minutes of uh requests and so we look at we we just looking at a previous one where it was like hey we have way too many of these now if we go ahead and sort this and look at our most recent traces we should sure enough be able to see oh I I also gave it the name that's uh that's like hey do drop drop this name so we're dropping two things here but our actual R dice is happening it it drops the routing span because again it has this name it doesn't like and then it also drops the sub like individual Ro once spans um okay folks that's been my stuff man 38 minutes wow usually when I'm doing something for like the first or second time I like time it out talk it all through I'm like 45 minutes and then when I actually get in front of people 11 minutes uh just zipping zipping but this one I guess there was there's a lot to say about this um final stuff uh the um functions are all out Are all uh fully supported the transform processor is still in Alpha and you're going to need your own build of The Collector to make sure you include it uh which I mean you need to do for for other you know collector contrib stuff so that's that's that's pretty standard but just be aware of it um you wouldn't want to base like your whole DIY observability stuff on transform processors like um do do some work ahead of time to to like get your stuff labeled right and don't don't be like completely relying on it because it is and outfit is going to shift a little bit and there are some standards conversations happening but it's definitely worth doing for stuff like this for like hey you have this filtering or or transformation task and you don't want to bother your product team okay folks that's been my time I don't know Adrian if you want to give like final stuff oh um while you're filtering out these spans not to be shipped to signas can you route them to something like S3 oh that's an interesting question so not with this set of tooling you cannot use like a filter processor to say now go to a different exporter Lane but you certainly can do that with the with um you know uh uh with with your import tools right so so with your um components where you're taking in data you can obviously say on your receivers oops don't do that you can say on your receivers like hey I want to uh pick up this these values and send them to this other endpoint yeah and then go go into a separate pipeline um I don't yeah filter filter is like Drop filter is like drop or remove data it's not route this data to another endpoint great question pretty sure you can do that routing by span name but also also uh should you do that should do that's a good question um that's a really good I should write something about that uh butare that's that's really good I'm gonna I'm gonna like a look at a future blog post about that that's a really good question any other questions for na Ren says in chat yeah there are a lot of tools out there that help with managing your pipeline uh including like especially you know uh we got three I think observability no four observability team people here who can talk a lot about how like there are times when I I think all of us want to be like hey we're your One-Stop shop to look at your data but there's going to be reasons that you're that you're sending it to multiple places uh examples of course be like uh sending stuff to cloudwatch but also sending it to a New Relic dashboard sending stuff to S3 as like hey this is we're going to Cold Storage this but we want to have it uh obviously classically logs right sending them to multiple places to one place where the retention is 10 days and another place where the retention is 10 years so yeah that's that's going to make a ton of sense so yeah there's there's something written about that Ren maybe I'll maybe I'll hit you up for like a we'll do like a a partner post or something on it because it's an interesting question and and not one where there's a ton written about it sure yeah that sounds good um most of the pipeline stuff I've seen is from pipeline companies it'd be interesting to break it down from a perspective of folks who are more neutral towards which pipeline you choose yeah yeah yeah really good page is like is like um uh Paige says in the last webinar I did they asked us how many absorbability tools they had in a significant portion of the groups at five plus I think if we were being honest everyone's gonna say at least two right every well no at least three they say hey I have an observability tool which is the the thing called observability that says observability on its um SEO filing then I have some kind of logging to and then I have a debugging tool that I use right I have my click Ops tool that I use to go in and look at what the heck's going on and so that's at least three right um if not then okay well now what's what do you use as like a pinger or other synthetic metrics right that can can so yeah now we're at five right with just with just what I would say like oh like you you list all five you're like yeah that that's normal that's normal I need all this it's like me looking in my purse I'm like uh I need to get some stuff out of here they're like no I need all of this I need two batteries for my phone uh okay rid of these oh man great chat thank you so much uh and and again big shout out to R you missed it right at the start when I was like really thanking you for uh doing the last minute promo uh I will be giving a a similar but not identical version of this talk because we can all admit there were parts that the dragg um I'll be a similar version of this talk at cgl next week um and then also we'll be doing it at another conference who Name Escapes Me in uh in January uh will will to follow uh coach smash Cod smash which is in January or something I'm gonna be giving the same talk um and also be a cubec con and if you want to come say hi cubec con uh come say hi and also tric signas and also thank you thank you so much na for uh for joining us today um and thank you everyone who was able to make it this was actually a really good turnout for uh for otel and practice so thank you for taking the time um also for um if you know anyone who um wanted to attend but couldn't make it let them know that we will be posting a recording of this on the open Telemetry YouTube channel um the handle is otel Das official if you don't haven't subscribed yet um we've got a bunch of uh previous videos from otel and practice otel Q&A and even some of it are in user discussion groups so definitely check it out out um please check out the Hazel weekly one from six weeks ago now was really really key stuff so that's that's really worth so if you want a reason to go check it out that's that's a really good one yeah it's it's next level it's very very good content and we've got an otel Q&A coming up at on November the 16th with uh Jennifer Moore where she talks about uh her experiences with uh with otel implementation at one of her previous companies so that will be also a really great session to look forward to it'll be in this time slot same time slot um yeah um does anyone else have anything else they want to share um reys Ren nope just a huge thank you to NAA yeah this was awesome diff --git a/video-transcripts/transcripts/2023-12-04T23:34:44Z-otel-q-a-feat-jennifer-moore.md b/video-transcripts/transcripts/2023-12-04T23:34:44Z-otel-q-a-feat-jennifer-moore.md index d667d1e..38cb1cb 100644 --- a/video-transcripts/transcripts/2023-12-04T23:34:44Z-otel-q-a-feat-jennifer-moore.md +++ b/video-transcripts/transcripts/2023-12-04T23:34:44Z-otel-q-a-feat-jennifer-moore.md @@ -10,101 +10,126 @@ URL: https://www.youtube.com/watch?v=ztrknOWIUh8 ## Summary -In this Otel Q&A session, host Jennifer Moore interviews a guest who previously worked at Screencastify, a screen recording application company. The discussion highlights the guest's experiences and challenges in implementing observability practices using OpenTelemetry. They share insights about their role as a DevOps lead, the technology stack used (including Typescript and Google Cloud Platform), and the initial state of observability at Screencastify. The guest explains how they improved the observability of the system by replacing ineffective instrumentation with OpenTelemetry, emphasizing the importance of understanding system behavior for effective incident management. They also discuss the learning curve associated with OpenTelemetry, the successful engagement of the DevOps team, and how management's perception of observability evolved through real incidents. Overall, the conversation underscores the value of observability in troubleshooting and system understanding, presenting a compelling case for adopting such practices in tech organizations. +In this YouTube Q&A session, host Jennifer Moore interviews Jennifer Moore, a DevOps lead who shares her experiences with implementing OpenTelemetry at her previous company, Screencastify, which specializes in screen recording applications. The discussion covers the application stack used at Screencastify, including TypeScript and Google Cloud Platform, and the initial struggles with observability practices. Moore explains how she identified the need for better observability tools and transitioned from using a proprietary SDK to OpenTelemetry for improved performance and understanding of the system. She emphasizes the importance of distributed tracing and how it facilitated better incident response and team engagement. The conversation also touches on the challenges of integrating observability tools, the need for consistent logging and metrics, and the cultural shift towards DevOps practices within the company. Overall, the session highlights the significance of observability in software development and the impact of proactive measures in improving system reliability. -# OTEL Q&A with Jennifer Moore +## Chapters -[Music] +00:00:00 Introductions and welcome to the stream +00:02:15 Background on Jennifer Moore and her previous company, Screencastify +00:04:30 Overview of Screencastify's application stack +00:06:50 Jennifer's role as DevOps lead and responsibilities +00:09:00 Discussion on the need for observability at Screencastify +00:12:30 Jennifer's initial assessment of existing observability practices +00:15:45 Motivation for improving observability in the system +00:18:20 Introduction of OpenTelemetry for better instrumentation +00:22:00 Challenges faced during implementation of OpenTelemetry +00:25:30 The impact of observability on team dynamics and management perception +00:30:00 Future applications of knowledge gained from previous experience at Influx Data +00:33:15 Feedback on OpenTelemetry and suggestions for improvement +00:37:00 Discussion on the correlation of logs and traces for better insights +00:40:30 Closing thoughts and encouragement for adopting observability practices -Welcome everyone to the OTEL Q&A. We are super excited to have Jennifer Moore with us today. Welcome, Jennifer! +# Otel Q&A with Jennifer Moore -**Jennifer:** Hi, thanks! +Welcome everyone to the Otel Q&A! We are super excited to have Jennifer Moore with us today. Welcome, Jennifer! -**Host:** Okay, so I guess first things first. I asked Jennifer to come on because I had talked to her previously about some experiences she had at a prior company around OTEL, and I thought it was a really interesting and compelling story. So, let's start with some background. This is not at your current employer but at a previous one. Can you give us some background on their application stack? +## Introduction -**Jennifer:** Sure! This was a whole job ago, and all of this was happening about a year ago, so some details may be fuzzy for me. The company is called Screencastify, and they make a screen recording application. They were in the early days of version two, which would include video hosting and editing services, whereas before it had just been a screen recorder that saved to whatever storage you had handy. +**Jennifer:** Hi, thanks! -The stack included lots of TypeScript, running in Google Cloud Platform (GCP) and containers. The main backend was a TypeScript server, likely Next.js, running in Google Cloud Run, which is a serverless container product. Everything being serverless will be relevant here. The backend handled routine web service tasks like account management and billing, as well as on-demand video encoding. +I asked Jennifer to join us because I previously discussed some compelling experiences she had at a prior company regarding Otel. I thought it would be interesting to share her story, so let’s start with some background. -We also had what we called a task system or job system that ran in Kubernetes. This consisted of multiple Kubernetes microservices that responded to video uploads and edits for transcoding and processing work, powered by a message queue. +## Company Background -**Host:** What was your role at the company? +**Interviewer:** This is regarding a previous employer, so could you give us some background on their application stack? -**Jennifer:** I joined as the DevOps lead. A lot of what I did there was to drive DevOps practices and handle the day-to-day tasks of DevOps, such as working on CI/CD pipelines, observability, internal tooling, and developer experience. Anything that wasn't primarily focused on building a feature in the application likely crossed my keyboard at some point. +**Jennifer:** Sure! This was a whole job ago, and everything was happening about a year ago, so some details may be a bit fuzzy. The company is called Screencastify; they make a screen recording application. They were in the early days of version two, which included video hosting and editing services. Previously, it was just a screen recorder that saved to whatever storage you had handy. -**Host:** That's a lot of hats to wear! You mentioned observability. Did the company recognize a big enough need for it when you joined? +The stack consisted of a lot of TypeScript, running in Google Cloud Platform (GCP) within containers. The main backend was a TypeScript server, utilizing Next.js, and it was running in Google Cloud Run, which is a sort of serverless containers product. Many components were serverless, which is relevant to our discussion. This backend handled routine web service tasks like account management and billing, as well as on-demand video encoding. -**Jennifer:** Yes, somewhat. They had encountered the concept and made some starts at observability. There were definitely people who recognized it as a good idea, and nobody was fighting against it. However, they didn't really know what to expect from it, and the efforts felt scattered. They had some tracing but it was disconnected and not useful, so most information came from logs before I took on that responsibility. +There was also what we called the task system, which ran in Kubernetes. It consisted of several microservices that responded to video uploads and edits, performing video transcoding and processing. This system was powered by a message queue, which helped manage the job processing. -**Host:** When you started implementing observability, what motivated you? +## Role at Screencastify -**Jennifer:** My motivation was to better understand what the system was doing. As a senior engineer, I needed to support these systems, and that's basically impossible without knowing what they are and how they behave. After a while, I grew uncomfortable lacking that understanding and decided to remove the ineffective auto-instrumentations and start instrumenting things more consistently. +**Interviewer:** And what was your role at Screencastify? -**Host:** So you went into the application code and started instrumenting? +**Jennifer:** I joined as the DevOps lead. A lot of what I did was drive DevOps practices and handle the day-to-day operations, like working on CI/CD pipelines, observability, internal tooling, and developer experience. Essentially, anything not primarily focused on building a feature in the application likely crossed my keyboard at some point. -**Jennifer:** Mostly, I removed the existing auto-instrumentations from tools like DataDog that weren't helpful for us and replaced them with OpenTelemetry auto-instrumentations, which worked for everything we wanted. Suddenly, my flame graphs filled out, and I could see what was going on from start to finish. +## Observability at Screencastify -**Host:** How was the learning curve with OpenTelemetry? Was it something you were familiar with initially? +**Interviewer:** You mentioned observability. Did the company recognize a need for it when you joined? -**Jennifer:** I was familiar with it conceptually. I spent a lot of time on Twitter talking to observability and resilience folks, so I understood the goals and concepts. However, the actual mechanics of turning on and using OpenTelemetry was new to me. +**Jennifer:** Yes, somewhat. They had encountered the concept and had started trying to implement some observability practices. There were definitely people who recognized that it was a good idea, and nobody was fighting against it. However, they didn’t fully understand what to expect from it and had made scattered attempts at instrumentation, which were not particularly useful. -**Host:** Did you have anyone from your team assisting you? +When I joined, there was some tracing that was very disconnected and not really aligned with what people needed. As a result, most of the useful information came from logs. -**Jennifer:** I didn't want to be the only person who knew how it worked, so I did the initial setup but made sure to distribute follow-up work to the rest of the DevOps team. I wanted others to understand what was going on. +**Interviewer:** When you started implementing observability, what was your motivation? -**Host:** How was that received within your team? +**Jennifer:** My motivation was to better understand what the system was doing. As a senior engineer, I needed to support these components, and it’s almost impossible to do that without understanding the system's behavior. After a while, I grew uncomfortable not having that understanding and decided to remove the ineffective instrumentation and start implementing things more consistently. -**Jennifer:** It went really well. Distributed tracing has some inherent complexity, so there was a learning ramp-up, but after going through that, everyone was pleased and happy to have it. +## Implementation of OpenTelemetry -**Host:** Did you see favorable results from this implementation? +**Interviewer:** Did you start instrumenting the application code yourself? -**Jennifer:** Yes, I started seeing favorable results, and while I noticed improvements, management was initially neutral but became more positive as we encountered incidents. They realized the value of having observability when problems arose. +**Jennifer:** Mostly, I removed the existing auto-instrumentation that was not helpful and replaced it with OpenTelemetry auto-instrumentations, which worked much better for us. Suddenly, my flame graphs filled out, and I could see what was going on from start to finish. It was exactly what I was looking for! -**Host:** Did this incentivize developers to do additional work, like manual tracing? +**Interviewer:** How was your learning curve with OpenTelemetry? -**Jennifer:** Yes, some engineers were very on board with the idea of tracing and wanted to dive deeper. While some were neutral, nobody was actively against it. Unfortunately, organizational changes limited the opportunities for more extensive manual tracing. +**Jennifer:** I was familiar with the concepts but had never implemented it before. The actual mechanics of turning it on and using it were new to me. -**Host:** Did you end up implementing an OTEL collector? +**Interviewer:** Did you have anyone from your team assisting you? -**Jennifer:** Yes, I wanted to do that. The company was already invested in DataDog, so moving toward an OpenTelemetry setup was a process. I started it, and my understanding is that everything is now OpenTelemetry, with a move away from DataDog to another vendor. +**Jennifer:** I didn’t want to be the only person who understood how it worked. I did the initial setup myself, but I made sure to distribute follow-up work to the rest of the DevOps team so everyone could get involved and understand what was going on. -**Host:** That’s great to hear! Have you been able to apply what you learned at your previous workplace to your current role? +## Team and Management Response -**Jennifer:** Yes and no. I am currently at Influx Data, where the engineering focus has been on getting the 3.0 version of the database ready. Now that we’ve achieved that, there’s an opportunity to think about observability and OpenTelemetry again, but the focus is more on making the database a better backend for collecting spans and telemetry data rather than instrumenting our applications. +**Interviewer:** How was this received within your team? -**Host:** What were the most challenging aspects of implementing OpenTelemetry in your previous organization? +**Jennifer:** It went really well! Distributed tracing has inherent complexity, so everyone had to ramp up, but they were pleased to have gone through that process and were happy with the results. -**Jennifer:** The most challenging part was that there had already been a start with a vendor's proprietary SDK, which confused things. I initially tried to leave that in place while I instrumented with OpenTelemetry, but it made traces more confusing. I wish I had removed the proprietary SDK sooner. +**Interviewer:** What about management? Did they see the value? -Another challenge was that there were areas without existing auto-instrumentations. For example, I had to write my own auto-instrumentation for BigQuery because it didn’t have one. That was a learning curve, but also a benefit because it allowed me to create something useful for us. +**Jennifer:** Initially, management was neutral to positive about it. They recognized it as a good idea, but they didn’t see the full value until we encountered some incidents that highlighted the need for better observability. -**Host:** How did your fellow DevOps engineers gain skills in OpenTelemetry? +## Developer Incentives and Future Work -**Jennifer:** The majority was through hands-on experience. After I set up the initial auto-instrumentation, I made a point to have others take it further. We also had a strong learning culture, including a technical book club where we read *Observability Driven Development*, which helped familiarize many people with the concepts. +**Interviewer:** Did your efforts incentivize developers to do additional work like manual tracing? -**Host:** Any feedback for the OpenTelemetry project as a whole? +**Jennifer:** Yes, several engineers were very on board with the idea of tracing. Some wanted to dive deeper and contribute additional instrumentation. However, some organizational issues arose, which limited opportunities for that to happen as much as I would have liked. Still, those who were interested in enhancing the observability found it rewarding. -**Jennifer:** The early onboarding and learning curve can be steep. Improving that would be great. Also, it would have been nice to have more stability in metrics and logging during my time, as a consistent interface for all telemetry signals would have been very convenient. +**Interviewer:** Did you implement an OpenTelemetry collector? -**Host:** Thank you, Jennifer! Does anyone else have questions? +**Jennifer:** Yes, I wanted to move away from the proprietary stack toward OpenTelemetry. The company was already invested in Datadog, which made the transition challenging, but I started the process before I left. My understanding is that they now fully transitioned to OpenTelemetry. -**Audience Member:** Can you elaborate on the consistent interface for telemetry signals? +## Current Experience at InfluxData -**Jennifer:** Sure! With OpenTelemetry, spans are easier to configure and send to a collector, compared to logs and metrics, which require separate configurations. It would be helpful to send logs and metrics to the collector in the same way spans are sent, without making those things application concerns. +**Interviewer:** Has your knowledge from Screencastify translated to your current role at InfluxData? -**Host:** I believe we're getting closer to that. +**Jennifer:** Yes and no. Currently, most engineering attention is focused on getting the new version of our database ready. My work intersects with observability, but it’s more about making the database a better backend for collecting spans and telemetry data rather than instrumenting our own applications for traces. -**Jennifer:** Yes, I think we are, especially with the correlation of logs to traces, which tells a fuller story. +## Challenges in Implementing OpenTelemetry -**Host:** Thank you, Jennifer, for sharing your inspiring story. It highlights that when things are broken, observability becomes essential. This has been a great session! +**Interviewer:** What were the most challenging aspects of implementing OpenTelemetry in your previous organization? -We have an upcoming Q&A on December 7th with someone from HashiCorp who has attempted to instrument Nomad. Stay tuned for that! +**Jennifer:** The biggest challenge was that there had already been an attempt with a proprietary SDK, which created confusion. I initially tried to keep that in place while adding OpenTelemetry, but it led to complications. I wish I had removed the proprietary SDK right away. Another challenge was areas without existing auto-instrumentations. I had to write my own for some libraries, which was a learning curve but also a valuable experience. -Thanks again, Jennifer, and we'll let you know once the video is posted! +## Final Thoughts and Feedback on OpenTelemetry -[Music] +**Interviewer:** Do you have any feedback for the OpenTelemetry project? + +**Jennifer:** I think the onboarding process is steep, and if that can be improved, it would be great. At the time, metrics and logging were not as stable as they are now, so having a consistent interface and SDK for all telemetry signals would be beneficial. + +**Interviewer:** Thank you for your insights! Does anyone else have questions for Jennifer? + +**Audience Member:** Could you elaborate on having a consistent interface in the SDK? + +**Jennifer:** With OpenTelemetry, spans are streamlined in terms of configuration, but logs and metrics often require different setups. It would be great to have a unified way to send all telemetry types to the collector without making it an application concern. + +**Interviewer:** Thank you, everyone, for attending. Jennifer, your story is incredibly inspiring. It highlights the importance of observability, especially when things break. We hope others draw inspiration from your experiences with OpenTelemetry. + +Stay tuned for our next Q&A on December 7th with someone from HashiCorp! Thank you, Jennifer, and we’ll let you know once the video is posted. ## Raw YouTube Transcript -[Music] so welcome everyone to otel Q&A we are super excited to have Jennifer Moore with us today welcome Jennifer uh hi thanks um okay so I guess first things first so I asked Jennifer to come on because um I had talked to her previously um about some experiences that she had had at a prior company around otel um and I thought it was a really really interesting and compelling story so um I guess we'll start with h why don't you give us some background I know this is it's not at your current employer it's at a previous employer why don't you give us some background of you know their uh for starters like what their their application stack was like um and um and then go from there sure um this was a whole job ago and all of this was happening like a year ago so um some of the details will have gotten fuzzy for me um by this point but that's cool um so the company um is called screencastify um they make a like screen recording application um and um the uh like the was um the kind of early days of a version two which would um have like be a you know video hosting and editing service um whereas before it had been just kind of a screen recorder and it was saved to whatever storage you happen to have handy um and that was the whole thing um so this stack um was uh lots of typescript um with uh um and all running in like gcp um and containers so the um like the main back end was a typescript um I think nextjs I forget some typescript um backend server um running in uh um Google Cloud run um this sort of like serverless containers um product that they have um and um that everything is serverless will or many things are serverless will be relevant um and then there was also um and so like that back end handled all of the uh like you know routine um you know we're hosting a web service um kind of stuff like account management and billing and Etc um and also some on demand video um en coding um and then there was um what we called the uh task system I think or job system um something like that um which was um like that ran in kubernetes um and it was um you know a bunch of um you know kubernetes uh microservices that would uh um you know respond to um videos being uploaded and um edited and things um to do um you know video transcoding and processing work on that stuff um and um that all um was powered by a bulm Q uh um a bulm q message Q um job Q um whatever and um yeah and I don't know otherwise ran fairly self-contained all right all right and and what was your role then um at this company um I joined as um the kind of like devops lead um it was like my uh like a lot of what I did there was to kind of Drive devops practices and um then also like do the um chopwood carry water parts of devops um so a lot of uh working on um the like cicd pipelines um a lot of workon observability um and uh internal tooling and um developer experience and things you know everything that um wasn't primarily focused on like building a feature in the application um was probably something that came across my keyboard at some point cool so that's a that's a lot of hats to wear then yeah awesome um okay so now you mentioned observability and we are here so tell us um tell us to what extent like when did you realize or or actually let's take a step back um did the company um see a need a big enough need for observability like had they seen that already when you had joined um yes somewhat I think um like I think that they you know they had encountered the concept um and they had you know made a start at like trying to do some observability things um you know there were definitely some people who like recognized that it was a good idea um and like nobody was fighting them about it um but they didn't I think they didn't really know what they should expect out of it um and so they'd kind of like done some like scattered starts at it and um instrumented some things um and but like you know some things in isolation and it wasn't in a very useful way um and so um you know when I got there there was uh like they had tracing um but uh like you know very disconnected um and um not really the things that people were interested in um and so a lot of like anything that anybody actually needed to get out of it um before I took that on um came from logs okay okay so then um you entered into the picture um and what was at that point when you started like implementing um or making making the systems more observable like what what was sort of your your um motivation for that um so my motivation like throughout for um making this system more observable was that I wanted to better understand what it was doing um because you know like one of the like one of the hats I was wearing was like Sr um and I needed to support these things um and that's basically impossible to do if you don't know like what it is or what it's doing um and certainly not you know like how it's behaving um and so um like after you know like after a little while um I grew very uncomfortable not having like that understanding of the system um and decided to just start like you know kind of remove the um false start instrumentations that um were already there and start instrumenting things more consistently so you actually went into the application code and started instrumenting things um somewhat um so mostly what I did was um like remove the auto instrument that were already there um from like there were I don't know data dogs like proprietary ones um and uh I'm sure they would work if they worked but they didn't actually Auto instrument several of the libraries we were using and so it wasn't helpful for us um and so I replaced those with um open elemetry auto instrumentations um which did have Auto instrument for all of the things we wanted um and suddenly like my flame flame graphs filled out and I could see what was going on from um start to finish more or less and that was exactly what I've been looking for magic um and and how was uh like how was your learning curve with with in terms of an in terms of open Telemetry is it something that you were famili familiar with initially or is that something that you had to teach yourself um so yeah I mean I was familiar with it like conceptually initially um I I don't know like back when I was on Twitter um I spent a lot of time talking to you know observability and resilience folks on Twitter um and um so you know like the um like the goals and the concepts and everything was all very um like very familiar to me by that point but then the actual mechanics of like how do you like turn on and start using open Telemetry um was not something i' had done before and so like was that did that end up end up being like a single-handed effort on your part like did you have anyone from your team assisting um yeah like I didn't I really did not want to be like the only person who knew what it was doing or how it was working um and so like I did do the initial setup myself but then like I made a point to kind of stop there and um like and like distribute followup work um from that to the rest of the devops team um at least so that you know there would be other people who knew what was going on and um knew what they were looking at whenever things needed detention and how was that received within your team um I mean I think that went really well um you know the um like distributed tracing is uh there's some inherent complexity to that domain and so there's um some sort of like learning and um ramp up um to get comfortable with the concepts there um and you know everybody had to go through that but I think like after having gone through that everybody was very pleased to have done it um and very happy to have it that's really great and so you know in terms of so did you would you say then that you were like starting to see um favorable results um like did or I mean you you mentioned that you know you started you yourself started seeing some favorable results like seeing the the things on the flame graphs um how would you say the the rest of your team took it and even a step further like about management How do they how did they take that did they did they see that as a benefit um I think uh you know at first management was kind of I don't know like like neutral um like neutral of positive about it um you know they like again it was a thing that they recognized um was a good thing to do but I don't think they really saw how much value it has at first um um you know but then we get to uh some stories with some incidents in them um and I think the value of it becomes a lot more obvious to them um at that point awesome and uh just uh circling back a little bit like you know you you it sounds like you did a lot of stuff around the um Auto instrumentation side of things um do you think like did this incentivize the developer of the actual application to go and do some additional work and add some like manual tracing for example um yes um like there were definitely some Engineers who were very on board with the idea of having tracing and you know wanted to um like wanted to go all in on that um and you know then there were others who didn't you know like again we're pretty neutral about it um I think you know no one was again like no one was fighting against it um a lot of it was you know like sure that seems fine you can do that um and a handful of boosters but were they okay with like going in and instrumenting themselves as well to kind of enhance um what the auto instrumentation was giving um yes I think so so like it's a little bit hard to say um because honestly um some things blew up organizationally um you know and so I like didn't uh I didn't have as much opportunity for that to actually happen as I would have liked but um people were very there were at least several people who were very like eager to have done it right right so at least it was they they at least got to like reap the rewards um of what you had at least set up right yeah which is always a good now in terms of your setup like um did you did you end up um like implementing an otel collector um they did yes um that was something that I had wanted to do um but you know so like the the company was sort of already on data dog and kind of invested in the data dog tooling ecosystem um and so moving like away from the like that proprietary stack and towards the you know open Telemetry um libraries and collector and things was a process um but you know it's one that like I had I tried to start and it continued after I left and um my understanding is that um like everything is open Telemetry there now um and I think they've even moved away from data dog to another vendor oh wow that's that's really cool to be able to like move to you know fully like vendor neutral sort of setup is is super awesome and I think especially like you had set the wheels in motion around that and that it continued even after you left yeah now how's that continued like um you know the the knowledge that you gained at your previous workplace is it something that you've been able to apply where you're at right now or uh yes and no um so right now um like I am at uh influx data um and so at influx um our like well when there's been um sort of all of the engineering attention um has been directed towards getting you know the 3.0 version of the database into um like uh to 3.0 um and uh um you know which is thing that like we have done now and that's going um pretty well um and so that you know frees up some people to think about other things again um and sort of where that intersects with observability and open Telemetry has been more about um like getting the database itself to be um a uh like a better better suited to work as a um backend for um collecting spans and Telemetry data um rather than like instrumenting our own things um with um for traces um and so we do have a lot of um we do have a lot of things that are very well instrumented um and like very uh like very very detailed instrumentation um and a lot of that is very manual um and that's done um with more of a performance um like um like with the objective of understanding performance um more than like Behavior or um sort of investigating um like errors or things like that and so um which is to say there's um like very deep small spans as opposed to like um you know wide spans which um I think are uh a little bit easier to work with if you're trying to like ask you know something weird is going on and you have no idea what um yeah yeah so you end basically like you have a slightly different goal compared to place but still using open Telemetry to achieve that goal yeah cool and um what would you say like were the most challenging aspects like given all your experience around open Telemetry what were the most challenging aspects in terms of implementing open Telemetry um in your specifically in your previous organization um yeah so like I think the most challenging um I think the most challenging was that there like had already been a like a start with a fender proprietary SDK um and so that um did a like that confused a lot of things um I had tried to like leave that in place um while I like at first and then instrument with open Telemetry but um you know then like these two libraries just started like um reporting on each other and um it made the whole um it made um traces like more confusing than they had been um without it um and so you know removing that proprietary SDK and just having the one um mechanism um for you know like in any given process for instrumentation was um you know something that I wish I had done right away um in retrospect um and I think the other thing was that um you know was in areas where like there did not already exist Auto instrumentations um so like I BQ earlier um which didn't have one um until I wrote it because I really needed it um and so um and like you know I I can say that's a challenge but also um a benefit because it like nobody else had one either um and with open Telemetry like it was a viable thing that I could go and write my own um Auto instrumentation package and um you know be able to get that for myself um with libraries that otherwise just didn't have it and how how was that experience of like writing the auto instrumentation yourself like was it a huge learning curve or was it that yeah that was a pretty substantial learning curve um and um I don't know it was a good thing that I started early because um you know then we got into some uh incidents and I really needed it um and I was able to get it you know like functional um in a relatively short time span from there so that I could start using it and actually see what was going on that's so cool and and you know you mentioned that you you uh your your fellow devops Engineers were were leveled up in the ways of otel like is that something that you help them out with like um what kind of like how is it that they gain the skills like leveled up so that they could also um you know have that knowledge of open Telemetry that you had um I mean I think the majority of that was you know just like Hands-On doing it um you know the like um again like after you know um I had set up like basically one um Auto instrumentation um and gotten that configured um and everything I like then um like I stopped there and I really did make a point to um have other people take that further um so that yeah like because I wanted them to have that experience um and then um like more broadly we did also have um like early on there there was a very strong learning culture which was one of my favorite things about that company um and so we had a like a sort of like technical book club um and we read the uh observability driven development um book for that book club and um you know I think that that was also very useful in terms of getting some a lot of people familiar with the like at least the concepts and you know what you should um sort of like clarifying expectations about what like this gets you um as you know like from an office perspective and from a engineering perspective and yeah and hopefully got them stoked too about it right um yeah I mean that it was um a very like the people who were inclined to be interested um definitely came away from that more interested nice nice it's really cool that this is a very like sort of operational driven um uh Hotel undertaking which I I guess it kind of makes sense because like when when things Break um it's always going to be more on the operational side of things and it's not necessarily the developers that are going to be the ones who are going to you know be oh why is this breaking right because by then it's like I hate to say it but less less their concern even though it is very much their concern but unfortunately I think I think our our culture is still kind of Shifting towards the not my problem because it's in production yeah it is um it is a little bit farther away um from them and like I have some sympathy for that i' spent a lot of time you know doing appdev um and um like but also only some sympathy because the whole time I was doing APD I was very frustrated by my inability to have any contact with production um and so I don't know and like that was also one of the things that I was trying to drive there like Beyond you know observability and open Telemetry was to like actually fulfill some of the promise of devops and have um you know have that not be such a bright line between the two do you think uh do you think you got closer to uh to achieving that goal I I think so yeah um one of the um and like one of the ways I did that was to like really emphasize um sort of how the tools are used um whenever um like whenever it came up um so you know like in doing incident um Retros the like um you know I made a point to ask people like what did you look at um that made you think that that was um you know the the concern um and you know like how did you execute those queries like let's actually open up the tool and um go through the UI and see how that's performed cool um I want to go back to um you know our chat earlier about like um Management's perspective on on using you know observability and I I believe you mentioned that it was kind of neutral until they're like oh this is this is useful um one of the things that I always find can be challenging in an organization is like bringing something new to management and and like you know you almost like need to need that permission from them um before before executing so is that something where you like did you ask for permission or did you end up asking for forgiveness in in that uh at that stage um so yeah like neither honestly um the way this worked out um so again like you know they had already like before I joined had whatever conversations they had about like getting better observability and like somebody I somebody signed off on doing something um and you know they started some instrumentation um and so that like that was clearly a thing that was like you know that was okay um so like um I didn't need to pitch them on having it um and like you know as a matter of fixing it like that was more of a um like like I don't know like a question for my fellow Engineers um like you know somebody put this in and somebody probably understands it um and I need to but like it's not you know satisfying the goals that we had for it I think and so I need to do something about that and like um but like management wasn't like wasn't going to say no to that um right because you already had some sort of an observability culture um and then in terms of like investing further in it um you know like we ran into an it um we you know launched and um a big part of this launch was or a big part of the reason for like going to the sort of 2.0 version of the product um was to enable some like account and billing um features that just weren't possible um with the way that like the um like those data schemas existed um and so like part of that and like they existed in Firebase um and they were moving to a like postgress to have you know real schemas um and that had I think some of the predictable problems um and um the migration kind of stalled out um and that was real bad um and things just stopped working for hours at a time because um the like the migration was kind of killing the um the post Chris um server without ever completing um but we didn't know that it just sort of like stopped working um and um like after some initial investigation um like instrumenting all the things to figure out what is even going on um was like the next thing and it wasn't something that I needed permission to do at that point because like you know production is down and it's going down for hours at a time um repeatedly so um it needs to be fixed um and um yeah like I don't know when the building's on fire you don't need permission to install spring clears anymore yeah absolutely absolutely so I guess at that point then even though like you were using like a different different tool set for observability and and you made the decision to like let's let's use open Telemetry Instead at that point as you said because things were not working then it became like less you know like put in put in the tool that works for us kind of thing is what it sounds like yeah um and we had already started that by the time we got to like that incident um and so like that was very fortunate because we were getting like we already had a better experience with open Telemetry like there were some parts of the system that we like could fairly well understand on what they were doing um but then like the rest of it you know that um like had not been instrumented and um so that like that had to be done quickly right so there that that's your incentive when things are burning when the house is burning open Telemetry to the rescue awesome um I think uh before I turn it over to the rest of the folks on this call um do you have any um feedback um for the open Telemetry project as a whole like good or bad in terms of like your experience um because I mean you know like we one of the one of the uh things that we do as as a group here is to um collect feedback from from users to help make the open Telemetry experience better so um um yeah I like I'm not sure that I don't know um I mean I think that like the and this was like a year and a half ago that I was doing it or whatever the like early like getting set up onboarding learning curve is pretty steep um I don't know exactly how to like how to improve that and I'm sure I'm not the first person to have said this either um I think you know like if that can be improved that would be great um also I think that like um metrics and logging you know got to like stable um since I was doing this um and it would have been really nice to have had that at the time I think um because it would be you know very convenient to have um like a consistent uh sort of like interface and um SDK for all of those Telemetry signals um and then being able to um you know like send them to a collector and then have the collector send them to wherever they're going um and um not making that like not making those things application concerns um is like would be great um I haven't had I haven't had the chance to actually do that um to see if it works or not cool thank you um does anyone else on this call have any questions for uh for Jennifer um I was just curious on the last piece you just said about having a consistent interface in the SDK for the Telemetry signals and not making them application concerns what does um I was just curious if you could elaborate a little bit more on that piece um yeah so like um so like you have open cemetry for your um like traces um and then you know like you're going to have some sort of logging solution for your logs and like hopefully you're doing structured logs um and like again some solution for metrics um assuming you're doing metrics um probably Prometheus or whatever um but like you know to get like to get those logs and those um metrics to go somewhere um you wind up having to like configure some um like vendors uh like logging Library um so that you can ship your logs to like your logging back end or um like or you log everything to the terminal or to a file or whatever and have some sort of like infrastructure magic pick it up and send it somewhere um but like and then you know Prometheus like you have to create your um scraping endpoint and let Prometheus scrape it and so you have to like have Prometheus configured and able to reach you and Etc um and um whereas like you know with spans um you like configure your exporter and you're done um and you're exporter just sends them to the collector um and um like and that can like The Collector address or whatever can come from config like after you're instrumented um you don't have to like there's not much to maintain there um and um you know like I would at the time I would really like have lik to be able to you know send my logs and my um like metrics to The Collector in the same way that I was doing with spans um rather than like having you know three solutions for three different kinds of telemetry gotcha I would see we're getting closer to that right I personally don't know I'm asking uh anyone else I think we're getting closer to it I think uh like definitely I like I think I think logs are fine like metrics are have definitely matured a lot in the last year um and I know a lot of observability backends are now ingesting metrics as well so in addition to traces which is nice um and then I think we're starting to see that with logs as well logs I know are a different Beast because there's like no um no logging SDK there's a logs Bridge um SDK so the idea is like use use one of the common logging libraries that is supported by open Telemetry and there's like a bridge um SDK for it and then it'll convert that format to OTP so then it can be ingested by whatever um whatever back end so which is kind of nice I guess they didn't want to reinvent uh reinvent wheel with logs because there's like so many yeah so many different things out there so yeah that's that's my understanding of it from my limited time playing with log so yeah yeah the which I like full sympathy to that strategy the um the momentum on the way people do logs is enormous it's not like that's not changing yeah um but how they get like collected and um organized you know you know that yeah I think the game changer is being able to correlate the logs to the traces so that you have that Fuller picture which I'm super excited about because I understand a lot of us grew up on logs but I think the traces will help tell the full story and so when you have that correlation then then you can get magic yeah I mean I think logs correlated with traces is like is the best most ideal outcome um like I know that Trac is have events and like can do point in time things but logs are just better at that like um and you know being able to see like point in time events um correlated to like where they happened within a span um and within a trace is exactly like that's what I want that's yeah yeah the dream yeah um anyone else have any other comments questions for Jennifer shall we break early then cool awesome well thank you Jennifer for sharing your story with us I think it I I really love love the story because I think it's the things are broken need observability so um I think it's such a compelling case for anybody who's like on the fence on whether or not it's a worthwhile investment um and it's it's awesome that your your company already kind of had its foot in the door um for that and and that you were able to kind of take it further to where it actually needed to be so that it was useful to uh to you and your team for troubleshooting so um I I think it's I think it's a great story I hope others will draw inspiration from that as well um as a compelling reason as to why observability with open Telemetry is a good combination um do we have I think we've got some upcoming events yeah on December the 7th we have another Q&A um with uh someone from Hashi Corp who has attempted many times to instrument Nomad so that'll be uh um I think that'll be a really interesting story as well so anyone who is following along stay tuned for that Q&A hopefully we'll get to see you live but if not we will see you on YouTube sounds good it was great to meet you all yeah thanks so much Jennifer and we'll let you know once the video is posted cool yeah um yeah I was gonna say thank you Jennifer for your story that's like really really inspiring because I feel like so many people want to always adopt it but then getting that buying from your leadership is always the problem I mean technically like it's not making their money until you sure that this is like causing problems and then we really need to do this yeah um yeah I don't know I don't know why like it's weird that it's such a fight so much of the time um because problems are going to happen and yeah like what this gets you is being able to point at where the problem is happening and I don't know why people don't want to be able to do that right so logical in my experience it only works either in teams that are excited about Progressive engineering practices or that are feeling so much pain from vendor lock in that they're ready to set everything on fire yes yes so true so basically means you need like a really nasty incident or or like really bad vendor lock in as your compelling reason well if you get that incident don't let it go to waste yeah totally right quote of the [Laughter] day awesome well thank you [Music] everyone +so welcome everyone to otel Q&A we are super excited to have Jennifer Moore with us today welcome Jennifer uh hi thanks um okay so I guess first things first so I asked Jennifer to come on because um I had talked to her previously um about some experiences that she had had at a prior company around otel um and I thought it was a really really interesting and compelling story so um I guess we'll start with h why don't you give us some background I know this is it's not at your current employer it's at a previous employer why don't you give us some background of you know their uh for starters like what their their application stack was like um and um and then go from there sure um this was a whole job ago and all of this was happening like a year ago so um some of the details will have gotten fuzzy for me um by this point but that's cool um so the company um is called screencastify um they make a like screen recording application um and um the uh like the was um the kind of early days of a version two which would um have like be a you know video hosting and editing service um whereas before it had been just kind of a screen recorder and it was saved to whatever storage you happen to have handy um and that was the whole thing um so this stack um was uh lots of typescript um with uh um and all running in like gcp um and containers so the um like the main back end was a typescript um I think nextjs I forget some typescript um backend server um running in uh um Google Cloud run um this sort of like serverless containers um product that they have um and um that everything is serverless will or many things are serverless will be relevant um and then there was also um and so like that back end handled all of the uh like you know routine um you know we're hosting a web service um kind of stuff like account management and billing and Etc um and also some on demand video um en coding um and then there was um what we called the uh task system I think or job system um something like that um which was um like that ran in kubernetes um and it was um you know a bunch of um you know kubernetes uh microservices that would uh um you know respond to um videos being uploaded and um edited and things um to do um you know video transcoding and processing work on that stuff um and um that all um was powered by a bulm Q uh um a bulm q message Q um job Q um whatever and um yeah and I don't know otherwise ran fairly self-contained all right all right and and what was your role then um at this company um I joined as um the kind of like devops lead um it was like my uh like a lot of what I did there was to kind of Drive devops practices and um then also like do the um chopwood carry water parts of devops um so a lot of uh working on um the like cicd pipelines um a lot of workon observability um and uh internal tooling and um developer experience and things you know everything that um wasn't primarily focused on like building a feature in the application um was probably something that came across my keyboard at some point cool so that's a that's a lot of hats to wear then yeah awesome um okay so now you mentioned observability and we are here so tell us um tell us to what extent like when did you realize or or actually let's take a step back um did the company um see a need a big enough need for observability like had they seen that already when you had joined um yes somewhat I think um like I think that they you know they had encountered the concept um and they had you know made a start at like trying to do some observability things um you know there were definitely some people who like recognized that it was a good idea um and like nobody was fighting them about it um but they didn't I think they didn't really know what they should expect out of it um and so they'd kind of like done some like scattered starts at it and um instrumented some things um and but like you know some things in isolation and it wasn't in a very useful way um and so um you know when I got there there was uh like they had tracing um but uh like you know very disconnected um and um not really the things that people were interested in um and so a lot of like anything that anybody actually needed to get out of it um before I took that on um came from logs okay okay so then um you entered into the picture um and what was at that point when you started like implementing um or making making the systems more observable like what what was sort of your your um motivation for that um so my motivation like throughout for um making this system more observable was that I wanted to better understand what it was doing um because you know like one of the like one of the hats I was wearing was like Sr um and I needed to support these things um and that's basically impossible to do if you don't know like what it is or what it's doing um and certainly not you know like how it's behaving um and so um like after you know like after a little while um I grew very uncomfortable not having like that understanding of the system um and decided to just start like you know kind of remove the um false start instrumentations that um were already there and start instrumenting things more consistently so you actually went into the application code and started instrumenting things um somewhat um so mostly what I did was um like remove the auto instrument that were already there um from like there were I don't know data dogs like proprietary ones um and uh I'm sure they would work if they worked but they didn't actually Auto instrument several of the libraries we were using and so it wasn't helpful for us um and so I replaced those with um open elemetry auto instrumentations um which did have Auto instrument for all of the things we wanted um and suddenly like my flame flame graphs filled out and I could see what was going on from um start to finish more or less and that was exactly what I've been looking for magic um and and how was uh like how was your learning curve with with in terms of an in terms of open Telemetry is it something that you were famili familiar with initially or is that something that you had to teach yourself um so yeah I mean I was familiar with it like conceptually initially um I I don't know like back when I was on Twitter um I spent a lot of time talking to you know observability and resilience folks on Twitter um and um so you know like the um like the goals and the concepts and everything was all very um like very familiar to me by that point but then the actual mechanics of like how do you like turn on and start using open Telemetry um was not something i' had done before and so like was that did that end up end up being like a single-handed effort on your part like did you have anyone from your team assisting um yeah like I didn't I really did not want to be like the only person who knew what it was doing or how it was working um and so like I did do the initial setup myself but then like I made a point to kind of stop there and um like and like distribute followup work um from that to the rest of the devops team um at least so that you know there would be other people who knew what was going on and um knew what they were looking at whenever things needed detention and how was that received within your team um I mean I think that went really well um you know the um like distributed tracing is uh there's some inherent complexity to that domain and so there's um some sort of like learning and um ramp up um to get comfortable with the concepts there um and you know everybody had to go through that but I think like after having gone through that everybody was very pleased to have done it um and very happy to have it that's really great and so you know in terms of so did you would you say then that you were like starting to see um favorable results um like did or I mean you you mentioned that you know you started you yourself started seeing some favorable results like seeing the the things on the flame graphs um how would you say the the rest of your team took it and even a step further like about management How do they how did they take that did they did they see that as a benefit um I think uh you know at first management was kind of I don't know like like neutral um like neutral of positive about it um you know they like again it was a thing that they recognized um was a good thing to do but I don't think they really saw how much value it has at first um um you know but then we get to uh some stories with some incidents in them um and I think the value of it becomes a lot more obvious to them um at that point awesome and uh just uh circling back a little bit like you know you you it sounds like you did a lot of stuff around the um Auto instrumentation side of things um do you think like did this incentivize the developer of the actual application to go and do some additional work and add some like manual tracing for example um yes um like there were definitely some Engineers who were very on board with the idea of having tracing and you know wanted to um like wanted to go all in on that um and you know then there were others who didn't you know like again we're pretty neutral about it um I think you know no one was again like no one was fighting against it um a lot of it was you know like sure that seems fine you can do that um and a handful of boosters but were they okay with like going in and instrumenting themselves as well to kind of enhance um what the auto instrumentation was giving um yes I think so so like it's a little bit hard to say um because honestly um some things blew up organizationally um you know and so I like didn't uh I didn't have as much opportunity for that to actually happen as I would have liked but um people were very there were at least several people who were very like eager to have done it right right so at least it was they they at least got to like reap the rewards um of what you had at least set up right yeah which is always a good now in terms of your setup like um did you did you end up um like implementing an otel collector um they did yes um that was something that I had wanted to do um but you know so like the the company was sort of already on data dog and kind of invested in the data dog tooling ecosystem um and so moving like away from the like that proprietary stack and towards the you know open Telemetry um libraries and collector and things was a process um but you know it's one that like I had I tried to start and it continued after I left and um my understanding is that um like everything is open Telemetry there now um and I think they've even moved away from data dog to another vendor oh wow that's that's really cool to be able to like move to you know fully like vendor neutral sort of setup is is super awesome and I think especially like you had set the wheels in motion around that and that it continued even after you left yeah now how's that continued like um you know the the knowledge that you gained at your previous workplace is it something that you've been able to apply where you're at right now or uh yes and no um so right now um like I am at uh influx data um and so at influx um our like well when there's been um sort of all of the engineering attention um has been directed towards getting you know the 3.0 version of the database into um like uh to 3.0 um and uh um you know which is thing that like we have done now and that's going um pretty well um and so that you know frees up some people to think about other things again um and sort of where that intersects with observability and open Telemetry has been more about um like getting the database itself to be um a uh like a better better suited to work as a um backend for um collecting spans and Telemetry data um rather than like instrumenting our own things um with um for traces um and so we do have a lot of um we do have a lot of things that are very well instrumented um and like very uh like very very detailed instrumentation um and a lot of that is very manual um and that's done um with more of a performance um like um like with the objective of understanding performance um more than like Behavior or um sort of investigating um like errors or things like that and so um which is to say there's um like very deep small spans as opposed to like um you know wide spans which um I think are uh a little bit easier to work with if you're trying to like ask you know something weird is going on and you have no idea what um yeah yeah so you end basically like you have a slightly different goal compared to place but still using open Telemetry to achieve that goal yeah cool and um what would you say like were the most challenging aspects like given all your experience around open Telemetry what were the most challenging aspects in terms of implementing open Telemetry um in your specifically in your previous organization um yeah so like I think the most challenging um I think the most challenging was that there like had already been a like a start with a fender proprietary SDK um and so that um did a like that confused a lot of things um I had tried to like leave that in place um while I like at first and then instrument with open Telemetry but um you know then like these two libraries just started like um reporting on each other and um it made the whole um it made um traces like more confusing than they had been um without it um and so you know removing that proprietary SDK and just having the one um mechanism um for you know like in any given process for instrumentation was um you know something that I wish I had done right away um in retrospect um and I think the other thing was that um you know was in areas where like there did not already exist Auto instrumentations um so like I BQ earlier um which didn't have one um until I wrote it because I really needed it um and so um and like you know I I can say that's a challenge but also um a benefit because it like nobody else had one either um and with open Telemetry like it was a viable thing that I could go and write my own um Auto instrumentation package and um you know be able to get that for myself um with libraries that otherwise just didn't have it and how how was that experience of like writing the auto instrumentation yourself like was it a huge learning curve or was it that yeah that was a pretty substantial learning curve um and um I don't know it was a good thing that I started early because um you know then we got into some uh incidents and I really needed it um and I was able to get it you know like functional um in a relatively short time span from there so that I could start using it and actually see what was going on that's so cool and and you know you mentioned that you you uh your your fellow devops Engineers were were leveled up in the ways of otel like is that something that you help them out with like um what kind of like how is it that they gain the skills like leveled up so that they could also um you know have that knowledge of open Telemetry that you had um I mean I think the majority of that was you know just like Hands-On doing it um you know the like um again like after you know um I had set up like basically one um Auto instrumentation um and gotten that configured um and everything I like then um like I stopped there and I really did make a point to um have other people take that further um so that yeah like because I wanted them to have that experience um and then um like more broadly we did also have um like early on there there was a very strong learning culture which was one of my favorite things about that company um and so we had a like a sort of like technical book club um and we read the uh observability driven development um book for that book club and um you know I think that that was also very useful in terms of getting some a lot of people familiar with the like at least the concepts and you know what you should um sort of like clarifying expectations about what like this gets you um as you know like from an office perspective and from a engineering perspective and yeah and hopefully got them stoked too about it right um yeah I mean that it was um a very like the people who were inclined to be interested um definitely came away from that more interested nice nice it's really cool that this is a very like sort of operational driven um uh Hotel undertaking which I I guess it kind of makes sense because like when when things Break um it's always going to be more on the operational side of things and it's not necessarily the developers that are going to be the ones who are going to you know be oh why is this breaking right because by then it's like I hate to say it but less less their concern even though it is very much their concern but unfortunately I think I think our our culture is still kind of Shifting towards the not my problem because it's in production yeah it is um it is a little bit farther away um from them and like I have some sympathy for that i' spent a lot of time you know doing appdev um and um like but also only some sympathy because the whole time I was doing APD I was very frustrated by my inability to have any contact with production um and so I don't know and like that was also one of the things that I was trying to drive there like Beyond you know observability and open Telemetry was to like actually fulfill some of the promise of devops and have um you know have that not be such a bright line between the two do you think uh do you think you got closer to uh to achieving that goal I I think so yeah um one of the um and like one of the ways I did that was to like really emphasize um sort of how the tools are used um whenever um like whenever it came up um so you know like in doing incident um Retros the like um you know I made a point to ask people like what did you look at um that made you think that that was um you know the the concern um and you know like how did you execute those queries like let's actually open up the tool and um go through the UI and see how that's performed cool um I want to go back to um you know our chat earlier about like um Management's perspective on on using you know observability and I I believe you mentioned that it was kind of neutral until they're like oh this is this is useful um one of the things that I always find can be challenging in an organization is like bringing something new to management and and like you know you almost like need to need that permission from them um before before executing so is that something where you like did you ask for permission or did you end up asking for forgiveness in in that uh at that stage um so yeah like neither honestly um the way this worked out um so again like you know they had already like before I joined had whatever conversations they had about like getting better observability and like somebody I somebody signed off on doing something um and you know they started some instrumentation um and so that like that was clearly a thing that was like you know that was okay um so like um I didn't need to pitch them on having it um and like you know as a matter of fixing it like that was more of a um like like I don't know like a question for my fellow Engineers um like you know somebody put this in and somebody probably understands it um and I need to but like it's not you know satisfying the goals that we had for it I think and so I need to do something about that and like um but like management wasn't like wasn't going to say no to that um right because you already had some sort of an observability culture um and then in terms of like investing further in it um you know like we ran into an it um we you know launched and um a big part of this launch was or a big part of the reason for like going to the sort of 2.0 version of the product um was to enable some like account and billing um features that just weren't possible um with the way that like the um like those data schemas existed um and so like part of that and like they existed in Firebase um and they were moving to a like postgress to have you know real schemas um and that had I think some of the predictable problems um and um the migration kind of stalled out um and that was real bad um and things just stopped working for hours at a time because um the like the migration was kind of killing the um the post Chris um server without ever completing um but we didn't know that it just sort of like stopped working um and um like after some initial investigation um like instrumenting all the things to figure out what is even going on um was like the next thing and it wasn't something that I needed permission to do at that point because like you know production is down and it's going down for hours at a time um repeatedly so um it needs to be fixed um and um yeah like I don't know when the building's on fire you don't need permission to install spring clears anymore yeah absolutely absolutely so I guess at that point then even though like you were using like a different different tool set for observability and and you made the decision to like let's let's use open Telemetry Instead at that point as you said because things were not working then it became like less you know like put in put in the tool that works for us kind of thing is what it sounds like yeah um and we had already started that by the time we got to like that incident um and so like that was very fortunate because we were getting like we already had a better experience with open Telemetry like there were some parts of the system that we like could fairly well understand on what they were doing um but then like the rest of it you know that um like had not been instrumented and um so that like that had to be done quickly right so there that that's your incentive when things are burning when the house is burning open Telemetry to the rescue awesome um I think uh before I turn it over to the rest of the folks on this call um do you have any um feedback um for the open Telemetry project as a whole like good or bad in terms of like your experience um because I mean you know like we one of the one of the uh things that we do as as a group here is to um collect feedback from from users to help make the open Telemetry experience better so um um yeah I like I'm not sure that I don't know um I mean I think that like the and this was like a year and a half ago that I was doing it or whatever the like early like getting set up onboarding learning curve is pretty steep um I don't know exactly how to like how to improve that and I'm sure I'm not the first person to have said this either um I think you know like if that can be improved that would be great um also I think that like um metrics and logging you know got to like stable um since I was doing this um and it would have been really nice to have had that at the time I think um because it would be you know very convenient to have um like a consistent uh sort of like interface and um SDK for all of those Telemetry signals um and then being able to um you know like send them to a collector and then have the collector send them to wherever they're going um and um not making that like not making those things application concerns um is like would be great um I haven't had I haven't had the chance to actually do that um to see if it works or not cool thank you um does anyone else on this call have any questions for uh for Jennifer um I was just curious on the last piece you just said about having a consistent interface in the SDK for the Telemetry signals and not making them application concerns what does um I was just curious if you could elaborate a little bit more on that piece um yeah so like um so like you have open cemetry for your um like traces um and then you know like you're going to have some sort of logging solution for your logs and like hopefully you're doing structured logs um and like again some solution for metrics um assuming you're doing metrics um probably Prometheus or whatever um but like you know to get like to get those logs and those um metrics to go somewhere um you wind up having to like configure some um like vendors uh like logging Library um so that you can ship your logs to like your logging back end or um like or you log everything to the terminal or to a file or whatever and have some sort of like infrastructure magic pick it up and send it somewhere um but like and then you know Prometheus like you have to create your um scraping endpoint and let Prometheus scrape it and so you have to like have Prometheus configured and able to reach you and Etc um and um whereas like you know with spans um you like configure your exporter and you're done um and you're exporter just sends them to the collector um and um like and that can like The Collector address or whatever can come from config like after you're instrumented um you don't have to like there's not much to maintain there um and um you know like I would at the time I would really like have lik to be able to you know send my logs and my um like metrics to The Collector in the same way that I was doing with spans um rather than like having you know three solutions for three different kinds of telemetry gotcha I would see we're getting closer to that right I personally don't know I'm asking uh anyone else I think we're getting closer to it I think uh like definitely I like I think I think logs are fine like metrics are have definitely matured a lot in the last year um and I know a lot of observability backends are now ingesting metrics as well so in addition to traces which is nice um and then I think we're starting to see that with logs as well logs I know are a different Beast because there's like no um no logging SDK there's a logs Bridge um SDK so the idea is like use use one of the common logging libraries that is supported by open Telemetry and there's like a bridge um SDK for it and then it'll convert that format to OTP so then it can be ingested by whatever um whatever back end so which is kind of nice I guess they didn't want to reinvent uh reinvent wheel with logs because there's like so many yeah so many different things out there so yeah that's that's my understanding of it from my limited time playing with log so yeah yeah the which I like full sympathy to that strategy the um the momentum on the way people do logs is enormous it's not like that's not changing yeah um but how they get like collected and um organized you know you know that yeah I think the game changer is being able to correlate the logs to the traces so that you have that Fuller picture which I'm super excited about because I understand a lot of us grew up on logs but I think the traces will help tell the full story and so when you have that correlation then then you can get magic yeah I mean I think logs correlated with traces is like is the best most ideal outcome um like I know that Trac is have events and like can do point in time things but logs are just better at that like um and you know being able to see like point in time events um correlated to like where they happened within a span um and within a trace is exactly like that's what I want that's yeah yeah the dream yeah um anyone else have any other comments questions for Jennifer shall we break early then cool awesome well thank you Jennifer for sharing your story with us I think it I I really love love the story because I think it's the things are broken need observability so um I think it's such a compelling case for anybody who's like on the fence on whether or not it's a worthwhile investment um and it's it's awesome that your your company already kind of had its foot in the door um for that and and that you were able to kind of take it further to where it actually needed to be so that it was useful to uh to you and your team for troubleshooting so um I I think it's I think it's a great story I hope others will draw inspiration from that as well um as a compelling reason as to why observability with open Telemetry is a good combination um do we have I think we've got some upcoming events yeah on December the 7th we have another Q&A um with uh someone from Hashi Corp who has attempted many times to instrument Nomad so that'll be uh um I think that'll be a really interesting story as well so anyone who is following along stay tuned for that Q&A hopefully we'll get to see you live but if not we will see you on YouTube sounds good it was great to meet you all yeah thanks so much Jennifer and we'll let you know once the video is posted cool yeah um yeah I was gonna say thank you Jennifer for your story that's like really really inspiring because I feel like so many people want to always adopt it but then getting that buying from your leadership is always the problem I mean technically like it's not making their money until you sure that this is like causing problems and then we really need to do this yeah um yeah I don't know I don't know why like it's weird that it's such a fight so much of the time um because problems are going to happen and yeah like what this gets you is being able to point at where the problem is happening and I don't know why people don't want to be able to do that right so logical in my experience it only works either in teams that are excited about Progressive engineering practices or that are feeling so much pain from vendor lock in that they're ready to set everything on fire yes yes so true so basically means you need like a really nasty incident or or like really bad vendor lock in as your compelling reason well if you get that incident don't let it go to waste yeah totally right quote of the day awesome well thank you everyone diff --git a/video-transcripts/transcripts/2023-12-18T23:04:41Z-otel-q-a-feat-luiz-aoqui-of-hashicorp.md b/video-transcripts/transcripts/2023-12-18T23:04:41Z-otel-q-a-feat-luiz-aoqui-of-hashicorp.md index 100d53d..1dc007d 100644 --- a/video-transcripts/transcripts/2023-12-18T23:04:41Z-otel-q-a-feat-luiz-aoqui-of-hashicorp.md +++ b/video-transcripts/transcripts/2023-12-18T23:04:41Z-otel-q-a-feat-luiz-aoqui-of-hashicorp.md @@ -10,68 +10,83 @@ URL: https://www.youtube.com/watch?v=HRIx9gJtECU ## Summary -In this YouTube video, Luiz Aoqui, a developer at HashiCorp working on the Nomad project, discusses his experiences with implementing OpenTelemetry for distributed tracing in Nomad. The session is formatted as a Q&A, with Luiz sharing insights into the challenges and methodologies adopted during his attempts to enhance visibility into Nomad's operations. He explains Nomad's architecture, including its server-client structure and the scheduling process, before diving into his various approaches to integrating OpenTelemetry. Luiz elaborates on his journey from initially trying to instrument every aspect of Nomad (which proved overly complex) to focusing on core functionalities, allowing users to leverage built-in telemetry capabilities without extensive modifications. He highlights the importance of semantic conventions in creating meaningful traces and shares lessons learned about managing overhead and user experience challenges. The conversation emphasizes the value of real-world implementations of OpenTelemetry in legacy systems and provides practical insights for developers interested in telemetry. +In this Otel Q&A session, Luiz Aoqui, a developer on HashiCorp Nomad, discusses his experiences with integrating OpenTelemetry (OTel) into the Nomad workload orchestrator. He shares insights from his four-year journey with Nomad and how he discovered the need for distributed tracing to improve observability and debugging. The conversation covers three main approaches he took: the first being an ambitious attempt to instrument all aspects of Nomad, which proved to be overly complex and generated excessive overhead; the second aimed at enhancing user applications running on Nomad by automatically injecting relevant metadata into traces; and the final approach, which focuses on creating meaningful spans for specific actions within Nomad without altering function signatures. The discussion highlights the challenges of working with legacy code and the importance of finding a balance in telemetry data to avoid overwhelming users. The session concludes with acknowledgment of the evolving conversation around OTel implementations and the shared learning experience. -# OpenTelemetry Q&A with Luiz Aoqui +## Chapters -Welcome everyone to the OpenTelemetry Q&A! I’m super excited to have Luiz Aoqui join me. He is a developer on HashiCorp Nomad. +Sure! Here are the key moments from the livestream along with their timestamps: -**Luiz:** Thank you! I’m really excited to be here. It’s always fun to talk OpenTelemetry. I haven't used it yet extensively, but I've dabbled a lot and will share more details about my adventures with OpenTelemetry. +``` +00:00:00 Introductions and welcome to Luiz Aoqui +00:02:15 Luiz shares his background and experience with Nomad +00:03:45 Overview of Nomad architecture and components +00:05:30 Explanation of distributed tracing and its relevance to Nomad +00:10:20 Discussion on initial attempts at implementing OpenTelemetry in Nomad +00:12:50 Challenges faced with the first approach to distributed tracing +00:17:00 Introduction of the second approach focusing on user experience +00:23:30 Discussion on environment variables and their limitations +00:30:15 Transition to a more focused approach on key processes within Nomad +00:35:45 Explanation of how semantic conventions aid in tracing +00:40:00 Conclusion and reflection on the journey of implementing OpenTelemetry +``` + +Feel free to let me know if you need any additional information! + +# Otel Q&A with Luiz Aoqui + +Welcome everyone to the Otel Q&A! I am super excited to have Luiz Aoqui join me. He is a developer on HashiCorp Nomad. + +**Luiz:** Thank you! Really excited to be here. It's always fun to talk OpenTelemetry. I mean, I haven't used it yet, but I dabble a lot, and I'll share more details of my adventures with OpenTelemetry. ### Introduction -Let me introduce myself first. As I mentioned, I'm Luiz, an engineer working at HashiCorp, specifically on the Nomad project. I've been working on Nomad for over four years now. At some point, I bumped into OpenTelemetry, particularly Distributed Tracing. It felt super interesting to me as a developer on Nomad because Nomad can be opaque to debug. Implementing and deploying it can be quite complex, so I thought it would be cool to incorporate some distributed tracing to better understand the internal workings. +So, as I mentioned, I'm Luiz, an engineer working at HashiCorp specifically on the Nomad project. I've been working on Nomad for over four years now. At some point, I bumped into OpenTelemetry, specifically Distributed Tracing, and it felt super interesting to me as a developer in Nomad because Nomad can be quite opaque to debug. Implementing, deploying, and using Nomad can be done in multiple different ways, so I thought it might be cool to do some distributed tracing to get a sense of what's going on internally. -After our releases, we usually have an internal hack week to cool down after the stressful moments. By the way, we just released Nomad 1.7, so if you want to try that out, it's available now! We’ll probably have a hack week soon, and maybe I’ll work on some extra OpenTelemetry integration then. +After our releases, we usually have an internal hack week just to cool down a bit after that stressful moment. By the way, we just released Nomad 1.7, so if you want to try that, it's out now! ### Nomad Overview -Let’s start with a quick introduction to Nomad since understanding its internals is essential for discussing the tracing aspect. +Now, I guess I should introduce Nomad briefly because understanding the internals is crucial for the distributed tracing aspect. + +**Nomad** is a workload orchestrator, similar to Kubernetes. You provide it with a configuration file, and that file becomes containers or other work. Internally, Nomad has a simple architecture with two main components: servers and clients. -**What is Nomad?** -Nomad is a workload orchestrator, similar to Kubernetes. You provide it with a configuration file, and it orchestrates the deployment of containers and other workloads. +- **Servers:** You typically have three to five servers that maintain the core state of the cluster. They communicate with each other to ensure that the state is replicated and available in case of a failure. The servers also handle scheduling, determining where to place jobs based on the requests they receive. -Internally, Nomad has a simple architecture consisting of two main components: **servers** and **clients**. +- **Clients:** These are the machines that actually run your workloads. You can have anywhere from one to thousands of clients in your Nomad cluster. -- **Servers:** Typically, you have three to five servers that maintain the core state of the cluster. They communicate with each other to ensure state consistency and to maintain the jobs running in the cluster. -- **Clients:** These are the machines that actually run your workloads. You can have anywhere from one to thousands of clients in a cluster. +The scheduling process involves creating a "plan" based on the job specification, which is then executed by the clients. -### Scheduling and Operations +### OpenTelemetry and Nomad -The servers are responsible for two main tasks: +I have tried three different work streams with OpenTelemetry in Nomad, each with its own challenges and benefits. My first attempt was to add distributed tracing throughout the entire process. The idea was to create a single trace for the entire process, so I wanted to map the process of running something end-to-end — from submitting a job to getting visibility into what happens inside the cluster. -1. **State Management:** They store the state of the cluster, similar to how etcd functions in the Kubernetes world. -2. **Scheduling:** The scheduling process occurs within the servers. When a job is submitted, the scheduler determines where to place the workload among the available clients. +This approach, which I refer to as "boil the ocean," aimed to trace everything at once. However, it created a lot of overhead and many tiny spans that weren't particularly useful. -Each client periodically queries the server for instructions on which allocations to run, similar to a pod in Kubernetes. +### Challenges Faced -### OpenTelemetry Integration Attempts +1. **Overhead:** Each span created was very short-lived, leading to excessive data transfer and storage. -Now, I want to dive into my attempts to integrate OpenTelemetry with Nomad. I'll go over three different approaches I took, each with its own challenges and lessons learned. +2. **Code Changes:** The implementation required significant changes to the Nomad codebase, including wrapping HTTP handlers, which made the process quite messy. -1. **First Attempt - Boil the Ocean Approach:** - My initial goal was to add distributed traces everywhere to provide comprehensive visibility. This approach resulted in excessive overhead due to too many spans and data being sent over the network. It became clear that monitoring at the functional level was not ideal for a monolithic application like Nomad. +3. **Function Boundaries:** Distributed tracing is usually designed for microservices, but Nomad operates more like a monolith, complicating the process of tracing across function calls. -2. **Second Attempt - User-Focused Instrumentation:** - In my second approach, I shifted my perspective to help users instrument their applications running on Nomad. I began by adding environment variables containing Nomad metadata that would automatically populate the spans generated by applications using OpenTelemetry. However, this approach also had limitations, primarily due to the static nature of environment variables. +4. **Consistency:** There was a lack of consistency in the metadata added to the spans, making it challenging to derive useful insights. -3. **Final Attempt - Targeted Instrumentation:** - In my last attempt, I focused on instrumenting key aspects of Nomad without trying to capture every single function call. I created spans for significant events while utilizing semantic conventions to standardize the attributes across different traces. This approach allowed for better filtering and analysis of the traces related to specific jobs or allocations. +### New Approaches -### Conclusion and Discussion +After some guidance from the OpenTelemetry community, I shifted my perspective. Instead of trying to instrument everything at once, I focused on core aspects of Nomad that would provide the most value. -This journey through integrating OpenTelemetry with Nomad has been quite insightful. I learned that it’s essential not to overwhelm the system with too many spans and that sometimes less is more when it comes to tracing. +- **Increased Granularity:** The new approach involved creating smaller, more meaningful spans rather than a single mega trace. This allowed for easier filtering of traces based on standardized attributes. -**Q&A Session:** -Feel free to ask any questions as we go. +- **Semantic Conventions:** I implemented semantic conventions for Nomad, allowing for better organization and filtering of trace data, making it easier for operators to find relevant information. -**Audience Member:** What kind of information does each span provide? +### Conclusion -**Luiz:** Each span can have different attributes depending on what’s being monitored. Some spans might have additional metadata like job IDs or eval IDs, while others might contain standard OpenTelemetry SDK data. +This journey has been enlightening, and I hope everyone gained insights from my attempts to integrate OpenTelemetry with Nomad. -Thank you for joining this session! I hope you found it helpful and insightful. We’ll be posting a recording of this Q&A on the official OpenTelemetry YouTube channel. Keep an eye out for that! +Thank you for joining this Q&A session! We will be posting a recording of this on the OTEL YouTube channel (OTEL - Official). Keep an eye out for the announcement on social media and Slack once it's up. -Thank you, everyone, for attending! +**Luiz:** Thank you for having me! I appreciate everyone's participation. Bye! ## Raw YouTube Transcript diff --git a/video-transcripts/transcripts/2023-12-20T04:44:51Z-the-humans-of-otel-kubecon-na-2023.md b/video-transcripts/transcripts/2023-12-20T04:44:51Z-the-humans-of-otel-kubecon-na-2023.md index b64b866..d71667e 100644 --- a/video-transcripts/transcripts/2023-12-20T04:44:51Z-the-humans-of-otel-kubecon-na-2023.md +++ b/video-transcripts/transcripts/2023-12-20T04:44:51Z-the-humans-of-otel-kubecon-na-2023.md @@ -10,115 +10,129 @@ URL: https://www.youtube.com/watch?v=coPrhP_7lVU ## Summary -In this YouTube video, various members of the OpenTelemetry community discuss their roles and experiences with the project, focusing on the concept of observability. Tyler Yahn, Amy Tobey, Carter Socha, and others share insights about their contributions and the evolution of OpenTelemetry, including discussions on the OTel CLI tool, the importance of integrating logs, metrics, and traces, and personal interpretations of observability. They highlight how modern observability allows for improved monitoring and understanding of systems, emphasizing the need for correlated telemetry data to efficiently troubleshoot issues. The conversation also touches on the community aspect of OpenTelemetry, with participants valuing collaboration across different vendors and the collective effort to enhance observability standards. Overall, they express excitement about the advancements in the project and the potential for new analysis tools as OpenTelemetry matures. +In this YouTube video, a group of OpenTelemetry contributors and maintainers, including Tyler Yahn, Amy Tobey, Carter Socha, and others, discuss the evolution and significance of Observability in software systems. The conversation covers the integration of logs, metrics, and traces as essential components of Observability, emphasizing the importance of having these signals connected to provide valuable insights during troubleshooting and system monitoring. The participants share their personal definitions of Observability, which range from understanding application behavior to being able to address issues swiftly in production environments. They also highlight the collaborative nature of the OpenTelemetry project, its journey from various predecessor projects, and the community's efforts to create a vendor-neutral standard for telemetry data. Throughout the discussion, the group expresses a strong preference for traces as the most effective telemetry signal, citing their depth and usefulness in operational contexts. -# OpenTelemetry Discussion Transcript +## Chapters -## Introduction +Here are the key moments from the livestream along with their timestamps: -**Tyler Yahn**: I'm Tyler Yahn. I am a maintainer for the OpenTelemetry Go SIG. We're working on some auto-instrumentation and specifications. +00:00:00 Introductions of speakers and their roles in OpenTelemetry +00:02:30 Discussion about the OTel CLI tool and its features +00:04:00 Overview of the importance of Observability in software systems +00:06:30 Different definitions of Observability from various speakers +00:10:00 The evolution of Observability and the integration of metrics, logs, and traces +00:13:45 The significance of having unified telemetry data for better analysis +00:17:00 Speakers share personal experiences that led them to OpenTelemetry +00:20:30 The community aspect of OpenTelemetry and its impact on the Observability landscape +00:25:00 Favorite telemetry signals and why traces are favored by many +00:28:00 Closing thoughts on the future of OpenTelemetry and Observability -**Amy Tobey**: I'm Amy Tobey, a senior principal engineer for digital interconnection at Equinix. I maintain a tool called OTel CLI. +Feel free to ask for more details or specific moments! -**Tyler**: Oh, you maintain the OTel CLI! +# OpenTelemetry Community Discussion -**Amy**: Yes, that's my project. It mostly has Traces right now. I've been meaning to implement Logs and Metrics for a while, and I think Logs just went GA recently, so it's time to do it. Traces have been so effective, and people really like it, that I haven't had much demand for them. +**Participants:** +- Tyler Yahn, Maintainer for OpenTelemetry Go SIG +- Amy Tobey, Senior Principal Engineer at Equinix +- Carter Socha, Product Manager and Maintainer of OpenTelemetry Demo +- Bogdan, Former Maintainer of Java and Collector +- Constance Caramanolis, Contributor to OpenTelemetry Collector +- Juraci, Software Engineer and Collector Developer +- Jacob Aronoff, Maintainer for OpenTelemetry Operator +- Alex, Contributor and Maintainer of OpenTelemetry +- Purvi, Senior Software Engineer -**Tyler**: Is the OTel CLI part of OpenTelemetry? +--- -**Amy**: It's not yet. I maintain it on our Equinix Labs GitHub account. There's not much process; mostly it's just me with a few folks like Alex and others who throw me a PR every now and then. I've thought about bringing it back to the community, but I'd have to maybe adhere more closely to the standards than I am right now because doing it in the command line, a lot of the standards don't really translate very well. So I've strayed a little bit away from the standards in a few places. +## Introductions -**Tyler**: That makes sense. I've talked to Austin about it. +**Tyler:** I'm Tyler Yahn, a maintainer for the OpenTelemetry Go SIG. We're working on some auto-instrumentation and specification. -## Meet the Team +**Amy:** I'm Amy Tobey, a senior principal engineer for digital interconnection at Equinix. I maintain a tool called OTel CLI. -**Carter Socha**: Hey, I got Ted Young with me! Hello! I'm Carter Socha. I work on a couple of different things. I'm one of the few product managers floating around, but I helped start the OpenTelemetry Demo, which I'm a maintainer of. I also work in the SIG Security, which helps the project improve its security response process. +**Carter:** Hi, I'm Carter Socha. I work on a couple of different things. I'm one of the few product managers floating around, but I helped start the OpenTelemetry Demo, which I'm a maintainer of. I also work in the SIG security to help improve the project's security response process. -**Bogdan**: My name is Bogdan. I took a break for parental leave, so I'm just jumping back. +**Bogdan:** My name is Bogdan. I took a break for parental leave, so I'm just jumping back in. I’ve done a lot of things, including being a member of the TC and GC, and I was a maintainer of the Collector. -**Carter**: What were you doing before? +**Constance:** Hi, I’m Constance Caramanolis. I’m one of the OG contributors to OpenTelemetry. I worked on the OpenTelemetry Collector and contributed to the configuration aspects. -**Bogdan**: I’ve done a lot of things, including being a member of TC, GC, and maintainer of Collector. I was a former maintainer of Java, so I've done a lot. +**Juraci:** My name is Juraci. I'm a software engineer and have been working with OpenTelemetry or Observability for a few years now. I was a maintainer on Jaeger and part of OpenTracing. -**Constance Caramanolis**: Hi, I'm Constance Caramanolis. I know that you were involved in OpenTelemetry, and you are one of the OG contributors. Tell us about that involvement. +**Jacob:** I’m Jacob Aronoff, a maintainer for the OpenTelemetry Operator project. -**Bogdan**: Yes, I worked on the OpenTelemetry Collector. I contributed to that. I did a lot of config things. I was also on the OpenTelemetry Governance Committee. I was involved in the incubation process and getting adoption. +**Alex:** Hi, I'm Alex, a contributor and maintainer in OpenTelemetry. I even wrote a book about it. -**Juraci**: My name is Juraci. I'm a software engineer, and I've been working with OpenTelemetry systems or Observability for a few years now. I come from a Tracing background; I was a maintainer on Jaeger and part of OpenTracing back in the day. I helped choose the name of the project that we have. Right now, I'm a Collector developer, and I help out on some components for OpenTelemetry Collector. I'm also part of the Governing Committee for OpenTelemetry, and I was just re-elected. +**Purvi:** Hey, my name is Purvi. I am a senior software engineer and have worked a lot with browsers and JavaScript. -**Jacob Aronoff**: My name is Jacob Aronoff. I am a maintainer for the OpenTelemetry Operator project. +--- -**Alex**: Hi, I'm Alex, and I'm a contributor and maintainer in OpenTelemetry. I wrote a book about OpenTelemetry. I do stuff with OTel. +## Observability Discussion -**Purvi**: Hey, my name is Purvi. I am a senior software engineer. I've worked a lot with browsers and JavaScript throughout my career. +**Purvi:** So, what does Observability mean to you? -## Discussion on Observability +**Carter:** To me, Observability means that when you wake up at 2:00 AM to fix a problem, you can resolve it. Ideally, you can revisit that code the next day and find a way to prevent the issue from occurring again. -**Purvi**: So what does Observability mean to you? +**Bogdan:** I think Observability is the capability of monitoring your production environment and determining when something goes wrong. -**Carter**: That's a great question. Personally, I think Observability means that when you wake up at 2:00 a.m. to fix a problem, you can do so. Ideally, the next day, you're able to look at that code again and find a way to prevent that problem from occurring again. It means being able to look at things coming out of the box and understand what's going on inside. +**Alex:** I view it as a tool to ensure things are working as intended. It's about gaining insight into both black and white boxes. -**Constance**: First of all, it’s monitoring. But really, Observability is a nebulous term. It represents a shift in how we think about monitoring our systems. Traditionally, we had different signals: logs, metrics, and tracing, which were siloed. Over the past few years, especially in the OpenTelemetry project, we’ve tried to integrate these tools. +**Constance:** I like to think of it as a murder mystery. You see clues and have many questions, and you use Observability to figure out what's going on. -**Juraci**: When you're using these tools, you want to move back and forth between them. You get an alert based on a metric, and when that alert goes off, you want to look at the logs related to the transactions causing those alerts. You want to have a trace ID associated with those logs to look them up. +**Juraci:** It allows us to understand problems in our systems without needing to know what's going wrong ahead of time. Observability is a spectrum; we won't achieve perfect Observability from day one, but we should have some telemetry to help us understand our systems. -**Alex**: For me, modern Observability is about having all this data connected into a graph. You need a graphical data structure where all these individual signals are part of the same system. This integration allows us to leverage technology to reduce the time we spend investigating issues. +**Amy:** Observability means understanding what's happening inside your applications, particularly in the code that matters to you. -**Purvi**: Observability means an application owner can see what's going on in their environment and answer pertinent questions about their business and how to improve their service. +**Jacob:** For me, Observability is essential. When something goes wrong, I can ask questions about my system and figure out what happened without needing to know exactly what to expect. -**Carter**: Observability is the capability of monitoring and determining when something goes wrong in your production environment. +--- -**Juraci**: I view it as getting insight into black boxes or even white boxes. I call it a "murder mystery." +## OpenTelemetry Insights -## The Nature of Observability +**Tyler:** When did you all get involved with OpenTelemetry? -**Constance**: I think Observability is a way for us to understand problems in our system. It doesn't matter if it comes from logs, metrics, or tracing, as long as we can tell what's going on. It's not a yes or no; it's a spectrum. +**Amy:** I got involved in 2019. I was brought in to instrument the entire stack for the Equinix Metal product. -**Amy**: My understanding of Observability is that it’s about understanding what's happening inside your applications, especially in the code you care about. +**Carter:** I was introduced to OpenTelemetry through my org at Microsoft, which was already doing a lot in that space. -**Juraci**: Observability means that when something goes wrong, I can ask questions about my system and get a sense of what happened without having to know ahead of time what to expect. +**Constance:** I was part of the early days when we were working on the incubation process. -**Carter**: I like what you said about not having perfect instrumentation. There is no such thing as perfect instrumentation, just like there is no such thing as done code or a network that never breaks. +**Jacob:** I got involved through working at Honeycomb, focusing on OpenTelemetry JavaScript, particularly on the browser side. -**Purvi**: Observability is about being curious with your data and having confidence in your production system. Testing in production is the best way to test your system. +--- -## Involvement with OpenTelemetry +## Perspectives on OpenTelemetry -**Amy**: When did you get involved with OpenTelemetry? +**Alex:** OpenTelemetry is about collaboration across the Observability space. It provides a vendor-neutral way to instrument systems. -**Carter**: I got involved in 2019, so quite early. I really love writing Go, which is where I started. I quickly jumped into the specification space because I wanted a better software solution for the pain points I encountered. +**Carter:** It makes my life easier by integrating with both open-source and proprietary components, allowing me to see traces across all products I use. -**Juraci**: I was hired into Equinix to instrument their entire stack for the Equinix Metal product. This was three years ago, before the fancy auto-instrumentation stuff was complete. +**Amy:** OpenTelemetry is a collection of standards that brings together various telemetry signals like metrics, logs, and traces. -**Jacob**: I got involved with OpenTelemetry through working at Honeycomb, focusing on OpenTelemetry JavaScript, especially the browser side. +**Juraci:** OpenTelemetry is a set of tools that helps extract telemetry data out of applications, gradually leading to better understanding of the systems. -**Constance**: OpenTelemetry is a standard; a collaboration across the Observability space. It’s a path forward for instrumentation without vendor lock-in. +**Constance:** It's a wonderful community that helps make Observability better by working across vendor boundaries. -**Bogdan**: OpenTelemetry makes my life easier because I can integrate it with open-source and proprietary components. +--- -## The Community Behind OpenTelemetry - -**Amy**: OpenTelemetry is a combination of different views finally coming together to make advancements easier. The hard part is making sense of the data we've gathered. - -**Purvi**: OpenTelemetry is a set of tools to get telemetry data out of applications in a vendor-neutral way. +## Favorite Telemetry Signals -**Carter**: It's like summer camp for maintainers. We collaborate to figure out what's needed in the coming months. +**Purvi:** What is your favorite telemetry signal? -**Juraci**: OpenTelemetry is about the community taking ownership of their telemetry data. Vendors should not determine the type of telemetry data sent to your systems. +**Jacob:** I’d probably say traces, as they provide a deeper understanding of operational behavior. -## Favorite Telemetry Signals +**Amy:** Traces are definitely my favorite too; they give you context. -**Carter**: What's your favorite telemetry signal? +**Carter:** Traces are the cooler version of logs. They give you more depth. -**Tyler**: That's a good question! I think traces are the cooler version of logs. They give you a lot more depth into operational behavior. +**Juraci:** I love traces. They allow you to quickly identify issues without overthinking. -**Bogdan**: My favorite signal is definitely traces. They are the easiest to work with, and they provide so much more context. +**Constance:** Traces number one! They are elegant and useful. -**Juraci**: Traces, of course! They are logs with context, and they make troubleshooting so much easier. +**Alex:** Traces are magic because they correlate metrics and logs with context. -**Carter**: For me, it's traces because they're beautiful and provide metrics and logs correlated with context. +--- -**Amy**: Traces are simply the best; they are magic! +In summary, this engaging discussion among various contributors and maintainers of OpenTelemetry reflects their shared passion for improving Observability and making it accessible and useful across different environments and applications. Their varied experiences and insights highlight the importance of community collaboration in advancing the field of Observability. ## Raw YouTube Transcript diff --git a/video-transcripts/transcripts/2024-01-19T04:34:21Z-observability-panel-with-charity-majors-amy-tobey-and-adriana-villela.md b/video-transcripts/transcripts/2024-01-19T04:34:21Z-observability-panel-with-charity-majors-amy-tobey-and-adriana-villela.md index 66e0a3c..4590167 100644 --- a/video-transcripts/transcripts/2024-01-19T04:34:21Z-observability-panel-with-charity-majors-amy-tobey-and-adriana-villela.md +++ b/video-transcripts/transcripts/2024-01-19T04:34:21Z-observability-panel-with-charity-majors-amy-tobey-and-adriana-villela.md @@ -10,101 +10,103 @@ URL: https://www.youtube.com/watch?v=Fuy3W5bro9k ## Summary -In this YouTube panel discussion on observability, hosted by an unnamed moderator, experts Adriana Vela (ServiceNow Cloud Observability), Charity Majors (Honeycomb.io), and Amy Tobey (Equinix) delve into the evolving landscape of observability within software development. They explore definitions of observability, emphasizing its role as a property of socio-technical systems rather than a standalone project, and discuss the transition from traditional metrics to more integrated, high cardinality observability tools. Key points include the importance of meaningful data, the necessity for engineers to engage with observability throughout the software lifecycle, and the potential for AI to enhance human capabilities in this field. The panelists also touch on misconceptions around observability, the challenges of sampling and data overload, and the need for better collaboration among engineering, QA, and support teams. Their collective wish for the future includes a deeper understanding of Service Level Objectives (SLOs) and a more holistic integration of observability into development practices. +In this panel discussion on observability, hosts Adriana Vela (ServiceNow), Charity Majors (Honeycomb.io), and Amy Toby (Equinix) explore the evolving landscape of observability tools and practices. They discuss the transition from traditional observability, characterized by fragmented metrics and logs, to a more integrated approach that emphasizes high cardinality data and meaningful insights. Charity defines observability as a property of socio-technical systems, while the panelists highlight the importance of understanding the context of data and the need for tighter feedback loops between engineering and operations. They also address misconceptions about observability being a standalone solution, emphasizing that it requires a cultural shift within organizations. The conversation touches on the role of AI in augmenting human decision-making and the challenges of managing data volume. As they conclude, the panelists express their hopes for a future where observability becomes more ingrained in the software development lifecycle, encouraging collaboration across teams to enhance visibility and improve the overall quality of software delivery. -# Observability Panel Transcript +## Chapters -[Music] +Here are the key moments from the livestream along with their timestamps: -Hello, everyone! We're very excited to have a panel today around the topic of **observability** for the observability community. Here with us today, we have **Adriana**, **Amy**, and **Charity**. We hope to have a fun, engaging conversation. +00:00:00 Introductions of the panelists +00:03:30 Adriana shares her background in observability +00:05:00 Charity discusses her definition of observability +00:09:45 Amy talks about her experience with observability at Equinix +00:14:00 Discussion on the evolution from observability 1.0 to 2.0 +00:21:00 Charity explains the importance of meaningful data in observability +00:28:30 The panel discusses the challenges of onboarding open Telemetry +00:35:00 Amy mentions the need for observability features in network services +00:40:00 Discussion on the role of AI in enhancing observability +00:50:00 Panelists share their thoughts on the biggest challenges in observability +00:55:00 Closing remarks and future discussions on observability -### Introductions +This format highlights the main topics covered during the livestream, making it easier for viewers to navigate through the content. -**Adriana:** -Hi, I’m Adriana Vela. I work with Anna at ServiceNow Cloud Observability, formerly known as Lightstep. I'm a Senior Staff Developer Advocate and have been doing the developer thing for almost two years. Before that, I oscillated between individual contributor engineering and management roles. I love observability! +# Observability Panel Discussion Transcript -**Charity:** -I’m Charity Majors, co-founder and CTO of Honeycomb.io. I identify as an Ops engineer, and although it’s been a long time since I held the pager, I still feel like I’ve got half my life on call. I started my first pager rotation when I was 19, so I’ve definitely paid my dues. +**Moderator:** Here we are! Hello, hello, hello, everyone! We're very excited to have a panel today around the topic of observability. We hope to have a fun and engaging conversation. Would you mind starting out, Adriana, by introducing yourself—who you are and what you do? -**Amy:** -Hi, I’m Amy Toby, a Senior Principal Engineer at Equinix, the largest data center company in the world. I’ve been doing observability since pretty much my first job back in the late '90s. My current work involves bringing open Telemetry into our operational stack at Equinix Metal, helping us pump metrics out to Honeycomb. +**Adriana:** Oh yeah! So, I am Adriana Vela. I work with Anna at ServiceNow Cloud Observability, formerly known as Lightstep. I’m a senior staff developer advocate and have been doing the dev ops thing for almost two years. Before that, I oscillated between individual contributor engineering and management roles. Yay! And I love observability. I knew you before you were a developer in observability! -It’s exciting to have you all on the panel today! +**Charity:** Yeah! When you were just trying to figure out what the hell this stuff was all about! Time definitely has passed by. -### Definition of Observability +**Moderator:** Charity, what about you? -Let’s start with one of the spicy questions: What is your definition of observability? Charity, would you like to kick it off? +**Charity:** I’m Charity Majors, co-founder and CTO of Honeycomb.io. I identify as an Ops engineer and probably always will, even though it's been a long time since I held the pager. I feel like I started picking up my first pager rotation when I was 19, so I feel like I've got half my life on call. -**Charity:** -Sure! Back in the days when observability was still new, we pushed for a technical definition—high cardinality, high dimensionality, explorability, and all that jazz. Nowadays, everyone seems to do observability, even if it’s just gathering some telemetry data. I’ve come around to the idea that observability is a property of a sociotechnical system. It exists on a continuum. +**Moderator:** You paid your dues! What about you, Amy? -I think there’s a noticeable difference between observability 1.0 and 2.0. Observability 1.0 relies on metrics, logs, and traces, where you gather data multiple times but don’t connect them effectively. You end up with a lot of data, but no meaningful insights because it’s all siloed. In contrast, observability 2.0 is based on a single source of truth, allowing for high cardinality data and better exploration of that data. +**Amy:** Hi, I'm Amy Toby. I am a senior principal engineer at Equinix, which is the largest data center company in the world. I’ve been doing observability pretty much since my first job, starting back at Solara Systems in the late '90s. I’ve been responsible for some things in networking and have been working on bringing OpenTelemetry into our entire Equinix Metal operational stack, pumping those metrics out to Honeycomb. -**Amy:** -I mostly agree with Charity. In my experience developing a new networking product, we’ve had to include observability features as customers demand them. They’re looking for insights about hyper-complex systems without having to hire expensive people to figure it all out. They want high-quality telemetry that’s easy to consume. +**Moderator:** Nice! Very exciting work that all of you are doing and creating community around observability and OpenTelemetry. I get to call you all friends, which is super exciting to have you all today on the panel. -**Adriana:** -I like the definition I heard recently from Hazel Weekly: “Observability is the ability to ask questions and get meaningful answers.” It emphasizes the importance of collecting relevant data. If you gather a bunch of useless data, you won’t be able to figure out what’s going on. Observability is a design problem, and it takes effort to determine what data is meaningful to you. +Let’s kick it off with one of the spicy questions. Charity, what is your definition of observability? -### The Role of AI in Observability +**Charity:** The correct one, of course! Back in the days when nobody else cared about observability, we really pushed the idea that there was a technical definition for it—high cardinality, high dimensionality, explorability, and all these things. Nowadays, everyone and their cat does observability. If you have any telemetry, you do observability. You know what? I’ve come around to this. I think it’s a great definition to say that observability is a property of sociotechnical systems. It exists on a continuum. -Let’s shift gears a bit. How do you think AI can help teams adopt observability? +I think it gives a lot of grace to people who are in the early days of their journey. I don’t want to be the person who says, “Well, actually, you're not really doing observability.” Nobody likes that person. -**Charity:** -I believe AI can augment humans in the observability pipeline rather than replace them. The opportunity lies in automating correlation and surfacing insights to humans. Instead of eliminating alerts, we can help people do more with less context and expertise. +That said, I do think there's a real kind of step function difference between what I'll call observability 1.0 and 2.0. Observability 1.0 is built off the primitives of metrics, logs, and traces. Every time you gather data, you have to gather it again in a different format and pay for it again. You gather it once for metrics, again for logs, again for traces, again for APM, and again for profiling. Talk about a cost crisis in observability tooling! -**Amy:** -Absolutely! AI can help reduce the cognitive load by summarizing data and highlighting potential issues. It’s about letting computers do what they do best so that humans can focus on what we do best—adding meaning to the data. +You're very limited in each one of those by the format you gathered it in, and the only thing that connects them is the poor human sitting in the middle guessing. So, let’s call that observability 1.0. -**Adriana:** -I agree! AI can alleviate some of the workload, allowing us to focus on more complex problems. It’s about using AI to support decision-making, not replace it. +Observability 2.0 tooling is based on a single source of truth. You can derive metrics from a structured data blob, and it handles high cardinality and provides a more explorable interface. It’s dynamic, where you can ask questions and follow breadcrumbs. -### Current Challenges in Observability +So, that’s my very long-winded answer. I think there are two generations of observability tooling out there right now, leading to very different outcomes. -What do you think is the current largest challenge in observability? +**Moderator:** What about you, Amy? -**Amy:** -Sampling is a big challenge. We’re generating a wealth of data, and managing that data so it remains meaningful and doesn’t overwhelm our systems is critical. +**Amy:** I think I agree with most of what Charity said. My work lately has involved developing a new networking product, and one of the feature sets we have to include is observability features. Customers demand this. We’ve been figuring out how much observability they need and how much we should build versus handing off to services like ServiceNow or Honeycomb. -**Charity:** -I think many people don’t realize how bad their current state is. They often don’t understand how much better it could be. There’s a need for a mindset shift. +What we've found is that customers are tired of hiring expensive people to carry the mental burden of tying context together with the metrics and figuring out what’s going on. They want observability telemetry because they need insights about these hyper-complex systems to fix them and get back to business. -**Adriana:** -Organizations often get in their own way. They understand the benefits of observability but fail to grasp its implementation fully, leading to missed opportunities. +**Adriana:** I think I started out with Charity's OG definition of observability, and I heard one recently from Hazel Weekley, which I quite like: "Observability is the ability for you to ask questions and get meaningful answers." It's up to you what is meaningful. What data are you collecting that’s meaningful to you? -### Misconceptions About Observability +I believe in trace-first observability, but we need to focus on meaningful data. You can instrument the heck out of a system, but if it’s emitting garbage, you won’t be able to figure out what’s going on. Observability is a design problem. -What do you think is the biggest misconception around observability right now? +I feel like we spent a long time figuring out the underlying plumbing, which is just a prerequisite. The real issue is that there’s so much data, and we need to equip ourselves to understand it. -**Adriana:** -People often think observability will solve all their problems without any additional work. They expect magical results without realizing the effort that goes into it. +**Charity:** I think we’re seeing that with OpenTelemetry. It’s an awesome standard, but the challenge will be making it easy for onboarding. Many organizations are scared of instrumenting their own code. -**Amy:** -That need for understanding doesn’t disappear; it just becomes more efficient. +**Amy:** Exactly. The metrics matter, and we often see folks reaching for tools to make sense of the noise they’ve built for themselves. They’re overwhelmed with alerts, and it’s challenging to know what’s important. -**Charity:** -Many see observability as a standalone project rather than an integral part of the software development lifecycle. It’s interwoven with everything we do in software development. +**Moderator:** That leads us to the next question: What do you think is the current largest challenge in observability? -### The Future of Observability +**Amy:** Sampling! We generate a wealth of data, and while we're not sampling a lot today, as we turn on tracing, we find that it tips over our collectors. We need to get control of the data we're generating that might not provide value. -As we wrap up, what do you see in store for observability in the upcoming year? +**Charity:** I think people don’t realize how badly they have it now. Many have spent their careers managing metrics without understanding how much better it could be. It’s a mental model shift that they have to make. -**Amy:** -I hope to see a deeper understanding of SLAs and SLOs among teams. It’s crucial for effective communication and management. +**Adriana:** I agree. Organizations often get in their own way because they don’t fully understand observability and how to implement it effectively. -**Adriana:** -I want to see more developers and project planners engage with observability, making it a part of the development conversation. +**Moderator:** What would you say is the biggest misconception around observability right now? -**Charity:** -I wish for more people to feel the grounding and confidence that comes from seeing telemetry feedback. Observability should enhance our experience in software development. +**Adriana:** People think it will solve all their problems without putting in the work. There’s a misconception that observability is magical fairy dust. -Thank you all for the amazing insights you’ve shared today! We're looking forward to our next discussion in June. +**Amy:** I would extend that. The need for understanding doesn’t disappear; it just gets more efficient. -Stay warm out there! +**Charity:** I think the biggest misconception is that observability is its own standalone project. It’s interwoven with software development and requires a holistic view. You can’t just check off an observability project; it requires continuous improvement in how we build software. -[Music] +**Moderator:** As we wrap up, what do you think is in store for observability in the upcoming year? + +**Amy:** My wish for the year would be that more people grasp SLAs and SLOs and realize the power they have in understanding their systems. + +**Adriana:** I want to see more people understanding that observability is everyone’s problem—not just the ops team. It should be part of the language across all teams involved in software delivery. + +**Charity:** I hope we can drive a deeper understanding of observability into our industry so that we don’t have to start at zero every time we discuss it. + +**Moderator:** Thank you all for the amazing insights today on our panel around observability. This group will be coming back later in June for another discussion, so stay tuned for those details. Thank you for watching, and stay warm out there! + +**Everyone:** Yay! ## Raw YouTube Transcript -[Music] here we are hello hello hello everyone we're very excited to have a panel today around the topic of observability plan for the observability community here with us today we have Adriana Amy and charity joining us we hope to have a fun build conversation would you mind starting out Adriana and introducing yourself who you are and what you do oh yeah so I am Adriana Vela I work with Anna at service now Cloud observability formerly known as lightstep say that three times fast um I uh yeah I'm a senior staff developer Advocate um been doing the devil thing for what almost two years but before that was you know oscillating between like IC um engineering and and management roles so yay and I love observability I knew you before you were a Deo before you in observability yeah yeah yeah when I was just like trying to figure out what the hell this stuff was all [Laughter] about time definitely has passed by charity what about you uh charity major co-founder and CTO of honeycomb.io I identify as an Ops engineer and probably always will even though it's been a long time since I held the pager but I feel like I started I feel like I you know I picked up my first page of rotation when I was 19 so I feel like I've got still like half my life on call so I still I I can still call myself an operations engineer you paid your dues I think so what about you Amy uh hi I'm Amy Toby I am a senior principal engineer at equinex which is a the largest data center company in the world um and I've been doing observability since pretty much my first job as well starting back on Solara systems in like 98 99 um S&M and and then into naio stuff I'm responsible for some things in naos I'm sorry um and like it have been doing it all along these days I do a little bit more kind of work in soot technical space but initial work at equinix was bringing open Telemetry into our entire equinix metal operational stack and pumping those uh metrics out to Honeycomb nice very exciting work that all of you are doing and creating Community around observability and open Telemetry and I get to call you all friends which is super exciting to have you all today on the panel sweet so I think we're going to start with one of the spicy questions and I'm going to kick it off with charity what is your definition of observability the correct one of course we'll see yes spice you know okay so back in the days when nobody else gave a about observability the housian days of I'm saying that even though we almost went out of business many times you know I think we really tried to push this this this idea that there was a technical definition for observability you know that it was high cardinality high dimensionality explorability and all these things and nowadays you know like every everyone and their cat does observability it's it's like if you have any Telemetry you do observability and you know what actually I've come around to this I'm all right I'm all right with this I feel like it's actually a great definition to say that observability is a property of soci technical system so it exists in a Continuum right uh I'm I'm totally down with that because I think that it it it gives a lot of Grace to people who are you know in the early days of their Journey like I don't want to be the person who like well actual you're not really doing observ like nobody likes that guy so I don't want to be that guy um that said I do think that there's a real kind of Step function difference in let's call it observability 1.0 toing that buil built off The Primitives of you know metrics logs and traces um where every time you gather the data you have to you have to gather again in a different format and pay for it again so like you're Gathering it once for metrics again for love again for traces again for APM again for profiling again for security you know like talk about a cost crisis and observability tooling like no wonder right uh and you're very limited in each one of those by The Format that you happen to gathered in and none of them can be connected to each other the only thing that connects them is the poor human sitting in the middle guessing right like guessing eyeball us you know so let's call that 1.0 and then observability like 2.0 tooling I think is based on a single Le single source of truth right these arbitrarily wide structured data blobs you could you can derive metrics you can dve traces you can dve all these things but you've got one source of Truth it handles high cardinality it handles you know it's a more explorable sort of interface so you don't have these static dashboards you have interface where you can ask questions and follow breadcrumb go and then what and then what um so that's my very long-winded answer I think that there's kind of two generations of observability tooling out out there right now and I think that they lead to two very different outcomes like one of the I know I'm taking a very long time here I'm almost done I promise I think that one of the most one of the soot technical factors that you really associate with observability 1.0 is it's for it's like a checklist thing you know you add it at the end before you put stuff in production like cool I can monitor this um and for observably 2.0 I think like the fundamental truth of it is it it underlies the entire software development life cycle and your ability to hook up these fast feedback loops rests on how well you can you know ask questions and explore your data and have this sort of constant conversation going with your code awesome what about you Amy I think I agree with most of what charity said but I've been I've been learning a lot from my my work lately in kind of developing a new networking product and one of the feature sets that we have to include in that today in a modern Network as a service kind of product is observability features and customers demand this and we've been scratching at this like what how much do they need like what how much should we build how much should we hand off to either you know service now or honeycomb you know and let those tools do what they're best at and so what what we've been finding with customers is like that that operational part that charity mentioned right that they are tired of this world where they have to go hire very expensive people who are often cranky um who who who can basically carry that mental burden of tying all the context together with the metrics and and divining what the heck is going on with the network and so what they're asking and what they're demanding from us and all of the other Cloud vendors is how give us the observability Telemetry which is mostly what they ask for because when we talk to customers like most the time when folks were talking to don't want to talk about cardinality um so they're asking for like how do we get all the stuff we need so we can feed it to our my my feelings about a apps aside but this is a common request feed it to our AI Ops tools feed it to our our our other observability systems that we already have so that we can start to leverage the skills we already have to get those insights but it always drives to how do I get insights about this hyper complex system so that we can fix it and get back to business and some sometimes folks don't even want to do the heavy lifting they're just like do it for nobody does right because like almost none of our customers any of us here their primary business is not Telemetry it's undifferentiated lifter they want they got a business to run they don't want to be messing with this stuff so they want me to provide really high quality Telemetry um and then they want you all to make that all super easy to consume yeah that's very true I love that it makes so much sense what is your definition AD so I think I I started out with you know Chari OG definition of observability which carried me well throughout the years and then I I heard one recently from uh Hazel Weekley what which I quite like which is observability is the ability for you to ask questions and get meaningful answers um which I really like because it's you know it's it's up to you as to what what is Meaningful right so what what data are you collecting that's meaningful to you and I think that's really important um in in our in our world um I will still say that I I think like Trace first for observability no matter what um but but I think like we really need to focus on on the meaningful data aspect of observability because like sure you can like instrument the crap out of a system and it's emitting a bunch of useless garbage right you know garbage in garbage out so you don't have good data that you're instrumenting well good luck trying to figure out what the hell is going on you can have the best observability tool out there um it won't do you any good so you're you're going to need um to have I it it takes a bit of you know there's there's that learning curve of of what's what's important to you what should you be collecting to be able to do the thing observability is a design problem like in a in a huge way like once you've s like I feel like we spent spent a long time like just sort of like figuring out the underlying Plumbing which is just a prerequisite to what really matters is this is a design problem because there's so much data and I think Amy and I share very uh similar uh thoughts on AI Ops and and like but like there's two kinds of tools right there are tools that um that pave over complexity and there are tools that try to help equip you to understand it and I will always be in the in the second Camp because at the end of the day we are legally liable for the software that we put out into the world somebody somewhere is going to have to understand that and the harder you've made it the more magical you've made it um the harder that Day of Reckoning is going to be definitely interestingly enough I I think we're we're seeing that a little bit um with um like open Telemetry right like open Telemetry is awesome it's you know this awesome standard uh that pretty much everyone has has aop opted vendor wise um but then now we're we're looking Beyond like that initial adoption of open Telemetry and like people really using it out in the field and and making sure I think open telemetry's biggest challenge I think in in the near future is going to be making making sure that it's something that is easy for onboarding because I think that can be really offputting um for organizations and and that can be really scary and then that leads to the you know you instrument code I don't want to instrument my own code kind of mentality I I think that the the place that the the metrics go really matters though because we talked about a Ops where I see folks really reaching for these tools um charity called it Paving over so what we've done in the network world for decades now and it's still the the case at most organizations is we have all these devices generating gobs of alerts and metrics nobody knows what any of it's doing there's there's a sense that we should be able to correlate all these events so like some Port fails on a a core rou or all the stuff behind it disappears we should be able to roll all that up and I'm using the S the should word the way my therapist calls it the s word right like we we believe these things that should happen but the real world is is is much harder than that so like these these alerts just flood through and people fight these alerts sometimes they've been fighting their whole career trying to get on top of this alert stream and so they reach for these tools to make sense of the morass that they built for themselves because they're not really empowered to go back to the Telemetry and like dial back the cardinality or retransform how they generate those metrics to get higher you know higher cardinality that they can process at the end um often they don't even have choices about what toet Tre they get because they're buying network devices that offer you get your SNMP or you're G and that's what you get oh my God yeah oh God world out there of Legions of people that are just fighting through the noise still um and it hasn't changed for them whereas they're in the software world we keep advancing the state-ofthe-art but it's it's going to be a challenge still to bring a lot of the classic systman and network administrators who are used to this world Along on that journey I feel like one of the characteristics that I often think about when we're talk when we're thinking about like transitioning from the One point0 World to the two point0 World which is going to take for forever um no doubt but but I feel like it's it's it's part of it is generating from a push world to a pull world where we're used to you like the only way you could make sense of like a naous based system is by looking at the cluster of things that just alerted you and go like ah it's probably that right and so you rely on you know all of these alerts but that's so noisy and it does not scale right but like as a as an Ops person I grew up thinking you only look at production when you get alerted you don't have to look at it otherwise and I feel like you know the the tradeoff that we we have to learn to make is okay we're only going to alert you when users are impacted right we set slos these are our agreements with ourselves each other and the world this is the level of service we are going to give you and we alert when user be when when the when when the outage is bad enough that users are being infected but in order to get to that world you have to agree that you're going to go look at your Telemetry affirmatively because most problems will never rise to the level of waking you up God I mean I hope right like most of the things most of the codes you write has a subtle bugs it affect a few people or it's a small thing right so like you in order to have a system in order to have a hygienic system that is not just like a hairball that the cat coughed up you know we can't just keep like in the old days like we were shipping code every day that we didn't really understand these systems we've never really understood and then we wonder why it's giant trash fire right like and the way that we start improving that is by you know not just making you know the Telemetry richer and better but also by committing to closing that loop as a software engineer my job is not done in when the test pass my job is done when I have looked at my Telemetry in production and gone it's doing what I expect it to do nothing else looks weird right so like that's asking it's like I like the technical debt metaphor here because it's asking you to like put in a little bit of effort up front in order to avoid the you know there's this great graph that shows that the cost of finding and fixing bugs and goes up exponentially from the moment that you write them so like you backspace good for you good job right you find it in your test good for you the next best time to find it is right after you deployed that after that God knows how long it's going to take days months years who knows but like the accumulation of that is what makes this this job a nightmare for so many people yeah it's so true like I I I think back to like you know my my old days of of having to hand off deployment instructions to like our Ops folks who theun they had no clue what was you know like what it was they were deploying and and you know I had to pray that my instructions were correct and then I had to pray that they read my instructions correctly and then they would deploy it to Pride and these quote unquote successful deployment like things went correctly was considered a successful project the machine we're done right right yeah and and and so I love the idea of like no like because otherwise it becomes a hit and run deployment right you know or driveby drive by deployment rather is is a better way of putting it where where like you know it's like deployed all right great um let's just leave it to fate but I I think like you know paying attention what's going on as soon as you deploy like there's something to be said because otherwise because you as the person writing the code you have context you had that nobody else in the world has or will ever have you know exactly what you're trying to do why you're doing it what you tried that worked what it what didn't work what the variables were named what the f like you know all this stuff and if you can close that Loop before you have to concept switch another it's so powerful like these feedback loops are what socio Technical Systems are all about like this is the one job of technical leadership in my opinion whether you're an IC or a director or VP one job is these feedback loops to make them as tight and short and functional as possible I think that like nailed down of like engineering responsibility has been so important and observability really brings it to a front but a lot of people are just not paying enough attention or like not willing to do it they're just like my job is to code and that's it and maybe I'm on call twice a year but it's someone else's problem there's not much we can do about that though like a lot of times right like if you are an engineering leader and you're trying to build a dynamic powerful engineering organization you have to either lead those people to care or lead them out right like because like the reality is is like there's the customers that y'all want that that have that engineering leadership and culture of like giving a crap yeah and then there's the vast majority of organizations out there where the most of the people who are writing the code that runs our world are trying to put bread on their table and so as soon as they are done with the with the specification that they were given for their job they e that code into to get and they run because like why should they care why should they metrics but this is this is a self-fulfilling prophecy because you know what makes people like check out tune the out not being connected to the results of their labor like there's something that's like what is it Dan pink says we all want in our job is autonomy Mastery and purpose the purpose of your work if you're so disconnected from it yeah that's all there is to it I'm like I'm not trying to suggest that this is easy or can be done in a one stop you know whatever but I do think that every step we make along this path that tightens up the feedback loops that connects people more with the consequences of their code is good and valuable and pays off interesting oh sorry go ahead I was going to say it's even letting Engineers understand customers like I've worked at many shops and with so many engineers that like that customer impact doesn't matter to them or they're unaware of how the customers are properly using their software or they themselves are not even using the software that they're building that's why like dog fooding is so important I go into help teams all the time where they're you know they're just churning this is like the most common thing that I go work help teams with right they just churning they're not making progress like usually I can guess that without even talking to anybody I can just like look at their J for two minutes and be like yeah they're turning so let's go figure out why right and then you go look and I lost my train of thought I'm sorry people are not connected to their work they're churning right I still forgot where I was going to connect that it's okay it's okay we can move on I I wanted to add one one thought to the mix because you know we we the the overarching theme here is you know like they they deploy and then they go or they write the code and then that's that's it and and I mean ultimately it we are failing the promise of devops at the end of the day right exact we did nothing to solve or to alleviate the problem because we are still continuing to do that yes we've got cicd pipelines that's awesome we've got automation but we still have not fulfilled the promise of devops at the end of the day and I think it's interesting though because I think observability is kind of you know forcing us to face that once again right um because now there like these are the consequences of now we know I I think the opportunity here is now that we have the data to show the business how the consequences of bad Engineering Management are harming our customers like this we didn't have back when we started devops right like we had no freaking idea how how our failed deploys or any of this were impacting our business right we were just guessing we're throwing dark birds and and like some of us were fairly accurate like yeah this is what's happening in the go talk to a customer and they would confirm it great now we can go do devops but there's tons of shops where they just never close that yeah right and then you could bring the data in though and this is one of the things I've been slowly doing right is I go talk to a team be like well the first thing you need to do is go talk to your product manager and I'll help them to find some freaking slos so you know what the acceptable abuse of your customers is because most people don't even know yeah let's start observing that let's start let's put in stick in s on it we don't have to alert or nothing just start looking getting the data now I can go back to the business and I can go tell your director of engineering or your VP or even your SVP and go yell at them be like look at look at this data you need to go invest in this team and change how they work because this is what your customers are going through and and I have actually data to show it and that's that's a game changer that changes the discussion it really does I side not I always get so much energy talking to Amy because like all these things I feel like I'm just like sitting here thinking about like she's out there in the field just doing like over and over and over and I'm just like oh my God you're so cool what talk sound cool but the dayto day is like meetings and waiting for the opportunity go like hey here yeah but the impact I mean I I just I just think it's so cool but like you say we we're failing the promise of devops I actually feel like devops is like it's like it's like the the the Band-Aid that we have on a self inflicted wounds that never should have happened like we never should have separated them from Ops like there's no possible World in which having one set of people write the code and another set of people understand the code makes any sense like and like I get it we were doing the best we could at the time you know we're like this is too complex we got to split this up somehow but like you know I I feel like the entire the entire idea of having different people do these jobs and and it's a little bit unfortunate I think that we've kind of settled on going oh well it's because op sucks and we're all going to be developers like whatever like okay fine I mean most of the time when this happens it's it's like cultural inertia because most of the corporate world they live in this world where they think about okay I have some people who make decisions about the product and then I go and I encapsulate that in some kind of requirements or design docs or whatever and I ship it off overseas to be manufactured so some other people do the labor some stuff shows up in a warehouse I probably never even see it and it and it goes out to distribute blah blah blah blah blah like this is like most the businesses in the world today are some kind of form of this kind of dis breaking the labor away from the design of it and and this is actually I think changing radically out even outside of the software world where where Now product design is speeding up you see kind of things coming faster so now they're starting to look at what we're doing in the devops and going like oh maybe there was something there maybe expertise matters maybe the idea that there's kind of work that isn't knowledge work is yeah I like that I I think one one thing that um is is also a you know a is to some effect a consequence of of this whole idea of separation of concerns right which um in you see especially in like large Enterprises like when I when I worked at at like a small like me it was like small to mediumsized Startup um you know we I had access to like the freaking PR database and then like I joined a bank right after right after I wrapped up that job and then they're like oh yeah we've got like a DBA for uat and a DBA for prod I'm like H you don't even want you to log into your laptop if yeah yeah that's right to do your job yeah that's changed a lot you know we borrowed so much of that stuff from the accounting world where things like you know the the concept of you shouldn't have the same person file the expense report and approve the expense report totally makes sense you know and I think that there are some Concepts that you know translate nicely and most of them really don't um I actually the most recent talk that I wrote was called um modern engineering is not incompatible with highly secure or like or or compliance environments or something like that yeah and it's just like because it's been and it was super like I've been wanting somebody to write this talk for like 10 years and I finally was like all right this is not my area Jes somebody's got to do this so I sat down and wrote it and I did a bunch of research and it was so interesting um because you know this is all probably old news to y'all but I I I learned a lot about how just like you know it's really Engineers should not have to think about security and compliance in their day-to-day work like if you have to think about it all the time something is very very wrong it should be baked into the architecture so that by default you do the right thing and you can use the accounting and like logging and like auditing principles of like your cicd system and like Engineers should just be do you know doing the work you really only when you need to like spin up a new data source or like build something new then you pull your security folks in you make those decisions from the get-go but like you know it it's just like the entire frame and what and what's so unfortunate is I think that security has this really unfortunate you know the security through security they laugh about but they all do it but it's like they think that talking about how they do things well is going to be bad um and it's not like I think that the only way we're going to make progress in this as an industry is if people start to realize that all their competitors are doing this well and they're about to be outcompeted by the fact that everyone else in the world is is so able to like write off for like a modern Team without being bogged down in the security theater so definitely I know you all touched about Ai and I wanted to to talk to ask a question about it how do you think AI can help like teams adopt observability or like how is it that AI can actually have some benefits to the observability world I think I I've been saying about AI for a few years now right like I think the real opportunities lie in joint cognitive systems research where the way we're using AI isn't necessary to replace humans in in the pipeline it's to augment humans in the pipeline and so for observability I really feel that that the strongest opportunities for AI are in kind of one kind of automated attention on doing correlation basically like Auto autocorrelation and and surfacing that to humans not this idea that I'm gonna eliminate all my cascading alerts through this because that that's a Fool's errand that we've been chasing for at least my entire career um I don't think AI really changes that game but the if we start to think about it instead of this crazy like we're going to make all the alerts go away but instead I'm going to help your people do more with less right less context less expertise um maybe less work um obvious sudden now it gets super duper powerful because that's the hardest problem to solve is how do I reason about this hyper complex system and draw out insights from it that's where I think AI can help by by augmenting the human not replacing it I couldn't agree more it's really about the inputs so much more than the outputs like we are all familiar with like the problems of spam filters right like false positives or brutal you know very high possibility for failure but like there's so much we can do you know honeycom we've been talking about bringing everyone up to the level of the best debugger right increasingly like the hard part is getting detail like these soci Technical Systems are made of just as much of our brains as they are the data like an example I often give is like the New York Times And The Washington Post if you just switch their people overnight nothing would work because so much of the system is in those people's heads and the more we can bring it out of the people's heads and put it somewhere that other people can add to it and ask questions of it you know then it it it like pools our knowledge and our resources like one of the biggest things that blew my mind after becoming a CTO was how many engineering Executives out there trust their vendors more than they trust their employees and they're constantly susceptible to vendors who come up and like so give me $10 million and your people will never have to understand your systems again we will tell you what to look at and we'll tell you what it means and that is and because because people come and go and vendors last forever but like come on like that is just not going to work but there are absolutely things that we can do to make your people more productive uh you know better you know to get better data in to like pull your knowledge I I like I wrote that down joint cognitive research because I think that I think that that's so true you know at honeycom we actually we built a little AI thing Bob which is using uh chat GPT to when you when you deploy some code you can ask questions about it using natural language just like what's slow you know which is super useful because using a query like Explorer is hard for everyone but like just being like what slow what changed is super valuable and it helps those Engineers that like just go around the room and it like like ask a question to the person next to them and it's like you're now winning back time in a way so much time yeah you know I I I agree with with everyone on on the stance on AI because I I think there's this overarching fear um out there where they're like everyone's like AI is gonna take our jobs no I hope so I I don't think we're at Skynet levels uh yet but but also I think like we like personally I like the idea of having you know something take away some of the cognitive load so I can focus on the cooler things and and also take away some of the biases that I might have um assuming that the AI is is is trained without the biases right I guess that's the cave um but you know I I like the idea of like something saying to me hey this is where you look further um you might be interested in blah so so you look and you're like sorry AI you're wrong but thanks for pointing it out anyway um and I think that that can be hugely helpful to just I don't know just take a load off right definitely or like you're getting paged about something it's a database you're not super familiar with and it's like hm something like this happened last year around this time and here's what they did to resolve it would this be useful to you yes exactly it's just your little help on your shoulder that's just telling you all the goodness that could be going on in your system anytime exactly anytime I think about something that I or people spend a lot of time trying to get information out whether that's like combing through data or trying to find stuff in the history or something that's an AI problem like they're so good at that let computers do what computers do best so that humans can do what we do best and I know that like there's this fear that we all have about um you know change and stuff and some jobs are absolutely going to be lost um and but many more will be changed and I think that the way that we make this good for people is not by resisting it fearing it but by being thoughtful about how we how we like involve it and how we adopt it and you know making a change Grace obviously now we're GNA start talking about government policies which is anyway but like but like yeah like Amy said I hope part of my job gets automated away because I could be using the cycles of something else well so there's something else I want to maybe shift to from when you were talking charity one of your favorite topics which is the unknown unknowns because I think AI can eat up most of the known knowns right like we go and ask ask the chat bot and be like hey this seems like it's happened before has this happened before and oh look here's all instent reports and what happened before those are known knowns we we kind of know what to do I think the place where today's efforts anyway we'll see if it changes but I don't think it's going to change drastically in the next decade um are are not are still not aimed at figuring out the the unknown unknown right this is where I think humans are still going to have an edge for a very long time um is basically like we'll use the AI to collate the data to do all the the undifferentiated lifting but it's still going to be people that peer into that space between the data and go I wonder if there's a packet law somewhere in you know deep in this network that's never happened before we've never seen it before and now you have a novel insight and that's the place where today's AI really like it can kind of make things that seem like novel insights because it's it's approximating what a human would do but those actual really tough novel insights I think are going to remain in our domain for a long time they they always come out with these demos that show this these like leaps of things you're like oh I never would have thought to look at that but like those are impressive because they're rare and and at the end of the day if it was the right thing you someone still has to go and like the thing that I keep coming back to that humans are good at is is attaching meaning to things right like any computer can tell you if there's a spike or how big it was but like only a person can come out and go this one really means something because of that that context that the computer won't necessarily always remember what do you think is the current largest challenge in observability I'll start with Amy sampling ask me again in five years it'll probably still be sampling like it's just you know we're generating a wealth of data um yeah you know there's a few places where we could probably tune things up and and generally we're not sampling a lot today but some of the systems that we want to turn on we're going to have to because they do a bunch of this is kind of circling back on like that Tech de under the covers like as soon as we turn on tracing in some of these systems we see that they're doing a bunch of busy work that doesn't make any sense but the first thing that happens is it tips over our our collectors and and overloads our our Upstream account and then we got to go turn it off and so we don't actually get to figure out what the heck is going on so I think like sampling and really just kind of getting control of all of the data we're generating that might not ever have any value and that might is super important because it's it's a by no means an easy problem to solve it's probably the hardest problem in observability right now it's like just how do we get this to a rational on of data so that my cost is rational so I can do more observability get deeper insights into my systems but you got to pay the bill and so like this is continually gonna be our struggle that's the answer of a super user true what do you think is the largest challenge shity um the people don't understand how badly they have it now and how much better it could be people who have spent their careers like slugging it out in the salt mines using metrics managing the overhead of metrics not having high cardinality data dancing around between metrics and logs and they genuinely have no idea that the way they're doing it is the hard way and they don't they don't actually they nobody's optimistic in computers because why would you be optimistic about computers but like until people see their data in our world it doesn't they don't understand how much easier it could be how much better it could be and this there's this mental model you know and and observ 2.0 is dramatically easier than 1.0 because it's just interesting shove it in interesting shove it in there's no like custom metrics to measure or or like manage and farm over time there's none but like the difficulty is that it's a mental model shift that people have to and and I am confident that 10 years from now 15 years from now uh people will be you know using the systems that you know we kind of sketched out a few years ago for observability to understand their systems because the rate of complexity is just going up you know too high we might be out of business by then like we might have been way too early but I am extremely confident that there will be no other way to understand your assistance 10 years from now but using something like this that's fair they are definely not getting any simpler or not and and ironically what that leads to is this increasingly this increas ing Reliance on Heroes and martyrs because if there's nothing knitting together all of your sources of the data except the people with the intuition and the scar tissue who can make good guesses you have to you don't have any other choice I call them loadbearing humans in my God because like when I run into that that's how I talk to the management tree about it I'm like you have a loadbearing human this one person is doing this task and if they go on a vacation or win the lottery we are going to have a bad time right and that's the way to surface the risk is like you wouldn't put a single you know firewall you would always do redundant right like so we need to make at least make that pun and that's usually a good way to start getting over the line and get people realizing that that loadbearing humans are holding the world together yes all over the world and even as our rhetoric has gotten better about this and our understanding has gotten better about this there's still these these Tech it's like the finger in the Dy so to speak you know the little Amsterdam yeah dude just like plugging the Dyke with his God I I got to stop using anecdote so terrible you say out loud but but like these people exist and then they're doing a hero's job but they shouldn't have to and it's bad for everyone it's also putting a lot of trust and retention oh my God very fragile yeah ad what about you what do you think is the biggest challenge um so I think the biggest challenges uh organizations getting in their own way um when it comes to observability um either because they don't they kind of get observability in much the same way that everyone kind of got like agile and devops is like important but they kind of totally missed the point at the same time and so we're you know we've made like a little bit of progress but also not um and and on a similar vein like with open Telemetry again it's like oh I understand the benefits of of observability let's get open telemetry into the OR and then again like not understanding like what they should do with that um either because you know of a vendors um who who oversell right where I we have a tracing tool we can totally take your traces and put them on a little Island by themselves with no food or water and nobody ever freaking looks at them because they're little island yeah we'll totally do that for you we'll take your money too thank you they'll be safe true exactly never use them yeah I you know we we we get uh especially in in large organizations um we we get very um tempted or or or drawn in by by the shiny shiny new thing that seems too good to be true and you're sold on the vaporware or like some reads an article about blah and they speak to a salesperson about at blah company and then all of a sudden they walk in all staryy so I feel like these are such huge barriers to to actual like actually getting done for observability True what would you say is the biggest misconception around observability right now and I'll start with you ad biggest misconception um I think right now I would say like people thinking that it'll solve all their problems without putting in the work I think there's a lot of like people assuming there's a lot of like magical fairy dust that comes in with observability and then when they realized that it's extra work it was like oh crap what about you Amy I I think I'll extend what Adriana said right like it's that extra work of like the the need for understanding doesn't disappear ideally as we get the data presentation into a better place it gets easier to achieve but the need for people to understand the world around them and be able to synthesize new information from from the observability data um that's that doesn't go away it just gets more efficient what do you think about a charity I think I think the biggest misconception I see is just people thinking that observability is its own sort of Standalone project that you can buy or do or check off when in fact it's so interwoven with the development of software iterating on software owning deciding what features you're going to build you know I mean deciding figuring out how your product is working figure out what your users are actually doing why you should care about it where you should spend your time like it has its tentacles everywhere and there are lots of things that we do that make observability better or worse have nothing to do with anything that's marked observability and this is both I think it should be both encouraging and reassuring to people and also a little bit depressing and demoralizing because but like it just it CS for a more holistic view of like this is not I don't think you could have like an observability project that you could just check off it's it requires us to get better at understanding the mission of building something which I think we're doing bit by bit over the years like we're understanding more about how to build software that is better for our users that is better for our Engineers that is more linked to the business you know and and like you were talking earlier about you know there there was this great discussion was kicked off by McKenzie of all people about measuring the work of software whether whether or not it's measurable you know and of course they're completely wrong and which meant that K Beck gave him a Smackdown and Gade did gave him a Smackdown and there's a great discussion about how you measure software development uh but in one of the pieces um someone said that people have got to start realizing that building software is not like building construction it's like it's more like it's more like the design phase and you would never say that a blueprint is better because it has more blue lines right which gets to your point earlier Amy about how this all work is knowledge work right and so I where was I going with all this I don't know I feel like it is it is indivisible from the act of getting better at delivering software so if I I think that a lot God I keep I I swear I've taken up 50% of the air time in this and I really apologize I made must have had too much coffee today but I feel like so many software orgs out there have kind of given up on saying that they're trying to get better at delivering software and so so instead they they they set these goals themselves kind of nibble around the edges we're going to get better at observ villing year we're going to get better at cicd we're going to get better at blah blah blah blah blah but like the great leaps forward that you make that you have to make in order to keep up with complexity because you can't just hire your way linearly out of this you have to make these big leaps and that requires taking a a much you know a much more bird eye view and being like in order to deliver software better this year it requires that we make these investments in observability we make these investments in testing and and and I feel like in conclusion in conclusion building software is an amazing job I thank my my lucky atheist Stars every day that I get to be in this industry not like picking weeds and getting dirt under my finger but like sitting here at a computer solving puzzles and playing with my friends and and I feel like I wish that everyone in software could have as magical of an experience in clear as I have had and I think that observability is a big part of how we get there that's true that actually brings me to my next question as we're getting ready to close up what do you think is in store for observability for this upcoming [Music] year the first person to jump in is okay I'm not gonna pick on one of you oh I haven't been keeping up on all the road maps and stuff but I guess so what is your wish what is your wish that this year brings fors for Ability I guess the thing that I'm working on is a lot of folks still don't really get slos and sis um as much as I think it will help both Engineering Management and Engineers on the ground just communicate better about what they we're all trying to accomplish like it's if I had a wish like it would be to drive kind of that understanding deeper in and just into our industry just so now so I can go into a room and talk about the power that's available here and just like having the metrics to understand and and not just get stared at like I just nailed a fish to the wall right like just you know it would just be nice to to not have to start at zero over and over um but I'll keep doing it until we're there that's fair I personally want to see more people like besides like you know we we we often um associate I guess observability with with like more the Y side of things and it is so not just that right I mean it's it's everybody's problem and so I want to see more conversations where like we get more developers giving a crap about observability and and not just developers like folks who are like planning Tech projects like software releases whatever like make sure it's part of make sure it's part of their language make sure that we Empower our our qas to use everybody misses that give it to your support team gosh like what a opportunity yeah well well even your your QA right like they're they're testing your software and they could use observability to figure out what the hell is wrong and tell developers hey this is why it's wrong you know um so that I want to see I want to see more of that um and and I think it it you know basically it puts us closer to you know Charities observability 2.0 where it's like more part more of a holistic part of the sdlc right I and I think that's where it needs to be so that it's it's you know part of it's baked in baked into how we deliver software and how we code software and and all that good stuff definitely everybody does a better job when they can see where they're going and see what they're doing when they are getting feedback back from the system for sure Amy I had this sticker that says that slos are apis for our engineering teams and and I completely agree like if if you don't have an agreement that you've all agreed upon you're down in the weeds negotiating over every single decision you make about how to spend your time like it's absurdly inex it's absurdly expensive um I I don't I don't know I expect that what's going to happen in 2024 is that everything's going to get even more muddled and people are getting even more confused and um and it's fine it's it's it's entropy it's how it goes um I guess my wish for Christmas would be that um my wish would be that more people just have that feeling in their gut that you get when you see Telemetry feedback to you about what you've done and just go that feeling that you get this like you feel more grounded you feel more certain you feel more confident what you've done observability is the way that we can move swiftly and with confidence and I feel like this is something we all know in our heads we need to learn in our bodies and the only way that we can learn it in our body is by seeing it and experiencing so I would wish that everyone had that experience most definely I love that I love that to keep that in store for 2024 as like find that fuzzy feeling when you start seeing your logs metrics start showing up on your vendor definitely well thank you all very much for the amazing insights you've been able to provide today on our panel around observability this group is going to be coming back later in June to have a little other chat around observability stay tuned to hear about those details later with that we'll like to sign off and say thank you all for watching stay warm out there [Music] yay +here we are hello hello hello everyone we're very excited to have a panel today around the topic of observability plan for the observability community here with us today we have Adriana Amy and charity joining us we hope to have a fun build conversation would you mind starting out Adriana and introducing yourself who you are and what you do oh yeah so I am Adriana Vela I work with Anna at service now Cloud observability formerly known as lightstep say that three times fast um I uh yeah I'm a senior staff developer Advocate um been doing the devil thing for what almost two years but before that was you know oscillating between like IC um engineering and and management roles so yay and I love observability I knew you before you were a Deo before you in observability yeah yeah yeah when I was just like trying to figure out what the hell this stuff was all about time definitely has passed by charity what about you uh charity major co-founder and CTO of honeycomb.io I identify as an Ops engineer and probably always will even though it's been a long time since I held the pager but I feel like I started I feel like I you know I picked up my first page of rotation when I was 19 so I feel like I've got still like half my life on call so I still I I can still call myself an operations engineer you paid your dues I think so what about you Amy uh hi I'm Amy Toby I am a senior principal engineer at equinex which is a the largest data center company in the world um and I've been doing observability since pretty much my first job as well starting back on Solara systems in like 98 99 um S&M and and then into naio stuff I'm responsible for some things in naos I'm sorry um and like it have been doing it all along these days I do a little bit more kind of work in soot technical space but initial work at equinix was bringing open Telemetry into our entire equinix metal operational stack and pumping those uh metrics out to Honeycomb nice very exciting work that all of you are doing and creating Community around observability and open Telemetry and I get to call you all friends which is super exciting to have you all today on the panel sweet so I think we're going to start with one of the spicy questions and I'm going to kick it off with charity what is your definition of observability the correct one of course we'll see yes spice you know okay so back in the days when nobody else gave a [ __ ] about observability the housian days of I'm saying that even though we almost went out of business many times you know I think we really tried to push this this this idea that there was a technical definition for observability you know that it was high cardinality high dimensionality explorability and all these things and nowadays you know like every everyone and their cat does observability it's it's like if you have any Telemetry you do observability and you know what actually I've come around to this I'm all right I'm all right with this I feel like it's actually a great definition to say that observability is a property of soci technical system so it exists in a Continuum right uh I'm I'm totally down with that because I think that it it it gives a lot of Grace to people who are you know in the early days of their Journey like I don't want to be the person who like well actual you're not really doing observ like nobody likes that guy so I don't want to be that guy um that said I do think that there's a real kind of Step function difference in let's call it observability 1.0 toing that buil built off The Primitives of you know metrics logs and traces um where every time you gather the data you have to you have to gather again in a different format and pay for it again so like you're Gathering it once for metrics again for love again for traces again for APM again for profiling again for security you know like talk about a cost crisis and observability tooling like no wonder right uh and you're very limited in each one of those by The Format that you happen to gathered in and none of them can be connected to each other the only thing that connects them is the poor human sitting in the middle guessing right like guessing eyeball us you know so let's call that 1.0 and then observability like 2.0 tooling I think is based on a single Le single source of truth right these arbitrarily wide structured data blobs you could you can derive metrics you can dve traces you can dve all these things but you've got one source of Truth it handles high cardinality it handles you know it's a more explorable sort of interface so you don't have these static dashboards you have interface where you can ask questions and follow breadcrumb go and then what and then what um so that's my very long-winded answer I think that there's kind of two generations of observability tooling out out there right now and I think that they lead to two very different outcomes like one of the I know I'm taking a very long time here I'm almost done I promise I think that one of the most one of the soot technical factors that you really associate with observability 1.0 is it's for it's like a checklist thing you know you add it at the end before you put stuff in production like cool I can monitor this um and for observably 2.0 I think like the fundamental truth of it is it it underlies the entire software development life cycle and your ability to hook up these fast feedback loops rests on how well you can you know ask questions and explore your data and have this sort of constant conversation going with your code awesome what about you Amy I think I agree with most of what charity said but I've been I've been learning a lot from my my work lately in kind of developing a new networking product and one of the feature sets that we have to include in that today in a modern Network as a service kind of product is observability features and customers demand this and we've been scratching at this like what how much do they need like what how much should we build how much should we hand off to either you know service now or honeycomb you know and let those tools do what they're best at and so what what we've been finding with customers is like that that operational part that charity mentioned right that they are tired of this world where they have to go hire very expensive people who are often cranky um who who who can basically carry that mental burden of tying all the context together with the metrics and and divining what the heck is going on with the network and so what they're asking and what they're demanding from us and all of the other Cloud vendors is how give us the observability Telemetry which is mostly what they ask for because when we talk to customers like most the time when folks were talking to don't want to talk about cardinality um so they're asking for like how do we get all the stuff we need so we can feed it to our my my feelings about a apps aside but this is a common request feed it to our AI Ops tools feed it to our our our other observability systems that we already have so that we can start to leverage the skills we already have to get those insights but it always drives to how do I get insights about this hyper complex system so that we can fix it and get back to business and some sometimes folks don't even want to do the heavy lifting they're just like do it for nobody does right because like almost none of our customers any of us here their primary business is not Telemetry it's undifferentiated lifter they want they got a business to run they don't want to be messing with this stuff so they want me to provide really high quality Telemetry um and then they want you all to make that all super easy to consume yeah that's very true I love that it makes so much sense what is your definition AD so I think I I started out with you know Chari OG definition of observability which carried me well throughout the years and then I I heard one recently from uh Hazel Weekley what which I quite like which is observability is the ability for you to ask questions and get meaningful answers um which I really like because it's you know it's it's up to you as to what what is Meaningful right so what what data are you collecting that's meaningful to you and I think that's really important um in in our in our world um I will still say that I I think like Trace first for observability no matter what um but but I think like we really need to focus on on the meaningful data aspect of observability because like sure you can like instrument the crap out of a system and it's emitting a bunch of useless garbage right you know garbage in garbage out so you don't have good data that you're instrumenting well good luck trying to figure out what the hell is going on you can have the best observability tool out there um it won't do you any good so you're you're going to need um to have I it it takes a bit of you know there's there's that learning curve of of what's what's important to you what should you be collecting to be able to do the thing observability is a design problem like in a in a huge way like once you've s like I feel like we spent spent a long time like just sort of like figuring out the underlying Plumbing which is just a prerequisite to what really matters is this is a design problem because there's so much data and I think Amy and I share very uh similar uh thoughts on AI Ops and and like but like there's two kinds of tools right there are tools that um that pave over complexity and there are tools that try to help equip you to understand it and I will always be in the in the second Camp because at the end of the day we are legally liable for the software that we put out into the world somebody somewhere is going to have to understand that [ __ ] and the harder you've made it the more magical you've made it um the harder that Day of Reckoning is going to be definitely interestingly enough I I think we're we're seeing that a little bit um with um like open Telemetry right like open Telemetry is awesome it's you know this awesome standard uh that pretty much everyone has has aop opted vendor wise um but then now we're we're looking Beyond like that initial adoption of open Telemetry and like people really using it out in the field and and making sure I think open telemetry's biggest challenge I think in in the near future is going to be making making sure that it's something that is easy for onboarding because I think that can be really offputting um for organizations and and that can be really scary and then that leads to the you know you instrument code I don't want to instrument my own code kind of mentality I I think that the the place that the the metrics go really matters though because we talked about a Ops where I see folks really reaching for these tools um charity called it Paving over so what we've done in the network world for decades now and it's still the the case at most organizations is we have all these devices generating gobs of alerts and metrics nobody knows what any of it's doing there's there's a sense that we should be able to correlate all these events so like some Port fails on a a core rou or all the stuff behind it disappears we should be able to roll all that up and I'm using the S the should word the way my therapist calls it the s word right like we we believe these things that should happen but the real world is is is much harder than that so like these these alerts just flood through and people fight these alerts sometimes they've been fighting their whole career trying to get on top of this alert stream and so they reach for these tools to make sense of the morass that they built for themselves because they're not really empowered to go back to the Telemetry and like dial back the cardinality or retransform how they generate those metrics to get higher you know higher cardinality that they can process at the end um often they don't even have choices about what toet Tre they get because they're buying network devices that offer you get your SNMP or you're G and that's what you get oh my God yeah oh God world out there of Legions of people that are just fighting through the noise still um and it hasn't changed for them whereas they're in the software world we keep advancing the state-ofthe-art but it's it's going to be a challenge still to bring a lot of the classic systman and network administrators who are used to this world Along on that journey I feel like one of the characteristics that I often think about when we're talk when we're thinking about like transitioning from the One point0 World to the two point0 World which is going to take [ __ ] for forever um no doubt but but I feel like it's it's it's part of it is generating from a push world to a pull world where we're used to you like the only way you could make sense of like a naous based system is by looking at the cluster of things that just alerted you and go like ah it's probably that right and so you rely on you know all of these alerts but that's so noisy and it does not scale right but like as a as an Ops person I grew up thinking you only look at production when you get alerted you don't have to look at it otherwise and I feel like you know the the tradeoff that we we have to learn to make is okay we're only going to alert you when users are impacted right we set slos these are our agreements with ourselves each other and the world this is the level of service we are going to give you and we alert when user be when when the when when the outage is bad enough that users are being infected but in order to get to that world you have to agree that you're going to go look at your Telemetry affirmatively because most problems will never rise to the level of waking you up God I mean I hope right like most of the things most of the codes you write has a subtle bugs it affect a few people or it's a small thing right so like you in order to have a system in order to have a hygienic system that is not just like a hairball that the cat coughed up you know we can't just keep like in the old days like we were shipping code every day that we didn't really understand these systems we've never really understood and then we wonder why it's giant trash fire right like and the way that we start improving that is by you know not just making you know the Telemetry richer and better but also by committing to closing that loop as a software engineer my job is not done in when the test pass my job is done when I have looked at my Telemetry in production and gone it's doing what I expect it to do nothing else looks weird right so like that's asking it's like I like the technical debt metaphor here because it's asking you to like put in a little bit of effort up front in order to avoid the you know there's this great graph that shows that the cost of finding and fixing bugs and goes up exponentially from the moment that you write them so like you backspace good for you good job right you find it in your test good for you the next best time to find it is right after you deployed that [ __ ] after that God knows how long it's going to take days months years who knows but like the accumulation of that [ __ ] is what makes this this job a nightmare for so many people yeah it's so true like I I I think back to like you know my my old days of of having to hand off deployment instructions to like our Ops folks who theun they had no [ __ ] clue what was you know like what it was they were deploying and and you know I had to pray that my instructions were correct and then I had to pray that they read my instructions correctly and then they would deploy it to Pride and these quote unquote successful deployment like things went correctly was considered a successful project the machine we're done right right yeah and and and so I love the idea of like no like because otherwise it becomes a hit and run deployment right you know or driveby drive by deployment rather is is a better way of putting it where where like you know it's like deployed all right great um let's just leave it to fate but I I think like you know paying attention what's going on as soon as you deploy like there's something to be said because otherwise because you as the person writing the code you have context you had that nobody else in the world has or will ever have you know exactly what you're trying to do why you're doing it what you tried that worked what it what didn't work what the variables were named what the f like you know all this stuff and if you can close that Loop before you have to concept switch another it's so powerful like these feedback loops are what socio Technical Systems are all about like this is the one job of technical leadership in my opinion whether you're an IC or a director or VP one job is these feedback loops to make them as tight and short and functional as possible I think that like nailed down of like engineering responsibility has been so important and observability really brings it to a front but a lot of people are just not paying enough attention or like not willing to do it they're just like my job is to code and that's it and maybe I'm on call twice a year but it's someone else's problem there's not much we can do about that though like a lot of times right like if you are an engineering leader and you're trying to build a dynamic powerful engineering organization you have to either lead those people to care or lead them out right like because like the reality is is like there's the customers that y'all want that that have that engineering leadership and culture of like giving a crap yeah and then there's the vast majority of organizations out there where the most of the people who are writing the code that runs our world are trying to put bread on their table and so as soon as they are done with the with the specification that they were given for their job they e that code into to get and they run because like why should they care why should they metrics but this is this is a self-fulfilling prophecy because you know what makes people like check out tune the [ __ ] out not being connected to the results of their labor like there's something that's like what is it Dan pink says we all want in our job is autonomy Mastery and purpose the purpose of your work if you're so disconnected from it yeah that's all there is to it I'm like I'm not trying to suggest that this is easy or can be done in a one stop you know whatever but I do think that every step we make along this path that tightens up the feedback loops that connects people more with the consequences of their code is good and valuable and pays off interesting oh sorry go ahead I was going to say it's even letting Engineers understand customers like I've worked at many shops and with so many engineers that like that customer impact doesn't matter to them or they're unaware of how the customers are properly using their software or they themselves are not even using the software that they're building that's why like dog fooding is so important I go into help teams all the time where they're you know they're just churning this is like the most common thing that I go work help teams with right they just churning they're not making progress like usually I can guess that without even talking to anybody I can just like look at their J for two minutes and be like yeah they're turning so let's go figure out why right and then you go look and I lost my train of thought I'm sorry people are not connected to their work they're churning right I still forgot where I was going to connect that it's okay it's okay we can move on I I wanted to add one one thought to the mix because you know we we the the overarching theme here is you know like they they deploy and then they go or they write the code and then that's that's it and and I mean ultimately it we are failing the promise of devops at the end of the day right exact we did nothing to solve or to alleviate the problem because we are still continuing to do that yes we've got cicd pipelines that's awesome we've got automation but we still have not fulfilled the promise of devops at the end of the day and I think it's interesting though because I think observability is kind of you know forcing us to face that once again right um because now there like these are the consequences of now we know I I think the opportunity here is now that we have the data to show the business how the consequences of bad Engineering Management are harming our customers like this we didn't have back when we started devops right like we had no freaking idea how how our failed deploys or any of this were impacting our business right we were just guessing we're throwing dark birds and and like some of us were fairly accurate like yeah this is what's happening in the go talk to a customer and they would confirm it great now we can go do devops but there's tons of shops where they just never close that yeah right and then you could bring the data in though and this is one of the things I've been slowly doing right is I go talk to a team be like well the first thing you need to do is go talk to your product manager and I'll help them to find some freaking slos so you know what the acceptable abuse of your customers is because most people don't even know yeah let's start observing that let's start let's put in stick in s on it we don't have to alert or nothing just start looking getting the data now I can go back to the business and I can go tell your director of engineering or your VP or even your SVP and go yell at them be like look at look at this data you need to go invest in this team and change how they work because this is what your customers are going through and and I have actually data to show it and that's that's a game changer that changes the discussion it really does I side not I always get so much energy talking to Amy because like all these things I feel like I'm just like sitting here thinking about like she's out there in the field just doing like over and over and over and I'm just like oh my God you're so cool what talk sound cool but the dayto day is like meetings and waiting for the opportunity go like hey here yeah but the impact I mean I I just I just think it's so cool but like you say we we're failing the promise of devops I actually feel like devops is like it's like it's like the the the Band-Aid that we have on a self inflicted wounds that never should have happened like we never should have separated them from Ops like there's no possible World in which having one set of people write the code and another set of people understand the code makes any [ __ ] sense like and like I get it we were doing the best we could at the time you know we're like this is too complex we got to split this up somehow but like you know I I feel like the entire the entire idea of having different people do these jobs and and it's a little bit unfortunate I think that we've kind of settled on going oh well it's because op sucks and we're all going to be developers like whatever like okay fine I mean most of the time when this happens it's it's like cultural inertia because most of the corporate world they live in this world where they think about okay I have some people who make decisions about the product and then I go and I encapsulate that in some kind of requirements or design docs or whatever and I ship it off overseas to be manufactured so some other people do the labor some stuff shows up in a warehouse I probably never even see it and it and it goes out to distribute blah blah blah blah blah like this is like most the businesses in the world today are some kind of form of this kind of dis breaking the labor away from the design of it and and this is actually I think changing radically out even outside of the software world where where Now product design is speeding up you see kind of things coming faster so now they're starting to look at what we're doing in the devops and going like oh maybe there was something there maybe expertise matters maybe the idea that there's kind of work that isn't knowledge work is [ __ ] yeah I like that I I think one one thing that um is is also a you know a is to some effect a consequence of of this whole idea of separation of concerns right which um in you see especially in like large Enterprises like when I when I worked at at like a small like me it was like small to mediumsized Startup um you know we I had access to like the freaking PR database and then like I joined a bank right after right after I wrapped up that job and then they're like oh yeah we've got like a DBA for uat and a DBA for prod I'm like H you don't even want you to log into your laptop if yeah yeah that's right to do your job yeah that's changed a lot you know we borrowed so much of that stuff from the accounting world where things like you know the the concept of you shouldn't have the same person file the expense report and approve the expense report totally makes sense you know and I think that there are some Concepts that you know translate nicely and most of them really don't um I actually the most recent talk that I wrote was called um modern engineering is not incompatible with highly secure or like or or compliance environments or something like that yeah and it's just like because it's been and it was super like I've been wanting somebody to write this talk for like 10 years and I finally was like all right this is not my area Jes somebody's got to do this so I sat down and wrote it and I did a bunch of research and it was so interesting um because you know this is all probably old news to y'all but I I I learned a lot about how just like you know it's really Engineers should not have to think about security and compliance in their day-to-day work like if you have to think about it all the time something is very very wrong it should be baked into the architecture so that by default you do the right thing and you can use the accounting and like logging and like auditing principles of like your cicd system and like Engineers should just be do you know doing the work you really only when you need to like spin up a new data source or like build something new then you pull your security folks in you make those decisions from the get-go but like you know it it's just like the entire frame and what and what's so unfortunate is I think that security has this really unfortunate you know the security through security they laugh about but they all do it but it's like they think that talking about how they do things well is going to be bad um and it's not like I think that the only way we're going to make progress in this as an industry is if people start to realize that all their competitors are doing this well and they're about to be outcompeted by the fact that everyone else in the world is is so able to like write off for like a modern Team without being bogged down in the security theater so definitely I know you all touched about Ai and I wanted to to talk to ask a question about it how do you think AI can help like teams adopt observability or like how is it that AI can actually have some benefits to the observability world I think I I've been saying about AI for a few years now right like I think the real opportunities lie in joint cognitive systems research where the way we're using AI isn't necessary to replace humans in in the pipeline it's to augment humans in the pipeline and so for observability I really feel that that the strongest opportunities for AI are in kind of one kind of automated attention on doing correlation basically like Auto autocorrelation and and surfacing that to humans not this idea that I'm gonna eliminate all my cascading alerts through this because that that's a Fool's errand that we've been chasing for at least my entire career um I don't think AI really changes that game but the if we start to think about it instead of this crazy like we're going to make all the alerts go away but instead I'm going to help your people do more with less right less context less expertise um maybe less work um obvious sudden now it gets super duper powerful because that's the hardest problem to solve is how do I reason about this hyper complex system and draw out insights from it that's where I think AI can help by by augmenting the human not replacing it I couldn't agree more it's really about the inputs so much more than the outputs like we are all familiar with like the problems of spam filters right like false positives or brutal you know very high possibility for failure but like there's so much we can do you know honeycom we've been talking about bringing everyone up to the level of the best debugger right increasingly like the hard part is getting detail like these soci Technical Systems are made of just as much of our brains as they are the data like an example I often give is like the New York Times And The Washington Post if you just switch their people overnight nothing would work because so much of the system is in those people's heads and the more we can bring it out of the people's heads and put it somewhere that other people can add to it and ask questions of it you know then it it it like pools our knowledge and our resources like one of the biggest things that blew my mind after becoming a CTO was how many engineering Executives out there trust their vendors more than they trust their employees and they're constantly susceptible to vendors who come up and like so give me $10 million and your people will never have to understand your systems again we will tell you what to look at and we'll tell you what it means and that is and because because people come and go and vendors last forever but like come on like that is just not going to work but there are absolutely things that we can do to make your people more productive uh you know better you know to get better data in to like pull your knowledge I I like I wrote that down joint cognitive research because I think that I think that that's so true you know at honeycom we actually we built a little AI thing Bob which is using uh chat GPT to when you when you deploy some code you can ask questions about it using natural language just like what's slow you know which is super useful because using a query like Explorer is hard for everyone but like just being like what slow what changed is super valuable and it helps those Engineers that like just go around the room and it like like ask a question to the person next to them and it's like you're now winning back time in a way so much time yeah you know I I I agree with with everyone on on the stance on AI because I I think there's this overarching fear um out there where they're like everyone's like AI is gonna take our jobs no I hope so I I don't think we're at Skynet levels uh yet but but also I think like we like personally I like the idea of having you know something take away some of the cognitive load so I can focus on the cooler things and and also take away some of the biases that I might have um assuming that the AI is is is trained without the biases right I guess that's the cave um but you know I I like the idea of like something saying to me hey this is where you look further um you might be interested in blah so so you look and you're like sorry AI you're wrong but thanks for pointing it out anyway um and I think that that can be hugely helpful to just I don't know just take a load off right definitely or like you're getting paged about something it's a database you're not super familiar with and it's like hm something like this happened last year around this time and here's what they did to resolve it would this be useful to you yes exactly it's just your little help on your shoulder that's just telling you all the goodness that could be going on in your system anytime exactly anytime I think about something that I or people spend a lot of time trying to get information out whether that's like combing through data or trying to find stuff in the history or something that's an AI problem like they're so good at that [ __ ] let computers do what computers do best so that humans can do what we do best and I know that like there's this fear that we all have about um you know change and stuff and some jobs are absolutely going to be lost um and but many more will be changed and I think that the way that we make this good for people is not by resisting it fearing it but by being thoughtful about how we how we like involve it and how we adopt it and you know making a change Grace obviously now we're GNA start talking about government policies which is anyway but like but like yeah like Amy said I hope part of my job gets automated away because I could be using the cycles of something else well so there's something else I want to maybe shift to from when you were talking charity one of your favorite topics which is the unknown unknowns because I think AI can eat up most of the known knowns right like we go and ask ask the chat bot and be like hey this seems like it's happened before has this happened before and oh look here's all instent reports and what happened before those are known knowns we we kind of know what to do I think the place where today's efforts anyway we'll see if it changes but I don't think it's going to change drastically in the next decade um are are not are still not aimed at figuring out the the unknown unknown right this is where I think humans are still going to have an edge for a very long time um is basically like we'll use the AI to collate the data to do all the the undifferentiated lifting but it's still going to be people that peer into that space between the data and go I wonder if there's a packet law somewhere in you know deep in this network that's never happened before we've never seen it before and now you have a novel insight and that's the place where today's AI really like it can kind of make things that seem like novel insights because it's it's approximating what a human would do but those actual really tough novel insights I think are going to remain in our domain for a long time they they always come out with these demos that show this these like leaps of things you're like oh I never would have thought to look at that but like those are impressive because they're rare and and at the end of the day if it was the right thing you someone still has to go and like the thing that I keep coming back to that humans are good at is is attaching meaning to things right like any computer can tell you if there's a spike or how big it was but like only a person can come out and go this one really means something because of that that context that the computer won't necessarily always remember what do you think is the current largest challenge in observability I'll start with Amy sampling ask me again in five years it'll probably still be sampling like it's just you know we're generating a wealth of data um yeah you know there's a few places where we could probably tune things up and and generally we're not sampling a lot today but some of the systems that we want to turn on we're going to have to because they do a bunch of this is kind of circling back on like that Tech de under the covers like as soon as we turn on tracing in some of these systems we see that they're doing a bunch of busy work that doesn't make any sense but the first thing that happens is it tips over our our collectors and and overloads our our Upstream account and then we got to go turn it off and so we don't actually get to figure out what the heck is going on so I think like sampling and really just kind of getting control of all of the data we're generating that might not ever have any value and that might is super important because it's it's a by no means an easy problem to solve it's probably the hardest problem in observability right now it's like just how do we get this to a rational on of data so that my cost is rational so I can do more observability get deeper insights into my systems but you got to pay the bill and so like this is continually gonna be our struggle that's the answer of a super user true what do you think is the largest challenge shity um the people don't understand how badly they have it now and how much better it could be people who have spent their careers like slugging it out in the salt mines using metrics managing the overhead of metrics not having high cardinality data dancing around between metrics and logs and [ __ ] they genuinely have no idea that the way they're doing it is the hard way and they don't they don't actually they nobody's optimistic in computers because why would you be optimistic about computers but like until people see their data in our world it doesn't they don't understand how much easier it could be how much better it could be and this there's this mental model you know and and observ 2.0 is dramatically easier than 1.0 because it's just interesting shove it in interesting shove it in there's no like custom metrics to measure or or like manage and farm over time there's none but like the difficulty is that it's a mental model shift that people have to and and I am confident that 10 years from now 15 years from now uh people will be you know using the systems that you know we kind of sketched out a few years ago for observability to understand their systems because the rate of complexity is just going up you know too high we might be out of business by then like we might have been way too early but I am extremely confident that there will be no other way to understand your assistance 10 years from now but using something like this that's fair they are definely not getting any simpler or not and and ironically what that leads to is this increasingly this increas ing Reliance on Heroes and martyrs because if there's nothing knitting together all of your sources of the data except the people with the intuition and the scar tissue who can make good guesses you have to you don't have any other choice I call them loadbearing humans in my God because like when I run into that that's how I talk to the management tree about it I'm like you have a loadbearing human this one person is doing this task and if they go on a vacation or win the lottery we are going to have a bad time right and that's the way to surface the risk is like you wouldn't put a single you know firewall you would always do redundant right like so we need to make at least make that pun and that's usually a good way to start getting over the line and get people realizing that that loadbearing humans are holding the world together yes all over the world and even as our rhetoric has gotten better about this and our understanding has gotten better about this there's still these these Tech it's like the finger in the Dy so to speak you know the little Amsterdam yeah dude just like plugging the Dyke with his God I I got to stop using anecdote so terrible you say out loud but but like these people exist and then they're doing a hero's job but they shouldn't have to and it's bad for everyone it's also putting a lot of trust and retention oh my God very fragile yeah ad what about you what do you think is the biggest challenge um so I think the biggest challenges uh organizations getting in their own way um when it comes to observability um either because they don't they kind of get observability in much the same way that everyone kind of got like agile and devops is like important but they kind of totally missed the point at the same time and so we're you know we've made like a little bit of progress but also not um and and on a similar vein like with open Telemetry again it's like oh I understand the benefits of of observability let's get open telemetry into the OR and then again like not understanding like what they should do with that um either because you know of a vendors um who who oversell right where I we have a tracing tool we can totally take your traces and put them on a little Island by themselves with no food or water and nobody ever freaking looks at them because they're little island yeah we'll totally do that for you we'll take your money too thank you they'll be safe true exactly never use them yeah I you know we we we get uh especially in in large organizations um we we get very um tempted or or or drawn in by by the shiny shiny new thing that seems too good to be true and you're sold on the vaporware or like some reads an article about blah and they speak to a salesperson about at blah company and then all of a sudden they walk in all staryy so I feel like these are such huge barriers to to actual like actually getting [ __ ] done for observability True what would you say is the biggest misconception around observability right now and I'll start with you ad biggest misconception um I think right now I would say like people thinking that it'll solve all their problems without putting in the work I think there's a lot of like people assuming there's a lot of like magical fairy dust that comes in with observability and then when they realized that it's extra work it was like oh crap what about you Amy I I think I'll extend what Adriana said right like it's that extra work of like the the need for understanding doesn't disappear ideally as we get the data presentation into a better place it gets easier to achieve but the need for people to understand the world around them and be able to synthesize new information from from the observability data um that's that doesn't go away it just gets more efficient what do you think about a charity I think I think the biggest misconception I see is just people thinking that observability is its own sort of Standalone project that you can buy or do or check off when in fact it's so interwoven with the development of software iterating on software owning deciding what features you're going to build you know I mean deciding figuring out how your product is working figure out what your users are actually doing why you should care about it where you should spend your time like it has its tentacles everywhere and there are lots of things that we do that make observability better or worse have nothing to do with anything that's marked observability and this is both I think it should be both encouraging and reassuring to people and also a little bit depressing and demoralizing because but like it just it CS for a more holistic view of like this is not I don't think you could have like an observability project that you could just check off it's it requires us to get better at understanding the mission of building something which I think we're doing bit by bit over the years like we're understanding more about how to build software that is better for our users that is better for our Engineers that is more linked to the business you know and and like you were talking earlier about you know there there was this great discussion was kicked off by McKenzie of all people about measuring the work of software whether whether or not it's measurable you know and of course they're completely wrong and which meant that K Beck gave him a Smackdown and Gade did gave him a Smackdown and there's a great discussion about how you measure software development uh but in one of the pieces um someone said that people have got to start realizing that building software is not like building construction it's like it's more like it's more like the design phase and you would never say that a blueprint is better because it has more blue lines right which gets to your point earlier Amy about how this all work is knowledge work right and so I where was I going with all this I don't know I feel like it is it is indivisible from the act of getting better at delivering software so if I I think that a lot God I keep I I swear I've taken up 50% of the air time in this and I really apologize I made must have had too much coffee today but I feel like so many software orgs out there have kind of given up on saying that they're trying to get better at delivering software and so so instead they they they set these goals themselves kind of nibble around the edges we're going to get better at observ villing year we're going to get better at cicd we're going to get better at blah blah blah blah blah but like the great leaps forward that you make that you have to make in order to keep up with complexity because you can't just hire your way linearly out of this you have to make these big leaps and that requires taking a a much you know a much more bird eye view and being like in order to deliver software better this year it requires that we make these investments in observability we make these investments in testing and and and I feel like in conclusion in conclusion building software is an amazing job I thank my my lucky atheist Stars every day that I get to be in this industry not like picking weeds and getting dirt under my finger but like sitting here at a computer solving puzzles and playing with my friends and and I feel like I wish that everyone in software could have as magical of an experience in clear as I have had and I think that observability is a big part of how we get there that's true that actually brings me to my next question as we're getting ready to close up what do you think is in store for observability for this upcoming year the first person to jump in is okay I'm not gonna pick on one of you oh I haven't been keeping up on all the road maps and stuff but I guess so what is your wish what is your wish that this year brings fors for Ability I guess the thing that I'm working on is a lot of folks still don't really get slos and sis um as much as I think it will help both Engineering Management and Engineers on the ground just communicate better about what they we're all trying to accomplish like it's if I had a wish like it would be to drive kind of that understanding deeper in and just into our industry just so now so I can go into a room and talk about the power that's available here and just like having the metrics to understand and and not just get stared at like I just nailed a fish to the wall right like just you know it would just be nice to to not have to start at zero over and over um but I'll keep doing it until we're there that's fair I personally want to see more people like besides like you know we we we often um associate I guess observability with with like more the Y side of things and it is so not just that right I mean it's it's everybody's problem and so I want to see more conversations where like we get more developers giving a crap about observability and and not just developers like folks who are like planning Tech projects like software releases whatever like make sure it's part of make sure it's part of their language make sure that we Empower our our qas to use everybody misses that give it to your support team gosh like what a opportunity yeah well well even your your QA right like they're they're testing your software and they could use observability to figure out what the hell is wrong and tell developers hey this is why it's wrong you know um so that I want to see I want to see more of that um and and I think it it you know basically it puts us closer to you know Charities observability 2.0 where it's like more part more of a holistic part of the sdlc right I and I think that's where it needs to be so that it's it's you know part of it's baked in baked into how we deliver software and how we code software and and all that good stuff definitely everybody does a better job when they can see where they're going and see what they're doing when they are getting feedback back from the system for sure Amy I had this sticker that says that slos are apis for our engineering teams and and I completely agree like if if you don't have an agreement that you've all agreed upon you're down in the weeds negotiating over every single decision you make about how to spend your time like it's absurdly inex it's absurdly expensive um I I don't I don't know I expect that what's going to happen in 2024 is that everything's going to get even more muddled and people are getting even more confused and um and it's fine it's it's it's entropy it's how it goes um I guess my wish for Christmas would be that um my wish would be that more people just have that feeling in their gut that you get when you see Telemetry feedback to you about what you've done and just go that feeling that you get this like you feel more grounded you feel more certain you feel more confident what you've done observability is the way that we can move swiftly and with confidence and I feel like this is something we all know in our heads we need to learn in our bodies and the only way that we can learn it in our body is by seeing it and experiencing so I would wish that everyone had that experience most definely I love that I love that to keep that in store for 2024 as like find that fuzzy feeling when you start seeing your logs metrics start showing up on your vendor definitely well thank you all very much for the amazing insights you've been able to provide today on our panel around observability this group is going to be coming back later in June to have a little other chat around observability stay tuned to hear about those details later with that we'll like to sign off and say thank you all for watching stay warm out there yay diff --git a/video-transcripts/transcripts/2024-02-23T19:09:30Z-otel-in-practice-opentelemetry-orientation-how-to-train-your-teams-on-observability.md b/video-transcripts/transcripts/2024-02-23T19:09:30Z-otel-in-practice-opentelemetry-orientation-how-to-train-your-teams-on-observability.md index cc7b9e3..3bec09d 100644 --- a/video-transcripts/transcripts/2024-02-23T19:09:30Z-otel-in-practice-opentelemetry-orientation-how-to-train-your-teams-on-observability.md +++ b/video-transcripts/transcripts/2024-02-23T19:09:30Z-otel-in-practice-opentelemetry-orientation-how-to-train-your-teams-on-observability.md @@ -10,77 +10,81 @@ URL: https://www.youtube.com/watch?v=o-1UE67l9q4 ## Summary -In this YouTube video, Paige Cruz, a developer advocate at Chronosphere, discusses the importance of effective training in observability, particularly focusing on tracing and OpenTelemetry (OTEL). She shares her experiences transitioning from Site Reliability Engineer (SRE) to advocating for open-source observability solutions, emphasizing the need to bridge knowledge gaps between developers and observability practices. Paige outlines her successful strategies for training, including hands-on workshops tailored to specific tech stacks, and the importance of engaging learners through clear goals and practical exercises. She highlights the challenges of ensuring developers take ownership of observability practices and how to create a culture of responsibility within teams. The session concludes with resources for further learning and a call to action for more inclusive observability practices. +In this presentation for OTEL in Practice, Paige Cruz, a developer advocate at Chronosphere, discusses her extensive experience in observability and shares insights on training developers to implement tracing effectively. Having transitioned from a Site Reliability Engineer (SRE) to focusing on open-source observability, Cruz emphasizes the importance of hands-on workshops tailored to an organization's specific tech stack to bridge knowledge gaps among developers. She explores the complexities of observability and the rising significance of OpenTelemetry (OTEL), noting that many developers lack fundamental knowledge about tracing and the tools available. Cruz also shares strategies for creating effective training programs, including identifying learners' needs, structuring content to promote engagement, and leveraging existing resources. The session concludes with discussions on the challenges of ownership in observability, highlighting the need for collaboration between operations and development teams. -# Observability Training Workshop Transcript +## Chapters -Thank you so much for having me, OTEL in Practice. I have been attending the last few sessions and I am excited to present a little bit about my world. I am Paige Cruz, and I work at Chronosphere. This is my third observability company. When I wasn't working for observability companies, I was an SRE at customers of those observability companies, working on observability. If you can't tell, I have thought about this stuff for a very long time and from many different perspectives. +Here are 10 key moments from the livestream with the corresponding timestamps: -**A Quick Tale from the Field** +00:00:00 Introductions and Paige Cruz's background +00:05:15 Overview of observability initiatives and challenges +00:10:20 Importance of training in the observability space +00:15:00 Discussing the complexity of introducing tracing +00:20:45 The significance of hands-on workshops for developer engagement +00:30:10 Strategies for effective training development +00:35:00 Identifying learners' knowledge gaps and tailoring content +00:45:30 Delivering training in various modalities for better reach +00:50:15 Sharing resources and further learning materials +00:58:00 Discussion on ownership boundaries in observability and training outcomes -I have retired from SRE today and I am a developer advocate focused on the open source side of observability. In my previous role, my team was always tasked with leading big observability initiatives. This included migrating from one vendor to another, transitioning from self-hosted open source to managed services from our cloud providers, and rolling out tracing. The logging bill was so high that we had to address it immediately. +# Observability Training Workshop -There are many things that fall under the observability umbrella that my team was tasked to manage. During these initiatives, which impacted every team, we had to re-instrument either to add traces or to switch from a proprietary protocol to open source. +Thank you so much for having me, OTEL in Practice. I have been attending the last few sessions and am excited to present a little bit about my world. -Yes, technically, if the development teams were too busy, my team could focus on this work for three or four quarters and deliver a migration by the end. However, we were then seen as the owners of observability and treated like "Ask Jeeves." It felt like the line of responsibility between ops and dev had become skewed. This was fair considering we moved everything to a new platform with a new UI and new libraries, resulting in many knowledge gaps to cover. +I am Paige Cruz, and I work at Chronosphere. This is my third observability company. Before working for observability companies, I was an SRE at customers of those companies working on observability. If you can't tell, I've thought about this stuff for a very long time and from lots of different perspectives. -I have steered quite a few migrations and tried many tactics to delegate the re-instrumentation work and pass the baton of ownership from solely ops to app devs. Out of everything I tried—linking documentation, sharing conference talks—the best tactic was hands-on workshops tailored specifically to my organization's tech stack. This approach highlighted the value of what we were trying to achieve. +### A Quick Tale from the Field -**Train the Trainer** +I have retired from SRE today and am now a developer advocate focusing on the open-source side of observability. During my time in SRE, I led my team through big observability initiatives, whether that was migrating from one vendor to another, transitioning from self-hosted open source to a managed service from one of our cloud providers, or rolling out tracing. The logging bill would often get so high that we had to take action. -This led me to bring forth the concept of "train the trainer," particularly in the context of tracing training, which requires participation from devs across the stack. To quote a famous work, "We’re all in this together." +There are many components that fall under the observability umbrella, and my team was tasked with being the stewards of these initiatives. I found that when we had to implement changes that touched every team, like re-instrumenting to add traces or moving away from a proprietary protocol to open-source tools, there were significant knowledge gaps. My team could do heads-down work for several quarters and successfully deliver a migration, but we were then seen as the owners of observability—almost like an “Ask Jeeves” for developers. This blurred the line of responsibility between operations and development. -**Why Training Matters** +While it's fair to say that we moved everything to a new platform with a new UI and libraries, there were many knowledge gaps to cover. I've steered numerous migrations and tried various tactics to delegate the re-instrumentation work and pass the baton of ownership from operations to application developers. -Now, let's talk about why training matters. There are several layers of complexity when introducing tracing. Observability is still an emerging field, and much of what we learn is done informally on the job. This varies based on the tech stack a company uses, the vendors or protocols they are using, and the companies they've worked for. +### Effective Training Techniques -Even two developers with the same years of experience could possess completely different sets of knowledge about observability. We must teach this stuff. While informal approaches are important for developing operational skills, we need to find a way to bridge the gap between informal knowledge and a strong foundation. +Out of everything I tried, the best tactic was hands-on workshops tailored to my organization's tech stack. This approach demonstrated the value of observability and the changes we wanted developers to implement. This is why I wanted to introduce the concept of "train the trainer," specifically focusing on tracing training, which requires participation from developers across the stack. To quote *High School Musical*, "We are all in this together." -This foundation is crucial so that developers can add tracing, manage costs, and understand how sampling affects viewing aggregate trace data. There is a lot of complicated information in this space, and we need to consider where our learners are, what they know, and what they might have seen. +#### Why Training Matters -In the realm of observability, we have open telemetry, which has evolved from just being known for tracing. However, many developers I speak with are unaware of OTEL unless they are specifically tasked with bringing it into their organization. While we are excited about new features and advancements, there remains a segment of our colleagues who may not have even heard of OTEL. +Observability is still an emerging field. Most learning occurs informally and depends on the tech stack, the vendors or protocols used, and individual experiences. Even two developers with the same number of years of experience can have completely different knowledge sets. -If we simply tell them, "Hey, add tracing to this service," and link to the OTEL docs, it won’t be helpful if they don’t understand what a span is or how to navigate OTEL’s components. Observability is new, OTEL is new and rapidly evolving, and tracing, in particular, is often seen as a tool for power users or the elite engineers. +While informal learning is beneficial for developing operational skills, we must find ways to bridge the gap between patchwork knowledge and a strong foundation. This enables developers to add tracing, manage costs effectively, and understand sampling effects when viewing aggregate trace data. The complexity of observability means we need to consider where our learners are, what they know, and what they’ve experienced. -**The Challenge with Tracing** +We also have to recognize that OpenTelemetry (OTEL) is rapidly evolving. Even if developers are familiar with OTEL, they may not know how to navigate its components or the differences between vendor implementations. Simply linking to the OTEL documentation may not suffice if they don’t understand basic terms like "span." -As someone who has worked on tracing products at New Relic and LightStep, I know tracing well. However, this is not a repeatable experience for everyone. We need all types of people to develop all sorts of things. This was the problem I focused on for my workshop: I want more people to use tracing. One barrier is that while partial traces can be helpful, the best scenario involves having a full trace from end to end across the system, which requires every service owner to do their part. +### Training Development -I aim to help people understand how to send and use trace data. Having the information on how to instrument isn’t enough if they don’t know what to do when they get to a trace waterfall. Some may argue that the documentation has everything they need, but while the OTEL documentation site is fantastic, people often have questions like, "Is this library instrumented with OTEL?" +To develop effective training, we need to find those "aha" moments of discovery. After all, the goal of training is to change behavior and encourage learners to take action. Just providing rote instructions is insufficient. -**Finding the Aha Moments** +We need to consider what our learners need to know and the skills they need to develop. This requires understanding the problem we’re trying to solve, whether it’s better visibility or deeper insights into a profitable system. -We need to provide more than just knowledge; we need to equip learners with skills. Many of us have experienced training that felt like a waste of time—training that was not tailored to our specific needs. Good training differs significantly from bad training, which may not match your expertise level or be relevant to your use case. +#### Identifying Learners’ Needs -So, why does training matter? Because many learners, as an anonymous user on Reddit said, "I don’t even know what I don’t know," making it difficult to answer their questions. This highlights the necessity of training: not everyone is at the same level of understanding. +If you're unsure of what your learners know, consider sending a survey or having informal coffee chats. It’s essential to include a variety of roles, as observability impacts everyone. -To develop effective training, we need to identify the gap between where learners are and where we want them to be. This involves understanding what knowledge they need and what skills they must acquire. +Understanding the consequences of missteps can also clarify the importance of training. If developers fail to instrument their applications correctly, it can lead to increased costs or loss of visibility, which highlights the need for ownership. -**Creating Effective Training** +### Scoping Training Content -When designing a training program, consider the problem you are trying to solve. Define the main drivers of the initiative, such as better visibility or ownership of data. Avoid the temptation to include everything you know about observability; focus on the essential aspects relevant to your audience. +When creating training material, focus on what is necessary for your specific audience. For instance, if you want attendees to leave the workshop able to instrument traces independently, ask yourself whether each piece of content contributes to that goal. -Engage a representative sample of your learners by conducting surveys or informal chats to gauge their backgrounds and current knowledge. It’s crucial to understand where they are coming from and tailor the training accordingly. +Additionally, consider the existing resources within your organization. Don’t reinvent the wheel; leverage the available tutorials, documentation, and community knowledge. -Once you have a draft, have both a novice and an expert go through it. Observing their experiences can reveal stumbling blocks and areas that need clarification. +### Delivery Methods -**Delivery and Modalities** +In delivering training, utilize various modalities, such as live sessions, self-guided learning, and recorded sessions. This approach accommodates the diverse learning styles and schedules of your audience. -When it comes to delivering your training, utilize existing resources and adapt them to your needs. Don’t reinvent the wheel; there are plenty of tutorials and workshops available. +### Conclusion -Consider the modalities of learning. Many people learn in different ways, so provide options for live guided sessions, self-guided learning, and recorded sessions. This increases the likelihood that more individuals will engage with the content. +In summary, effective training in observability requires: -In summary, think about the problem you are trying to solve, understand where your learners are, and narrow the training gap as much as possible. +- Clearly defining the problem and identifying learner needs. +- Bridging the gap between current knowledge and desired outcomes. +- Tailoring content to the specific tech stack and organizational context. +- Utilizing existing resources and encouraging ownership among developers. -**Final Thoughts and Resources** - -I want to share resources that can help you on your journey. Two recommended books are **"Design for How People Learn"** and **"Better Onboarding"** from A Book Apart. Both provide valuable insights into learning and training. - -For open observability resources, the OTEL documentation has a section specifically for developers. The CNCF Glossary is also helpful for understanding acronyms and terminology. - -Lastly, the OpenTelemetry demo is an excellent resource for those seeking a more realistic example. - -Thank you for your attention! If you have any questions or want to discuss your training needs, feel free to reach out. I’m here to help, and I’m excited to see more people embracing tracing! +Thank you for your attention! I'm looking forward to hearing your thoughts and discussing your training needs. ## Raw YouTube Transcript diff --git a/video-transcripts/transcripts/2024-03-05T19:49:24Z-otel-collector-user-feedback-panel.md b/video-transcripts/transcripts/2024-03-05T19:49:24Z-otel-collector-user-feedback-panel.md index 647684f..2f2747b 100644 --- a/video-transcripts/transcripts/2024-03-05T19:49:24Z-otel-collector-user-feedback-panel.md +++ b/video-transcripts/transcripts/2024-03-05T19:49:24Z-otel-collector-user-feedback-panel.md @@ -10,131 +10,133 @@ URL: https://www.youtube.com/watch?v=LL8v_B417ok ## Summary -In this YouTube video, a panel of OpenTelemetry (OTEL) collector practitioners discusses their experiences and insights on using the OTEL collector. The panelists include Adriana Villela (the host), Iris (a senior observability engineer at Miro), Juraj Michalik (a senior logging and monitoring engineer at Swissery), Greg Eales (from Ocado Technology), and Joel (from Open Systems). The discussion covers various aspects of the OTEL collector, including usage, deployment challenges, debugging difficulties, scaling strategies, and desired features. Key points include the importance of debugging tools, the need for better handling of metrics, and the desire for specific features such as profiling and a Prometheus remote write receiver. The panelists share their experiences with scaling the collector using Kubernetes and HPA, and express a collective wish for the tool to mature and improve in usability. The session concludes with an invitation to an upcoming KubeCon event where further discussions will take place. +In this YouTube video, a panel discussion was held featuring practitioners of the OpenTelemetry (OTEL) Collector, including Adriana Villela (moderator), Iris, Juraj Michalik, Greg Eales, and Joel from various companies. The panel focused on gathering feedback about the OTEL Collector to inform its development roadmap. Each panelist shared their experiences with the Collector, discussing its applications in their respective organizations, challenges faced during deployment and debugging, and the use of custom builds. Key topics included the need for better handling of metrics, issues with scaling and memory management, and desired features such as profiling and cumulative metrics processing. The panelists expressed a desire for the OTEL community to address these challenges and enhance the Collector's functionality. The discussion concluded with announcements about upcoming events at KubeCon in Paris, where further user feedback sessions will take place. + +## Chapters + +Here are the key moments from the livestream along with their timestamps: + +00:00:00 Introductions and event overview +00:02:30 Panelist introductions - Iris +00:04:00 Panelist introductions - Juraj +00:05:30 Panelist introductions - Greg +00:06:50 Panelist introductions - Joel +00:08:30 Discussion on how teams are using the OpenTelemetry Collector +00:10:10 Iris discusses current usage of the Collector +00:12:00 Juraj shares experiences with past and current implementations +00:14:00 Greg talks about migrating from legacy systems to the Collector +00:16:30 Joel explains their multi-layered approach to using the Collector +00:20:00 Challenges faced with debugging and troubleshooting the Collector +00:25:00 Panelists share experiences with the Collector Builder +00:30:00 Wishlist for improvements and desired features in the Collector +00:35:00 Discussion on scaling the Collector and experiences with auto-scaling +00:40:00 Wrap-up and information on future events, including KubeCon + +Feel free to adjust any of the descriptions or timestamps as needed! # OTEL Collector Panel Discussion Transcript -Thank you everyone for joining. This is a really great turnout, and we're excited to have assembled this panel of OTEL collector practitioners. One of the missions of the OTEL and User Working Group is to collect feedback on OTEL to deliver back to the SIGs. This started because Judah C, who works in the OTEL collector SIG, reached out to us to put together a survey to collect some information on the OTEL collector to see how it's being used and drive the roadmap and vision for the collector in the near future. We also thought it would be beneficial to have a panel for a lively discussion with a handful of OTEL collector practitioners. +**Adriana Villela**: Thank you everyone for joining. This is a really great turnout, and we're excited to have assembled this panel of OTEL collector practitioners. One of the missions of the OTEL and User Working Group is to collect feedback on OTEL to deliver back to the SIGs. This started because Judah C., who works in the OTEL collector SIG, reached out to us to put together a survey to collect some information on the OTEL collector and see how it's being used. This information will help drive the roadmap and vision for the collector in the near future. We thought it would be great to extend this and also have a panel for a lively discussion with a handful of OTEL collector practitioners. So, here we are! -## Introductions +Let’s introduce the panelists. I’ll introduce myself first. My name is Adriana Villela. I work with Reese and Dan in the OTEL end user working group. Our numbers are getting a little bit bigger, so I’m very excited to see that. We’re hosting this event. Let’s go over to our panelists. I’m going to pick names. Iris, how about you go first? -Let’s introduce the panelists. I’ll start with myself. My name is **Adriana Villela**, and I work with Reese and Dan in the OTEL end user working group. Our group numbers are growing, which is exciting. We’re hosting this event today. +**Iris**: Hello everyone! My name is Iris. I'm a senior observability engineer at Miro, and I’ve been working with OpenTelemetry for a little over a year now, across two companies. I have a lot of experience and use cases since my two experiences are completely different from each other. I’m happy to be here. Nice to meet you! -**Iris**, would you like to introduce yourself? +**Adriana**: Thank you, Iris. Okay, next we have Juraj. Did I pronounce that correctly? Please correct me if I'm wrong. -**Iris**: Hello everyone! My name is Iris. I'm a senior observability engineer at Miro. I've been working with OpenTelemetry for a little over a year now in two different companies, giving me a lot of experience and use cases, as my two experiences are completely different from each other. I'm happy to be here. Nice to meet you! +**Juraj Michalik**: Yes, you did! Hi, my name is Juraj Michalik. I’m based in Madrid and currently working for a company called Swissery in the insurance space. Before that, I worked for two different companies where we used OpenTelemetry. In the last one, it was used very sparingly, but in the previous one, we fully migrated to OpenTelemetry for the collection of logs, metrics, and traces. I’m a Senior Logging and Monitoring Engineer in my current role. -**Adriana**: Thank you, Iris. Next, we have **Juraj**. Did I pronounce that correctly? +**Adriana**: Awesome, thanks. Next we have Greg. -**Juraj**: Yes, you did! Hi, my name is Juraj Michalik. I'm based in Madrid and recently started working for a company called Swissery in the insurance space. Before this, I worked for two different companies where we used OpenTelemetry. In the last one, we migrated fully to OpenTelemetry for the collection of logs, metrics, and traces. I'm a Senior Logging and Monitoring Engineer in my current role. +**Greg Eales**: Hi! Can you hear me? My name is Greg Eales. I work for Ocado Technology, which started as an online supermarket and now sells online supermarket technology to other supermarkets. My team has been using the collector for just over a year, but I’ve only been on the team for a bit less than a year. -**Adriana**: Awesome, thanks! Next, we have **Greg**. +**Adriana**: Oh, awesome! And finally, we have Joel. -**Greg**: Hi! Can you hear me? My name is Greg Eales. I work for Ocado Technology, which started as an online supermarket and now sells online supermarket technology to other supermarkets. My team has been using the collector for just over a year, but I've only been on the team for less than a year. +**Joel**: Hi! I’m Joel from a company called Open Systems. We’re based in Switzerland but have a global presence with offices in San Francisco, London, and Germany. We offer managed SASE, which includes managed connectivity and VPN for our customers. I’m on the observability team, responsible for ensuring we get telemetry from our devices, of which we have 10,000 across the world. We’re also dealing with data coming from Kubernetes, including Prometheus metrics and logs. We started using the collector about a year ago, probably in April last year, and we quickly decided to fully migrate everything to it since we had various service teams doing things in their own way. We found that the collector and OpenTelemetry in general was a great way to consolidate everything. -**Adriana**: Great! And finally, we have **Joel**. +**Adriana**: Awesome! Thank you again to all of you for joining us. Let’s get into the questions. First things first, Iris, how about you start by telling us how you and your team are primarily using the Collector? -**Joel**: Hi, I'm Joel from a company called Open Systems. We're based in Switzerland but have a global presence with offices in San Francisco, London, and Germany. We offer managed SASE, which includes managed connectivity and VPN for our customers. I'm on the observability team, responsible for ensuring we get telemetry from our 10,000 devices worldwide. We have also started dealing with data from Kubernetes, like Prometheus metrics and logs. We’ve been using the collector for just under a year and have quickly decided to migrate everything to it as we’re consolidating different service teams. +**Iris**: Sure! I can talk a little bit about how we’re using it now and in my previous company as well since I’m very new in my current position. We are using the Collector mostly as a transport layer at the moment. We have already migrated traces and are in the process of migrating logging and metrics. In my previous company, we were a bit more advanced. We were using the collector itself and the operator. Currently, we are only using it for deployment, basically transporting data from our legacy components, and we’re slowly moving to collect everything through the OpenTelemetry Collector. In the past, we utilized the presets offered by the collector itself, running it as a daemon set, collecting everything, especially in our Kubernetes cluster. The goal now is to have the collector take as much of the information collection as possible and get rid of the legacy solutions like Jaeger and Prometheus. -Thank you all for joining us. Let’s get into the questions! +**Adriana**: Great! Same question for you, Juraj. How is it that you and your company are using the collector? -## How Are You Using the Collector? +**Juraj**: In my current role, which I just joined this month, we're just starting with OpenTelemetry and traces in general. But in my previous role, I worked for a SaaS startup where there was some concern about costs. We started migrating from a vendor’s proprietary collection pipeline and client libraries to OpenTelemetry, which enabled us to do a POC of other alternatives. We settled on a self-hosted solution based on the Grafana stack, hosted in Kubernetes. We used typical Kubernetes deployment with a daemon set for collecting logs, metrics, and traces from everything running on the nodes. We also had several stateful sets dedicated to collecting Kubernetes-related metadata and pulling data from AWS CloudWatch. -**Adriana**: First things first, let's start in the order in which folks were introduced. Iris, can you tell us how you and your team are primarily using the Collector? +**Adriana**: Awesome! And how about you, Greg? -**Iris**: Sure! In my current company, we are using the Collector mostly as a transport layer. We have already migrated traces and are in the process of migrating logging and metrics. In my previous company, we were more advanced; we used the collector itself and the operator. Currently, we’re only using it for deployment, basically transporting from our legacy components while slowly moving to collect everything through OpenTelemetry Collector. We also heavily used the presets offered by the collector itself, running it as a daemon set in our Kubernetes cluster. Our goal is to have the collector take as much of the collecting of information as possible, moving away from legacy solutions like Jaeger and Prometheus. +**Greg**: We use the collector for traces. We had a legacy trace system called Request Viewer based on logs, which was quite heavy. We’re migrating everyone to send traces via the collector. We do some processing, sanitization, and tail sampling to allow sampling by whole traces. We don’t do anything with metrics apart from our own metrics, which we gather from the collector and forward them to Grafana using Prometheus remote write. We deploy it in ECS, not Kubernetes, and we have one central cluster that scales in and out with the load of the collector. -**Adriana**: Great! Juraj, how is it that you and your company are using the collector? +**Adriana**: Interesting! Finally, Joel, how do you use the collector? -**Juraj**: In my current role, which I just joined this month, we’re just starting with OpenTelemetry and traces. In my previous role, we were a SaaS startup that migrated from a vendor's proprietary collection pipeline to OpenTelemetry. We did a proof of concept for alternatives and settled on a self-hosted solution based on the Grafana stack. We used the collector in a Kubernetes deployment, utilizing daemon sets for collecting logs, metrics, and traces from everything running on the nodes, and also deployed it as sidecars for various services. +**Joel**: We have collectors deployed all over the place, running on all our hosts. They act as egress gateways for telemetry. Primarily, the data we’re shipping is mostly log data. We have pipelines enabled for metrics and traces, but currently, it’s just the collector's own metrics that we ship. Very few applications are using traces at the moment. So primarily, we’re using it as a log telemetry mesh to lay the groundwork for full migration in the future. Our egress collectors run on our hosts, with an ingress collector in our central Kubernetes cluster and multiple layers of collectors behind that for different backends like Loki, Thanos, and Tempo. We self-host everything and aim to ensure all telemetry has a uniform set of labels for better correlation between signals in the backends. -**Adriana**: Thank you! And Greg, how about you? +**Adriana**: Thank you all! Let’s move on to the next question. Starting with Iris, are you using a vanilla collector, a custom-built one, or a vendor distribution? -**Greg**: We use the collector for traces. We had a legacy system called request viewer, which was based on logs. We’re migrating to use the collector for sending traces. We do some processing and sanitization, adding attributes and doing tail sampling. We don’t do much with metrics beyond gathering our own metrics from the collector and forwarding them to Grafana through Prometheus remote write. We deploy it in ECS, and we have one central cluster that scales with the load of the collector. +**Iris**: We’re using the vanilla one. So far, we haven’t had the need for any custom builds or vendor distributions; the open-source version has worked great for us. -**Adriana**: Interesting! And finally, Joel? +**Adriana**: Perfect! Same question for you, Juraj. -**Joel**: We have collectors deployed on all our hosts, acting as egress gateways for telemetry. The primary data we’re shipping is log data, but we have pipelines for metrics and traces as well. Most of the applications are not using traces yet, so we’re mainly using it as a log telemetry mesh and laying the groundwork for future migration. We have egress collectors running on our hosts and an ingress collector in our central Kubernetes cluster, along with multiple layers of collectors for different backends like Loki and Thanos. +**Juraj**: We use a variety. We have multiple custom distributions built. One reason for this is that we wanted to minimize the size of the binary for the OTEL collectors exposed to the Internet. We only include the necessary components for security. We also created a special distribution with a gold debugger for debugging issues. All of this is based on the upstream OpenTelemetry collector contrib with custom configurations. -## Collector Distribution +**Adriana**: That’s interesting! Greg, what about you? -**Adriana**: Thank you all. Next question: are you using a vanilla collector, collector contrib, a custom-built one, or a vendor distro? Iris? +**Greg**: We build our own. We forked the contrib repo because we needed a feature to add AWS Cloud Map service discovery to the load balancer exporter. We’ve contributed that back and have an open pull request that’s still pending. We’ve discovered some bugs and have been able to debug the code locally, which has been beneficial for learning more about Go development. -**Iris**: We’re using the vanilla one. So far, we haven't had the need for any custom builds, as the vanilla collector works great in our case. +**Adriana**: Great! And finally, Joel? -**Adriana**: Same question, Juraj? +**Joel**: We’re running a custom build. We use the builder, and it was not a problem to get it running in our build pipelines. We’ve had a very positive experience with it. However, there’s always a concern when maintaining your own distribution, especially with the fast release cadence, as you can run into deprecated features that can blindside you during deployment. -**Juraj**: We use a variety of custom distributions. We've built multiple custom distributions to minimize the size of the binary and the number of components, especially from a security perspective. Everything is based on the upstream OpenTelemetry collector contrib with custom configurations. +**Adriana**: Thank you all! What’s been the biggest challenge or challenges that you’ve faced with the Collector? Iris, let’s start with you. -**Adriana**: Great! And Greg, what about you? +**Iris**: Deployment has been straightforward, but debugging has been a challenge. I recently had an issue enabling ingress. I spent more than two days figuring out that I needed to pass the right paths in the receiver. Even after enabling debug logs, there wasn’t enough information to help us troubleshoot effectively. -**Greg**: We build our own. We forked the contrib repo and based our version on that. We had specific features we needed, like adding ECS CloudMap service discovery to the load balancer exporter, which we contributed back to the community. +**Adriana**: Juraj, how about you? -**Adriana**: And Joel? - -**Joel**: We are running our custom build. The reason is that we have service teams who built their own processes, and it was one of the big draws for adopting OpenTelemetry. We use the builder and had a good experience getting it running in our build pipelines. - -## Challenges with the Collector - -**Adriana**: Moving on to challenges, starting with Iris, what’s the biggest challenge or challenges you’ve faced with the Collector? - -**Iris**: Debugging is definitely a challenge. Deploying it is pretty straightforward, but debugging can be overwhelming. I had a situation where I was using the collector with ingress for the first time, and it took my coworker and me over two days to figure out an issue with passing the right paths. The debugging logs did not provide enough information, and we ended up deep in the code before we found the solution. - -**Adriana**: Juraj, what about you? - -**Juraj**: We ran into bugs that were hard to debug, especially under large loads. We had to build our own distribution with a debugger to see what was happening. It’s challenging to test complex pipelines without deploying them to full-blown environments. +**Juraj**: Debugging has been difficult, especially when bugs only occur under heavy loads. We built our own distribution with the gold debugger in the dev environment to find the root causes. Also, late-breaking changes have caused incidents, like stopping data from being sent to Datadog due to label changes that we didn’t catch in lower environments. -**Adriana**: Greg? - -**Greg**: Our biggest challenge has been the quality of the platform we provide to the organization. There have been bugs or misconfigurations causing traces and spans not to be available when users looked for them. Initially, there was also naivety about the collector’s reliability, and it caused anxiety among the team. +**Adriana**: Greg, what’s your experience? -**Adriana**: And Joel? +**Greg**: Our biggest challenge has been the quality of the platform we provide. When users look for requests they’re trying to debug and can’t find them, it creates anxiety. We didn’t know how to monitor the collector properly, which led to doubts about where the problem lies in the pipeline. We’ve improved on that, but we still have areas of uncertainty. -**Joel**: I echo a lot of what has been said. One challenge is the handling of cumulative metrics. We heavily rely on logs, and there’s no general way to generate metrics from logs within the collector currently. This is something we’re working on internally. +**Adriana**: Finally, Joel? -## Wishlist for Improvements +**Joel**: The handling of cumulative metrics has been a challenge. We ran into issues with the count connector, which didn’t work as expected. It’s disappointing to inform users that a proposed solution isn’t going to work, and we need to rely on upstream development for fixes. -**Adriana**: What’s on your collector wishlist for feature improvements? Iris? +**Adriana**: Thank you all for sharing your experiences! What’s on your collector wishlist as a feature improvement? Iris? -**Iris**: I'm looking forward to profiling. I’ve seen some promising improvements and I believe it will be groundbreaking. The collector has been very complete for our use cases, so I’m excited about the future developments. +**Iris**: I’m looking forward to profiling. I’ve seen some promising developments and am excited about its potential. -**Adriana**: Juraj? +**Adriana**: Juraj, what’s on your wishlist? -**Juraj**: I’d like to see improvements around reducing network usage, like what Apache Arrow is promising. There are also some bugs that need fixing, especially around the gRPC layer. +**Juraj**: I’d like to see the Apache Arrow project mature, as it could reduce network usage. Additionally, there are still silent ways to lose data with metrics. I’d also like OpenTelemetry to adopt a more unified approach to metric production to avoid compatibility issues. **Adriana**: Greg? -**Greg**: I hope the collector matures more. I want features like a dead-letter queue for exporters and better handling of service discovery to avoid split traces when scaling. +**Greg**: I’d like the platform to mature more. I want to be able to assure users that features will work as expected. Additionally, I’d like to see a dead letter queue integration point for exporters to handle errors more effectively. **Adriana**: And Joel? -**Joel**: I would like to see a Prometheus remote write receiver. Additionally, a general connector for creating metrics from any telemetry would be beneficial. +**Joel**: I echo many of these points. A Prometheus remote write receiver would be on my wishlist, as well as a more generic connector for building metrics from logs. -## Scaling the Collector +**Adriana**: Great insights! Lastly, let’s discuss scaling the collector. Iris, any feedback or pain points around scaling? -**Adriana**: Lastly, let’s talk about scaling the collector. Starting with Iris, any feedback or pain points around scaling the collector? - -**Iris**: We’ve used the HPA effectively for scaling, and in my previous company, we also explored KEDA, which worked well for auto-scaling based on queue size. +**Iris**: We’re currently using a horizontal auto-scaler, which works great for us. We’ve also tried KEDA, and it worked well for auto-scaling based on queue size. **Adriana**: Juraj? -**Juraj**: For our deployments, we also used HPA based on memory and CPU, and we looked at VPA for daemon sets. +**Juraj**: We’ve also used HPA based on memory and CPU. It worked nicely, and we were looking into using VPA for better resource management. **Adriana**: Greg? -**Greg**: We have a conservative auto-scaling policy based on memory as the tail sampler holds a lot of memory. We haven’t put much effort into optimization yet, as we’re focused on processing data properly. - -**Adriana**: Joel? - -**Joel**: We run everything on one cluster per environment and have been using HPA with no issues. I need to talk to Juraj about gRPC issues we’ve encountered with scaling. +**Greg**: We’re in the early stages of our journey with scaling. We’ve over-provisioned for now to avoid losing data. We’re also exploring exporting internal metrics to CloudWatch to inform our auto-scaling policies. -## Conclusion +**Adriana**: Finally, Joel? -That brings us to the end of our panel discussion. Thank you to all our panelists for sharing their insights on the OTEL Collector and how you use it in the wild. This is incredibly valuable. Thank you to everyone else who joined today as well. +**Joel**: We run everything on one cluster per environment and have found that HPA with memory limits works well. I need to investigate gRPC scaling issues further, as it sometimes leads to parts not receiving telemetry. -Don’t forget to check out our OTEL YouTube channel for the recording of this session. For those attending KubeCon in Paris next month, many of us from the OTEL end user working group will be there. We’ll have various activities lined up, including user feedback sessions, demos, and more. Keep an eye on the OTEL blog for further details. +**Adriana**: Thank you all for your valuable insights. We’re at the end of our time. Thanks to our panelists for sharing their thoughts on the OTEL Collector and how you use it in the wild. Thank you to everyone who joined us today. We appreciate it! -Thank you all for your time, and we hope to see many of you in Paris! +For those who couldn’t make it, a recording will be available on the OTEL YouTube channel, called OTEL Official. If you’re attending KubeCon in Paris next month, many of us from the OTEL end user working group will be there, and we hope to see you at the OTEL Observatory. Thank you so much again, and see you next time! ## Raw YouTube Transcript diff --git a/video-transcripts/transcripts/2024-05-01T19:33:21Z-otel-prometheus-interoperability-user-feedback-panel.md b/video-transcripts/transcripts/2024-05-01T19:33:21Z-otel-prometheus-interoperability-user-feedback-panel.md index f4031ee..a8d6700 100644 --- a/video-transcripts/transcripts/2024-05-01T19:33:21Z-otel-prometheus-interoperability-user-feedback-panel.md +++ b/video-transcripts/transcripts/2024-05-01T19:33:21Z-otel-prometheus-interoperability-user-feedback-panel.md @@ -10,79 +10,102 @@ URL: https://www.youtube.com/watch?v=9a3ctZhJj-o ## Summary -The YouTube video features a panel discussion on the interoperability between Prometheus and OpenTelemetry (OTEL), moderated by David Ashpole from Google. The panelists, including Iris from Miro, Nikos Takas from Corelogs, Vijay Samuel, and Dan Bromitt from Home Depot, share their experiences and challenges integrating these two observability tools in their respective organizations. Key topics discussed include the architecture of their observability stacks, the transition from Prometheus to OpenTelemetry, performance issues, configuration challenges, and the need for consistent metric naming conventions. The panelists express excitement for upcoming features in Prometheus, such as OTLP support and enhanced resource efficiency, while also emphasizing the importance of maintaining backward compatibility to avoid disruptions in large-scale deployments. Overall, the discussion highlights the collaborative efforts needed to improve interoperability and usability between the two systems. +The YouTube video features a panel discussion on Prometheus and OpenTelemetry (OTEL) interoperability, moderated by David Ashpole from Google. The panelists include Iris, Nikos Takas, Vijay Samuel, and Dan Bromitt, who share their experiences with using Prometheus and OTEL in their observability stacks. The discussion covers various topics, including challenges faced in integrating these tools, the importance of maintaining compatibility to avoid breaking changes, and the need for consistent metric collection practices. The panelists express a desire for improved features in future versions, such as enhanced OTLP support and standardized naming conventions to streamline the user experience. Overall, the conversation highlights the complexities of observability in large organizations and the ongoing efforts to enhance the integration between Prometheus and OpenTelemetry. -# Prometheus OTEL Interoperability Panel Transcript - -**Moderator:** Welcome everyone to the Prometheus OTEL interoperability panel. David reached out to us to gather end user feedback on ensuring that Prometheus and OpenTelemetry (OTEL) work harmoniously together, which is always a great initiative. We have four panelists here today, and David will be our moderator, guiding the discussion and asking questions. Let's start with introductions. David, would you like to go first? - -**David Ashpole:** Sure! I'm David Ashpole, and I work for Google. Currently, I'm involved with the Prometheus OTEL work group, focused on ensuring that the protocols, exporters, and integrations between OpenTelemetry and Prometheus function well and consistently. Our goal is to enhance the user experience for those utilizing both technologies. +## Chapters -**Panelist Introductions:** +00:00:00 Introductions and panelist overview +00:05:00 Dan discusses his observability stack and challenges with store environments +00:08:30 Vijay explains instrumentation and Prometheus/OpenTelemetry setup +00:12:00 Nicholas shares current use of OpenTelemetry and challenges with resource consumption +00:14:30 Iris talks about using Victoria Metrics and experimenting with OpenTelemetry +00:20:00 Discussion on the challenges of adopting OpenTelemetry and Prometheus together +00:26:00 Panelists share feedback on upcoming features in Prometheus 2.0 and 3.0 +00:30:00 Discussion about the impact of breaking changes in large organizations +00:37:00 Panelists provide thoughts on the benefit of OTLP support and configuration simplicity +00:42:00 Audience participation and final remarks from panelists -- **Iris:** Hello everyone! I'm Iris, a senior observability engineer at Miro. My work revolves around Prometheus and OpenTelemetry, and I'm excited to be here. - -- **Nikos Takas:** Hi, I'm Nikos Takas, an engineer at Corelogs, where I'm working on observability units. I'm a Prometheus Operator Maintainer and also a Persis Maintainer. My work involves contributing code to Prometheus to assist with Prometheus 3.0 and OTLP ingestion. +# Prometheus OTEL Interoperability Panel Transcript -- **Vijay Samuel:** Hi, I'm Vijay Samuel, and I help with observability. It's great to see many familiar faces here again. +**Welcome and Introductions** -- **Dan Bromitt:** Hi, I'm Dan Bromitt, Senior Manager of SRE at Home Depot, overseeing the Store and Payments Systems. We utilize many OTEL and observability tools. +Everyone who's able to join for this Prometheus OTEL interoperability panel, David reached out to us to see if we could get some end-user feedback on ensuring that Prometheus and OpenTelemetry (OTEL) work well together, which is always awesome. We have four panelists and David is our moderator who will be asking questions. Let's have everyone introduce themselves, starting with David. -**Discussion on Observability Stack:** +**David Ashpole:** +Sure. I'm David Ashpole. I work for Google and right now I'm working with the Prometheus OTEL work group. We're trying our best to ensure that the protocols, exporters, and all components that integrate OpenTelemetry and Prometheus work well and consistently, helping users who use both to have a good experience. -**David:** Great! Let's go around and discuss what your observability stack looks like and how you incorporate Prometheus and OpenTelemetry. Dan, let's start with you. +**Iris:** +Hello everyone, I'm Iris. I work as a senior observability engineer at Miro. My life revolves around Prometheus and OpenTelemetry recently. It’s nice to be here! -**Dan:** Sure! We monitor services across various compute environments, including on-prem virtual machines, GCP (primarily GKE), and small clusters inside our stores. We run OpenTelemetry collectors alongside Prometheus installations. In the cloud, we share metrics across GCP projects, but we've faced challenges in our store environments, where network connectivity can be unreliable. Backfilling metrics can be problematic due to timestamps, especially when devices are offline. Configuring the fleet of services is also a challenge, though we're improving on the Kubernetes side. +**Nikos Takas:** +Hello everyone. My name is Nikos Takas, and I'm an engineer at Corelogs, currently working on observability units. I'm a Prometheus Operator Maintainer and also a Persis Maintainer. I've been working on getting some code into Prometheus to help with the Prometheus 3.0 and OTLP ingestion as well. My daily focus is on Prometheus, OpenTelemetry, metrics, and related topics. Pleasure to be here! -**Vijay:** Our setup involves standardizing instrumentation using a mix of the Prometheus Java client and Micrometer for Java applications. We also use the community-supported Prometheus client for JavaScript. These expose metrics that are scraped by OpenTelemetry collectors deployed on our Kubernetes clusters. Most of our metrics come from these sources, with a smaller group sent directly to our gateway APIs. We aim to encourage more use of the OpenTelemetry SDK for metrics, as we already use it for tracing. +**Vijay Samuel:** +Hi, my name is Vijay Samuel. I help with observability. I think I've met most of you, it's nice to see you all again. -**Nikos:** Our setup currently involves using OpenTelemetry Collector for traces and recently starting with logs. In metrics, we’re considering replacing our Prometheus agents with the OpenTelemetry Collector, though resource consumption remains a concern. We hope to see Prometheus natively ingesting OTLP in the future. +**Dan Bromitt:** +Hi, I'm Dan Bromitt, Senior Manager of SRE at Home Depot for the Store and Payments Systems. We use many OTEL and observability-adjacent tools. Great to have you all here! -**Iris:** Our architecture is straightforward; we primarily use Prometheus clients and Victoria Metrics for custom metrics. We're experimenting with OpenTelemetry, especially for host metrics, as it provides more convenience for our needs. We aim to transition to OpenTelemetry for a more uniform data collection approach. +**Observability Stack Overview** -**David:** It seems many of you are using Prometheus libraries and are considering moving to OpenTelemetry for consistency. What drives this desire for consistency? +Let's go person by person to discuss what your observability stack looks like and how you're using Prometheus and OpenTelemetry in that stack. We'll start with Dan. -**Iris:** The OpenTelemetry SDKs for tracing have proven superior to previous SDKs. Given our existing use of OpenTelemetry for tracing, it makes sense to unify our metrics collection under the same framework. +**Dan:** +We observe services across a wide range of compute environments, including on-prem virtual machines in our data centers, primarily using GCP and GKE, and within our stores, where we run OpenTelemetry collectors alongside Prometheus installations. -**Dan:** I want to add that we are using the OpenTelemetry Java agent in production, although some of our legacy systems are still using various libraries. +Most use cases with OpenTelemetry and Prometheus involve Kubernetes in the cloud, which is fairly standard. However, our challenges arise in store environments where we have miniature data centers. We need to ensure everything is functioning correctly, as network connectivity can be spotty, resulting in some stores being offline daily. This makes backfilling metrics challenging due to timestamps and other issues. -**Challenges Faced:** +**Vijay:** +Starting with instrumentation, we've standardized on a mix of the Prometheus Java client and Micrometer for Java applications. We use community-supported Prometheus clients for JavaScript, which expose metrics for everything not considered managed. These endpoints are scraped by OpenTelemetry collectors deployed on all our internal Kubernetes clusters. -**David:** Let's discuss challenges. Nicholas, can you share what struggles you've faced while adopting Prometheus and OpenTelemetry? +For about 90-95% of our metrics, we send data directly to our gateway APIs, which can handle proprietary formats. We're also encouraging the use of the OpenTelemetry SDK for metrics to streamline our processes. The backend is predominantly Prometheus-first, utilizing the Prometheus storage engine with Thanos for long-term storage. Over time, we aim to have a unified system for metrics, traces, and eventually logs. -**Nikos:** A significant challenge has been the performance of the OpenTelemetry Collector compared to the Prometheus agent. We find that the Collector consumes more resources, which is a bottleneck since we manage a large volume of metrics. We're working toward replacing the Prometheus agent with the Collector, but resource constraints are currently holding us back. +**Nikos:** +Our setup is quite simple. We're leveraging OpenTelemetry Collector for traces and are starting to explore logs. For metrics, we currently use a mix of Prometheus servers and agents but aim to replace the Prometheus agent with the OpenTelemetry Collector to utilize its pipeline capabilities. However, resource consumption is a challenge that is currently blocking us from fully transitioning. -**Dan:** For us, one major struggle was integrating the OpenTelemetry Java agent with legacy microservices that run on unsupported versions of Java and Tomcat. Configuration complexity for telemetry transmission over messaging frameworks has also been challenging. Additionally, timestamps for offline locations can lead to issues when they come back online. +**Iris:** +For us, the architecture is straightforward. We're mainly using Prometheus clients and Victoria Metrics for custom metrics, with everything in Prometheus format. We're experimenting with OpenTelemetry, especially for host metrics since it’s more convenient for collection. We also run a monolith alongside Kubernetes, using the Prometheus receiver and adding script jobs to OpenTelemetry for areas challenging to cover with Victoria Metrics. Our goal is to have a uniform way of collecting all data, especially since tracing is already handled by OpenTelemetry. -**Vijay:** Our biggest struggle has been achieving scrape parity. We encountered various scenarios where Prometheus differs from how the OpenTelemetry collector handles metrics. While some issues have been resolved through PRs, we remain cautious about any changes that might disrupt our systems. +**Challenges in Using Prometheus and OpenTelemetry** -**Iris:** Our challenges stem from the size of our organization. With around 700 engineers relying on our observability platform, any significant changes could disrupt their dashboards and alerts. We need to proceed cautiously to avoid overwhelming our teams during the transition. +**Nicholas:** +My struggles primarily revolve around user experience and understanding how to set up metrics properly. It took time to grasp the nuances of configuring the collector and managing attributes. The biggest issue was switching from the Prometheus receiver to the OpenTelemetry operator and figuring out how to run the target allocator without it. -**Future Developments:** +**Dan:** +One major struggle has been the lack of support for certain legacy systems. We tried to leverage the Java agent for observability on older microservices that ran on unsupported versions of Java and Tomcat. The configuration of OpenTelemetry for telemetry over a messaging framework was also a challenge for our engineers. -**David:** Let's talk about upcoming features and improvements. What are you looking forward to in future releases? +**Vijay:** +Our biggest struggle was scrape parity. Initially, there were significant differences in how Prometheus and OpenTelemetry Collector handled scenarios like label names starting with an underscore. Thankfully, many of these issues have been addressed through PRs we filed. -**Iris:** I'm excited about the Prometheus remote write support for OTLP and the improvements to the Prometheus exporter and receiver. Uniformity across our metrics collection will greatly benefit our engineers. +**Iris:** +Our challenges mostly stem from the size of our organization. With 700 engineers using our observability platform, any change can disrupt their dashboards or alerts. We’re moving slowly to ensure that transitions don’t negatively impact user experience. -**Nikos:** I agree! The remote write improvements will help save resources, which is crucial for our operations. I'm also looking forward to OTLP ingestion becoming GA. +**Looking Ahead: Future Developments** -**Dan:** The OTLP integration is promising, as it could simplify our configurations. However, we need to understand how late data will be handled in terms of storage. +**Iris:** +I'm excited about the upcoming features, especially the Prometheus 2.0 exporter and receiver. We’re testing it heavily to prepare for our next move. The support for dots in metric names and the uniformity it brings to our engineers is also something I’m looking forward to. -**Vijay:** I'm looking forward to OTLP ingestion and maintaining the integrity of our metric names. Ensuring consistency between what is instrumented and what is queried is essential for our operations. +**Nicholas:** +For me, the remote write 2.0 is significant as it promises better resource consumption. I'm also looking forward to the OTLP ingestion in 3.0, which will facilitate more seamless integration between OpenTelemetry and Prometheus. -**David:** Thank you all for your insights. We appreciate the feedback, and it's clear that balancing the existing Prometheus standards with the evolving OpenTelemetry conventions will be crucial for the community's success. +**Dan:** +The OTLP support will simplify our configuration as we centralize our metrics. We’re also curious about handling late data with OTLP and how that will integrate with our existing infrastructure. -**Open Floor for Audience Feedback:** +**Vijay:** +I'm interested in the OTLP ingestion and how it will allow for a seamless experience where the instrumentation matches the backend metrics without unnecessary transformations. It’s crucial that we preserve names and avoid breaking changes. -**David:** Now, I'd like to open the floor for any audience feedback or questions. +**Audience Interaction** -**Audience Member:** I'd like to emphasize the importance of keeping conventions consistent in querying. If we can maintain familiarity while introducing UTF-8 support, it would greatly help engineers adapt. +If there's anyone from the audience who would like to share feedback or thoughts, the floor is open. -**David:** Thank you for that input! It’s been a fruitful discussion, and I appreciate everyone’s participation. We look forward to more community discussions in the future. +**Audience Member:** +I would like to emphasize the importance of keeping conventions consistent across both ecosystems to minimize the learning curve for engineers. -**Panelists:** Thank you! +**Moderator:** +Thank you, everyone for your valuable insights. This discussion has been incredibly beneficial. We look forward to more community discussions in the future. -**David:** Thank you, everyone! +**Thanks and Farewell** +Thank you for joining us! ## Raw YouTube Transcript diff --git a/video-transcripts/transcripts/2024-06-05T21:36:59Z-the-humans-of-otel-kubecon-eu-2024.md b/video-transcripts/transcripts/2024-06-05T21:36:59Z-the-humans-of-otel-kubecon-eu-2024.md index 1c2121f..728928b 100644 --- a/video-transcripts/transcripts/2024-06-05T21:36:59Z-the-humans-of-otel-kubecon-eu-2024.md +++ b/video-transcripts/transcripts/2024-06-05T21:36:59Z-the-humans-of-otel-kubecon-eu-2024.md @@ -10,136 +10,138 @@ URL: https://www.youtube.com/watch?v=bsfMECwmsm0 ## Summary -In this video, various experts in the field of observability come together to discuss their perspectives on observability and the significance of OpenTelemetry. Key participants include Iris Durish from Meo, SE Nman from Cisco, Kayla Riel from New Relic, and Morgan Mlan from Swun, among others. The main topics explored include the evolving definition of observability, the transition from traditional monitoring to a more comprehensive understanding of system behavior, and the role of OpenTelemetry as a unifying standard for collecting telemetry signals. The speakers share personal anecdotes about their professional journeys into observability, emphasize the importance of community collaboration, and highlight the impact of OpenTelemetry on improving observability practices across various industries. Each participant also expresses their favorite telemetry signals, showcasing their diverse experiences and preferences within the observability landscape. +In this YouTube video, various experts in the field of observability share their insights and experiences related to OpenTelemetry, a project aimed at improving observability in software systems. The participants include Iris Durish (Meo), SE Nman (Cisco), Kayla Riel (New Relic), Morgan Mlan (Swun), Henrik Reet (Dan Race), Vijay Samuel (eBay), Daniel Gomez Blanco (Skyscanner), and others, each discussing their roles and contributions to the observability landscape. They explore the significance of observability, its evolution from traditional monitoring, and how OpenTelemetry serves as a vital tool for gathering telemetry data across diverse systems. The discussion highlights the importance of community collaboration, the future of open standards, and the integration of various signals like metrics, logs, and traces to enhance system understanding and performance. The participants express their enthusiasm for the advancements in observability and the transformative impact of OpenTelemetry on engineering practices. -# Observability Insights +## Chapters -**Iris Durish** -I’m Iris Durish, a Senior Observability Engineer at Meo. My professional life revolves around observability. I build an observability platform that provides tools for engineering teams at Meo to monitor, observe, and get the best out of their applications. +Sure! Here are the key moments from the livestream with the corresponding timestamps: -**SE Nman** -My name is SE Nman, and I work at Cisco in the Open Source Program Office. I’m a member of the OpenTelemetry Governance Committee and one of the co-maintainers of the OpenTelemetry documentation. +00:00:00 Introductions of participants +00:02:15 What is observability? +00:05:30 The evolution of observability from APM +00:09:00 Observability as peace of mind +00:12:45 Observability versus monitoring +00:15:00 How observability impacts user experience +00:19:00 The role of OpenTelemetry in modern observability +00:23:30 Community and collaboration in OpenTelemetry +00:27:15 Personal journeys into OpenTelemetry +00:31:00 Discussion on favorite telemetry signals (traces, metrics, logs) -**Kayla Riel** -I’m Kayla Riel, and I work for New Relic, contributing to the OpenTelemetry Ruby project. +These moments encapsulate the main ideas and discussions shared during the livestream. -**Morgan Mlan** -I’m Morgan Mlan, the Director of Product Management at Swun. I have been with OpenTelemetry since day one and am one of the co-founders of the project. Currently, I’m involved with the release of traces, metrics 1.0, logs 1.0, and I’m working on profiling and extending OpenTelemetry into mainframe computers. +# Observability Engineers Panel Discussion Transcript -**Henrik Reet** -My name is Henrik Reet, and I am a Cloud Native Advocate at Dan Race. I’m passionate about observability and performance, and I try to help the community by providing content on getting started with various solutions. +**Iris Durish**: +I'm Iris Durish, a senior observability engineer at Meo. My professional life revolves around observability. I build an observability platform that provides the tools for engineering teams at Meo to monitor, observe, and get the best out of their applications. -**Vijay Samuel** -I’m Vijay Samuel, and I help architect the observability platform at eBay. +**SE Nman**: +My name is SE Nman, and I work at Cisco in the open source program office. I'm a member of the OpenTelemetry governance committee and one of the co-maintainers of the OpenTelemetry documentation. -**Daniel Gomez Blanco** -I’m Daniel Gomez Blanco, a Principal Engineer at Skyscanner and also a member of the OpenTelemetry community. +**Kayla Riel**: +I'm Kayla Riel, and I work for New Relic. I contribute to the OpenTelemetry Ruby project. -**Doug Odard** -My name is Doug Odard, a Senior Solutions Architect with ServiceNow Cloud Observability, which is also part of Lightstep. I’ve been a customer of OpenTelemetry for several years. +**Morgan Mlan**: +I'm Morgan Mlan, a director of product management at SWUN. I've been with OpenTelemetry since day one and am one of the co-founders of the project. I'm on the governance committee. Early on, I worked on a bit of everything, but more recently, I've been involved in the release of traces, metrics 1.0, and logs 1.0. Right now, I’m focusing on profiling and expanding OpenTelemetry into mainframe computers. -**Adnan** -I’m Adnan, and I work at TraceTest as a Developer Advocate. I contribute to various aspects regarding OpenTelemetry, including documentation for the blog and the demo. +**Henrik Reet**: +My name is Henrik Reet, and I am a cloud-native advocate at Dan Race. I'm passionate about observability performance and am trying to help the community by providing content on getting started with observability solutions. -**Ren Manuso** -I’m Ren Manuso, and I work for Honeycomb.io as one of the maintainers of the OpenTelemetry project. +**Vijay Samuel**: +I’m Vijay Samuel, and I help with architecture for the observability platform at eBay. ---- +**Daniel Gomez Blanco**: +I'm Daniel Gomez Blanco, a principal engineer at Skyscanner and also a member of the OpenTelemetry community. -## What Does Observability Mean to Me? +**Doug Odard**: +I'm Doug Odard, a senior solutions architect with ServiceNow Cloud Observability, which is also formerly Lightstep. I was a previous customer using OpenTelemetry for several years prior. -**Iris Durish** -To me, observability is the biggest passion of my life and my professional career. It’s one of those areas that you’re not initially interested in when you start your career because it’s not taught in school. However, once you discover it, you realize how amazing it is to help teams get the best out of their products. +**Adnan**: +Hey, I’m Adnan, and I work at TraceTest as a developer advocate. I handle various tasks related to OpenTelemetry and contribute to the documentation, blog, and demos. -**SE Nman** -I believe observability is a significant evolution from what we have done, especially in Application Performance Management (APM). It is making tools available for everyone, moving away from just adding a post-compilation agent to creating native observability that developers and operations teams can use across all organizations. +**Ren Manuso**: +My name is Ren Manuso, and I work for Honeycomb.io. I am one of the maintainers of the end-user documentation. -**Kayla Riel** -To me, observability means having peace of mind. It’s a reliable way to see what happened and what went wrong, allowing us to feel more connected to our customers and users. +--- -**Morgan Mlan** -Observability transcends just the computing industry. It’s about understanding how systems work and how to fix them quickly if they break. OpenTelemetry expands visibility beyond infrastructure and applications, allowing us to see any systems that impact our business. +### What Observability Means -**Henrik Reet** -Observability is more than just monitoring. It’s about having enough information to understand a given situation. If you only look at metrics, you may have a guess that something is wrong, but with logs, events, traces, and profiling, you can combine those dimensions to pinpoint your problem. +**Ren Manuso**: +Observability means a lot to me; it’s a significant passion in my life and professional career. It’s one of those areas you may not initially be interested in when starting your career because it’s not taught in school or heavily discussed in tech communities. However, once you discover it, you realize how amazing it is to help teams get the best out of their products. -**Vijay Samuel** -In my role at eBay, observability means knowing if the site is running fine or not, ensuring high availability. +**Morgan Mlan**: +I think observability is a game changer. It represents an evolution from what we’ve done, particularly in APM over the years. My background is in selling APM agents, which provided many of the benefits observability promises today. The big change is that observability is becoming accessible to everyone, moving away from the need to add a post-compilation agent to applications towards native observability that developers and operations teams can utilize across all organizations. -**Daniel Gomez Blanco** -For me, observability is crucial for understanding complex systems to deliver a good experience for our end users. +**SE Nman**: +To me, observability means having peace of mind. It’s about having a reliable system to understand what happened and what went wrong. It’s also a way to feel connected to your customers and users, seeing how they interact with your software. -**Doug Odard** -As a full-stack developer, I understand the need for observability. It’s the way to see what’s happening in your system without being up all night trying to figure things out. +**Kayla Riel**: +Observability transcends just the computing industry. It's the ability to peer into something and understand how it works, how it’s performing right now, and how to quickly fix it if it breaks. -**Adnan** -With the rise of OpenTelemetry and tracing, the possibilities for observability have reached an all-time high, and I’m excited about being part of this growth. +**Henrik Reet**: +In the context of OpenTelemetry, observability means visibility into back-end infrastructure and applications. We’re expanding into client applications and mainframes, providing visibility into any systems that impact your business. -**Ren Manuso** -Observability is about asking deeper questions of our systems and not just relying on alerts for emergencies. It allows us to explore the unknown. +**Vijay Samuel**: +For me, observability is about understanding everything happening on the site to ensure high availability. It’s knowing if the site is running fine or not. ---- +**Doug Odard**: +Observability allows us to understand complex systems so we can deliver a good experience for our end users. -## The Role of OpenTelemetry +--- -**Morgan Mlan** -OpenTelemetry is the tool that is making observability accessible again. It centralizes telemetry signals and allows for semantic conventions, enabling observability teams and engineering teams to focus more on building and improving observability. +### The OpenTelemetry Community -**Henrik Reet** -To me, OpenTelemetry is the vehicle for observability, enabling it for everyone. It fosters an incredible community where we can share ideas and discuss the future of observability. +**Adnan**: +OpenTelemetry means a community effort to take the best practices from existing instrumentation and consolidate them so that everyone can benefit. -**Kayla Riel** -OpenTelemetry brings together the best of what has already been out there for instrumentation, allowing everyone to benefit from it. +**Morgan Mlan**: +OpenTelemetry is the vehicle for observability. I joined the community a few years back out of curiosity. It’s been an amazing experience, with a great community where we not only discuss OpenTelemetry but also share thoughts about the future of observability. -**Doug Odard** -OpenTelemetry is my passion. It extracts signals, metrics, traces, and logs from infrastructure and services, making them observable and processable on the backend. +**Kayla Riel**: +OpenTelemetry has a unique ability to bring together experts in observability and programming languages to create something meaningful for everyone. -**Vijay Samuel** -It signifies the future for observability, providing a common standard that opens up various use cases across the industry. +**SE Nman**: +OpenTelemetry is my baby. I’ve put a lot of effort into creating this project, and it means a lot to me. It enables us to extract metrics, traces, logs, and profiles from our services and infrastructure, making them observable. --- -## Personal Experiences with OpenTelemetry - -**Adnan** -I got involved with OpenTelemetry because I was curious about its potential. I began contributing to the documentation and realized how much I enjoyed working within this community. +### Future of Observability -**Morgan Mlan** -I got involved in OpenTelemetry through a merger of projects. My experience with various programming languages and my role at Google led me to contribute to this open-source initiative. +**Henrik Reet**: +OpenTelemetry represents the future of observability. By having an open standard, we can ensure a common format across the industry, which opens up new use cases. -**Kayla Riel** -I joined OpenTelemetry after being asked to evaluate the current status of the Ruby project. My work has focused on stabilizing logs and metrics for Ruby applications. +**Doug Odard**: +OpenTelemetry is becoming a standard, allowing for centralization of telemetry signals and helping observability teams focus on building better observability practices. -**Daniel Gomez Blanco** -I began exploring OpenTelemetry at Skyscanner to simplify how we approach infrastructure and data collection, leading to a stronger adoption of open standards. +**Morgan Mlan**: +OpenTelemetry allows us to ask deeper questions about our systems and go beyond just alerting. It’s about understanding complex systems' performance. -**Vijay Samuel** -I became involved in OpenTelemetry while working to introduce it within my company, driven by the need for a unified approach to observability. +**Vijay Samuel**: +OpenTelemetry is the future because it enables product teams and infrastructure teams to collaborate more effectively, improving the overall customer experience. --- -## Favorite Signals in Observability +### Favorite Signals -**Morgan Mlan** -My favorite signal is profiling because it closes a significant gap in observability, providing code-level visibility that APM products often lack. +**Morgan Mlan**: +My favorite signal is profiling because it closes a significant gap in observability. -**Kayla Riel** -I love traces for their storytelling ability, but I also have a soft spot for logs, as they provide essential connections to spans and traces. +**Kayla Riel**: +I love traces because they tell stories in meaningful ways. However, I also have a soft spot for logs. -**Adnan** -My favorite signal is distributed traces because they help in understanding system behavior, especially during incidents. +**Doug Odard**: +I’m partial to distributed traces since that’s where the project started, but I’m also excited about logs and their potential. -**Doug Odard** -While I appreciate all signals, I’m particularly fond of metrics for their potential in anomaly detection and machine learning applications. +**Adnan**: +I enjoy all signals, but I’m particularly excited about profiling. It helps us understand performance optimizations. -**Henrik Reet** -I’m eager for continuous profiling to become mainstream, as it can significantly enhance our understanding of system performance. +**Henrik Reet**: +I’m a fan of metrics because of their power in anomaly detection and machine learning applications. --- -This collective perspective from various professionals in the observability and OpenTelemetry community highlights the evolving landscape of observability, its significance across different platforms, and the collaborative spirit driving its future. +This concludes the panel discussion on observability and OpenTelemetry. Thank you for joining us! ## Raw YouTube Transcript -[Music] well I'm Iris durish I'm a senior observability engineer at Meo and my life uh my professional life is all about the observability I build an observability platform that provides the tools for engineering teams at Meo to monitor to observe and get the best of their obligations my name is SE nman uh I'm working at Cisco at the open source program office and I'm a member of the open Telemetry governance committee and I'm one of the co- maintainers of the open Telemetry documentation my name is Kayla Riel I work for New Relic and I'm contributing to the open Telemetry Ruby project my name is Morgan mlan I'm a director of product management and swun I've been with open cementry since day one I'm one of the co-founders of the project I'm on the governor committee uh wow what do I work on with an mtel uh bit of everything I mean early on it was literally everything I mean myself Ted and various others were doing many many jobs uh more recently uh I've involved in the release of traces and then metrics 1.0 logs 1.0 last year right now I'm working on profiling uh as well as openet sus spansion into Mainframe computer my name is Henrik Reet I am a cloud native Advocate at Dan race uh and I'm passionate about observability performance uh and I'm yeah I'm trying to help the community by uh providing content on getting started on any solutions that out my name is uh Vijay Samuel and uh I help uh do architecture for the observ platform at eBay I'm Daniel Gomez Blanco I'm a principal engineer and Skys scanner and also member of the open my name is Doug odard I'm a Senior Solutions architect with service now Cloud observability which is also formula light step um I'm also a previous customer uh of using open Telemetry for several years prior to that hey I am Adnan I work at Trace test as a developer Advocate which is you you can guess better than me what that is pretty much do bunch of everything regarding open Elementry I'm one of the contributors for the documentation for the Vlog and the uh demo my name is Ren manuso I work for honeycom Io and um I am one of the maintainers of the end [Music] user what does observability mean to me uh observability to me is the biggest fashion of my life and also my professional career career uh it is one of those areas that you are not very interested when you start your career because you don't know anything about it it's not taught in school uh it's not preached by the tech communities a lot but then you discover it and you say wow this this is amazing we're actually making a change and uh we're helping the teams make the best of their product so yeah that's all for I think observability is a is a big game changer right so it's Evolution from from what we have done in especially APM over the last few years so I worked uh I worked for a very long time at EP Dynamics and we sold APM agents to customers and and and we gave them a lot of the things that observability is promising today as well but the big change I see with observability that it's coming down let's say to everybody right so this is um this is making um the things that we did there available for everybody and even more we're moving away from this hey let's add a post compilation agent into your application to like yeah let's make Native observability let's make this a thing that developers that operation teams are using across all the organizations so to me observability means having peace of mind it means having something that you can rely on in order to see what happened and what went wrong um I think observability is also Al a way to feel more technically connected to your customers and your users so that you can see the ways that they're interacting with your software instead of just the ways that that you might interact with it I mean observability to me transcends just the Computing industry it's the ability to peer into something and understand how it works what it's doing right now uh and thus if it breaks how to fix it more quickly uh certainly when we think of open Telemetry in this industry what observability classically is me is visibility to back in infrastructure and applications kind of excitingly I think it's expanding right now right we both Telemetry we're pushing into client applications we're pushing into main frames I mentioned earlier and so it's really visibility into any systems that impact your business any technical system observa usually when you people mention of observ they say hey it's about uh it's a replacement of the old name monitoring but in fact for me is a uh it's more than monitoring because monitor is like you just look at something and OBS observa it for me is like having enough information to understand a giv situation so if you just look at metrics then okay you you you get you have a guess that something is going wrong but you don't understand so having the options to get more information like logs events exceptions traces profiling then at the end combine all those Dimensions together then you say okay I got it this is my problem and I can res so what does observability mean to me uh I belong to what is called the site engineering organization inside of eBay and uh our goal is to make sure that we can uh observe everything that's going on in the site uh and ensure that we have high availability so basically observability means knowing if the site is uh running fine or not because that's that's why I'm there what does oability mean to me as a way for us to understands what's happening within our systems because we run quite a complex system so like we need to understand what goes on inside of them so we can deliver a good experience for our end users at the end of the day oh my God so observability is to me I've been an a full stack developer for years and so as we observe I actually I ended up on an incident Response Team uh doing um tracking of incident but also trying to figure out what was wrong and it pointed out to me how much we need this uh how much how hard it was to look at so many different screens and so durability for me is the way to actually see what's happening in your system it's the Pinnacle of not being up the whole night trying to figure out what went wrong and with with open Telemetry and with the rise of tracing the last couple of years it has hit an all-time high with regards to the possibilities that we have right now so I'm just really really happy to be part of the project I'm also really happy that um it's growing at that pace that it's growing right now and I can't see what how that's going to evolve within the next couple of years for me observability is about being able to ask deeper questions of our systems being able to um demand I think more or than just alerting on things that are emergencies things we've seen before but actually being able to go out into the unknown and understand how complex systems are [Music] performing op Telemetry is the tool that is making observability great again uh I would say that observability is seeing the surge now that open Telemetry is becoming so popular it's allowing Central ation of telemeter signals it's allowing semantic conventions and it's generally helping observability teams and the engineering teams uh take more attention to the observability and building it and making it better what does open tement mean to me um I think it's the vehicle for observability right I mean it's it's enabling that and uh I I joined open telet Community a few years back because I was curious is about like this idea to bring observability to everybody um and I think we are doing a really good job and what it also means to be now is that it's an amazing Community right so we're at cucon here and I meet so many people I just know from those digit conversations and now I can talk to them in person and we talk a lot about open Telemetry but we also talk a lot about other things than open Telemetry we talk about observability of course about what we think about is going to happen in a few years and and all those other things and and that's what's open Telemetry me so open Telemetry to me seems like it's a community effort to take the best of what's already been out there for instrumentation and collect it in one group so that everyone can benefit from it I think that we've learned so much as different agent Engineers but there's also so much to learn from users of products themselves and open Telemetry does a great job of bringing both people who are you know experts in observability and experts in languages to make something it's really great and meaningful for everyone I mean open cemetry is my baby put so much effort into creating this project um what does it mean to me I mean there's the boring answer which is it extracts signals metrics traces logs profiles everything else from your infrastructure from your services from your clients makes those observable processable on the back end but I think to a lot of us who've been in this community so long and a lot of us like yourself and Henrik here and others who participated in the community so much I mean hotel is also just a really nice open source Community to participate in it's a thing I just enjoy working on uh I know that's abstract and kind of like a sort of squishy thing to say but but I don't know hotel has a lot of meaning to me in many different ways all very positive open chetry for me is uh means the future uh because at the end uh by having an open standard we have uh the luxury I would say uh to to have a common standard for common format for all the solution of the markets and and having that common formats for all the industry and all the vendors and all the solutions it will just open use cases I think testing used to rely uh on I don't know uh feedback from users and now with observable data we can be so much efficient in the way with testing uh we could be so much efficient in replacing marketing tools uh business analytics tools I think it's the future and one thing that also uh a lot of people talk about AI everywhere machine learning blah blah blah blah but I think it's the same thing as a Testa I mean Tesla when you drive your car uh it takes decisions based on the sensors that you measure and if you don't have those sensors and those measurements then you can have a smart you can have the smartest systems but without the data you canot take the right decisions so I think it's an enabler also for the future implementations of modern applications open telary is the standard for observability uh going forward uh and it's very important because as we have gone through the Journey of observability over the past few years we have had to hunt for Open Standards in Prometheus and few others uh now at least with uh ingestion and collection uh it's a single standard for everyone to adopt and I think that's uh pretty powerful uh for the long run what does open t mean to me that I think is bringing people together bringing everyone together under one single language and one single way of like thinking about Telemetry I think human languages are difficult enough for us to understand each other and I think you know open Telemetry is bringing the technology together under one single way of like thinking about tety thinking about how we observe our systems to me open Telemetry is bringing um the ability to have product teams infrastructure teams helping their jobs make it easier um and also just improve the customer experience and just make it overall a better experience to do our jobs elemetry is the I'm going to say the future of of observability we've seen so many companies so many vendors move to an open Telemetry first mindset and the way that you can use open Telemetry to to generate them to actually gather all Telemetry signals with one set of libraries with one tool is just the way it was supposed to be you're not locked into one to one vendor one cloud provider anymore you can do basically whatever you want and you can use both the metric logs and traces for basically anything you want to do so really happy to see it bro open Telemetry is a instrumentation protocol um that um helps us ask more detailed questions about observability because it collects multiple signals um from many flexible types of systems um every folks monitor everything from the control plane um in kubernetes all the way up to um physical on Prem systems it's a really flexible language and it's a beautiful community of humans that came together over the pandemic to build something really [Music] special I was working in a very fast pacing observability team and we were maintaining a lot of tools and we really did not have conventions there we did not have centralizations and we really were not flexible when it came to beend uh and vendor agnostic in general so we discovered this amazing tool called open Telemetry we said okay let's give it a try it worked great for us and here I am today uh one year later more than one year later and uh uh let's say pushing the migration to open Telemetry in my second project how how did I got involved into open tele so yeah I mentioned that so so I got curious a few years ago so I was um I was at up Dynamics working as a so-called domain architect and I was an expert for noas python and a lot of those other languages and there was always this conversation around like hey there's this thing now called open Telemetry and should we not integrate this into our product and I was like okay I want to learn more then I was like what is a good way to learn something new about an open source technology yeah get involved into that so I was involved into JavaScript at some point and then at some point I realized like yeah but if I really want to get a good view into open Telemetry doing documentation is a good way into that and that's how I ended up being a maintainer for the documentation I got involved in open Telemetry uh last spring when a New Relic asked me to take a look at what the current status was of the op Elementry Ruby project I also work as an engineer on the New Relic Ruby agent team and that gave me an opportunity to start to contribute to the project and I noticed that a lot of the signals for Ruby weren't yet stable so a lot of my work so far has been going into trying to bring logs and metrics to stability and M I was working at Google on Google's observability products like tracing profiling debugging that's whole thing uh and one of the challenges we had tracing was getting data from people's applications really really hard you need Integrations of hundreds of thousands of pieces of software no one team no one company is going to maintain that it's just infeasible uh and so we wanted to do something open source there were other open source standards there was one that had started I think roughly around the same time we were doing this called open tracing restart open census at some points uh especially amongst the more social media Savvy members of the team which I am not one of uh there was some contention between those projects uh about uh where you know people who maintain databases and langage front time things should actually spend their integration efforts uh and was limiting the success of both projects so I was leading open census Ted and Van and others were leading over tracing and uh in late 2018 early 2019 we we finally sort of brought things to a head and decided to merge those into what is now called open Slumber TR uh so that's that's sort of you know I've been involved since then I've been now I work at squ different company but still on the same of things but that's how my M started and it's just grown and grown and snowball when I started the adventure in oiv of course I joined diant TR and dant TR has their their vendor agent called one agents and uh I saw this movement of open Telemetry and coming from the performance background I looked at it and I said wow an open stereo that sounds quite exciting because I had a performance a gig for a customer where I implemented like a collecting logs and processing it and putting machine learning and I told myself at that time I said I would be so wonderful to have one common standard so then instead of doing a custom implementation I could have something for everyone and when I looked at the uh just definition of the project and and the things behind the project I was so excited I said oh gosh I want to be involved in a project and that's where I started to build content to help the community get started uh I'm I used to be a developer but I'm a bad developer for sure so that's why I'm trying to help the project in other way in other directions uh and yeah my my goal is uh increase the adoptions of the Open Standards uh making sure that it's been uh adopted everywhere so then we can move forward by fling even more exciting uh implementations I started a few years ago uh for two reasons one we were uh looking to introduce ing inside the company uh and at that time uh open tracing and open sensus was converging into open Telemetry uh we started evaluating um open Telemetry for that and given that we were moving into open Telemetry for tracing we I also uh went through the Journey of migrating our uh metrics collection uh into open Telemetry that's that's that's basically how I got uh involved how did I get involved in open I got involved through uh I work as sky scanner as an end user I was uh driving adoption of Open Standards open Telemetry uh during Co there was a a need for simplification and how we approach an infrastructure how we approach how we collect how we process how we export Le of data and also basically to um to basically lead the adoption of of uh Open Standards and that simplification as so as an ability Le I go more involved into the community as Telemetry decid to interact with all end users and meeting people that want to solve the same problems they want to find a solution that works for so M Telemetry I um actually for several years in my previous position I was hired to actually develop um observability software um I was writing my own thing uh we were doing a lot of alert management and various things it was so much work and I thought this has got to be easy plus I wanted to make sure that it could be future future proof there I use that term um but also extensible and when I discovered open Telemetry I was just like oh thank you uh because it's something that the the company could carry forward and also we didn't have to worry about storing the data um as much and so it it's really provided a really excellent platform so that we can focus on the task at hand versus how how to do the job so how I got involved in the project was actually first as a customer um I was it was about three close to four years ago kind of the infancy of open Telemetry and I would go online I would look at the documentation or I would be in the code a lot but I wanted to learn more so I would go to a s call and um there would be someone from Google and Microsoft and other companies and then there was this guy from this uh small fintech in the US and at at first it was a little awkward but they were so excited to have me in the call because they I was an end user and so it really was it was a wonderful experience to begin that way to realize that I could could contribute to this um rather than simply be a consumer of it so it was great and then I transitioned my career into working for a vendor uh and we Implement these systems now for customers like myself that I was years ago so it's it's kind of a pay forward give back type of thing how did we get involved into the open project we started contributing more to the blog with you guys started contributing a bit to the the dogs as well and yeah it's just been a wholehearted effort in the team to always kind of dedicate a few a few minutes of of each day to to check out the op project and to find a way to contribute to I got involved in the open Telemetry project honestly I was working at one observability company in marketing and um they didn't see the point they didn't want me to get involved and um the I really believed in open source I'd worked at Mozilla and Wikipedia and really believe that like this was the way forward um strategic perspective so the second I could switch to a company that did let me get involved that's what I did and now I'm at honeycomb and I'm glad to say within the first three months I Made Project member and started working with the end user working group and worked to grow it into a Sig into into all the programs that it has today together with [Music] others racing is my favorite signal my favorite signal now is profiling because I think this is really closing a big gap that was missing in observability right so I mentioned before right I come from the APM space and now for me APM observability it's very hard to to make like difference here but one thing that when I talk with people using APM products right now is they're like Hey where's code level visibility with open televis right my commercial agent is giving me that line of code that is breaking something and this is what we get with profiling and that's why I'm really really excited about it to decide a favorite signal is kind of difficult for me I really love the power of traces I think that traces can tell stories in ways that are very meaningful but on the same like on the other hand uh I've been so immersed in logs and trying to allow logs to have more connections to spans and traces but I definitely have a sof spot for logs as well uh I mean I'm partial to distributed traces because that's where this project got it start and I think early on that's where a lot of the value was no one else was really doing standardized distributed Trace collection right there were some open source examples of it attached to like zipin and jger but I think the reason open slage we got so much traction so quickly is that it was providing that I'm also partial to logs which we launched last year just because that's one where like I've been involved in a lot of parts of hotel but that's one where like I was involved in a lot of the core specific early on and driving that and so it was really exting to see that ship also logs are just a thing that throughout my career before working on any of this I just get frustrated with because they're never standardized they slow to process they're expensive hotel's going to bring a lot of changes there for the better for everyone who uses Vlogs and finally I guess profiles cuz I'm working on that now and it's new I when I was in Google many years ago I launched what I think was the world's first distributed like continuous profiling product at least publicly available one which was Google Cloud profiling stack profiling they still support it I still think it's free it's very powerful uh and but profiling has always been a bit of a niche thing like I I know like it's spunk and other companies we support it but it's not as well known as metrics and traces block I think with otel starting to later this year we're going to launch like full support for profiles that's really going to change things like I we had customers at Google who would spend an hour of our profiler and save like 20 30% of their aggre CU cuz they found some really poorly optimized code really quickly for more people to have that ability and speed to speed things up and for developers actually to get insight on how things work that's super exciting like the tech has been there a long time and hotel bringing this mainstream is huge when people ask me who is your favorite kid usually I say I don't have a favorite kid you know uh all my kids are wonderful uh they all have a h i don't know a a great thing you know out of it so I think I love traces because sometimes it help you to understand where it's slowed down I love metrics because as a performance engineer I used to use metrics a lot and I love logs because logs at the end there's no sing so if you just do analytics on logs wow you you are so much precise so I I don't think I'll have a a favorite signals I'll just say that depending on on what I needs and pick and choose there's clearly one signal that will help me more uh there's one thing that I'm I'm very eager and waiting since Valencia is continues profiling uh because it's uh I love profiling and I think traces is great but if there is a problem somewhere profiling would be so much helpful so I think uh yeah I don't answer any questions but I say yeah I love all the the signals provided by open I am thoroughly biased towards uh metrics uh I feel metrics uh are the most powerful signal uh as long as you are uh thinking through your instrumentation and making sure that you have the right granularity SL cardinality uh being sent in to the platform uh you can do powerful powerful things uh with regards to anomal detection machine learning and many other things so uh I love metrics I mean I have to say traces because they give you the content tracers give you the backbone corelation for all the other signals right but I do saying that the the current like design of the API design of metric is so powerful that I'm like falling in love again with metrics because of that way that we decouple uh instrumentation and measurement from aggregation of those metrics so powerful and so much uh richness to basically give us a way to describe our system that I'm calling back again in love mentions my favorite signal I have to say I'm arshal the traces uh because I'm I've been doing software development for so long that that was the first thing that really turned me on to it was the ability to see that um especially because I know what it's like to debug but it's also I also know what it's like in an incident to have to focus in very quickly so yes traces are my favorite uh but I do also like to send that Trace ID and and span ID into the logs now it's kind of becoming my next favorite uh my favorite singal is traces I'm going to say traces definitely uh my favorite singer is Ed sh what is my favorite signal I mean I work for honeycom so I am constitutionally obliged to say traces are my favorite signal [Music] +well I'm Iris durish I'm a senior observability engineer at Meo and my life uh my professional life is all about the observability I build an observability platform that provides the tools for engineering teams at Meo to monitor to observe and get the best of their obligations my name is SE nman uh I'm working at Cisco at the open source program office and I'm a member of the open Telemetry governance committee and I'm one of the co- maintainers of the open Telemetry documentation my name is Kayla Riel I work for New Relic and I'm contributing to the open Telemetry Ruby project my name is Morgan mlan I'm a director of product management and swun I've been with open cementry since day one I'm one of the co-founders of the project I'm on the governor committee uh wow what do I work on with an mtel uh bit of everything I mean early on it was literally everything I mean myself Ted and various others were doing many many jobs uh more recently uh I've involved in the release of traces and then metrics 1.0 logs 1.0 last year right now I'm working on profiling uh as well as openet sus spansion into Mainframe computer my name is Henrik Reet I am a cloud native Advocate at Dan race uh and I'm passionate about observability performance uh and I'm yeah I'm trying to help the community by uh providing content on getting started on any solutions that out my name is uh Vijay Samuel and uh I help uh do architecture for the observ platform at eBay I'm Daniel Gomez Blanco I'm a principal engineer and Skys scanner and also member of the open my name is Doug odard I'm a Senior Solutions architect with service now Cloud observability which is also formula light step um I'm also a previous customer uh of using open Telemetry for several years prior to that hey I am Adnan I work at Trace test as a developer Advocate which is you you can guess better than me what that is pretty much do bunch of everything regarding open Elementry I'm one of the contributors for the documentation for the Vlog and the uh demo my name is Ren manuso I work for honeycom Io and um I am one of the maintainers of the end user what does observability mean to me uh observability to me is the biggest fashion of my life and also my professional career career uh it is one of those areas that you are not very interested when you start your career because you don't know anything about it it's not taught in school uh it's not preached by the tech communities a lot but then you discover it and you say wow this this is amazing we're actually making a change and uh we're helping the teams make the best of their product so yeah that's all for I think observability is a is a big game changer right so it's Evolution from from what we have done in especially APM over the last few years so I worked uh I worked for a very long time at EP Dynamics and we sold APM agents to customers and and and we gave them a lot of the things that observability is promising today as well but the big change I see with observability that it's coming down let's say to everybody right so this is um this is making um the things that we did there available for everybody and even more we're moving away from this hey let's add a post compilation agent into your application to like yeah let's make Native observability let's make this a thing that developers that operation teams are using across all the organizations so to me observability means having peace of mind it means having something that you can rely on in order to see what happened and what went wrong um I think observability is also Al a way to feel more technically connected to your customers and your users so that you can see the ways that they're interacting with your software instead of just the ways that that you might interact with it I mean observability to me transcends just the Computing industry it's the ability to peer into something and understand how it works what it's doing right now uh and thus if it breaks how to fix it more quickly uh certainly when we think of open Telemetry in this industry what observability classically is me is visibility to back in infrastructure and applications kind of excitingly I think it's expanding right now right we both Telemetry we're pushing into client applications we're pushing into main frames I mentioned earlier and so it's really visibility into any systems that impact your business any technical system observa usually when you people mention of observ they say hey it's about uh it's a replacement of the old name monitoring but in fact for me is a uh it's more than monitoring because monitor is like you just look at something and OBS observa it for me is like having enough information to understand a giv situation so if you just look at metrics then okay you you you get you have a guess that something is going wrong but you don't understand so having the options to get more information like logs events exceptions traces profiling then at the end combine all those Dimensions together then you say okay I got it this is my problem and I can res so what does observability mean to me uh I belong to what is called the site engineering organization inside of eBay and uh our goal is to make sure that we can uh observe everything that's going on in the site uh and ensure that we have high availability so basically observability means knowing if the site is uh running fine or not because that's that's why I'm there what does oability mean to me as a way for us to understands what's happening within our systems because we run quite a complex system so like we need to understand what goes on inside of them so we can deliver a good experience for our end users at the end of the day oh my God so observability is to me I've been an a full stack developer for years and so as we observe I actually I ended up on an incident Response Team uh doing um tracking of incident but also trying to figure out what was wrong and it pointed out to me how much we need this uh how much how hard it was to look at so many different screens and so durability for me is the way to actually see what's happening in your system it's the Pinnacle of not being up the whole night trying to figure out what went wrong and with with open Telemetry and with the rise of tracing the last couple of years it has hit an all-time high with regards to the possibilities that we have right now so I'm just really really happy to be part of the project I'm also really happy that um it's growing at that pace that it's growing right now and I can't see what how that's going to evolve within the next couple of years for me observability is about being able to ask deeper questions of our systems being able to um demand I think more or than just alerting on things that are emergencies things we've seen before but actually being able to go out into the unknown and understand how complex systems are performing op Telemetry is the tool that is making observability great again uh I would say that observability is seeing the surge now that open Telemetry is becoming so popular it's allowing Central ation of telemeter signals it's allowing semantic conventions and it's generally helping observability teams and the engineering teams uh take more attention to the observability and building it and making it better what does open tement mean to me um I think it's the vehicle for observability right I mean it's it's enabling that and uh I I joined open telet Community a few years back because I was curious is about like this idea to bring observability to everybody um and I think we are doing a really good job and what it also means to be now is that it's an amazing Community right so we're at cucon here and I meet so many people I just know from those digit conversations and now I can talk to them in person and we talk a lot about open Telemetry but we also talk a lot about other things than open Telemetry we talk about observability of course about what we think about is going to happen in a few years and and all those other things and and that's what's open Telemetry me so open Telemetry to me seems like it's a community effort to take the best of what's already been out there for instrumentation and collect it in one group so that everyone can benefit from it I think that we've learned so much as different agent Engineers but there's also so much to learn from users of products themselves and open Telemetry does a great job of bringing both people who are you know experts in observability and experts in languages to make something it's really great and meaningful for everyone I mean open cemetry is my baby put so much effort into creating this project um what does it mean to me I mean there's the boring answer which is it extracts signals metrics traces logs profiles everything else from your infrastructure from your services from your clients makes those observable processable on the back end but I think to a lot of us who've been in this community so long and a lot of us like yourself and Henrik here and others who participated in the community so much I mean hotel is also just a really nice open source Community to participate in it's a thing I just enjoy working on uh I know that's abstract and kind of like a sort of squishy thing to say but but I don't know hotel has a lot of meaning to me in many different ways all very positive open chetry for me is uh means the future uh because at the end uh by having an open standard we have uh the luxury I would say uh to to have a common standard for common format for all the solution of the markets and and having that common formats for all the industry and all the vendors and all the solutions it will just open use cases I think testing used to rely uh on I don't know uh feedback from users and now with observable data we can be so much efficient in the way with testing uh we could be so much efficient in replacing marketing tools uh business analytics tools I think it's the future and one thing that also uh a lot of people talk about AI everywhere machine learning blah blah blah blah but I think it's the same thing as a Testa I mean Tesla when you drive your car uh it takes decisions based on the sensors that you measure and if you don't have those sensors and those measurements then you can have a smart you can have the smartest systems but without the data you canot take the right decisions so I think it's an enabler also for the future implementations of modern applications open telary is the standard for observability uh going forward uh and it's very important because as we have gone through the Journey of observability over the past few years we have had to hunt for Open Standards in Prometheus and few others uh now at least with uh ingestion and collection uh it's a single standard for everyone to adopt and I think that's uh pretty powerful uh for the long run what does open t mean to me that I think is bringing people together bringing everyone together under one single language and one single way of like thinking about Telemetry I think human languages are difficult enough for us to understand each other and I think you know open Telemetry is bringing the technology together under one single way of like thinking about tety thinking about how we observe our systems to me open Telemetry is bringing um the ability to have product teams infrastructure teams helping their jobs make it easier um and also just improve the customer experience and just make it overall a better experience to do our jobs elemetry is the I'm going to say the future of of observability we've seen so many companies so many vendors move to an open Telemetry first mindset and the way that you can use open Telemetry to to generate them to actually gather all Telemetry signals with one set of libraries with one tool is just the way it was supposed to be you're not locked into one to one vendor one cloud provider anymore you can do basically whatever you want and you can use both the metric logs and traces for basically anything you want to do so really happy to see it bro open Telemetry is a instrumentation protocol um that um helps us ask more detailed questions about observability because it collects multiple signals um from many flexible types of systems um every folks monitor everything from the control plane um in kubernetes all the way up to um physical on Prem systems it's a really flexible language and it's a beautiful community of humans that came together over the pandemic to build something really special I was working in a very fast pacing observability team and we were maintaining a lot of tools and we really did not have conventions there we did not have centralizations and we really were not flexible when it came to beend uh and vendor agnostic in general so we discovered this amazing tool called open Telemetry we said okay let's give it a try it worked great for us and here I am today uh one year later more than one year later and uh uh let's say pushing the migration to open Telemetry in my second project how how did I got involved into open tele so yeah I mentioned that so so I got curious a few years ago so I was um I was at up Dynamics working as a so-called domain architect and I was an expert for noas python and a lot of those other languages and there was always this conversation around like hey there's this thing now called open Telemetry and should we not integrate this into our product and I was like okay I want to learn more then I was like what is a good way to learn something new about an open source technology yeah get involved into that so I was involved into JavaScript at some point and then at some point I realized like yeah but if I really want to get a good view into open Telemetry doing documentation is a good way into that and that's how I ended up being a maintainer for the documentation I got involved in open Telemetry uh last spring when a New Relic asked me to take a look at what the current status was of the op Elementry Ruby project I also work as an engineer on the New Relic Ruby agent team and that gave me an opportunity to start to contribute to the project and I noticed that a lot of the signals for Ruby weren't yet stable so a lot of my work so far has been going into trying to bring logs and metrics to stability and M I was working at Google on Google's observability products like tracing profiling debugging that's whole thing uh and one of the challenges we had tracing was getting data from people's applications really really hard you need Integrations of hundreds of thousands of pieces of software no one team no one company is going to maintain that it's just infeasible uh and so we wanted to do something open source there were other open source standards there was one that had started I think roughly around the same time we were doing this called open tracing restart open census at some points uh especially amongst the more social media Savvy members of the team which I am not one of uh there was some contention between those projects uh about uh where you know people who maintain databases and langage front time things should actually spend their integration efforts uh and was limiting the success of both projects so I was leading open census Ted and Van and others were leading over tracing and uh in late 2018 early 2019 we we finally sort of brought things to a head and decided to merge those into what is now called open Slumber TR uh so that's that's sort of you know I've been involved since then I've been now I work at squ different company but still on the same of things but that's how my M started and it's just grown and grown and snowball when I started the adventure in oiv of course I joined diant TR and dant TR has their their vendor agent called one agents and uh I saw this movement of open Telemetry and coming from the performance background I looked at it and I said wow an open stereo that sounds quite exciting because I had a performance a gig for a customer where I implemented like a collecting logs and processing it and putting machine learning and I told myself at that time I said I would be so wonderful to have one common standard so then instead of doing a custom implementation I could have something for everyone and when I looked at the uh just definition of the project and and the things behind the project I was so excited I said oh gosh I want to be involved in a project and that's where I started to build content to help the community get started uh I'm I used to be a developer but I'm a bad developer for sure so that's why I'm trying to help the project in other way in other directions uh and yeah my my goal is uh increase the adoptions of the Open Standards uh making sure that it's been uh adopted everywhere so then we can move forward by fling even more exciting uh implementations I started a few years ago uh for two reasons one we were uh looking to introduce ing inside the company uh and at that time uh open tracing and open sensus was converging into open Telemetry uh we started evaluating um open Telemetry for that and given that we were moving into open Telemetry for tracing we I also uh went through the Journey of migrating our uh metrics collection uh into open Telemetry that's that's that's basically how I got uh involved how did I get involved in open I got involved through uh I work as sky scanner as an end user I was uh driving adoption of Open Standards open Telemetry uh during Co there was a a need for simplification and how we approach an infrastructure how we approach how we collect how we process how we export Le of data and also basically to um to basically lead the adoption of of uh Open Standards and that simplification as so as an ability Le I go more involved into the community as Telemetry decid to interact with all end users and meeting people that want to solve the same problems they want to find a solution that works for so M Telemetry I um actually for several years in my previous position I was hired to actually develop um observability software um I was writing my own thing uh we were doing a lot of alert management and various things it was so much work and I thought this has got to be easy plus I wanted to make sure that it could be future future proof there I use that term um but also extensible and when I discovered open Telemetry I was just like oh thank you uh because it's something that the the company could carry forward and also we didn't have to worry about storing the data um as much and so it it's really provided a really excellent platform so that we can focus on the task at hand versus how how to do the job so how I got involved in the project was actually first as a customer um I was it was about three close to four years ago kind of the infancy of open Telemetry and I would go online I would look at the documentation or I would be in the code a lot but I wanted to learn more so I would go to a s call and um there would be someone from Google and Microsoft and other companies and then there was this guy from this uh small fintech in the US and at at first it was a little awkward but they were so excited to have me in the call because they I was an end user and so it really was it was a wonderful experience to begin that way to realize that I could could contribute to this um rather than simply be a consumer of it so it was great and then I transitioned my career into working for a vendor uh and we Implement these systems now for customers like myself that I was years ago so it's it's kind of a pay forward give back type of thing how did we get involved into the open project we started contributing more to the blog with you guys started contributing a bit to the the dogs as well and yeah it's just been a wholehearted effort in the team to always kind of dedicate a few a few minutes of of each day to to check out the op project and to find a way to contribute to I got involved in the open Telemetry project honestly I was working at one observability company in marketing and um they didn't see the point they didn't want me to get involved and um the I really believed in open source I'd worked at Mozilla and Wikipedia and really believe that like this was the way forward um strategic perspective so the second I could switch to a company that did let me get involved that's what I did and now I'm at honeycomb and I'm glad to say within the first three months I Made Project member and started working with the end user working group and worked to grow it into a Sig into into all the programs that it has today together with others racing is my favorite signal my favorite signal now is profiling because I think this is really closing a big gap that was missing in observability right so I mentioned before right I come from the APM space and now for me APM observability it's very hard to to make like difference here but one thing that when I talk with people using APM products right now is they're like Hey where's code level visibility with open televis right my commercial agent is giving me that line of code that is breaking something and this is what we get with profiling and that's why I'm really really excited about it to decide a favorite signal is kind of difficult for me I really love the power of traces I think that traces can tell stories in ways that are very meaningful but on the same like on the other hand uh I've been so immersed in logs and trying to allow logs to have more connections to spans and traces but I definitely have a sof spot for logs as well uh I mean I'm partial to distributed traces because that's where this project got it start and I think early on that's where a lot of the value was no one else was really doing standardized distributed Trace collection right there were some open source examples of it attached to like zipin and jger but I think the reason open slage we got so much traction so quickly is that it was providing that I'm also partial to logs which we launched last year just because that's one where like I've been involved in a lot of parts of hotel but that's one where like I was involved in a lot of the core specific early on and driving that and so it was really exting to see that ship also logs are just a thing that throughout my career before working on any of this I just get frustrated with because they're never standardized they slow to process they're expensive hotel's going to bring a lot of changes there for the better for everyone who uses Vlogs and finally I guess profiles cuz I'm working on that now and it's new I when I was in Google many years ago I launched what I think was the world's first distributed like continuous profiling product at least publicly available one which was Google Cloud profiling stack profiling they still support it I still think it's free it's very powerful uh and but profiling has always been a bit of a niche thing like I I know like it's spunk and other companies we support it but it's not as well known as metrics and traces block I think with otel starting to later this year we're going to launch like full support for profiles that's really going to change things like I we had customers at Google who would spend an hour of our profiler and save like 20 30% of their aggre CU cuz they found some really poorly optimized code really quickly for more people to have that ability and speed to speed things up and for developers actually to get insight on how things work that's super exciting like the tech has been there a long time and hotel bringing this mainstream is huge when people ask me who is your favorite kid usually I say I don't have a favorite kid you know uh all my kids are wonderful uh they all have a h i don't know a a great thing you know out of it so I think I love traces because sometimes it help you to understand where it's slowed down I love metrics because as a performance engineer I used to use metrics a lot and I love logs because logs at the end there's no sing so if you just do analytics on logs wow you you are so much precise so I I don't think I'll have a a favorite signals I'll just say that depending on on what I needs and pick and choose there's clearly one signal that will help me more uh there's one thing that I'm I'm very eager and waiting since Valencia is continues profiling uh because it's uh I love profiling and I think traces is great but if there is a problem somewhere profiling would be so much helpful so I think uh yeah I don't answer any questions but I say yeah I love all the the signals provided by open I am thoroughly biased towards uh metrics uh I feel metrics uh are the most powerful signal uh as long as you are uh thinking through your instrumentation and making sure that you have the right granularity SL cardinality uh being sent in to the platform uh you can do powerful powerful things uh with regards to anomal detection machine learning and many other things so uh I love metrics I mean I have to say traces because they give you the content tracers give you the backbone corelation for all the other signals right but I do saying that the the current like design of the API design of metric is so powerful that I'm like falling in love again with metrics because of that way that we decouple uh instrumentation and measurement from aggregation of those metrics so powerful and so much uh richness to basically give us a way to describe our system that I'm calling back again in love mentions my favorite signal I have to say I'm arshal the traces uh because I'm I've been doing software development for so long that that was the first thing that really turned me on to it was the ability to see that um especially because I know what it's like to debug but it's also I also know what it's like in an incident to have to focus in very quickly so yes traces are my favorite uh but I do also like to send that Trace ID and and span ID into the logs now it's kind of becoming my next favorite uh my favorite singal is traces I'm going to say traces definitely uh my favorite singer is Ed sh what is my favorite signal I mean I work for honeycom so I am constitutionally obliged to say traces are my favorite signal diff --git a/video-transcripts/transcripts/2024-06-17T19:00:22Z-otel-q-a-feat-daniel-dias-and-oscar-reyes-of-tracetest.md b/video-transcripts/transcripts/2024-06-17T19:00:22Z-otel-q-a-feat-daniel-dias-and-oscar-reyes-of-tracetest.md index eef6d8e..ae9dc74 100644 --- a/video-transcripts/transcripts/2024-06-17T19:00:22Z-otel-q-a-feat-daniel-dias-and-oscar-reyes-of-tracetest.md +++ b/video-transcripts/transcripts/2024-06-17T19:00:22Z-otel-q-a-feat-daniel-dias-and-oscar-reyes-of-tracetest.md @@ -10,111 +10,155 @@ URL: https://www.youtube.com/watch?v=KR8j11FECgc ## Summary -In this OTEL Q&A session, Daniel Dias and Oscar Reyes from Tracetest discuss the integration of trace-based testing into the OpenTelemetry (OTEL) community demo. The conversation covers the concept of trace-based testing, which utilizes emitted traces from distributed systems to perform assertions about their state, enhancing testing efficacy. Daniel and Oscar explain how Tracetest implements this testing method through a three-step process: triggering the system, generating traces, and conducting tests on the telemetry data. They also highlight the benefits of this integration, including improved testing reliability and ease of identifying issues across service boundaries. The discussion touches on the challenges faced during the integration, such as ensuring accurate context propagation and adapting existing tests into trace-based formats. The session concludes with insights into how the community has embraced this testing approach, enhancing the overall quality of the OTEL demo. +In this OTEL Q&A session, hosted by Adriana, Daniel Dias and Oscar Reyes from Tracetest discussed the integration of trace-based testing into the OpenTelemetry (OTEL) community demo. Daniel, a Brazilian software developer and CNCF ambassador, and Oscar, a lead software engineer from Mexico, explained the concept of trace-based testing, which utilizes emitted traces to assert system states and validate interactions between microservices. They outlined Tracetest's implementation process, which involves triggering a system, generating traces, and then testing those traces using assertions. The duo shared their experience of transitioning from traditional integration tests to trace-based tests in the demo, highlighting the advantages of traceability in identifying issues and improving developer experience. They also noted the positive reception from the OTEL community and the increased reliance on trace tests in CI/CD pipelines. The session concluded with a discussion on the challenges faced during integration and the ongoing evolution of trace-based testing practices within the community. + +## Chapters + +Here are 10 key moments from the livestream, along with their timestamps: + +00:00:00 Introductions and welcome to the livestream +00:02:30 Introduction of Daniel Dias and Oscar Reyes +00:04:00 Explanation of trace-based testing +00:06:15 Importance of trace-based testing for distributed services +00:08:00 Overview of how Tracetest implements trace-based testing +00:10:45 Discussion on creating trace-based tests using YAML +00:12:30 Introduction to the OTEL community demo +00:15:00 Explanation of the architecture of the OTEL demo +00:18:30 Integration of Tracetest into the OTEL demo +00:24:15 Benefits and challenges of integrating trace-based tests in the demo +00:30:00 Audience Q&A about context propagation and baggage in traces + +Feel free to reach out if you need further details or additional information! # OTEL Q&A with Daniel Dias and Oscar Reyes -Welcome everyone who has been able to join us today! This is super exciting to have both Daniel Dias and Oscar Reyes from Tracetest with us for an OTEL Q&A. We haven't had an OTEL Q&A for a while, so I'm glad that both of you were able to make it. +Welcome, everyone, who has joined us today. This is super exciting! We have both Daniel Dias and Oscar Reyes from Tracetest joining us for an OTEL Q&A. We haven't had an OTEL Q&A for a while, so I’m really glad that both of you could make it today. -For those of you here today, thank you for joining! Please let your friends know that we will be posting a recording of this on the OTEL YouTube channel (OTEL official) for anyone who couldn't make it. We'll also provide links on social media once the recording is available. +For those of you who are here, thank you for joining! Also, feel free to tell your friends that we will be posting a recording of this on the OTEL YouTube channel (OTEL Official) for anyone who couldn't make it. We will also provide the links on social media once the recording is available. -## Introductions +With that, let's get started! Why don't I get you to introduce yourselves? -With that, let's get started. Why don't I get you to introduce yourselves? +## Introductions -**Daniel:** -Hello everyone! I'm Daniel Dias, a Brazilian living and complaining in Argentina right now because of the weather. I am a software developer at Tracetest, and while I'm not helping the team evolve with new features, I'm bragging about those features somewhere. I'm also a newly minted CNCF ambassador! +**Daniel Dias:** +Hello, everyone! I'm Daniel Dias, a Brazilian living and complaining in Argentina right now because of the weather. I am a software developer at Tracetest, and while I'm not helping the team evolve with new features, I'm bragging about the features somewhere. I'm also proud to be a newly minted CNCF ambassador! -**Oscar:** -For my part, I'm Oscar Reyes. I'm Mexican, living in Mexico, and I am a lead software engineer here at Tracetest. I'm really happy to be here to talk about Tracetest and OpenTelemetry. +**Oscar Reyes:** +For my part, I'm Oscar Reyes. I'm Mexican, I live in Mexico, and I'm a lead software engineer at Tracetest. I'm really happy to be here and talk about Tracetest and OpenTelemetry. ## What is Trace-Based Testing? -First things first, for folks on the call who might not be familiar, can you tell us what trace-based testing is? +For folks on the call who might not be familiar, can you explain what trace-based testing is? **Daniel:** -The main idea behind trace-based tests is that we use traces emitted by systems to assert the state of a trace. We call a component in the system, wait for the trace to be generated, and then make assertions to ensure everything is correct. This is especially helpful for distributed services where there are many moving pieces. If the entire system is instrumented, we can make numerous assertions around the generated OpenTelemetry data or traces. +The main idea behind trace-based tests is that we use traces emitted by systems to assert the state of the trace. We call some component in the system, wait for a while until the trace is generated, and then start making assertions to guarantee everything is correct. This is particularly helpful for distributed services with many moving pieces. If the entire system is instrumented, you can perform various assertions around the generated OpenTelemetry data or the traces. -You can validate that certain spans exist, and go deeper to validate attributes or execution times of those spans. It's a multi-layered level of testing across all the spans or traces your system generates. +You can validate that certain spans exist and also go deeper to validate attributes or execution time of the spans. It's a multi-layer level of testing for all the spans or traces generated by the system. **Oscar:** -What I love about trace-based testing is that we're already emitting traces in our code, so why not take advantage of that? It's like a two-for-one deal! +What I love about trace-based testing is that we’re already emitting traces in our code, so why not take advantage of that? It feels like a two-for-one deal! -## Implementing Trace-Based Testing in Tracetest - -How does Tracetest implement trace-based testing? +## How Does Tracetest Implement Trace-Based Testing? **Oscar:** -Tracetest follows a two-step process. First, we trigger the system under test. Currently, we support HTTP, gRPC requests, Kafka, and other integrations like Cypress or Playwright. This triggers the system to generate traces and spans, which are stored in a tracing backend like Jaeger or Tempo. +So, how does Tracetest implement trace-based testing? The way that trace-based testing works is through a two-step process. + +1. **Trigger the System:** The first step is triggering the system under test. Currently, we support things like HTTP and gRPC requests, as well as Kafka, which was integrated by Daniel. We also have integrations for systems like Cypress and Playwright. -The second half of what Tracetest does is pull those traces based on your configuration. Depending on how your system works, the generation of the entire trace could take a few seconds or even several minutes. We support various timing scenarios and pull the trace into the Tracetest system based on the trace ID generated in the first step. This will act as the parent ID for the assertions on the spans and telemetry data. +2. **Pull the Traces:** After the system generates the instrumented traces and spans, they are stored in a tracing backend, like Jaeger or Tempo. The second part of what Tracetest does is pull those traces based on your configuration. We wait for the trace to be ready, depending on how the system works—some generate all the traces in seconds, while others might take longer. We pull the data into the Tracetest system based on the generated trace ID, which acts as the parent ID for our assertions on the spans and telemetry data. -So, it’s essentially a three-step process: trigger, get the trace, and then test. +So, in summary, we have a three-step process: trigger the system, get the trace, and then conduct the tests. **Daniel:** -I’d like to add that Tracetest allows you to create your trace-based tests using YAML, which everyone loves to hate! We’ve been working on simplifying the introduction of trace-based testing to systems, whether through Git actions, CI/CD processes, or cron jobs. With our CLI and TypeScript library, you can automate your tests and manage them through the UI or CLI. +Do you have anything to add to that? + +**Oscar:** +No, I think that covers it! + +## Creating Trace-Based Tests + +**Oscar:** +If I remember correctly, you can create your trace-based tests using YAML, right? -## The OTEL Community Demo Integration +**Daniel:** +Yes! One of the things we have been pushing in the past few months is not only allowing users to have these three steps but also automating and simplifying how to introduce trace-based testing into their systems. You can use our CLI and different tools, like our TypeScript library, to graph the specifications in YAML or even JSON files. You can export them from the UI or CLI and execute everything from there, which gives you multiple ways to interact with the system and the test definitions. -The main reason we called you both for this Q&A is because you did something super cool last fall by integrating trace-based testing from Tracetest into the OTEL community demo. Can one of you explain what the OTEL community demo is for those unfamiliar? +## Integration with the OTEL Community Demo **Oscar:** -The OpenTelemetry demo emulates a telescope shop, but with a twist. Under this shop, we have various microservices that interact with each other, each with its own responsibility. We have an API for payments, another for shipping, and so forth. The main goal of this demo is to demonstrate how you can integrate OpenTelemetry into your system and visualize the data across multiple microservices. +One of the reasons we called Daniel and Oscar for this Q&A is that they did something super cool last fall: they integrated trace-based testing into the OTEL community demo. Can one of you explain what the OTEL community demo is for folks who aren't familiar? + +**Daniel:** +The main idea of the OpenTelemetry demo is that it emulates a telescope shop with a bunch of microservices that interact with each other, each serving one responsibility in the system. We have one API for payments, another for shipping, and so on. The demo showcases how you can integrate OpenTelemetry into your system and see the data integrated across various microservices. -## How the Integration Happened +**Oscar:** +It's a great way to get started with OTEL, especially if you find it overwhelming. It's not too difficult to try out the OTEL demo, and I recommend it for those who haven't yet. -So, now that we understand the OTEL demo, how did you integrate Tracetest into it? +## Integrating Tracetest into the OTEL Demo **Oscar:** -About a year and a half ago, we were looking for ways to contribute to the community and push OpenTelemetry further. I started by migrating the existing front-end application, which was built entirely in Go with server-side rendering, to a front-end application using Next.js, incorporating React and OpenTelemetry instrumentation. +Now that we understand what the OTEL demo is, let's talk about how you integrated Tracetest into it. **Daniel:** -After Oscar's work, we realized that while the OpenTelemetry demo was great, it only had integration tests at the API level. We needed to test how services acted together, especially when background processes were involved. So, we started discussing the idea of trace-based tests. +About a year and a half ago, we were looking for partnerships and ways to contribute to the community and push OpenTelemetry further. We discovered the OTEL demo, and I started working on migrating the existing front-end application built in Go to Next.js, which uses React. I also introduced front-end instrumentation using browser-side OpenTelemetry libraries, which connected the browser all the way to the backend services. -We began writing traceability tests for the integration tests, not only checking API responses but also ensuring the APIs called behind the scenes were functioning correctly. This proved invaluable when we encountered issues, like when the Kafka SDK changed ownership, and we used trace tests to ensure everything was still working correctly. +**Oscar:** +After Daniel joined, we started looking at the testing side of the OpenTelemetry demo. Initially, the demo had integration tests, but they were only at the API level. We realized we needed to test how the services worked together, especially for background processes. So we discussed the idea of implementing trace-based tests to enhance the existing integration tests. -## Challenges and Benefits +**Daniel:** +We saw that the integration tests were useful, but we needed a way to ensure that one service's output was correctly processed by another service. By creating trace-based tests, we could validate both the API responses and the background processes. -What challenges did you face during the integration of trace-based testing into the OTEL demo? +## Challenges Encountered **Oscar:** -One major challenge was ensuring our tests did not impact the entire ecosystem. We decided to work exclusively with Jaeger, making it easier to manage the tests. Additionally, understanding how the services interacted was crucial, which sometimes led to discovering discrepancies, like mismatched JSON formats. +What were some of the big challenges or unexpected things you encountered during this integration? **Daniel:** -We had a couple of unexpected challenges, such as needing to embed protobuffer files in tests, which complicated things. However, we simplified our tests and improved the developer experience based on feedback from the OpenTelemetry demo team. +One of the first challenges was figuring out how to run these tests without impacting the entire ecosystem. We decided to use Jaeger since the demo already utilized it. This made it easier to test without affecting the overall system performance. -What benefits did you start seeing after the integration of trace-based testing? +Another challenge was understanding the services and how they worked. For example, we discovered that the JSON sent to one service was in snake_case, not camelCase, which required some tweaking. + +## Benefits of Trace-Based Testing **Oscar:** -One of the biggest changes was seeing the OpenTelemetry demo maintainers integrate trace tests into their CI/CD pipeline. They could now validate PRs more effectively, catching issues earlier in the process. +What benefits have you seen since integrating trace-based testing? **Daniel:** -We even received a shoutout from Josh Lee, which was a huge affirmation for us! The integration has allowed for greater reliability and has shifted how testing is approached within the community demo. +One of the significant changes was when the OpenTelemetry demo maintainers started integrating trace tests into their CI/CD pipeline. Previously, they had to run manual tests, and sometimes they would forget to test specific services. With trace tests in place, they could validate PRs and catch issues earlier, ensuring that everything was working correctly. -## Future Considerations +We also received a shoutout from Josh Lee, which was great! His support for trace-based testing and its importance to the OpenTelemetry community helped raise awareness. -Now that trace-based testing has been integrated for several months, have you made any changes to your approach? +## Future Changes and Improvements + +**Oscar:** +Now that trace-based testing has been integrated into the demo, have there been any significant changes in how you approach it? **Daniel:** -Yes, we've noticed that other developers are starting to write their own trace-based tests, which is fantastic. We've also had discussions about adding more tests and exploring interesting use cases to further enhance the demo. +One significant change was that the maintainers began building their own traceability tests. This was a big step, as they realized that with trace tests, they could validate their systems more effectively. They even decided to simplify their testing approach by reducing reliance on other testing libraries, focusing instead on trace-based testing. + +**Oscar:** +If you had a redo, would you change anything about how you integrated trace-based testing with the OTEL demo? + +**Daniel:** +At first glance, I wouldn’t change anything because it felt like a perfect match! However, I would like to add more tasks and think about more interesting use cases to showcase. ## Audience Questions -We do have a question from the audience: "Can trace tests help identify where context is being broken when it shouldn't?" +**Daniel (from the audience):** +One of the biggest pains for distributed systems is ensuring correct context propagation across service boundaries. Can trace tests help identify where context is being broken? **Oscar:** -Absolutely! One of our goals is to enable Trace-Based Driven Development (TBDD). Users can create assertions based on expected traces. If a span is missing, it indicates that context propagation may not be functioning correctly. - -And a follow-up: "Can Tracetest make assertions on baggage?" +Yes! We have all felt that pain. Trace-Based Driven Development (TDD) allows users to create assertions based on expected traces. If a span doesn’t exist as expected, it indicates that context propagation may have failed. This technique can help identify those breaks. **Daniel:** -Currently, we can test custom attributes in spans that you define. We continually look to the OpenTelemetry specification for more metadata to integrate, such as error codes and span statuses. +Another question we received was whether Tracetest can make assertions on baggage. -## Closing Remarks +**Oscar:** +Currently, we can test custom attributes in spans. We are continuously looking at the OpenTelemetry specification to integrate more metadata, like error codes and span statuses. -Thank you both, Daniel and Oscar, for joining today! This has been a wonderful discussion about the integration of trace-based testing into the OTEL community demo. I’m excited to see how this evolves and appreciate your insights into the power of OpenTelemetry and Tracetest. +## Conclusion -Thank you, everyone, for joining us, and a special thanks to Adriana for organizing this session! +Thank you both, Daniel and Oscar, for joining today! It’s been great to discuss trace-based testing and its integration with the OTEL community demo. I'm a big fan of the demo and see how this integration adds a powerful and tracing-native approach to integration testing. Thank you, everyone, for participating! ## Raw YouTube Transcript diff --git a/video-transcripts/transcripts/2024-08-01T05:59:23Z-otel-in-practice-otel-for-mobile-apps-with-hanson-ho-and-eliab-sisay.md b/video-transcripts/transcripts/2024-08-01T05:59:23Z-otel-in-practice-otel-for-mobile-apps-with-hanson-ho-and-eliab-sisay.md index aaadfac..fd2c526 100644 --- a/video-transcripts/transcripts/2024-08-01T05:59:23Z-otel-in-practice-otel-for-mobile-apps-with-hanson-ho-and-eliab-sisay.md +++ b/video-transcripts/transcripts/2024-08-01T05:59:23Z-otel-in-practice-otel-for-mobile-apps-with-hanson-ho-and-eliab-sisay.md @@ -10,125 +10,117 @@ URL: https://www.youtube.com/watch?v=mTcdwdWIFgI ## Summary -In this episode of "Otel and Practice," hosts Hansen and Eliah from Embrace discuss the challenges and opportunities of implementing data modeling and observability in mobile app design, particularly through the lens of OpenTelemetry (otel). Eliah introduces the concept of mobile observability, emphasizing the need for developers to monitor app performance in real-time to enhance user experience. Hansen elaborates on the unique complexities of mobile environments, such as diverse hardware and software configurations, session tracking issues, and the limitations of traditional observability tools. The presentation highlights the differences between mobile and backend systems, the importance of user-centric metrics, and ongoing efforts to standardize mobile observability practices. The session concludes with a call to action for developers interested in contributing to the mobile observability ecosystem through various OpenTelemetry Special Interest Groups (SIGs). +In this episode of "Otel and Practice," hosts Hansen and Eliab from Embrace discuss the challenges and opportunities of implementing data modeling and application performance (AP) design in mobile apps using OpenTelemetry (OTel). Eliab, a product manager at Embrace, introduces the topic by highlighting the significance of mobile observability, emphasizing that mobile apps are not distributed systems but rather installed software that interacts with distributed services. Hansen, an Android architect at Embrace, delves into the unique challenges of mobile observability, including the dynamic runtime environment, issues with session tracking, and the fragmentation of mobile devices. He explains how OTel's assumptions may not always hold true in mobile contexts, stressing the need for tailored instrumentation and user experience-focused metrics. The presentation concludes with a call to action for developers interested in mobile observability to engage with OTel working groups and contribute to developing robust standards for mobile applications. -# Otel in Practice: Mobile Data Modeling and Observability +## Chapters -[Music] +Here are the key moments from the livestream along with their timestamps: -**Welcome to another edition of Otel in Practice!** Today, we have Hansen and Eliah from Embrace to discuss the challenges and opportunities of data modeling and application design in mobile apps. While OpenTelemetry often focuses on backend systems, we will explore its application in client-side environments. +00:00:00 Introductions and Overview of the Session +00:01:30 Introduction to Eliah and Hansen from Embrace +00:03:40 Importance of Mobile Observability in App Performance +00:05:30 Challenges in Implementing OpenTelemetry in Mobile Apps +00:10:00 Unique Characteristics of Mobile Runtime Environments +00:15:00 Fragmentation and Device Variability in Mobile Ecosystem +00:20:00 The Role of OpenTelemetry in Mobile Observability +00:25:00 Key Differences Between Mobile and Backend Observability +00:30:00 Discussion on User Experience and Golden Signals in Mobile Apps +00:35:00 Q&A Session on Mobile Instrumentation Challenges -### Presentation Overview +These timestamps capture significant moments in the presentation and discussion, providing a clear overview for viewers. -We will begin with a presentation followed by a Q&A session. Please feel free to drop your questions in the chat, or add them to the Agile Coffee board I’ve set up. We will review the questions at the end of the talk. +# OpenTelemetry in Mobile Apps: Challenges and Opportunities -Without further ado, I'll hand it over to Hansen, who will kick things off. +**Welcome everyone to another edition of OpenTelemetry in Practice!** Today, we have Hansen and Eliah from Embrace with us. They will discuss the challenges and opportunities associated with data modeling and API design in mobile apps. While OpenTelemetry is often focused on backend systems, we will explore its application on the client side as well. -**Hansen:** Thanks! I'm actually going to pass it off to Eliah to get us started. +We'll start with a presentation, followed by a Q&A session. There’s a link in the chat where you can drop your questions, and I've set up an Agile Coffee board for you to add your queries. Feel free to vote for the questions you’d like us to address at the end. -### Introduction by Eliah Cisi +Without further ado, let's kick things off. I’ll hand it over to Hansen. -Hi everyone! My name is Eliah Cisi, and I am a product manager at Embrace. We are a mobile observability company that provides developers with SDKs to integrate into their mobile apps for real-time performance monitoring. Our main goal is to help developers keep their apps running smoothly by proactively identifying and resolving issues, ultimately improving the overall user experience. - -We are heavily invested in OpenTelemetry, having begun our transition about nine months ago. This transition has helped us provide our customers with a unified and comprehensive view of app performance, from what happens client-side on the user's mobile device all the way to the backend services that power those experiences. - -While the benefits of OpenTelemetry are numerous, we have encountered specific challenges when implementing it in a mobile environment, which is what we will discuss today. - -On the call, we also have Hansen, an Android architect at Embrace and a former mobile performance engineer at Twitter. Hansen will be doing most of the presentation today. - -### Mobile App Usage Statistics - -Let’s set some context. In 2022, the average mobile user spent a little over four hours a day on their phones, with over 90% of that time spent in native mobile apps. By the end of this year, mobile apps are expected to generate over $900 billion in revenue. If you think about your own mobile use, it’s likely that the brands or companies you interact with have a native app that you use consistently. - -Over the past 5 to 10 years, we’ve worked to improve observability for backend systems, creating various standards, notably OpenCensus and OpenTracing, which form the foundation of OpenTelemetry. However, mobile apps differ significantly from distributed systems. They are installed software running on distributed compute resources that interact with distributed systems, presenting unique challenges. - -For instance, session tracking in the mobile world can range from several seconds to several hours and may be interrupted by phone calls, network changes, or the app running in the background. Furthermore, the mobile ecosystem is highly fragmented, with multiple operating systems and countless device manufacturers. - -One customer of ours reported over 42,000 different combinations of device models and chipsets that could exist, making it clear how complex mobile observability can be. - -Historically, monitoring and observability in the mobile ecosystem have largely been proprietary systems, often starting with Firebase Crashlytics, which, while free, is highly sampled and limited in features. Serious app developers often seek more robust solutions. - -When discussing customer experience, we find that many customer-impacting Service Level Objectives (SLOs) are tied to the mobile device. Unfortunately, many developers lack adequate information that correlates this data with their work on backend reliability and resiliency. +--- -### Challenges and Opportunities in Mobile Observability +## Introduction by Eliah -Now, I’ll hand it off to Hansen to discuss some challenges and opportunities we face today. +Hi everyone! My name is Eliah Cisi, and I’m a Product Manager at Embrace. We are a mobile observability company providing developers with SDKs to integrate into their mobile apps for real-time performance monitoring. Our main goal is to help developers keep their apps running smoothly by proactively identifying and resolving issues, ultimately improving the overall user experience. -**Hansen:** Thank you, Eliah. I’m here to talk about how mobile is different in terms of observability. The crux of the problem is often about moving tooling onto mobile platforms and making them run effectively. +We are heavily invested in OpenTelemetry, having begun our transition about nine months ago. This has enabled us to provide our customers with a unified and comprehensive view of app performance, from what's happening client-side on mobile devices to the backend services that power those experiences. -There are fundamental differences in the assumptions we make about durability and the questions we ask of our tooling. Without identifying these differences, we can’t start improving the specifications or tools. +However, we discovered that while the benefits of OpenTelemetry are numerous, several specific challenges arise when implementing it in a mobile environment. That’s what we’re here to talk about today. -### Key Differences in Mobile Observability +I’ll pass it off to Hansen, who will cover most of the presentation. -1. **Dynamic Runtime Environment**: Mobile devices have unique, unpredictable environments with over 42,000 device combinations. The runtime is dynamic; for example, losing a mobile connection while in an elevator can affect app performance. +--- -2. **Fragility of Data Capture and Transmission**: Transmitting data from mobile devices to servers can be fragile. Data can be lost due to crashes or unstable network connections, leading to incomplete information. +## Presentation by Hansen -3. **User Experience-Centric Data**: Observability on mobile focuses on the user experience. Each measurement corresponds to a unique user instance, making it more than just a system health check. +Thank you, Eliah. I’m here to discuss how mobile is different in terms of observability. The crux of the problem lies in moving tools onto mobile platforms and making them run effectively. There are fundamental differences embedded in the assumptions we make regarding durability and the questions we ask of our tooling. Without identifying these differences, we cannot improve specs or tooling. -### OpenTelemetry Considerations +### Key Differences in Mobile Observability: -Now, let’s talk about OpenTelemetry. It works well as a lingua franca of observability, but its backend roots mean it was designed for a specific context. When we apply it to mobile, some parts don’t fit well. +1. **Runtime Environment**: Mobile devices have dynamic and heterogeneous runtime environments. There are 42,000 unique chipsets and device combinations, which can lead to unpredictable behavior. Factors such as losing network connection or background activity can affect app performance. -- **Spans**: While spans are great for measuring application operations in predictable environments, they may not be effective in mobile contexts where operations can be disrupted by crashes or interruptions. +2. **Capture and Transmission Pipeline**: Data capture in mobile environments is fragile. Data can be lost at multiple stages before being transmitted to servers due to crashes or unstable network connections. Even when data reaches the server, it may be delayed or out of order. -- **Telemetry Reliability**: OpenTelemetry assumes telemetry is reliably recorded and transmitted. However, that’s not necessarily the case in mobile environments. +3. **User-Centric Data**: The data we capture must center around user experience. Observations from millions of independent app instances should provide insights into individual user experiences rather than just system health. -- **Resource Strain**: Mobile devices, especially lower-end models, may struggle with the resource strain that telemetry recording and transmission can cause. +### Challenges with OpenTelemetry: -- **Understanding of Low-Level Concepts**: Many mobile engineers may not understand low-level concepts like threads or context propagation, making it important to adapt APIs for broader usability. +OpenTelemetry generally works well as a lingua franca of observability, but it was designed to solve specific backend problems. Here are some challenges we face when adapting it for mobile: -- **Execution Boundaries**: Mobile app operations can span several modules owned by different teams, making instrumentation brittle without the right infrastructure. +- **Spans**: While they are great for measuring durations in predictable environments, they fall short in mobile where operations can be interrupted or lost entirely. +- **Protocols and APIs**: Assumptions that telemetry is recorded and transmitted reliably do not hold true in mobile environments. Tools must account for unreliable data transmission and the strain telemetry puts on system resources. +- **Contextual Understanding**: Mobile engineers may not be familiar with low-level concepts like threads or context propagation. Therefore, APIs must be adapted for broader understanding. +- **Execution Boundaries**: Mobile operations can span multiple modules owned by different teams, making instrumentation brittle without proper infrastructure. -- **Independent Instances**: Mobile devices are independent instances, which complicates how we aggregate metrics and could limit their usefulness. +Lastly, mobile devices represent millions of independent instances, which complicates how we approach metrics. Metrics need to be grounded in the context of the system to be useful. -### Conclusion and Call to Action +--- -In conclusion, we’re working on addressing these challenges and would love to involve more people interested in mobile observability. Hansen is involved in the Android and client-side special interest groups (SIGs), and we have people working on Swift SIG as well. If you’re interested, we welcome your contributions! +## Call to Action -Thank you for your time! We’ll now open the floor for questions. +We encourage everyone involved in the mobile ecosystem to contribute. Hansen is involved in the Android SIG and the Client-Side SIG, and we are working on OpenTelemetry contributions for React Native and Flutter. We want to make mobile observability as robust and standardized as backend systems. --- -### Q&A Session +### Questions -**Question 1:** In web, Core Web Vitals have become a standard for measuring user experience. Do you see an emerging standard for mobile apps? +**Q: In web, Core Web Vitals have become a standard to measure user experience. Do you see an emerging standard for mobile apps?** -**Hansen:** Yes, we are working on modeling certain metrics for mobile applications, particularly around sluggishness and performance. We want to collaborate with the SIGs to define these standards. +Yes, we are working on semantic conventions for mobile that can capture metrics like ANRs and sluggishness. Once we figure out the best approach, we'll collaborate with the SIGs to submit conventions. -**Question 2:** What about applications using Kotlin Multiplatform (KMP)? What are the instrumentation options available? +**Q: What about applications using Kotlin Multiplatform? What instrumentation options are available?** -**Hansen:** KMP is similar to React Native. Currently, there isn’t a native SDK for KMP that emits OpenTelemetry telemetry. We’re evaluating how best to approach this and see potential in KMP, but we need an SDK to get started. +Currently, there is no native SDK for Kotlin Multiplatform that emits OpenTelemetry telemetry. There is potential for a native SDK or a bridge to existing SDKs. We are evaluating how best to approach this. -**Question 3:** How closely related is mobile instrumentation to browser instrumentation in tracking user journeys? +**Q: Is the variability of runtime environments an open problem?** -**Hansen:** There are more similarities between mobile and browser apps than between mobile and backend distributed traces. We should consider best practices in both areas. +Yes, it's an ongoing challenge. We are working with the entities working group to help capture external mutable states related to apps. We have some workarounds, but we're still exploring the best approaches. -**Question 4:** Do different types of mobile apps require different kinds of data models? +**Q: How closely related are mobile instrumentation and browser instrumentation?** -**Hansen:** Yes, different apps may require specific semantic conventions. For example, gaming apps may need to track frame rates closely, while social media apps may focus more on engagement metrics. +There are many similarities between mobile apps and web apps. We should consider web instrumentation when designing mobile instrumentation due to overlapping environmental factors. -**Question 5:** Do you have ideas on golden signals for mobile apps, irrespective of device type or OS version? +**Q: Do different types of mobile apps require different data models?** -**Hansen:** The ultimate golden signal is whether the user is happy. Metrics like user return rates and operation success rates are critical. +Yes, different apps have unique characteristics that require tailored semantic conventions. For instance, gaming apps may focus on frame rates, while financial apps prioritize transaction speeds. -**Question 6:** Any thoughts on resource utilization metrics like CPU or memory for mobile apps? +**Q: Are there golden signals for mobile irrespective of device type?** -**Hansen:** Resource utilization can be tricky on mobile, as other apps and the OS can affect performance. It’s crucial to capture contextual information for it to be useful. +The ultimate golden signal is user happiness, which can be assessed through user engagement and retention metrics. Tracking success rates and abandonment rates for key operations is also crucial. -**Question 7:** When might OpenTelemetry mobile instrumentation be ready for production use? +**Q: Is OpenTelemetry mobile instrumentation ready for production use?** -**Hansen:** It’s ready now, depending on your specific needs. There are SDKs available, and some apps are already in production using these solutions. +Yes, various SDKs are available that you can integrate into your mobile apps. Depending on your needs, you can roll your own implementation or use existing ones. -**Closing Thoughts:** +--- -We encourage everyone to get involved with mobile observability discussions in the SIGs. The ecosystem is continuously evolving, and your insights can help shape its future. +**Closing Thoughts** -Thank you for joining us today, and we hope to see you in future editions of Otel in Practice! +We want to get more people involved in the working groups and SIGs for mobile observability. The ecosystem is still emerging, and we need diversity in our approaches. If you're using OpenTelemetry or interested in mobile, please join us in shaping its future. -[Music] +Thank you for joining us today, and we hope to see you in the next edition of OpenTelemetry in Practice! ## Raw YouTube Transcript -[Music] well hello everyone uh welcome to another edition of otel and practice and today we've got Hansen and Eliah from Embrace uh they're going to take us through some of the challenges and some of the opportunities as well of doing a data modeling AP design in mobile apps so in open Telemetry we talk a lot about back end but uh we'll see how we can apply open Telemetry to the client site as well so um we will do a presentation first and then there will be time for Q&A and dropp in a link in the chat and then I've created a agile coffee board you can go there you can add your questions and then at the end of a talk we'll go through the Q&A can vote for your questions as well so feel free to add your questions as you're as you see in the presentation and then we'll go through them at the end so without further Ado um I'll introduce Hansen I think you're going to be picking off um so take it away great I'm actually going to pass it off to elab to kick it off nice uh hi everyone yeah like uh it was mentioned my name is elab cisi I am a product manager at Embrace uh you can go next slide Hansen we are a mobile observability company um we provide developers with sdks that they can integrate into their mobile apps to monitor performance in real time and our main goal really is to help developers keep their apps running smoothly by proactively identifying and resolving issues really with the objective of improving the overall user experience um we're also heavily uh invested in open Telemetry uh we began that transition about nine months ago and it's really helped us provide our customers with a more unified and and comprehensive view of app performance from what's Happening client side on the user's mobile device all the way to the backend services that power those experiences um and in doing that migration we discovered that while the benefits of open Telemetry are numerous there are some specific challenges that occur when trying to implement it in a mobile environment and that's kind of what we're here to to talk about today and I'll be honest and say that the term we is very generous um also on the call is Hansen um who we talked about he's an Android architect at Embrace formerly a mobile performance engineer at Twitter so I'm just here to do kind of a brief intro and then I'll hand it off to Hansen who will be doing uh a majority of of the presentation um next slide thank you um one more so I want to start by just setting a little bit of context so in 2022 uh the average mobile user spent a little over four hours a day on their phone and of that over 90% of of it or about 90% of it was within native mobile apps and by the end of this year mobile apps are expected to generate over 900 billion dollars in in revenue and if you yourself think about your own mobile use I'm willing to bet that the brand or companies that you interact with most have a native app that you use consistently now as we think about the observability ecosystem we've spent the last 5 to 10 years really trying to figure out how we do observability better for for backend systems and there are a variety of standards that were created uh most notably open census and open tracing which form the foundation for where we are today with open Telemetry to provide the visibility into the health of distributed systems um but mobile apps are not a distributed system they are installed software running on distributed compute resources that interact with distributed systems and there's a lot of unique challenges with that environment um take session tracking for instance uh in the mobile World a user session can be anything from several seconds to several hours and it can be interrupted by calls or network changes or the app running in the background which is really quite different from what you see server side mobile apps themselves are also extremely different like a gaming app for example needs to track things like frame rate and rendering times while a financial app might focus more on transaction speeds and and security events add to that the fragmentation in the mobile ecosystem where you've got multiple operating systems and countless device manufacturers and it's really easy to see how complex that becomes um we uh were looking at a number at the number of unique devices for just one customer and we saw that there was like 42,000 plus different combinations of device models and chipsets that could potentially exist which is you know crazy um and for most of its life cycle like observability and monitoring in the mobile ecosystem has been mostly proprietary systems designed by vendors like the typical starting point is Firebase crash lytics which is free but highly sampled and extremely Limited in its feature set so if you're a serious app developer you're not going to use that and that's led to a bunch of vendors building their own proprietary standards and trying to convince people that they should be serious about observability with their solution but mostly it's just crash reporting and error tracking and so the result is that the Paradigm looks something like this and the unfortunate thing is that a lot of mobile Engineers actually consider that to be kind of acceptable um but when we talk to customers who truly care about the user experience that their customers are having on their mobile devices they tell us that all of their customer impacting slos are directly tied to the mobile device but they don't have a good source of information that correlates that data to the work they're doing to build reliability and resiliency in their backend systems um and so that's what Hansen's going to be talking about which is some of the challenges in this Paradigm that we're facing today um and some of the opportunities and and how we're working to address that um but more than anything I think today is really a call to action for the people in the zoom for the people watching if you yourself or if you have co-workers or people that are interested in the mobile ecosystem we would love for you guys to get involved Hansen's involved in the Android Sig and the client side Sig we have people working in the Swift Sig uh we're working on otel contributions for react native and flutter and we're really committed to making mobile observability as robust and standardized as it is for for backend systems and we'd love to involve as many of you in that effort as possible um so uh with that I'll hand it off to to Hansen thanks aab so I am here to talk about how make how mobile is different in terms of observability I think the the first thing to approach this is that the the Crux of the problem is actually about simply moving uh tooling onto these mobile platforms and making them run because there are fundamental differences that are embedded in the assumptions that we make uh when we have durability and the questions that we ask of our tooling and without first identifying and acknowledging these differences we can't start improving the specs or or tooling because we don't know what it should look like you know to make the front of the horse uh look like the back of the horse we got to First figure out what the the face should be and and here is what I'm trying to do is is just to to go over a little bit what the differences may be uh I can spend a couple hours here talking about this but hey we don't have that much time so I'm just going to concentrate this in in three general areas where mobile is different first is the runtime environment we're talking about Dynamic heterogeneous runtime environments running you know devices that are unpredictable 42,000 unique chipsets and and device combinations not only that the environment that they run in is extremely dynamic as well you walk into an elevator and you lose your mobile connection you lose your network connection you back around the app to answer a notification that affects how the operating system runs the app not only that the actual Hardware that is running your app is quite severely limited phones exists 10 years ago still even Ed today pH are released last year that cost $99 and well the hardware uh mirrors that cost and the OS not only limits what you could do with that already limited Hardware it also does it in a very unpredictable way so what you have is is a dynamic environment where you don't know what is actually executing your app and not only that objective performance which is typically what we measure is only part of the equation because what makes nap slow differs in terms of where I'm using it or whether somebody else is using it perceived performance is also part of the equation and slos have to somehow take that into account another aspect of mobile that makes it challenging and different than backend is the fragility of the capture and transmission pipeline how to get data from these devices onto servers I could do something with it data could be lost in multiple stages uh before it gets captured because it was a crash uh before you persist the data or after we persist the data uh because the network connection is unstable and we can't get the data over to the server and even when the server gets the data it could be delayed or out of order so what you get may not be the full picture until several hours later when more data comes in thank you network connections and lastly the data we capture has to Center the user experience because operational runtime in device state is useful only in how they provide insight into individual user experiences end to end otherwise it's just trivia on mobile we're observing millions of independent app instances but each one of the measurements we get maps back to user it is not merely a state of health of the system if something is slow somebody is looking at a very slow phone it's like a P99 what does that mean well P99 is 1% of all measurements are that slow or slower so if you think that's an outlier well 1% of of of interactions being that slow means it is more than an outlier it is something that we have to capture now let's talk about otel so in general otel tell it works really well as a lingual franka of observability uh its backend Roots though mean that it was designed to solve a problem with a very specific context and when we change that context to mobile you know some part starts to not fit very well and these are the ones I'm going to talk about so first let's talk about spans spans are great on hotel if you want to measure the duration of operations of applications that have a very fix uh runtime environment um predictable runtime code path uh so you can actually measure and calculate deviations based on some baselines they are not so great if say duration is not an indicator of performance if you want to measure a period of time um well otel signal to use looks like it's spans but is it because duration is not an indicator of performance and you start aggregating and say hey what's P95 that starts to not make sense also operations run for a long time in otel well unfortunately you don't get to know what happened until it ends either in a failure uh or success but what if it gets interrupted middle uh through a crash that we don't know about uh mobile apps could be killed without actually alerting the app so operations like that are gone or lost uh and with otel itself we wouldn't know about it because it's not done and that's kind of challenging for mobile also operations that need to be contextualized with a lot of mutable State um that are expensive to obtain well that's a challenge because in in otel you write the data into the spans of span events or uh or attributes um but sometimes the act of getting these types of State takes a while uh for the mobile app to actually get from the OS and for us to basically block the the ending of a of a trace um just the right nute that's not fantastic the second point I want to talk about is that the protocols and apis of votel make certain assumptions that are just not true on mobile now for example assumption one is that Telemetry is recorded and transmitted reliably and you could trust that data that gets recorded will make make it to the server the collector in a in a reasonable amount of time but as we mentioned before that is not true in Mobile and it could also may not be true in a number of fascinating ways so tooling that that is usable on mobile has to take that into account and build a resilience like dis persistence for instance before export um you know thanks to Cesar for doing that on uh open Java or the open cry Android extension it's extremely helpful to in production um in the future perhaps there's a be place for ways for us to automatically transmit uh a group of related Telemetry so that we don't get into this you know uh uh weird state in the back end wondering if more data is coming that'd be nice another assumption is that recording and transmitting Telemetry doesn't put a strain on system resources the SDK are well written um and they perform well so on the back end use them it's no problem however on low-end devices and meter networks uh it means that every signal captured reduces and potentially reduces performance or cost users money so we have to be ex a lot more careful in how we uh record data and how we transmit data and even even simply the act of getting the data to record can be expensive so the consideration that we have to you know go into what to record is I think a lot greater and depends on the use case another assumption is that Engineers that use the API understand lowlevel Concepts like threads or or or context propagation um even the idea of tracing uh and what spans are um may not be universally understood if you change the audience to mobile Engineers simply because the background the variety of background of folks building mobile apps changes quite a bit it could be somebody who's just done a six-month boot camp who is building you know a mobile app for the local grocery store um and they want observability too they want to know why their stuff is slow but they're used to higher level constructs uh like Coe routines when you're using threads but not really they don't know about it and how do you explain propagation when they don't understand the the existence of a thread so adapting the apis to be Ematic is one thing but making those Concepts understandable by those without CS backgrounds that's a another challenge right there and lastly another assumption is that traced operations have a clear execution boundary um and code ownership so you have a service that creates a span and the runtime you know generally a team owns that you know you know uh front to back and you transmit the context via context propagation etc etc as you would in distributed tracing unfortunately things are not as clean on mobile because a single span for an operation can go through several modules owned by several teams and not all of them may be aware of all these problems so instrumentation can be extremely brittle if you don't have the right infrastructure in place to catch you know regressions where implementation changes but the instrumentation doesn't um it's not as modular nicely connected cleanly connected as you would for distributed Trace just because of of how mobile apps are architected and the last otel thing I'm going to talk about is that mobile devices are millions of independent instances uh an otel generally assumes that the system being monitored and observed is is one connected system um it could be made of uh dozens of microservices deployed in various ways but effectively they all roll up um and metrics in that context makes a lot of sense otel metrics but for mobile when we have disconnected app instances running that starts to break down because metrics need to be grounded in the context of the system that Ms them in order for the baselines to be created and compared and munging together uh metrics from phones of various models into one limits how useful those generated metrics are what does P95 of uh Heap size of an app at one minute mean when you look at the entire you know Fleet of devices I don't know did it when it changes when increases or decreases is that good or bad well I don't know because we don't know the reasons because these are different systems and not that metrics aren't useful on mobile they're extremely useful but you tend to need to have like to like comparison Apples to Apples and it tends to involve Dimensions that are high in cardinality and unfortunately that just doesn't work fantastically well with lotel metrics um and also just simply the strict time aligned aggregation is not really suitable for apps that have operations that have variable duration and have data come in at any time uh it it just wasn't meant to do what mobile wants it to do but that's okay because these are not insurmountable challenges these are these are ongoing things that we could s that we could work with uh the the protocol and and the sigs and the folks to to kind of you know improve so we're working with the in in with various sigs to try to you know get some of these problems surface and address um we are also you know you know at Embrace uh having workaround to go around otel or perhaps use otel in um uh I would say non-standard ways in order to get the data to to do what we wanted to do um but that's not the end State the end State we want to see is a diverse ecosystem and tooling that builds on uh not only what folks in Nel have done before but also extends support the plethora of use cases that we're bringing up and and frankly this is just the beginning because mobile apps that I'm familiar with run on very small restricted you know uh set of circumstances and phones and tablets is what I'm I'm used to using I I haven't worked on cars or TVs iot um but those apps deserve observability as well and I'm sure as we as we take the tooling and the in the spec forward more use cases will come on the board and say hey hey I want to I want to connect my data from my TV to my backend data well what what challenges additional challenges are there when I don't even know how a TV Works in terms of uh how it uses Android but I'm sure it's different and I'm sure there are new things um and at this point we're just exploring ourselves we're we're trying to ask questions we're trying to figure out if our assumptions are correct are there things we could change about how we used to do things and order to fit more into otel but at the end of the day we want everyone to kind of come up with their use cases and and help ask these questions um of mobile and client use cases for otel um great work has been done in the Android and Swift and and client Sig already but I think there could be more going forward um and folks listening maybe the converted the Bing otel but hopefully other people also watch this and say hey you know what I looked at otel it didn't really fit but after this presentation maybe maybe I can make it fit maybe I can use it in a way that is productive and actually truly have this become the lingual franer of observability for both the back end and the front end so that's enough of us talking um any questions uh I'm gonna end the presentation if I can there we go thank you very much Hansen um for those that have joined a bit later maybe haven't seen this message we've got an agile coffee board you can your questions in there um think we've got a couple of questions uh we'll start with one uh related to uh so we've got um in web cwe vitals have become a standard to measure user experience for better Awards do you see an emerging standard to measure equivalent Concepts in mobile apps for sluggishness or speed yes uh so you know with with Motel everything is defined effectively as semantic conventions built on the existing signals um and on and embrace we've kind of modeled some of the T Rec capture for things like anrs on Android uh uh and various other kind of Mobile you know slowness sluggishness um we did it in a certain way that you know it works but is it the best we don't know um but once once we figure it out we'll want to you know work with the sigs to to submit conventions uh to to model that and uh I believe especially with the introduction of these um you know emerging specs of of profile profiling and and even entities a lot of these problems that we have are going to be addressed um so it's it's a matter of uh figuring out what we have and then defining them and if you're interested in you know defining certain uh uh slowness uh metrics and or you know metrics Telemetry uh you know for mobile you know let's talk uh there's one that's I'm in the middle of putting together with the help of a lot of folks from the the client s um a crash uh uh stic convention um that spans platforms that's different from exceptions uh that we have many more down the pipe and is limited by the the abandonments that we have uh but anything that can be should be standardized in as semantic inventions so it's a very long way of saying yes and please help good stuff uh let's go on to another one um there are some options available for otel instrumenting iOS or Android code what about applications using cotlin multiplatform what instrumentation options are available any additional considerations when instrumenting KMP so C multiplatform is interesting uh I mean for those unfamiliar it's it's similar to to you know react native or things like that where you write it once and it kind generates na native apps um for the various platforms um I'm sure I'm getting something wrong in the technicality there but the idea is that you have one codebase multiple platforms um there isn't an SDK that I'm aware of uh that works in such a way that is built natively into cotland multiplatform um that will emit Hotel Telemetry uh whenever we have iOS or Android Android we use a Java SDK uh you know at the core and iOS uses the Swift SDK um to have cotland multiplatform either there has to be a native SDK for cot multiplatform in cotlin only that will you know transpile into the native platforms um so not only IOS and Android there's you know web and whole bunch of different platforms or there has to be a way of getting um a bridge built to the other sdks um we are we are evaluating you know at Embrace at least we're evaluating um how best to approach this um we think the native way is is the best way um but we don't know we're working through some of these problems um not specifically multiplatform but with react native um which is I think why more well known and more well adopted um so uh there's definitely uh opportunity in C multiplatform um but first we got to have an SD before we got to have a way of getting otel Telemetry working on that platform like recording first before anything else can happen so if if you're working on it you know you should start talking to people think about that because that' be really I'd be interested as well thank you I think the next question is about one of your challenges sounds like one of the core challenges for mobile observability is the variability of runtime environments software and Hardware is this entirely an open problem or are there suggestions on how to start tackling this so I think I to do it natively in otel um I think entities will have to if for for those who are who may not be familiar with that that particular working group The entities working group is is a way of I guess capturing um external mutable transition of of States um that independent from but related to apps um and I think for us we could see it capturing a lot of this variable State um without having to directly you know uh well actually the implementation is not yet I don't know maybe it could work um we have done certain things that we're not super uh uh I mean we've done certain things to work around this um that may not be accept or or enatic uh so we're using uh spans uh to log uh I guess durations of of interesting things happening um and then on the back end kind of you know merging the stuff all together uh that's not great but also say it also allows us to capture the stuff independently um and not have uh Telemetry recording be blocked um and also not have to like encode every change of network condition onto every piece of telemetry sent and have race conditions that will you know especially on mobile apps uh make the the edges blurry um so we have something working um but we're not sure if it's the best quite yet and we're really looking for entities um you know being able to do that um or help us does that answer your question I think there were two parts I might have only answer one part I'm assuming so oh uh I think we've got a few questions so I'm going to move to the next one um I know that you're part of the uh the the client side instrumentation seg so this was quite interesting one how closely or not is mobile instrumentation to browser instrumentation as far as tracking user Journeys for example if the client side instrumentation work gear towards one or the other or are the data models for both browser and mobile considered the same or similar enough I think there are differences but certainly there are a lot more similarities with uh uh web uh web apps and mobile apps then I would say uh between mobile apps and backend distributed traces um I think the differences are are more nuanced and I think there's enough similarity that anything that we should consider for web we should consider for mobile and vice versa uh there may be cases where you know it is immediately you know un uh you know this doesn't fit um but honestly you can you could run a mobile application on a mobile browser honorable device so all the environmental changes uh are are effectively uh similar that they have to deal with um so I think in that respect um they are very very similar so yeah we work very closely with the web folks moving on um what different types of mobile apps require different K kinds of data models um for example an online gaming like magic the Gathering versus a social media app like Instagram or a messaging app oh so the question is um do does it or yeah do they do they require different kinds of data models those different types of apps certainly different semantic conventions I would say um and also there are certain characteristics about the runtime that is more interesting to uh High frame rate apps like games for instance um if you using a you know I don't know Salesforce online uh frame rate you know scroll Jank is bad but you know it doesn't deter from the experience that much and also it's not as sensitive to say mobile device capabilities but if you're running a un game and your frame rate drops by half uh in critical instances you want to know about it so having refined having more um detailed data for that kind of stuff will be applicable to certain domains and not others um but I think that becomes more of a a challenge of tooling and instrumentation rather than the the spec um I think the spec as it is you know with with you know logs and spans well events actually specifically I think um and and and spans and hopefully a way of classifying spans in the future um they give us enough of the building blocks to to model these you know different use cases um so the same challenges that we have in terms of transmission things like that as long as those dealt with um I think a good portion of that stuff is is is going to be taken care of now there may be additional things that I I'm not aware of that will certainly need to to to to have different types of consideration um but I think I think before we run let's just crawl and I think getting uh unity and others those those different apps that that we may not typically think about console apps for instance you have McDonald's and you have that thing that opens for 24 hours that's a different use case than a mobile app that you background all the time after seconds um I think I think with with with the the work that we're doing now hopefully we we we'll move things forward enough so that uh we can start looking at some of these you know more more challenging uh issues like uh frame rates um on on on unity and things like that I think this one this next one relates back to some of the um earlier questions but you mention that duration is not an indicator of performance are there golden signals for mobile irrespective of device type an OS version or do we always need to take these factors into consideration I think the golden signal is whether the user is happy or not um and on on mobile uh there are indicators of whether user is happy in terms of uh whether they actually come back and use the the app again um so things like you know simply da whether the user comes back a usage rate um because just because one operation is slow it the effects could be multiplied if you have many many many many instances these slow operations you basically get fed up with the app so you know at the at the very end the very highest level whether the app is being used is a good indicator but um going lower level because that that that that level is is almost too late sometimes whether an operation succeeded um users tend to give up on operations if they things take too slow uh to load they they you know they background the app or they uh or they you know hop to another page or whatever it is so looking at abandoned rate um of of of uh of operations as usful if you're tracking you know whether particular operation is taking too long um so looking at duration in those cases even if duration is important um abandon rate is actually super important as well because you can actually have a case where your p50 p 5 goes down because more people give up so your success rate reduces uh and and you're and you're uh and your and your performance increases you're like what's that well it's because people are it's so slow people are giving up so the population is is different underneath you're you're losing people already so all the people are remaining are the fast people and in fact this is actually a key problem in Mobile performance is that people look at the current state and you're like oh yeah P99 that looks great well you haven't considered people who are for whom your app is way too slow and you can't even use it or they use it they install it it's too slow they uninstall it so you naturally filter out a whole bunch of users that would otherwise want to use your app but it's just too darn slow so this this kind of self-filling Prophecy of oh I don't have to invest um in apps because people experience uh uh them experience some high quality fast well it's because you've lost all the people and the inability to trap those to track those you've lost is is a huge I think Miss um in in Mobile observability and I think otel we have facilities um to track it uh we can't end a spand you know with with a unsuccessful ending so data can be collected to to take care of this use case and I think that to me is the most important thing to track is is not just duration it's whether or not it actually succeeded can I can I add just one other thing there just from my product hat here I think it the the golden signals that are asked in that question really depend on the type of app that you have so Hansen talk about startup and abandon rate and slow frame rates all of those can have different impacts on your users depending on the intent of your app whether it's like we talked about this earlier whether it's a gaming app or a financial services app or you know whatever that may be so if I were you know PM or working on a on a mobile app you should really go back to what are the primary key workflows that we need our users to do to make this a successful transaction and I use that term very broadly for them and then you go instrument those based on whatever that is so you know for a gaming app your golden signal may be frame rate because you know if your frame rate is as high as possible and your users are having um a really frictionless experience you see gameplay time increase you see all these other second order effects um really turn into uh uh really have positive impacts on those second order effects whereas retail app it's really about checkout right like how fast is that checkout flow for me are the operations that are that blay are checkout flow are they performant um are they reliable Etc so it really really depends on what your app is is trying to do and also the audience is important too like are they using your app because they have to um or are they using your app because they choose to if you're a game and you're slow while I can just play another game if if your if your workplace uses Microsoft oh no shouldn't say uh uses certain apps that you have no choice but to use um you know what yeah slow but you don't you don't have a choice so the golden signal there may be a little bit less than than if if folks are have have more options to go um next question I think this is yeah specific type of app as well uh do you know of some uh use cases of otel working on video streaming apps like Netflix not that I'm aware of um and if they exist it would be I think the instrumentation would be fairly bespoke because o no not that I'm aware of but it would be very interesting use case um I think we've got two more um resource utilization like CPU or memory is normally uh represented as a gauge in a Time metric um time CDs format um on a backend system do time series make sense in mobile apps at all certain time series uh I would say utilization less so um because the app is not the only utilizer of resources um uh and you could be running very you know expectedly and something higher priority the OS decides to schedule somebody you know starts a video in the corner of their tablet they have multiscreen enabled and suddenly the the fast cores are now going to the uh to the uh mobile tablet video and well your app is running and suddenly things are slow SL suddenly you are getting issues or you didn't get before um it's good to to track the the utilization of the CPU but there are probably other things you would want to know about like that that the utilization is supposed to tell you so I think the key is is can the utilization tell you information that you can't get otherwise um and I think on mobile there are just so many different things that could affect utilization um that you almost need to capture so much other contextual information for that to be useful that if you're going to do that you may as well go direct and say Hey you know uh are are we seeing lag are we seeing failures are we seeing unexpected occurrences um in in in the app um in certain instances and knowing that your app is not being prioritized that's important but you know how big of the Heap is for instance you know the Dos changes that on you so much it it you it'll GC you know out of in the most inappropriate places um simply because it wants more uh resources for other apps it'll kill your app in the background uh because another app is running and needs it and it's no fault of yours that your thing got killed faster it's something else that affects it so res socialization there may be use cases where it's useful but I think for me uh it's it's harder to use the data makes sense I think this this is the last question we've got um do you have some idea of when otel mobile instrumentation might be ready for use and production is it months is it years it's ready now depends on what you want to do uh there is uh sdks out there um you know directly the Swift SDK and the and the the various JavaScript SDS so if you want to kind of roll your own it works um the Android uh open your Android project is something Dr drag you know just drop it in your app and it'll kind of do some monitoring for you uh the Embrace sdks uh you can actually use it without using embrace you can just drop it in um configure your uh your exporters to go to your own collectors um use our implementation of of the tracing API uh to you know have instrumentation libraries you like data sessions and have it all sent to your your own your own servers without Embrace being involved uh there are I mean I'm sure there are other implementations out there as well but it depends on what you want and what you need you there are things off the shelf you can use for free and you can also roll your own with the sdks um all the challenges I was talking about really is is to build a platform that is encompassing of of all the corner cases that we want to support for our you know all our customers um if you have a specific app with a specific use case with a specific thing you want to measure um you may not need any of this other stuff and and and so also if you not share in that code if you're just kind of using it on your own well who cares about stantic conventions if no one else is going to look at that data um now you're you're quite locked in to your own instrumentation which is not a good thing but if it works for you it works for you so I would go ahead and and Fork some of these repos and just try it out they should work they they do work there are apps that are in in production with all of these sdks So Co I think that's so that's all we had so um thank you very much Hansen leab um is there any closing thoughts anything you would like to to add to finish off um I mean I just go back to kind of where we started this and I think as Hansen's been talking about through this at all is that um we really really really want to get more people involved who care about mobile um in uh the working groups in the sigs like we are opinionated and we come to it with our own experiences and the work that we've done but we're by no means the the uh you know final Arbiter of what is right and wrong in Mobile and the the only way we can work through those is having a variety of use cases um and I think that Netflix question is a good example of one that we don't deal with a lot and so we may not be close to all the intricacies that come with running a video service at that scale or or other services at that scale and so um if you uh care about mobile leave even if you not be working directly on it or if you have teams that work directly on it we would love for you guys to get involved in the sigs and and provide your perspective and input yeah the ecosystem is is is just IM merging I think for mobile there I think isn't enough diversity there AR enough kind of you know folks leveraging the sdks and and then build instrumentations for uh for Mobile use cases and mobile libraries that are popular um I think the closing thought is that if you're using otel or or if your backend is using otel and you're not um you could actually get a lot of mileage just by having talking the same language and using the same signals um I used to not think there was a ton of overlap um in terms of like well just mobile folks can just have that data and then backend folks all you have to do is encode all your your your your uh your your conditionals and conditions and your requests in the back has all that data right well it's not so easy to ingest all that crap um in a performant way at every request so I've come around to understanding the utility of having two separate sets of data that can be connected um and I think this is what it does um so if you're if you're a backend person um and your company has mobile apps um that you know use unnamed uh observability companies that may be free or maybe very not free um but that don't really talk to your backend signals well have a look at otel and see what you can get in terms of of of things that that are frankly even better especially if it's provided by folks who only do mobile um there are a lot of things that are not captured um if you just use you know folks that well I forget it I'm not gonna never blah blah blah yes join us I guess that's the thing join us on the cncf LA I'll post the link in the in the channel otel client site Telemetry we also have the otel seg and user Channel if you're an end user and you want to discuss more things about how you're approaching open Telemetry so uh yeah thanks again uh both Hansen and eliab and uh we'll hopefully see you in the next edition of fotel and practice thank you bye-bye [Music] +well hello everyone uh welcome to another edition of otel and practice and today we've got Hansen and Eliah from Embrace uh they're going to take us through some of the challenges and some of the opportunities as well of doing a data modeling AP design in mobile apps so in open Telemetry we talk a lot about back end but uh we'll see how we can apply open Telemetry to the client site as well so um we will do a presentation first and then there will be time for Q&A and dropp in a link in the chat and then I've created a agile coffee board you can go there you can add your questions and then at the end of a talk we'll go through the Q&A can vote for your questions as well so feel free to add your questions as you're as you see in the presentation and then we'll go through them at the end so without further Ado um I'll introduce Hansen I think you're going to be picking off um so take it away great I'm actually going to pass it off to elab to kick it off nice uh hi everyone yeah like uh it was mentioned my name is elab cisi I am a product manager at Embrace uh you can go next slide Hansen we are a mobile observability company um we provide developers with sdks that they can integrate into their mobile apps to monitor performance in real time and our main goal really is to help developers keep their apps running smoothly by proactively identifying and resolving issues really with the objective of improving the overall user experience um we're also heavily uh invested in open Telemetry uh we began that transition about nine months ago and it's really helped us provide our customers with a more unified and and comprehensive view of app performance from what's Happening client side on the user's mobile device all the way to the backend services that power those experiences um and in doing that migration we discovered that while the benefits of open Telemetry are numerous there are some specific challenges that occur when trying to implement it in a mobile environment and that's kind of what we're here to to talk about today and I'll be honest and say that the term we is very generous um also on the call is Hansen um who we talked about he's an Android architect at Embrace formerly a mobile performance engineer at Twitter so I'm just here to do kind of a brief intro and then I'll hand it off to Hansen who will be doing uh a majority of of the presentation um next slide thank you um one more so I want to start by just setting a little bit of context so in 2022 uh the average mobile user spent a little over four hours a day on their phone and of that over 90% of of it or about 90% of it was within native mobile apps and by the end of this year mobile apps are expected to generate over 900 billion dollars in in revenue and if you yourself think about your own mobile use I'm willing to bet that the brand or companies that you interact with most have a native app that you use consistently now as we think about the observability ecosystem we've spent the last 5 to 10 years really trying to figure out how we do observability better for for backend systems and there are a variety of standards that were created uh most notably open census and open tracing which form the foundation for where we are today with open Telemetry to provide the visibility into the health of distributed systems um but mobile apps are not a distributed system they are installed software running on distributed compute resources that interact with distributed systems and there's a lot of unique challenges with that environment um take session tracking for instance uh in the mobile World a user session can be anything from several seconds to several hours and it can be interrupted by calls or network changes or the app running in the background which is really quite different from what you see server side mobile apps themselves are also extremely different like a gaming app for example needs to track things like frame rate and rendering times while a financial app might focus more on transaction speeds and and security events add to that the fragmentation in the mobile ecosystem where you've got multiple operating systems and countless device manufacturers and it's really easy to see how complex that becomes um we uh were looking at a number at the number of unique devices for just one customer and we saw that there was like 42,000 plus different combinations of device models and chipsets that could potentially exist which is you know crazy um and for most of its life cycle like observability and monitoring in the mobile ecosystem has been mostly proprietary systems designed by vendors like the typical starting point is Firebase crash lytics which is free but highly sampled and extremely Limited in its feature set so if you're a serious app developer you're not going to use that and that's led to a bunch of vendors building their own proprietary standards and trying to convince people that they should be serious about observability with their solution but mostly it's just crash reporting and error tracking and so the result is that the Paradigm looks something like this and the unfortunate thing is that a lot of mobile Engineers actually consider that to be kind of acceptable um but when we talk to customers who truly care about the user experience that their customers are having on their mobile devices they tell us that all of their customer impacting slos are directly tied to the mobile device but they don't have a good source of information that correlates that data to the work they're doing to build reliability and resiliency in their backend systems um and so that's what Hansen's going to be talking about which is some of the challenges in this Paradigm that we're facing today um and some of the opportunities and and how we're working to address that um but more than anything I think today is really a call to action for the people in the zoom for the people watching if you yourself or if you have co-workers or people that are interested in the mobile ecosystem we would love for you guys to get involved Hansen's involved in the Android Sig and the client side Sig we have people working in the Swift Sig uh we're working on otel contributions for react native and flutter and we're really committed to making mobile observability as robust and standardized as it is for for backend systems and we'd love to involve as many of you in that effort as possible um so uh with that I'll hand it off to to Hansen thanks aab so I am here to talk about how make how mobile is different in terms of observability I think the the first thing to approach this is that the the Crux of the problem is actually about simply moving uh tooling onto these mobile platforms and making them run because there are fundamental differences that are embedded in the assumptions that we make uh when we have durability and the questions that we ask of our tooling and without first identifying and acknowledging these differences we can't start improving the specs or or tooling because we don't know what it should look like you know to make the front of the horse uh look like the back of the horse we got to First figure out what the the face should be and and here is what I'm trying to do is is just to to go over a little bit what the differences may be uh I can spend a couple hours here talking about this but hey we don't have that much time so I'm just going to concentrate this in in three general areas where mobile is different first is the runtime environment we're talking about Dynamic heterogeneous runtime environments running you know devices that are unpredictable 42,000 unique chipsets and and device combinations not only that the environment that they run in is extremely dynamic as well you walk into an elevator and you lose your mobile connection you lose your network connection you back around the app to answer a notification that affects how the operating system runs the app not only that the actual Hardware that is running your app is quite severely limited phones exists 10 years ago still even Ed today pH are released last year that cost $99 and well the hardware uh mirrors that cost and the OS not only limits what you could do with that already limited Hardware it also does it in a very unpredictable way so what you have is is a dynamic environment where you don't know what is actually executing your app and not only that objective performance which is typically what we measure is only part of the equation because what makes nap slow differs in terms of where I'm using it or whether somebody else is using it perceived performance is also part of the equation and slos have to somehow take that into account another aspect of mobile that makes it challenging and different than backend is the fragility of the capture and transmission pipeline how to get data from these devices onto servers I could do something with it data could be lost in multiple stages uh before it gets captured because it was a crash uh before you persist the data or after we persist the data uh because the network connection is unstable and we can't get the data over to the server and even when the server gets the data it could be delayed or out of order so what you get may not be the full picture until several hours later when more data comes in thank you network connections and lastly the data we capture has to Center the user experience because operational runtime in device state is useful only in how they provide insight into individual user experiences end to end otherwise it's just trivia on mobile we're observing millions of independent app instances but each one of the measurements we get maps back to user it is not merely a state of health of the system if something is slow somebody is looking at a very slow phone it's like a P99 what does that mean well P99 is 1% of all measurements are that slow or slower so if you think that's an outlier well 1% of of of interactions being that slow means it is more than an outlier it is something that we have to capture now let's talk about otel so in general otel tell it works really well as a lingual franka of observability uh its backend Roots though mean that it was designed to solve a problem with a very specific context and when we change that context to mobile you know some part starts to not fit very well and these are the ones I'm going to talk about so first let's talk about spans spans are great on hotel if you want to measure the duration of operations of applications that have a very fix uh runtime environment um predictable runtime code path uh so you can actually measure and calculate deviations based on some baselines they are not so great if say duration is not an indicator of performance if you want to measure a period of time um well otel signal to use looks like it's spans but is it because duration is not an indicator of performance and you start aggregating and say hey what's P95 that starts to not make sense also operations run for a long time in otel well unfortunately you don't get to know what happened until it ends either in a failure uh or success but what if it gets interrupted middle uh through a crash that we don't know about uh mobile apps could be killed without actually alerting the app so operations like that are gone or lost uh and with otel itself we wouldn't know about it because it's not done and that's kind of challenging for mobile also operations that need to be contextualized with a lot of mutable State um that are expensive to obtain well that's a challenge because in in otel you write the data into the spans of span events or uh or attributes um but sometimes the act of getting these types of State takes a while uh for the mobile app to actually get from the OS and for us to basically block the the ending of a of a trace um just the right nute that's not fantastic the second point I want to talk about is that the protocols and apis of votel make certain assumptions that are just not true on mobile now for example assumption one is that Telemetry is recorded and transmitted reliably and you could trust that data that gets recorded will make make it to the server the collector in a in a reasonable amount of time but as we mentioned before that is not true in Mobile and it could also may not be true in a number of fascinating ways so tooling that that is usable on mobile has to take that into account and build a resilience like dis persistence for instance before export um you know thanks to Cesar for doing that on uh open Java or the open cry Android extension it's extremely helpful to in production um in the future perhaps there's a be place for ways for us to automatically transmit uh a group of related Telemetry so that we don't get into this you know uh uh weird state in the back end wondering if more data is coming that'd be nice another assumption is that recording and transmitting Telemetry doesn't put a strain on system resources the SDK are well written um and they perform well so on the back end use them it's no problem however on low-end devices and meter networks uh it means that every signal captured reduces and potentially reduces performance or cost users money so we have to be ex a lot more careful in how we uh record data and how we transmit data and even even simply the act of getting the data to record can be expensive so the consideration that we have to you know go into what to record is I think a lot greater and depends on the use case another assumption is that Engineers that use the API understand lowlevel Concepts like threads or or or context propagation um even the idea of tracing uh and what spans are um may not be universally understood if you change the audience to mobile Engineers simply because the background the variety of background of folks building mobile apps changes quite a bit it could be somebody who's just done a six-month boot camp who is building you know a mobile app for the local grocery store um and they want observability too they want to know why their stuff is slow but they're used to higher level constructs uh like Coe routines when you're using threads but not really they don't know about it and how do you explain propagation when they don't understand the the existence of a thread so adapting the apis to be Ematic is one thing but making those Concepts understandable by those without CS backgrounds that's a another challenge right there and lastly another assumption is that traced operations have a clear execution boundary um and code ownership so you have a service that creates a span and the runtime you know generally a team owns that you know you know uh front to back and you transmit the context via context propagation etc etc as you would in distributed tracing unfortunately things are not as clean on mobile because a single span for an operation can go through several modules owned by several teams and not all of them may be aware of all these problems so instrumentation can be extremely brittle if you don't have the right infrastructure in place to catch you know regressions where implementation changes but the instrumentation doesn't um it's not as modular nicely connected cleanly connected as you would for distributed Trace just because of of how mobile apps are architected and the last otel thing I'm going to talk about is that mobile devices are millions of independent instances uh an otel generally assumes that the system being monitored and observed is is one connected system um it could be made of uh dozens of microservices deployed in various ways but effectively they all roll up um and metrics in that context makes a lot of sense otel metrics but for mobile when we have disconnected app instances running that starts to break down because metrics need to be grounded in the context of the system that Ms them in order for the baselines to be created and compared and munging together uh metrics from phones of various models into one limits how useful those generated metrics are what does P95 of uh Heap size of an app at one minute mean when you look at the entire you know Fleet of devices I don't know did it when it changes when increases or decreases is that good or bad well I don't know because we don't know the reasons because these are different systems and not that metrics aren't useful on mobile they're extremely useful but you tend to need to have like to like comparison Apples to Apples and it tends to involve Dimensions that are high in cardinality and unfortunately that just doesn't work fantastically well with lotel metrics um and also just simply the strict time aligned aggregation is not really suitable for apps that have operations that have variable duration and have data come in at any time uh it it just wasn't meant to do what mobile wants it to do but that's okay because these are not insurmountable challenges these are these are ongoing things that we could s that we could work with uh the the protocol and and the sigs and the folks to to kind of you know improve so we're working with the in in with various sigs to try to you know get some of these problems surface and address um we are also you know you know at Embrace uh having workaround to go around otel or perhaps use otel in um uh I would say non-standard ways in order to get the data to to do what we wanted to do um but that's not the end State the end State we want to see is a diverse ecosystem and tooling that builds on uh not only what folks in Nel have done before but also extends support the plethora of use cases that we're bringing up and and frankly this is just the beginning because mobile apps that I'm familiar with run on very small restricted you know uh set of circumstances and phones and tablets is what I'm I'm used to using I I haven't worked on cars or TVs iot um but those apps deserve observability as well and I'm sure as we as we take the tooling and the in the spec forward more use cases will come on the board and say hey hey I want to I want to connect my data from my TV to my backend data well what what challenges additional challenges are there when I don't even know how a TV Works in terms of uh how it uses Android but I'm sure it's different and I'm sure there are new things um and at this point we're just exploring ourselves we're we're trying to ask questions we're trying to figure out if our assumptions are correct are there things we could change about how we used to do things and order to fit more into otel but at the end of the day we want everyone to kind of come up with their use cases and and help ask these questions um of mobile and client use cases for otel um great work has been done in the Android and Swift and and client Sig already but I think there could be more going forward um and folks listening maybe the converted the Bing otel but hopefully other people also watch this and say hey you know what I looked at otel it didn't really fit but after this presentation maybe maybe I can make it fit maybe I can use it in a way that is productive and actually truly have this become the lingual franer of observability for both the back end and the front end so that's enough of us talking um any questions uh I'm gonna end the presentation if I can there we go thank you very much Hansen um for those that have joined a bit later maybe haven't seen this message we've got an agile coffee board you can your questions in there um think we've got a couple of questions uh we'll start with one uh related to uh so we've got um in web cwe vitals have become a standard to measure user experience for better Awards do you see an emerging standard to measure equivalent Concepts in mobile apps for sluggishness or speed yes uh so you know with with Motel everything is defined effectively as semantic conventions built on the existing signals um and on and embrace we've kind of modeled some of the T Rec capture for things like anrs on Android uh uh and various other kind of Mobile you know slowness sluggishness um we did it in a certain way that you know it works but is it the best we don't know um but once once we figure it out we'll want to you know work with the sigs to to submit conventions uh to to model that and uh I believe especially with the introduction of these um you know emerging specs of of profile profiling and and even entities a lot of these problems that we have are going to be addressed um so it's it's a matter of uh figuring out what we have and then defining them and if you're interested in you know defining certain uh uh slowness uh metrics and or you know metrics Telemetry uh you know for mobile you know let's talk uh there's one that's I'm in the middle of putting together with the help of a lot of folks from the the client s um a crash uh uh stic convention um that spans platforms that's different from exceptions uh that we have many more down the pipe and is limited by the the abandonments that we have uh but anything that can be should be standardized in as semantic inventions so it's a very long way of saying yes and please help good stuff uh let's go on to another one um there are some options available for otel instrumenting iOS or Android code what about applications using cotlin multiplatform what instrumentation options are available any additional considerations when instrumenting KMP so C multiplatform is interesting uh I mean for those unfamiliar it's it's similar to to you know react native or things like that where you write it once and it kind generates na native apps um for the various platforms um I'm sure I'm getting something wrong in the technicality there but the idea is that you have one codebase multiple platforms um there isn't an SDK that I'm aware of uh that works in such a way that is built natively into cotland multiplatform um that will emit Hotel Telemetry uh whenever we have iOS or Android Android we use a Java SDK uh you know at the core and iOS uses the Swift SDK um to have cotland multiplatform either there has to be a native SDK for cot multiplatform in cotlin only that will you know transpile into the native platforms um so not only IOS and Android there's you know web and whole bunch of different platforms or there has to be a way of getting um a bridge built to the other sdks um we are we are evaluating you know at Embrace at least we're evaluating um how best to approach this um we think the native way is is the best way um but we don't know we're working through some of these problems um not specifically multiplatform but with react native um which is I think why more well known and more well adopted um so uh there's definitely uh opportunity in C multiplatform um but first we got to have an SD before we got to have a way of getting otel Telemetry working on that platform like recording first before anything else can happen so if if you're working on it you know you should start talking to people think about that because that' be really I'd be interested as well thank you I think the next question is about one of your challenges sounds like one of the core challenges for mobile observability is the variability of runtime environments software and Hardware is this entirely an open problem or are there suggestions on how to start tackling this so I think I to do it natively in otel um I think entities will have to if for for those who are who may not be familiar with that that particular working group The entities working group is is a way of I guess capturing um external mutable transition of of States um that independent from but related to apps um and I think for us we could see it capturing a lot of this variable State um without having to directly you know uh well actually the implementation is not yet I don't know maybe it could work um we have done certain things that we're not super uh uh I mean we've done certain things to work around this um that may not be accept or or enatic uh so we're using uh spans uh to log uh I guess durations of of interesting things happening um and then on the back end kind of you know merging the stuff all together uh that's not great but also say it also allows us to capture the stuff independently um and not have uh Telemetry recording be blocked um and also not have to like encode every change of network condition onto every piece of telemetry sent and have race conditions that will you know especially on mobile apps uh make the the edges blurry um so we have something working um but we're not sure if it's the best quite yet and we're really looking for entities um you know being able to do that um or help us does that answer your question I think there were two parts I might have only answer one part I'm assuming so oh uh I think we've got a few questions so I'm going to move to the next one um I know that you're part of the uh the the client side instrumentation seg so this was quite interesting one how closely or not is mobile instrumentation to browser instrumentation as far as tracking user Journeys for example if the client side instrumentation work gear towards one or the other or are the data models for both browser and mobile considered the same or similar enough I think there are differences but certainly there are a lot more similarities with uh uh web uh web apps and mobile apps then I would say uh between mobile apps and backend distributed traces um I think the differences are are more nuanced and I think there's enough similarity that anything that we should consider for web we should consider for mobile and vice versa uh there may be cases where you know it is immediately you know un uh you know this doesn't fit um but honestly you can you could run a mobile application on a mobile browser honorable device so all the environmental changes uh are are effectively uh similar that they have to deal with um so I think in that respect um they are very very similar so yeah we work very closely with the web folks moving on um what different types of mobile apps require different K kinds of data models um for example an online gaming like magic the Gathering versus a social media app like Instagram or a messaging app oh so the question is um do does it or yeah do they do they require different kinds of data models those different types of apps certainly different semantic conventions I would say um and also there are certain characteristics about the runtime that is more interesting to uh High frame rate apps like games for instance um if you using a you know I don't know Salesforce online uh frame rate you know scroll Jank is bad but you know it doesn't deter from the experience that much and also it's not as sensitive to say mobile device capabilities but if you're running a un game and your frame rate drops by half uh in critical instances you want to know about it so having refined having more um detailed data for that kind of stuff will be applicable to certain domains and not others um but I think that becomes more of a a challenge of tooling and instrumentation rather than the the spec um I think the spec as it is you know with with you know logs and spans well events actually specifically I think um and and and spans and hopefully a way of classifying spans in the future um they give us enough of the building blocks to to model these you know different use cases um so the same challenges that we have in terms of transmission things like that as long as those dealt with um I think a good portion of that stuff is is is going to be taken care of now there may be additional things that I I'm not aware of that will certainly need to to to to have different types of consideration um but I think I think before we run let's just crawl and I think getting uh unity and others those those different apps that that we may not typically think about console apps for instance you have McDonald's and you have that thing that opens for 24 hours that's a different use case than a mobile app that you background all the time after seconds um I think I think with with with the the work that we're doing now hopefully we we we'll move things forward enough so that uh we can start looking at some of these you know more more challenging uh issues like uh frame rates um on on on unity and things like that I think this one this next one relates back to some of the um earlier questions but you mention that duration is not an indicator of performance are there golden signals for mobile irrespective of device type an OS version or do we always need to take these factors into consideration I think the golden signal is whether the user is happy or not um and on on mobile uh there are indicators of whether user is happy in terms of uh whether they actually come back and use the the app again um so things like you know simply da whether the user comes back a usage rate um because just because one operation is slow it the effects could be multiplied if you have many many many many instances these slow operations you basically get fed up with the app so you know at the at the very end the very highest level whether the app is being used is a good indicator but um going lower level because that that that that level is is almost too late sometimes whether an operation succeeded um users tend to give up on operations if they things take too slow uh to load they they you know they background the app or they uh or they you know hop to another page or whatever it is so looking at abandoned rate um of of of uh of operations as usful if you're tracking you know whether particular operation is taking too long um so looking at duration in those cases even if duration is important um abandon rate is actually super important as well because you can actually have a case where your p50 p 5 goes down because more people give up so your success rate reduces uh and and you're and you're uh and your and your performance increases you're like what's that well it's because people are it's so slow people are giving up so the population is is different underneath you're you're losing people already so all the people are remaining are the fast people and in fact this is actually a key problem in Mobile performance is that people look at the current state and you're like oh yeah P99 that looks great well you haven't considered people who are for whom your app is way too slow and you can't even use it or they use it they install it it's too slow they uninstall it so you naturally filter out a whole bunch of users that would otherwise want to use your app but it's just too darn slow so this this kind of self-filling Prophecy of oh I don't have to invest um in apps because people experience uh uh them experience some high quality fast well it's because you've lost all the people and the inability to trap those to track those you've lost is is a huge I think Miss um in in Mobile observability and I think otel we have facilities um to track it uh we can't end a spand you know with with a unsuccessful ending so data can be collected to to take care of this use case and I think that to me is the most important thing to track is is not just duration it's whether or not it actually succeeded can I can I add just one other thing there just from my product hat here I think it the the golden signals that are asked in that question really depend on the type of app that you have so Hansen talk about startup and abandon rate and slow frame rates all of those can have different impacts on your users depending on the intent of your app whether it's like we talked about this earlier whether it's a gaming app or a financial services app or you know whatever that may be so if I were you know PM or working on a on a mobile app you should really go back to what are the primary key workflows that we need our users to do to make this a successful transaction and I use that term very broadly for them and then you go instrument those based on whatever that is so you know for a gaming app your golden signal may be frame rate because you know if your frame rate is as high as possible and your users are having um a really frictionless experience you see gameplay time increase you see all these other second order effects um really turn into uh uh really have positive impacts on those second order effects whereas retail app it's really about checkout right like how fast is that checkout flow for me are the operations that are that blay are checkout flow are they performant um are they reliable Etc so it really really depends on what your app is is trying to do and also the audience is important too like are they using your app because they have to um or are they using your app because they choose to if you're a game and you're slow while I can just play another game if if your if your workplace uses Microsoft oh no shouldn't say uh uses certain apps that you have no choice but to use um you know what yeah slow but you don't you don't have a choice so the golden signal there may be a little bit less than than if if folks are have have more options to go um next question I think this is yeah specific type of app as well uh do you know of some uh use cases of otel working on video streaming apps like Netflix not that I'm aware of um and if they exist it would be I think the instrumentation would be fairly bespoke because o no not that I'm aware of but it would be very interesting use case um I think we've got two more um resource utilization like CPU or memory is normally uh represented as a gauge in a Time metric um time CDs format um on a backend system do time series make sense in mobile apps at all certain time series uh I would say utilization less so um because the app is not the only utilizer of resources um uh and you could be running very you know expectedly and something higher priority the OS decides to schedule somebody you know starts a video in the corner of their tablet they have multiscreen enabled and suddenly the the fast cores are now going to the uh to the uh mobile tablet video and well your app is running and suddenly things are slow SL suddenly you are getting issues or you didn't get before um it's good to to track the the utilization of the CPU but there are probably other things you would want to know about like that that the utilization is supposed to tell you so I think the key is is can the utilization tell you information that you can't get otherwise um and I think on mobile there are just so many different things that could affect utilization um that you almost need to capture so much other contextual information for that to be useful that if you're going to do that you may as well go direct and say Hey you know uh are are we seeing lag are we seeing failures are we seeing unexpected occurrences um in in in the app um in certain instances and knowing that your app is not being prioritized that's important but you know how big of the Heap is for instance you know the Dos changes that on you so much it it you it'll GC you know out of in the most inappropriate places um simply because it wants more uh resources for other apps it'll kill your app in the background uh because another app is running and needs it and it's no fault of yours that your thing got killed faster it's something else that affects it so res socialization there may be use cases where it's useful but I think for me uh it's it's harder to use the data makes sense I think this this is the last question we've got um do you have some idea of when otel mobile instrumentation might be ready for use and production is it months is it years it's ready now depends on what you want to do uh there is uh sdks out there um you know directly the Swift SDK and the and the the various JavaScript SDS so if you want to kind of roll your own it works um the Android uh open your Android project is something Dr drag you know just drop it in your app and it'll kind of do some monitoring for you uh the Embrace sdks uh you can actually use it without using embrace you can just drop it in um configure your uh your exporters to go to your own collectors um use our implementation of of the tracing API uh to you know have instrumentation libraries you like data sessions and have it all sent to your your own your own servers without Embrace being involved uh there are I mean I'm sure there are other implementations out there as well but it depends on what you want and what you need you there are things off the shelf you can use for free and you can also roll your own with the sdks um all the challenges I was talking about really is is to build a platform that is encompassing of of all the corner cases that we want to support for our you know all our customers um if you have a specific app with a specific use case with a specific thing you want to measure um you may not need any of this other stuff and and and so also if you not share in that code if you're just kind of using it on your own well who cares about stantic conventions if no one else is going to look at that data um now you're you're quite locked in to your own instrumentation which is not a good thing but if it works for you it works for you so I would go ahead and and Fork some of these repos and just try it out they should work they they do work there are apps that are in in production with all of these sdks So Co I think that's so that's all we had so um thank you very much Hansen leab um is there any closing thoughts anything you would like to to add to finish off um I mean I just go back to kind of where we started this and I think as Hansen's been talking about through this at all is that um we really really really want to get more people involved who care about mobile um in uh the working groups in the sigs like we are opinionated and we come to it with our own experiences and the work that we've done but we're by no means the the uh you know final Arbiter of what is right and wrong in Mobile and the the only way we can work through those is having a variety of use cases um and I think that Netflix question is a good example of one that we don't deal with a lot and so we may not be close to all the intricacies that come with running a video service at that scale or or other services at that scale and so um if you uh care about mobile leave even if you not be working directly on it or if you have teams that work directly on it we would love for you guys to get involved in the sigs and and provide your perspective and input yeah the ecosystem is is is just IM merging I think for mobile there I think isn't enough diversity there AR enough kind of you know folks leveraging the sdks and and then build instrumentations for uh for Mobile use cases and mobile libraries that are popular um I think the closing thought is that if you're using otel or or if your backend is using otel and you're not um you could actually get a lot of mileage just by having talking the same language and using the same signals um I used to not think there was a ton of overlap um in terms of like well just mobile folks can just have that data and then backend folks all you have to do is encode all your your your your uh your your conditionals and conditions and your requests in the back has all that data right well it's not so easy to ingest all that crap um in a performant way at every request so I've come around to understanding the utility of having two separate sets of data that can be connected um and I think this is what it does um so if you're if you're a backend person um and your company has mobile apps um that you know use unnamed uh observability companies that may be free or maybe very not free um but that don't really talk to your backend signals well have a look at otel and see what you can get in terms of of of things that that are frankly even better especially if it's provided by folks who only do mobile um there are a lot of things that are not captured um if you just use you know folks that well I forget it I'm not gonna never blah blah blah yes join us I guess that's the thing join us on the cncf LA I'll post the link in the in the channel otel client site Telemetry we also have the otel seg and user Channel if you're an end user and you want to discuss more things about how you're approaching open Telemetry so uh yeah thanks again uh both Hansen and eliab and uh we'll hopefully see you in the next edition of fotel and practice thank you bye-bye diff --git a/video-transcripts/transcripts/2024-08-08T05:55:11Z-otel-q-a-feat-eliab-sisay-and-austin-emmons-of-embrace.md b/video-transcripts/transcripts/2024-08-08T05:55:11Z-otel-q-a-feat-eliab-sisay-and-austin-emmons-of-embrace.md index 28a617b..6abb826 100644 --- a/video-transcripts/transcripts/2024-08-08T05:55:11Z-otel-q-a-feat-eliab-sisay-and-austin-emmons-of-embrace.md +++ b/video-transcripts/transcripts/2024-08-08T05:55:11Z-otel-q-a-feat-eliab-sisay-and-austin-emmons-of-embrace.md @@ -10,113 +10,104 @@ URL: https://www.youtube.com/watch?v=A-t1hZdh7JY ## Summary -In this YouTube video, Elli and Austin from Embrace discuss mobile observability and their integration of OpenTelemetry into their software development kits (SDKs). Embrace is a mobile observability company that provides tools for developers to monitor app performance in real-time, aiming to enhance user experience by proactively identifying issues. They share insights on the architecture of their SDKs, which are built using Swift for iOS and Kotlin for Android, and emphasize the importance of OpenTelemetry in creating a unified observability framework. The conversation touches on challenges encountered during the integration process, such as managing dependencies and ensuring adherence to semantic conventions. They also discuss the educational efforts required to help developers understand the importance of observability in mobile applications and highlight their contributions to the OpenTelemetry community. Key topics include the differences between mobile and backend observability, the chaotic nature of mobile environments, and the need for clear communication and documentation in the observability space. +In this YouTube video, hosts Elli and Austin from Embrace discuss mobile observability and their integration of OpenTelemetry into their SDKs. Embrace, a mobile observability company, focuses on providing developers with tools to monitor app performance in real-time, ultimately enhancing user experience. They explain their architecture, which utilizes Swift for iOS and Kotlin for Android, and discuss the transition to OpenTelemetry, emphasizing its importance in creating a unified observability framework. The conversation covers challenges with mobile environments, the necessity of education on observability for developers, and the collaborative nature of contributing to the OpenTelemetry community. Key points include the benefits of a common language in observability, the unique challenges of mobile app development, and ongoing efforts to improve crash reporting standards within the OpenTelemetry ecosystem. -# Embrace Q&A Session Transcript +## Chapters -[Music] +Here are the key moments from the YouTube livestream along with their timestamps: -Welcome to the Embrace Q&A session! We have some folks from Embrace here, so let’s get started. +00:00:00 Introductions and team roles at Embrace +00:03:15 Overview of what Embrace does as a mobile observability company +00:05:30 Description of Embrace's architecture and tech stack +00:09:15 Discussion on the integration of OpenTelemetry into Embrace's SDKs +00:13:00 Explanation of the challenges faced when adopting OpenTelemetry +00:17:30 Overview of how Embrace uses OpenTelemetry for mobile observability +00:21:45 Benefits of using OpenTelemetry, including common language for observability +00:26:00 Embrace's contributions to the OpenTelemetry community +00:30:00 Discussion on resources for learning OpenTelemetry +00:35:00 Audience Q&A session on mobile observability and crash reporting -**Introduction** +Feel free to use these timestamps to navigate through the livestream! -- **Elli**: My name is Elli, and I lead product for our data collection and ingestion team at Embrace. My team is responsible for a suite of mobile SDKs across iOS, Android, Unity, Flutter, and React Native. - -- **Austin Emmans**: I’m Austin Emmans, a developer on the iOS team. I lead part of the iOS SDK development and collaborate with other teams regarding OpenTelemetry. +# Embrace Q&A Transcript Clean-Up -**What Does Embrace Do?** +Welcome to the Embrace Q&A session! We have a team from Embrace here to discuss their mobile observability products and integration with OpenTelemetry. -Elli: We are a mobile observability company that provides developers with SDKs to monitor performance in real time. Our main goal is to help developers keep their apps running smoothly by proactively identifying and resolving issues, ultimately improving the user experience. We are heavily invested in OpenTelemetry, and we began that transition about nine months ago, which has allowed us to provide our customers with a more unified view of app performance—everything from client-side events on the user's mobile device to the backend services powering those experiences. +## Introductions -We open-sourced all of our SDKs, and we encourage everyone to check them out in our GitHub repo, play around with them, and provide feedback. +**Elli:** +Hi, I'm Elli. I lead product for our data collection and ingestion team at Embrace. My team is responsible for a suite of mobile SDKs across iOS, Android, Unity, Flutter, and React Native. -Austin: It's cool that as a mobile observability company, we're essentially dogfooding by using OpenTelemetry. It’s exciting that we didn’t start in that direction but moved toward it. +**Austin Emmans:** +I'm Austin Emmans, a developer on the iOS team, focusing on the iOS SDK development and communication with other teams regarding OpenTelemetry. My main focus is on Apple platforms. -**Technical Architecture** +## What Does Embrace Do? -Elli: Can you describe the architecture that Embrace uses, including the programming languages and deployment environment? +Elli: +We are a mobile observability company providing developers with SDKs that integrate into their mobile apps to monitor performance in real time. Our main goal is to help developers keep their apps running smoothly by proactively identifying and resolving issues to improve user experience. We are heavily invested in OpenTelemetry, which we've transitioned to over the last nine months. This has allowed us to provide our customers with a unified view of app performance from the client's mobile device to the backend services. We've open-sourced all of our SDKs, and we encourage everyone to check them out in our GitHub repository, play around with them, and provide feedback. -Austin: Sure! On the iOS side, we rewrote the SDK with a Swift-first approach, employing a Swift Package Manager project structure. The Android SDK takes a Kotlin-first approach, but it still has a lot of existing Java code to maintain familiarity for those transitioning from Java. Our backend follows a standard microservice architecture, deploying using Argo CD and primarily running in a Kubernetes cluster. We also utilize different data stores like ClickHouse and Cassandra, with the backend code primarily written in Go, supplemented by some Python. +## Embrace Architecture and Technology -**OpenTelemetry Integration** +Elli: +I can describe our architecture. I focus mostly on the iOS side, where we rewrote the iOS SDK with a Swift-first approach, using a Swift Package Manager project structure. The Android side has a Kotlin-first approach, with a lot of existing Java. We ensure that the interface is familiar for developers transitioning from Java. -Elli: I’m excited to hear about your OpenTelemetry integration. How did you learn about OpenTelemetry, and why did you decide to integrate it into your product? +Our backend follows a standard microservice architecture, deploying using Argo CD and primarily running in a Kubernetes cluster. We use various data stores depending on how we're accessing the data, including ClickHouse and Cassandra. The backend code is mostly written in Go, with some Python sprinkled in. -Austin: About five or six years ago, there were various open standards leading to OpenTelemetry, most notably OpenCensus and OpenTracing. When those combined, we saw the community rally around OpenTelemetry as the standard for modern observability practices. We decided to adopt OpenTelemetry because it aligned perfectly with our vision of modernizing observability through open standards. It provides a transparent and flexible way to collect data, essential for creating a unified observability framework. +## OpenTelemetry Integration -Conversations with customers revealed that one of the biggest challenges was integrating insights from user-facing web and mobile applications into their overall observability practices. In an ideal world, frontend teams would collect data on user experience while backend teams would monitor infrastructure health. However, companies often use separate tools that don’t share a common telemetry framework, which is counterproductive. +Elli: +I’m excited to discuss our integration with OpenTelemetry. We were inspired by earlier open standards like OpenCensus and OpenTracing, which led to OpenTelemetry. We saw the community coalesce around this standard for modern observability practices, fitting perfectly with our vision of modernizing observability through open standards. -We wanted to solve this problem by collaborating with the OpenTelemetry community, focusing on mobile observability. Our goal is to provide developers with a comprehensive view of their app's performance. +We decided to use OpenTelemetry because it provides a transparent, portable, and flexible way to collect data, which is essential for creating a unified observability framework that ties the frontend and backend of applications together. We noticed that one of the biggest challenges developers face is integrating insights from user-facing applications with their observability practices. -**SDKs and Core Products** +By collaborating with the OpenTelemetry community, we aim to drive the future of open standards for observability, specifically within mobile. Our goal is to provide developers with a comprehensive view of their app's performance. Today, we collect detailed technical and behavioral information for every user session with our OpenTelemetry-compatible SDKs. -Elli: Do you have OpenTelemetry integrated into your core product as well? +Austin: +One of the significant benefits we’ve seen is that by using OpenTelemetry, we can help developers avoid vendor lock-in. Our SDKs allow users to integrate with any backend of their choice or other observability tools. -Austin: Yes, our SDKs are our core product. We also have a frontend dashboard. One of the features is called Data Destinations, where we collect metrics in our backend and export them as OpenTelemetry signals. This allows users not using the SDK to send data to us and receive signals from our backend to integrate with other providers. +## Challenges with OpenTelemetry -**Telemetry Interaction** +Elli: +When we first decided to implement OpenTelemetry in our SDKs, we faced several challenges. One was the unfamiliarity with the OpenTelemetry Swift SDK and the hesitation around adding a new dependency. We had to do due diligence to ensure it fit our expectations and could handle edge cases effectively. -Elli: How do you interact with the telemetry coming from applications and services in your organization? +Austin: +A unique challenge in mobile observability is the chaotic environment. Mobile apps are affected by many factors beyond our control, such as battery life and network status. We need to ensure our SDKs can recover gracefully from these issues. -Austin: We have internal dashboards, including our own Grafana instance. Our DevOps team runs reports and monitors standard metrics like uptime, response times, and throughput. As an SDK developer, I can pull reports and generate visualizations to analyze how our customers use the product, which helps us improve documentation and identify areas for support. +## Contributions to OpenTelemetry -**Observability and User Interaction** +Austin: +I started attending the client-side SIG meetings a few months ago. I volunteered to represent iOS after noticing a lack of representation. The discussions have been great for sharing ideas and proposals, such as formalizing user session events and how to handle crash telemetry. -Elli: It’s fascinating how observability aids not just in troubleshooting but also in understanding user interactions with the system. +## Audience Questions -Austin: Absolutely! Observability can provide insights into user behavior, which is often overlooked in discussions about performance. +**Paige Cruz:** +What learning resources do you recommend for understanding OpenTelemetry? -**Challenges with OpenTelemetry Integration** +Elli: +We have a book club around the "Learning OpenTelemetry" book by Ted Young and Austin Parker. This book is excellent for understanding observability concepts, especially as they relate to mobile. -Elli: When you first implemented OpenTelemetry in your SDKs, what challenges did you encounter? +Austin: +From a visual perspective, seeing a trace tree laid out can help make observability concepts more tangible. -Austin: It was a process to integrate OpenTelemetry, primarily due to the uncertainty of bringing in a new dependency. We faced some resistance from the team regarding whether this was a dependency we wanted to manage. We had to ensure that it fit our expectations, test edge cases, and verify performance. +**Audience Member:** +What’s being developed in terms of crash events in OpenTelemetry? -Another challenge was recognizing prior art and adhering to existing semantic conventions. Ensuring our instrumentation aligned with OpenTelemetry standards required thorough discussions and careful consideration. +Austin: +We're currently working on a proposal to represent crashes as events, which is particularly challenging in the mobile space due to the variability in crash reports across different platforms. We aim to formalize this to make it easier for developers to send and parse crash data. -**Rewrites and Modernization** +## Conclusion -Elli: Did you update your existing SDK or completely rewrite it? +Elli: +Thank you, everyone, for joining us today! It was great discussing mobile observability and OpenTelemetry with you all. -Austin: For the iOS team, we seized the opportunity to rewrite it in Swift, moving away from the previous Objective-C SDK. The Android team maintained their core codebase while modernizing the architecture to incorporate OpenTelemetry. +Austin: +We appreciate your time and interest. Looking forward to more discussions in the future! -**Advantages of Embrace SDKs** +--- -Elli: Why should developers choose Embrace SDKs over existing OpenTelemetry SDKs tailored for mobile? - -Austin: Our SDKs simplify the integration process, managing a lot of the setup and minimizing the overhead for developers. We also provide mobile-specific instrumentation, automating aspects that developers wouldn’t have to worry about. If we create useful instrumentation, we aim to contribute that back upstream to benefit the community. - -**Benefits of OpenTelemetry** - -Elli: What benefits have you seen since switching to OpenTelemetry? - -Austin: The main benefit has been the establishment of a common language, making discussions about telemetry concepts much easier. This has helped break down silos internally, allowing us to focus on the insights we provide to customers. - -**Engagement with the Community** - -Elli: Austin, can you share your involvement with the OpenTelemetry community? - -Austin: I started attending the client-side SIG meetings about five or six months ago. I volunteered to represent the iOS perspective, and it has been a great experience to share ideas and collaborate on proposals. - -**Resources for Learning OpenTelemetry** - -Paige Cruz: What learning resources do you recommend for someone looking to learn OpenTelemetry? - -Elli: The "Learning OpenTelemetry" book by Ted Young and Austin Parker is required reading at Embrace. We’ve started a book club around it to help everyone in the company understand observability concepts. - -Austin: Visual representations of traces were particularly enlightening for me. Seeing my code in action made the concepts more tangible. - -**Challenges in Mobile Observability** - -Austin: The biggest challenge in mobile observability is the chaotic environment—app developers do not control external factors like battery life or network status. We need to handle these variables and ensure performance remains optimal. - -**Wrap-Up** - -Elli: Thank you, Austin, for sharing your insights, and thanks to everyone who joined us today. - -[Music] +Thank you for attending this Q&A session on mobile observability and OpenTelemetry! ## Raw YouTube Transcript -[Music] welcome to otel Q&A we've got folks from Embrace here so um I guess let's uh why don't you introduce yourselves uh I guess I can start my name is Elli I lead product for our data collection and ingestion team here at Embrace uh my team is responsible for a suite of mobile sdks so across iOS Android Unity flutter and react native uh and I'm Austin emmans I'm uh developer on the iOS team or uh you know the leads part of the iOS SDK development um and you know talks to the other teams just about open Telemetry in general but uh my main focus is is Apple platforms awesome and can uh can one of you tell us uh like what what does embrace do yeah totally um so we are a mobile observability company um we provide developers with sdks that integrate into their mobile apps to monitor performance in real time um our main goal is to help developers keep their apps running smoothly by proactively identifying and resolving issues uh really with the objective of improving the overall uh user experience we're heavily invested in open Telemetry which I know we'll kind of dig into a little bit later uh but we began that transition about nine months ago and it's really helped us provide our customers with a more unified comprehensive view of app performance from what's Happening client side on the user's mobile device all the way to um the backend services that power those experiences and so by doing that we really enabled develop to understand the full impact of any issues and optimize uh their apps more effectively we've op sourced all of our sdks that I talked about earlier and we would encourage everyone to go check those out in our GitHub repo um play around with them and we'd love feedback awesome and it's really cool that you know as a mobile observability company you are basically dog fooding I guess uh by by using open Telemetry and it's cool that it sounds like you didn't start in that direction but you moved in that direction and I can't wait to dig into that a little bit uh more um before we get into that um can uh can you describe the architecture um that Embrace uses and any like any of the programming languages that are being used like deployment environment like what's your Tech landscape look like uh I could take that one so um being a SD I I focus mostly on on the iOS side of things um and as we took on this journey we uh kind of rewrote the the iOS SDK with a Swift first approach and that that takes on um you know a swift package manager project structure um the Android side is similar it takes on the cotlin first approach um there's a lot of existing Java in that in that SDK and and they um do a really good job at making sure that the interface on the Android side kind of if you're a cin developer you should be familiar with it if you're coming from calling in from a Java uh a piece of java code then then that interface should feel like Java there as well um and that's you know the sdks our backend though um it's standard microservice architecture um we deploy using Argo CD and um primarily run in kubernetes cluster um and then we have a couple of different data stores depending on um how we're accessing the data that include uh click house and Cassandra um and then the the code for the back end is mostly go um with some python you know sprinkled sprinkled in here and there cool awesome and then and and now that uh we've got some of the background I'm I'm really excited to hear about um your open Telemetry integration so first off how did you learn about open Telemetry and then why did you decide to like integrate open Telemetry into the product yeah um so I think as like recently as five six years ago there's Pro there's a variety of Open Standards that kind of led to otel I think most notably open census and open tracing and once those combined into open Telemetry um we just saw the community really coales around that as the standard for modern observability practices and it really um and we decided to use open Telemetry because that fit perfectly with our vision of modernizing observability through Open Standards so open Telemetry providing a transparent portable flexible way to collect data we felt was really essential for creating a unified observability framework that ties both the front end and the back end um of applications um as we were talking to customers we kept hearing that one of the biggest challenges that sres and developer teams consistently faced was marrying uh insights from their user-facing web and mobile applications to into their observability practice all up um and so I think in an Ideal World your front end teams are collecting data about what's happening uh on to the end user the health of your end user experiences and your backend teams are collecting data about the uh health of infrastructure and services and today it's common for those to be entirely separate tools they don't share a common set of tele Telemetry aren't interoperable and really prevent engineering teams from speaking the same language um and companies really want to work on what matters most and it requires understanding where to invest their engineering resources to deliver the best user experiences and so we really saw that as an opportunity um we wanted to help solve that challenge by collaborating with the open Telemetry Community to drive the future of Open Standards for observability specifically within our expertise of mobile um and our goal is to provide developers with a comprehensive view of their apps performance so today we collect the full Technical and behavioral details of every user session with our open Telemetry compatible sdks and users can even extend that instrumentation to any custom library in their app using open Telemetry signals and then leverage our platform to contextualize the added instrumentation and because we're using open Telemetry at the core of our sdks they can easily integrate with any back end of their choice or other observability tools really giving users of our sdks more flexibility to help them avoid being to one vendor now of course we think our product as some pretty cool stuff for uh mobile development teams in terms of workflow and helping them understand data and resolve things quickly but really we think you know contributing to the community and helping mobile engineering teams modernized and helping Sr and devops teams be able to take that data and make sense of it as part of their entire system will ultimately Empower them to build better more resilient mobile apps now as as a followup question um so you mentioned that you have like open Telemetry built into to your sdks do you have open Telemetry built into your core product as well we uh so I would say the sdks are our core product that that are customers use we also have a frontend dashboard yeah so front end yeah front end is what I'm thinking of yeah um I don't know Austin can you speak to that I'm more involved side um the are some like features that we have like uh it's called Data destinations where we would collect metrics um or some of the metrics that we aggregate in our backend we then export as open Telemetry signals um and so if you're not using the SDK and exporting the open Telemetry signals directly from you know a user's device um you might be sending data to us and then getting signals sent from from our back end to uh you know wherever you configure what whatever other provider you might be using uh to hopefully get the mobile data and your backend data uh you know in the same place gotcha gotcha and um similar on a similar vein like how do you interact with the Telemetry that's coming from the from the applications and services in in your organization yeah so we have a um a bunch of internal dashboards uh and so we host our own grafana instance um and can you know run reports um monitor the standard things that you would on a on a back end uptime um you know response times throughput um and we have all of that uh you know our our devops team and backend teams have those monitors and then me as a front- end developer um and kind of uh or front end an SDK developer um looking at how our customers use the product um I I can pull reports and generate my own visualizations just to see you know is this feature being used um how is it being used how how dirty is this data if they're if they're you know adding custom properties to um their Telemetry then um is that you know are they using it in in quote unquote the correct way or the the way we'd expect um and uh that is really insightful just in terms of okay what documentation do we need to provide how should we help people um because uh sometimes if if you're not looking at this data then then you're kind of at a loss or you're just guessing um and so yeah for internal visibility it's it's mostly internal grafana dashboards um and and running our own custom queries against those um awesome and you know you touched upon something that's so interesting and that we don't hear too much about in the context of observability which is because we always think like observability helps us with troubleshooting but observability also has that added benefit of like understanding how users interact with the system and it's really really cool that um that is one of the things that that you're doing with the with the Telemetry data that you're Gathering so I wanted to call that out because I think that's very interesting and it's not very often talked about um all right well um on to some meteor stuff um when when you first decided to implement open Telemetry in your SDK um what were some of the challenges that you encountered um yeah I mean it was a process not that we really had to convince ourselves I think there was a a big upwell and a big push um but just just from a software point of view it's tough to bring in a dependency to a project um because it's there it's an unknown um at that point in time we we were un pretty unfamiliar with the open Telemetry Swift SDK itself um in that package and so you have to bring it in and and there was some push back on the team on is this a dependency that we want to manage um we had to do our due diligence and some very tedious work to make sure that it fit how we expected it to um make sure that any edge cases that we could think of uh we could test and verify that it it would work how we expected it to and then also make sure that the performance um you know matched the the scale that we were expecting um and so that's just very tedious work especially when you're compressing it into some deadlines um but that's that's not really open Telemetry specific that's just software and dependencies uh in general so a a challenge that was probably more specific to open telem and open and and observability is their um you know our first instinct is when we instrument something okay let's go instrument and and get it done um and you kind of have to catch yourself and say oh there's actually some prior art here and so let's go search for these um and so you know open cemetry has the specification and the shape of the data model and what it should be and then on top of that there are all these existing semantic conventions on how to use something and how to fit into a system and you know when you record a span that's a network request what attributes you should use for that Network request where the URL goes um what the name of that span is and so you know because we are you know Allin we want to make sure we adhere to those um to to those semantic conventions and and it's tough when there's something very similar to what you're instrumenting uh and you kind of have have to have the discussions okay is this something new is this the same should we leverage some of that and and massage it a little bit to fit our our needs um and that's also just kind of a a you know a tedious process to to make sure that uh you're not stepping on any toes but you're innovating and you're you're ultimately getting what you need done yeah and in in so when you when you did this was it like were you um just updating your existing SDK or was this like a complete rewrite when you made this decision to use open Telemetry um so for the iOS team um we we took the opportunity to rewrite into Swift um and so we had an existing SDK observability SDK that was mostly Objective C um and being a larger shift we were able to take this opportunity to really modernize it um and and you know present it to Apple developers in the year 2024 our our Android team um their SDK was uh a little younger if if that's a term you can use for SDK but um uh they were able to just maintain that that same you know the core code base and then start shifting uh just the data model piece and and I think they also took the opportunity to to modernize a lot of the architecture because um you know it changes a bunch when you're pulling in uh something like the open Telemetry St K and uh there's a lot of Concepts in there that are very useful that uh you know you either massage to to fit or you just completely real place um and so it it it was different for each platform but uh but yeah cool now some I'm sure some folks who are listening in or who will listen to this in the future might be wondering also like you know open Telemetry does have already some sdks tailored for mobile so why uh what's what's the advantage of of using like the Embrace sdks in in that case yeah it's it's really just to to simplify um we use the sdks uh as dependencies they underly like what we do and and we're trying to add a layer on top of that um to either make it a little more accessible you know we manage a lot of the setup process for uh the SDK uh and try to simplify and streamline that but then we also just try to minimize the the overhead the mental overhead of okay when I'm creating a span what do I need to do and and hopefully that's one simple call in into irdk instead of you know getting your a tracer provider building a tracer from that taking that um and creating a span Builder and then and then once you have that span so there's there's a couple of steps that the the sdks themselves have and that's part of the spec and and there's good separation there's reason for that separation um but we want to streamline that and then that's just you know if you're manually instrumenting things we also want to add our own instrumentation that is mobile specific um that is the space that we're we're really in and and we are experts of and so when you're on an iOS device there are things that occur there that is very specific to the iOS system or just users of that application um that doesn't really apply to a backend system uh and so we want to make sure that we can automate that instrumentation um so that the users uh you know don't need to reimplement that or uh don't need to worry about it and there's definitely a push for us to if we create that instrumentation to get it Upstream um because the goal is it benefits the community and then it's kind of off our maintenance burden um and so I mentioned there were two layers of the spec and those semantic conventions the Third on top of that is the Embrace semantic conventions that we're hopefully keeping as as thin as possible and we're pushing down into those otel semantic conventions when when we do create or think of new things that's awesome so basically like your SDK serve as a rapper but then also like any any sort of things that you you come up with that would benefit the community then you you uh contribute them back Upstream which is very cool and I I think you know that really speaks to the power of community because I think open Telemetry success is is due to the fact that we have so much support from from various vendors who have all decided no we're not going to like try to do our own thing and and work on an island we're going to collaborate so I think I I really want to emphasize that stuff like this is is extremely beneficial to the community um now you know when when you switched your sdks to using open Telemetry what kind of benefits did you uh did you start seeing as a uh for me it was the common language and um it became very easy to discuss um with people what open Telemetry is using you know a trace a span uh a log and what those you know glossery items are uh and one thing that uh really bugged me before is pretty much every vendor models these Concepts but calls them kind of things um and we you know were doing that before too with so we're at fault as well um and so being able to use just the common envelope and uh a common language that was a massive benefit when we started switching to open Telemetry just you know at the non-technical level um can I can I add one thing to that I think uh from like an organizational perspective it also helped to break down some silos for us internally in that even across our sdks across the different platforms prior to open Telemetry our Swift SDK our Android SDK react native Unity Etc we're all kind of built with specific uh implementation details for those platforms in mind and so kind of lived a little bit in a silo but when you think about our customers a customer of ours that has an iOS or an Android app they don't they build it specific to those platforms but in terms of performance their mobile teams want to understand that a consumer is having a great experience regardless of whether they're on an iOS device or an Android device and I think building the underlying sdks in silos internally um or I should say the other way when we move to open Telemetry being able to use those common signals helped us break down those silos such that we could really focus on the insights that we're giving um to our customers which is really the ultimate value that that we aim to provide that's awesome yeah that's a really great point um now I want to switch gears a little bit and because you know um Austin you mentioned that uh there you make some contributions Upstream um for the um to the client side Telemetry say what uh so you know tell us about uh your involvement with with that Sig and how how did you start getting involved with that Sig as well yeah um so I started attending the client side Sig um probably five or six months months ago um right at the beginning of 2024 uh my coworker Hansen was had been joining those before me and and um uh I had been joining the otel Swift Sig uh meetings before that as well and and somebody just popped their head into the a swift Sig meeting and just said hey we have this client side Sig there's not really any iOS representation would somebody please like volunteer to come hang out with us uh and I volunteered and um they' been part of that that Sig since um and joining every every week um and it's been great I mean it's it's Community ideas are shared we talk about what's going on what proposals um are happening or or what problems people are trying to solve and um you know when the time comes it's that could just be you know unmuting yourself on this Zoom call and and sharing an idea or even just sharing well this is how it works for me um that could be going into the GitHub repos for some of the semantic conventions and just adding a comment and starting the discussion there or responding and continuing the discussion there um or could be going into those semantic convention repos and proposing um some new things and so uh Hansen has done a better job than I have but uh he's LED some semantic prevention proposals for things that are mobile specific or client side specific some things like like um a user session is as something that's a little formalized and and standardized um a little bit more than it is or uh a big thing for mobile apps is when a mobile app crashes um how do we model that as as Telemetry um and now that the events um the Standalone event kind of data model has has taken shape in the spec um there's a semantic invention around uh semantic invention proposal around modeling uh client side crashes as as an event um and so that'll be very useful because we deal with crash reporting as well at Embrace um and to formalize that and and say okay here's how other people interpret crash reports um that gives that flexibility to to send that data anywhere um and so it's it's again ridding ourselves of that vendor lock in um because it's it's burdensome when when you have it yeah that's so awesome and so when uh like in terms of like starting to attend those those otel uh meetings um did you did you start doing that when you uh when the decision was made to like start using otel internally like to move the SDK to using otel like when when when did that come about I I would say the decision was made when we started switch is like okay we're we want to take this on we want to be part of this the community um I procrastinated a little bit just because I'm a person and I I didn't actually start joining until the beginning of the year but but that's just you know the realistic approach I guess um uh and and it's tough because you don't think that you have anything to contribute um and so there's a uh impostor syndrome that happens and uh it takes a bit to rid yourself of that and I'm here to tell you rid yourself of that please join um we love hearing voices from from all types of you know people using the the the tools that that are provided and so um you know come even just come and watch you can just hang out and uh chat um and then uh if if something that comes up that that you're comfortable with then then unmute yourself and and and join the chat verbally and so um yeah that that was kind of how it came about I guess that's great and I thank you for calling that out as well specifically because I I I think it's so common um when people join open source projects like the you're right the impostor syndrome is is very real it's very scary you are being so vulnerable there and and especially when you join a group that seems to be like very well established and is it appears to be full of very intelligent people and then you start thinking oh my God am I you know am I worthy of this and and the answer as you said it's yes you are you are worthy of it um yeah everyone everyone has a contribution to make and it it can be any kind of contribution I mean you know I can't emphasize enough like so so often um I have past me has has been like you know using something from an open source project and I'll like be following whatever in the documentation and and the docs are wrong and and my first reaction is to be mad oh my God these idiots don't know what they're doing and you know what like turn that frown upside down because like what you should be doing if you if if you notice an issue in the documentation if there's a boo boo um something is wrong there's nothing that prevents you from filing a pull request to fix that documentation to prevent other people from getting frustrated from it being wrong so I mean that is the simplest thing that you can do and it makes such a huge impact so it doesn't have to necessarily be code it can be documentation and it can even it can be a minor correction so that's that's my little PSA there yeah that that was exactly our approach too so it was good like awesome we want some first issues let's go read the documentation is there anything that we can adjust or make more clear so exactly exactly yeah that's so great I I love hearing these stories and and I feel like these are the types of things that we just need to keep repeating over and over in in our community and and and tell people yes this this is a great way to contribute um you know we we talked about the benefits that you um that you you saw with um integrating open Telemetry um into your sdks what are some of the challenges um that you've seen um in open Telemetry especially like in in the in the mobile space like now now that you've you know U dipped your toes a little bit more more into that in in the community um I would say the the biggest challenge that we faced is really um you know we we this layer above the open Telemetry SDK and we are trying to streamline the process of of getting into observability and it's mostly because mobile developers just aren't familiar with this territory um they might what's interesting is a lot of developers are and there are a lot of existing tools out there that that will log events and things but um we're still trying to just flatten the learning curve uh for observability for people and so a lot of the challeng is just educating and figuring out okay what what clicks for people why is this um why is this valuable why is this useful um and then going from there and then you know educating them on how to you use our our product specifically um and so that's probably the one of the biggest non-technical challenges um the technically the the biggest challenge is that in the mobile space the environment is just chaotic it's it's something an app developer does not control um this is not a a server on in a box on some you know air conditioned shelf um that you have on Key and Lock uh it is in somebody's pocket hopefully um it could be on a nightstand you have no idea on the network status you have no idea how much battery is left on that device um or how angry that operating system is and whether or not it's just going to kill your app for who knows what reason and and so um the variability is is uh everpresent and and we need to be able to recover if any anything goes wrong so we have to handle things like okay we're trying to write Stu locally to dis so we can recover this data but there's no dis dis space What do we do um you know the battery is at 2% what do we do uh and it's just it's just a little more hectic than than you would expect or hope for if you're an app developer and and really the worst the worst thing is if if you're an app developer that's worried about performance um these things are out of your control but the user is going to blame you if if the app is not performing well and so uh if they are on 2% battery and they're you know rushing to to call a car to get to wherever they need to be um you need to make sure that they can complete that operation before the system just like okay I'm done time to sleep uh and so it's uh yeah it's just it's just crazy with with what we do yeah it's definitely a whole different ball game uh in in the mobile space compared to like the non-mobile space where where I guess it's it's a little less chaotic it's it's it feels like more unknowns in the in the mobile space that you're you're having to deal with right yeah and we're we're monitoring more than just the applications performance as well there's just how the user is interacting with with this um this application and what their behavior is is is something that you don't really have to deal with if it's you know if you are in a microservice architecture there's very consistent entry points very consistent uh exit points um in in any client side application not just mobile specific but even in a web browser um you know the user might be doing something on the page that that you just you don't know how they got there they came in with state that you didn't expect uh and so now you need to understand um you know how did you actually get to this issue I I can't reproduce it works on my machine but that's that's not an excuse can use so y yeah absolutely um now uh switching back to like you know um talking about contributing to open Telemetry um you know you you mentioned um getting getting started in in contributing um what you know now that that you've been doing it for for several months um how how has it been like you know one of the things that are um our Sig does is we want to understand what the contribution experience is for open Telemetry so we can provide that kind of feedback back to the sigs um to really um to really improve that right because if if we don't if we if if they're not aware of the feedback what what what can they do to improve so what are what are some of the some of the you know tell us the good the bad the ugly uh that's that you've experienced um with with uh you know becoming a a contributor um so it's it's good I I don't really have any bad or ugly um which uh maybe that's just me but um with joining the Swift Sig I think it was like a week or two in and these are weekly meetings so my second or third meeting um we were discussing an issue and and um nacho I think just said oh hey Austin can you take a stab and look at this um and uh I was like like o okay sure like you trust me to do that that that's that's awesome um and so that was kind of a a good like uh a sign of good faith and and confidence um and uh and then you kind of go from there it's like okay now what can I do to help out um and so some of the things that we're trying to do to help out are are push um proposals Upstream into the semantic conventions um make updates to the documentation to to try to flatten that learning curve so that you know you have specific examples on how to instrument uh you know on iOS what you know maybe what this view controller is doing or um you know making a network request with with URL session um these are very common patterns for iOS developers and um it just it makes it very easy if there's a snippet out there that you can just go grab and then and then tweak into in your application and so those are the the types of contributions that were um trying to make and and you know in terms of like feedback for the the sigs it's I don't know be be less friendly I I don't have much feedback it's um it's been great so that's awesome that that's really great to hear um and in terms of like I I know you said like it's been it's been a great experience so far do you have anything like that you'd want to uh improve that you you could see like maybe this could use a little tweak um I I don't know if it's an improvement but I It Like H finding that prior art is tough sometimes um and and mostly because you know some of these are proposals that are ongoing um a lot of things are just marked experimental and you don't know um how experimental that thing is um and and it's supposed to go I think experimental beta stable there there are two or three stages for for every proposal or or specification change and um and so making sure that those signals are are up to date um I think would be very useful and helpful as a proposal goes through the process but um you know that's just bookkeeping so it if you're joining and you're you're talking about things that's that's something that if you just ask a question someone would would help you out very quickly as well awesome awesome if I can add one thing there I think Austin Austin touches on something that took me a little bit to to kind of get used to as we started working with open Telemetry which is um it does take a bit longer than you would normally expect to get contributions accepted and move through that process like Austin is talking about but I think one thing to be aware of is that is by Design and I think when you're used to working like we're at a startup and so for us we try to make really quick decisions that concept of of oneway versus two-way doors and Implement them and see what happens and so working in a community like this it does take more consensus because we want that obviously the goal of this is to be a unified um solution that anyone can come use and and by design that means decisions take a little longer than you might be used to coming into this as someone that's new um and so I think that's just something to be aware of but but realize that that is part of what it means to create a standard that anyone can come in and use and really understand and is Thoroughly vetted so um that's something that people should probably be um aware of there yeah that's actually a really great call out and thanks for pointing that out um so we're we're finished now with the main questions uh portion so does anyone from uh from our audience have questions that they would like to ask yes Paige hello I am really curious what learning resources you would recommend if it was a particular blog or metaphor or tutorial that helped have that aha moment for you both um and then the kind of followup is any helpful metaphors is that you use for teaching your customers and kind of end users paig real quick before we answer that can can you tell us just who you are and and what your goal is yeah let me let me come on hello I'm Paige Cruz I work as a principal developer Advocate over at Kronos year um I used to be in Sr mostly focused on those beautiful backend systems the microservices running in beautiful air condition data centers and so I'm looking to learn more about mobile um I'm very interested in making sure observability is useful for everybody up and down the stack and mobile is a area that I need to brush up on and y'all are the experts so I was here to learn yeah thank you just wanted to um make sure I knew who I was talking to so appreciate that um actually it's funny that you mention blogs and books and resources it is required reading it embrace the learning open Telemetry book we literally have that oh you can't see it now but because my background but by uh Tedd young and Austin Parker um awesome we've started a book club around that at Embrace this is everyone in the company and I think it addresses particularly in mobile one of the other challenges that we face that's not technical which is um when you think about observability Concepts in open Telemetry it's been primarily focused on backend to date and so taking those Concepts and introducing them to front-end and mobile developers and mobile teams and why they're important and inand observability as a whole is a challenge so before we can even get them to start implementing our Solutions or understanding or uh or helping them understand why they're important they need to understand the fundamental concepts first and so before we can help them with that we need to understand those as well and so in our journey over the last nine months it's been various levels of Education through our entire org not just myself and Austin who are involved deeply in the sdks but our CSM teams our go to market teams our um sdrs all of them so we can all kind of speak the same language and understand the benefits of of open Telemetry not just for backend observability but for client side and mobile as well um Austin I don't know if you have other on oh and we intend a bunch of conferences too so that's great just being in the same space as people and and learning what they're going through has been super helpful yeah I I can't remember like when it flicked um but I think I'm a very visual person and a very Hands-On learner and so I think it probably the first time I saw a trace as a trace tree like a visualized out in in a timeline um I think that's when it was just like oh I this is this is my code running right now and I can see it and I can see the differences here you know if I'm comparing two that that should be identical but you know aren't because of stuff um you know that's when it was like oh okay this is this is interesting and useful awesome uh do anyone have any other questions yeah I was curious you were talk Austin I think you were talking about you know a lot of the challenges um with the mobile environment just being chaotic and you know a lot of things being out of control and um I think related to that you mentioned the data model for crash events and I was just kind of curious like what um what's going on like with all that like are you like yeah what's in development and kind of what are the current topics in that area um so I uh it was open to Elementry day this week so I didn't attend the client Sig this week uh because most people were were in person um but the latest I heard which would be as of last week um you know uh nine days ago um The Proposal is up in the GitHub repo um and uh the pr has been made I believe and we're maybe it's just in a Google doc but there's a document shared that describes a crash report as an event um the the big thing events are built on top of logs and log records Hanson's in chat he says proposal not up and PR not up yet but soon winky face so uh uh heads and nose um so yeah we we're discussing and there will be a proposal very soon um to use an event events are built on logs um but the key difference is there's event name um attribute that you can key off of and and so the value of that attribute attribute is is meant to represent kind of the schema of the log body and then I think also um the other attributes in in in that log record um and so if you start calling an event with a name you know this is a client crash um then the body can take a shape that is a little more formal um and uh what's interesting and what we're kind of working through is not all crash reports are the same um in a mobile environment um you know on on Android you have the the jvm which is you know the Java virtual machine and that runtime um then there's the native crashes below that um and those are mostly like C C++ um crash reports iOS you have Swift uh crash reports there's some C++ crash reports that are kind of weird and then we're started we we have these Hy apps react native and um uh the others that are are interesting because you actually you might get two crash reports and two stack traces uh for a single crash and and there's some weirdness there just because you know if a react native crashes it it might crash at the bridge layer um or in the JavaScript and and then it throws an exception down to the native code um and so you want to capture kind of all of this stuff and it's it's very different um and so what part of what's going into the proposal is okay how do we um how do we represent this in in a format that's consumable um but also understand that it can be very different um and when when a an otel collector receives this data um you know how should they start even trying to parse this out uh and so definitely recommend checking out the proposal as soon as it's up um sorry Hansen for putting the pressure on you but uh but uh uh we're looking forward to it because it it it's one of the biggest things that when we came into otel we we kind of asked ourselves well how are we going to represent crashes um and we we struggled with that um but uh we found a way we just kind of shove everything in as a log record right now and we're like this isn't good enough this could be better let's let's try to make this better interesting okay yeah I guess I didn't um think about you know these were things like low battery as being something that would affect my apps of course I just you know like you said stupid app and not yeah so well and low battery is one where where the systems will start throttling uh the processor so instead of a lot of mobile devices now there are these high efficiency processor cores and then the like you know the AI Crazy Fast processor cores uh and so when the system sees that the battery is getting low it's it starts throttling things and pushing things to those high efficiency uh CPUs and so um that has a downstream effect of your logic is now less performant um because it's it's either bottlenecked and waiting behind something or just the processor is slower it's just naturally not um not as quick as it would if it was you know charging or or had having a full battery interesting yeah thank you for sharing I'm sure I'll have more questions once I've processed this more but yeah this is this is good thanks for the question oh no problem well we prepare to wrap up we we will turn the tables and allow you to ask us questions okay I'm curious uh I think you know Paige Reese I think both of you are uh I think Reese your developer relations engineer page you work as an Sr to the other attendees as well I'm not sure exactly your roles but I'd be curious to what extent mobile observability has is part of your current observability practice um and how that fits in or how you marry that with your backend observability um that you have in place o yeah I mean I don't personally monitored any mobile applications um and you know I work with Adriana in the andies Sig and just you know anecdotally having heard a lot of end users being interested in this space like you know like Paige I want to learn more about it because I just don't really know much about mobile observability and I'm really interested yeah honestly I think it was um learning that Embrace existed kind of I'm like oh of course you'd want to do like observability for your mobile applications as well so that was you know honestly it wasn't until that was when I made the mental connection of like duh we should but why why w why aren't we talking about it so I I see this as an opportunity also to like really bring awareness to folks that of course you should be doing observability for your mobile apps yeah that triggers you said something earlier that I meant to that noted down for this question section what like what tactics have you found work best in that regard so for instance we have deep expertise in Mobile um we've just started working in open Telemetry just like over the last last like six to nine months and we want to bring that mobile expertise to the open Telemetry Community but I think there's a there's a thing around education from the mobile perspective that we also have to do that I think relate to a lot of the questions that we talked about today what have you guys found tactics that you guys have found to be particularly impactful in that regard whether it's something you guys have done or you've seen done um in terms of just um educating others on the kind of nuances of for us it'll be mobile but it can be whatever platform right or it could be open Telemetry or whatever um C would be interested to hear about that I would say like talks blogs um um yeah I mean apply to talk at conferences that'll bring bring awareness for sure blog posts YouTube videos get on people's podcasts um is another way to just bring that awareness yeah I don't know if uh any of the other folks have any additional thoughts for me it was really eye openening um at an org we sent all of our application data to the same observability platform and I was mostly focused on the kind of website website and I just for funsies looked at the mobile side just to see what's going on there just poking around and I saw oh my God like 90% of our requests came in through our IOS and Android apps and I was like I should just honestly ignore the website because to the business the impact was would be so much greater if we were to make these optimizations and that business observability is not always talked about inside org so really even just asking folks what's the split between um entry points from Mobile versus web that can really be a thread to start pulling on and then you also get your managers and other parts of the business like PMS um also interested and invited to the party and then last thing is at srecon this year I posted a link in the chat Isabelle gave a great talk called The Invisible front door reliability gaps in the front end and touched a bit on um mobile monitoring and how sres really need to turn I um towards this space so that's awesome thank you for sharing that that's great thanks Paige well we are almost out of time so um I thank you uh Austin and iLab for uh for joining us today this has been really great really educational and um thank you everyone who who joined us as well um see you for otel and practice [Music] +welcome to otel Q&A we've got folks from Embrace here so um I guess let's uh why don't you introduce yourselves uh I guess I can start my name is Elli I lead product for our data collection and ingestion team here at Embrace uh my team is responsible for a suite of mobile sdks so across iOS Android Unity flutter and react native uh and I'm Austin emmans I'm uh developer on the iOS team or uh you know the leads part of the iOS SDK development um and you know talks to the other teams just about open Telemetry in general but uh my main focus is is Apple platforms awesome and can uh can one of you tell us uh like what what does embrace do yeah totally um so we are a mobile observability company um we provide developers with sdks that integrate into their mobile apps to monitor performance in real time um our main goal is to help developers keep their apps running smoothly by proactively identifying and resolving issues uh really with the objective of improving the overall uh user experience we're heavily invested in open Telemetry which I know we'll kind of dig into a little bit later uh but we began that transition about nine months ago and it's really helped us provide our customers with a more unified comprehensive view of app performance from what's Happening client side on the user's mobile device all the way to um the backend services that power those experiences and so by doing that we really enabled develop to understand the full impact of any issues and optimize uh their apps more effectively we've op sourced all of our sdks that I talked about earlier and we would encourage everyone to go check those out in our GitHub repo um play around with them and we'd love feedback awesome and it's really cool that you know as a mobile observability company you are basically dog fooding I guess uh by by using open Telemetry and it's cool that it sounds like you didn't start in that direction but you moved in that direction and I can't wait to dig into that a little bit uh more um before we get into that um can uh can you describe the architecture um that Embrace uses and any like any of the programming languages that are being used like deployment environment like what's your Tech landscape look like uh I could take that one so um being a SD I I focus mostly on on the iOS side of things um and as we took on this journey we uh kind of rewrote the the iOS SDK with a Swift first approach and that that takes on um you know a swift package manager project structure um the Android side is similar it takes on the cotlin first approach um there's a lot of existing Java in that in that SDK and and they um do a really good job at making sure that the interface on the Android side kind of if you're a cin developer you should be familiar with it if you're coming from calling in from a Java uh a piece of java code then then that interface should feel like Java there as well um and that's you know the sdks our backend though um it's standard microservice architecture um we deploy using Argo CD and um primarily run in kubernetes cluster um and then we have a couple of different data stores depending on um how we're accessing the data that include uh click house and Cassandra um and then the the code for the back end is mostly go um with some python you know sprinkled sprinkled in here and there cool awesome and then and and now that uh we've got some of the background I'm I'm really excited to hear about um your open Telemetry integration so first off how did you learn about open Telemetry and then why did you decide to like integrate open Telemetry into the product yeah um so I think as like recently as five six years ago there's Pro there's a variety of Open Standards that kind of led to otel I think most notably open census and open tracing and once those combined into open Telemetry um we just saw the community really coales around that as the standard for modern observability practices and it really um and we decided to use open Telemetry because that fit perfectly with our vision of modernizing observability through Open Standards so open Telemetry providing a transparent portable flexible way to collect data we felt was really essential for creating a unified observability framework that ties both the front end and the back end um of applications um as we were talking to customers we kept hearing that one of the biggest challenges that sres and developer teams consistently faced was marrying uh insights from their user-facing web and mobile applications to into their observability practice all up um and so I think in an Ideal World your front end teams are collecting data about what's happening uh on to the end user the health of your end user experiences and your backend teams are collecting data about the uh health of infrastructure and services and today it's common for those to be entirely separate tools they don't share a common set of tele Telemetry aren't interoperable and really prevent engineering teams from speaking the same language um and companies really want to work on what matters most and it requires understanding where to invest their engineering resources to deliver the best user experiences and so we really saw that as an opportunity um we wanted to help solve that challenge by collaborating with the open Telemetry Community to drive the future of Open Standards for observability specifically within our expertise of mobile um and our goal is to provide developers with a comprehensive view of their apps performance so today we collect the full Technical and behavioral details of every user session with our open Telemetry compatible sdks and users can even extend that instrumentation to any custom library in their app using open Telemetry signals and then leverage our platform to contextualize the added instrumentation and because we're using open Telemetry at the core of our sdks they can easily integrate with any back end of their choice or other observability tools really giving users of our sdks more flexibility to help them avoid being to one vendor now of course we think our product as some pretty cool stuff for uh mobile development teams in terms of workflow and helping them understand data and resolve things quickly but really we think you know contributing to the community and helping mobile engineering teams modernized and helping Sr and devops teams be able to take that data and make sense of it as part of their entire system will ultimately Empower them to build better more resilient mobile apps now as as a followup question um so you mentioned that you have like open Telemetry built into to your sdks do you have open Telemetry built into your core product as well we uh so I would say the sdks are our core product that that are customers use we also have a frontend dashboard yeah so front end yeah front end is what I'm thinking of yeah um I don't know Austin can you speak to that I'm more involved side um the are some like features that we have like uh it's called Data destinations where we would collect metrics um or some of the metrics that we aggregate in our backend we then export as open Telemetry signals um and so if you're not using the SDK and exporting the open Telemetry signals directly from you know a user's device um you might be sending data to us and then getting signals sent from from our back end to uh you know wherever you configure what whatever other provider you might be using uh to hopefully get the mobile data and your backend data uh you know in the same place gotcha gotcha and um similar on a similar vein like how do you interact with the Telemetry that's coming from the from the applications and services in in your organization yeah so we have a um a bunch of internal dashboards uh and so we host our own grafana instance um and can you know run reports um monitor the standard things that you would on a on a back end uptime um you know response times throughput um and we have all of that uh you know our our devops team and backend teams have those monitors and then me as a front- end developer um and kind of uh or front end an SDK developer um looking at how our customers use the product um I I can pull reports and generate my own visualizations just to see you know is this feature being used um how is it being used how how dirty is this data if they're if they're you know adding custom properties to um their Telemetry then um is that you know are they using it in in quote unquote the correct way or the the way we'd expect um and uh that is really insightful just in terms of okay what documentation do we need to provide how should we help people um because uh sometimes if if you're not looking at this data then then you're kind of at a loss or you're just guessing um and so yeah for internal visibility it's it's mostly internal grafana dashboards um and and running our own custom queries against those um awesome and you know you touched upon something that's so interesting and that we don't hear too much about in the context of observability which is because we always think like observability helps us with troubleshooting but observability also has that added benefit of like understanding how users interact with the system and it's really really cool that um that is one of the things that that you're doing with the with the Telemetry data that you're Gathering so I wanted to call that out because I think that's very interesting and it's not very often talked about um all right well um on to some meteor stuff um when when you first decided to implement open Telemetry in your SDK um what were some of the challenges that you encountered um yeah I mean it was a process not that we really had to convince ourselves I think there was a a big upwell and a big push um but just just from a software point of view it's tough to bring in a dependency to a project um because it's there it's an unknown um at that point in time we we were un pretty unfamiliar with the open Telemetry Swift SDK itself um in that package and so you have to bring it in and and there was some push back on the team on is this a dependency that we want to manage um we had to do our due diligence and some very tedious work to make sure that it fit how we expected it to um make sure that any edge cases that we could think of uh we could test and verify that it it would work how we expected it to and then also make sure that the performance um you know matched the the scale that we were expecting um and so that's just very tedious work especially when you're compressing it into some deadlines um but that's that's not really open Telemetry specific that's just software and dependencies uh in general so a a challenge that was probably more specific to open telem and open and and observability is their um you know our first instinct is when we instrument something okay let's go instrument and and get it done um and you kind of have to catch yourself and say oh there's actually some prior art here and so let's go search for these um and so you know open cemetry has the specification and the shape of the data model and what it should be and then on top of that there are all these existing semantic conventions on how to use something and how to fit into a system and you know when you record a span that's a network request what attributes you should use for that Network request where the URL goes um what the name of that span is and so you know because we are you know Allin we want to make sure we adhere to those um to to those semantic conventions and and it's tough when there's something very similar to what you're instrumenting uh and you kind of have have to have the discussions okay is this something new is this the same should we leverage some of that and and massage it a little bit to fit our our needs um and that's also just kind of a a you know a tedious process to to make sure that uh you're not stepping on any toes but you're innovating and you're you're ultimately getting what you need done yeah and in in so when you when you did this was it like were you um just updating your existing SDK or was this like a complete rewrite when you made this decision to use open Telemetry um so for the iOS team um we we took the opportunity to rewrite into Swift um and so we had an existing SDK observability SDK that was mostly Objective C um and being a larger shift we were able to take this opportunity to really modernize it um and and you know present it to Apple developers in the year 2024 our our Android team um their SDK was uh a little younger if if that's a term you can use for SDK but um uh they were able to just maintain that that same you know the core code base and then start shifting uh just the data model piece and and I think they also took the opportunity to to modernize a lot of the architecture because um you know it changes a bunch when you're pulling in uh something like the open Telemetry St K and uh there's a lot of Concepts in there that are very useful that uh you know you either massage to to fit or you just completely real place um and so it it it was different for each platform but uh but yeah cool now some I'm sure some folks who are listening in or who will listen to this in the future might be wondering also like you know open Telemetry does have already some sdks tailored for mobile so why uh what's what's the advantage of of using like the Embrace sdks in in that case yeah it's it's really just to to simplify um we use the sdks uh as dependencies they underly like what we do and and we're trying to add a layer on top of that um to either make it a little more accessible you know we manage a lot of the setup process for uh the SDK uh and try to simplify and streamline that but then we also just try to minimize the the overhead the mental overhead of okay when I'm creating a span what do I need to do and and hopefully that's one simple call in into irdk instead of you know getting your a tracer provider building a tracer from that taking that um and creating a span Builder and then and then once you have that span so there's there's a couple of steps that the the sdks themselves have and that's part of the spec and and there's good separation there's reason for that separation um but we want to streamline that and then that's just you know if you're manually instrumenting things we also want to add our own instrumentation that is mobile specific um that is the space that we're we're really in and and we are experts of and so when you're on an iOS device there are things that occur there that is very specific to the iOS system or just users of that application um that doesn't really apply to a backend system uh and so we want to make sure that we can automate that instrumentation um so that the users uh you know don't need to reimplement that or uh don't need to worry about it and there's definitely a push for us to if we create that instrumentation to get it Upstream um because the goal is it benefits the community and then it's kind of off our maintenance burden um and so I mentioned there were two layers of the spec and those semantic conventions the Third on top of that is the Embrace semantic conventions that we're hopefully keeping as as thin as possible and we're pushing down into those otel semantic conventions when when we do create or think of new things that's awesome so basically like your SDK serve as a rapper but then also like any any sort of things that you you come up with that would benefit the community then you you uh contribute them back Upstream which is very cool and I I think you know that really speaks to the power of community because I think open Telemetry success is is due to the fact that we have so much support from from various vendors who have all decided no we're not going to like try to do our own thing and and work on an island we're going to collaborate so I think I I really want to emphasize that stuff like this is is extremely beneficial to the community um now you know when when you switched your sdks to using open Telemetry what kind of benefits did you uh did you start seeing as a uh for me it was the common language and um it became very easy to discuss um with people what open Telemetry is using you know a trace a span uh a log and what those you know glossery items are uh and one thing that uh really bugged me before is pretty much every vendor models these Concepts but calls them kind of things um and we you know were doing that before too with so we're at fault as well um and so being able to use just the common envelope and uh a common language that was a massive benefit when we started switching to open Telemetry just you know at the non-technical level um can I can I add one thing to that I think uh from like an organizational perspective it also helped to break down some silos for us internally in that even across our sdks across the different platforms prior to open Telemetry our Swift SDK our Android SDK react native Unity Etc we're all kind of built with specific uh implementation details for those platforms in mind and so kind of lived a little bit in a silo but when you think about our customers a customer of ours that has an iOS or an Android app they don't they build it specific to those platforms but in terms of performance their mobile teams want to understand that a consumer is having a great experience regardless of whether they're on an iOS device or an Android device and I think building the underlying sdks in silos internally um or I should say the other way when we move to open Telemetry being able to use those common signals helped us break down those silos such that we could really focus on the insights that we're giving um to our customers which is really the ultimate value that that we aim to provide that's awesome yeah that's a really great point um now I want to switch gears a little bit and because you know um Austin you mentioned that uh there you make some contributions Upstream um for the um to the client side Telemetry say what uh so you know tell us about uh your involvement with with that Sig and how how did you start getting involved with that Sig as well yeah um so I started attending the client side Sig um probably five or six months months ago um right at the beginning of 2024 uh my coworker Hansen was had been joining those before me and and um uh I had been joining the otel Swift Sig uh meetings before that as well and and somebody just popped their head into the a swift Sig meeting and just said hey we have this client side Sig there's not really any iOS representation would somebody please like volunteer to come hang out with us uh and I volunteered and um they' been part of that that Sig since um and joining every every week um and it's been great I mean it's it's Community ideas are shared we talk about what's going on what proposals um are happening or or what problems people are trying to solve and um you know when the time comes it's that could just be you know unmuting yourself on this Zoom call and and sharing an idea or even just sharing well this is how it works for me um that could be going into the GitHub repos for some of the semantic conventions and just adding a comment and starting the discussion there or responding and continuing the discussion there um or could be going into those semantic convention repos and proposing um some new things and so uh Hansen has done a better job than I have but uh he's LED some semantic prevention proposals for things that are mobile specific or client side specific some things like like um a user session is as something that's a little formalized and and standardized um a little bit more than it is or uh a big thing for mobile apps is when a mobile app crashes um how do we model that as as Telemetry um and now that the events um the Standalone event kind of data model has has taken shape in the spec um there's a semantic invention around uh semantic invention proposal around modeling uh client side crashes as as an event um and so that'll be very useful because we deal with crash reporting as well at Embrace um and to formalize that and and say okay here's how other people interpret crash reports um that gives that flexibility to to send that data anywhere um and so it's it's again ridding ourselves of that vendor lock in um because it's it's burdensome when when you have it yeah that's so awesome and so when uh like in terms of like starting to attend those those otel uh meetings um did you did you start doing that when you uh when the decision was made to like start using otel internally like to move the SDK to using otel like when when when did that come about I I would say the decision was made when we started switch is like okay we're we want to take this on we want to be part of this the community um I procrastinated a little bit just because I'm a person and I I didn't actually start joining until the beginning of the year but but that's just you know the realistic approach I guess um uh and and it's tough because you don't think that you have anything to contribute um and so there's a uh impostor syndrome that happens and uh it takes a bit to rid yourself of that and I'm here to tell you rid yourself of that please join um we love hearing voices from from all types of you know people using the the the tools that that are provided and so um you know come even just come and watch you can just hang out and uh chat um and then uh if if something that comes up that that you're comfortable with then then unmute yourself and and and join the chat verbally and so um yeah that that was kind of how it came about I guess that's great and I thank you for calling that out as well specifically because I I I think it's so common um when people join open source projects like the you're right the impostor syndrome is is very real it's very scary you are being so vulnerable there and and especially when you join a group that seems to be like very well established and is it appears to be full of very intelligent people and then you start thinking oh my God am I you know am I worthy of this and and the answer as you said it's yes you are you are worthy of it um yeah everyone everyone has a contribution to make and it it can be any kind of contribution I mean you know I can't emphasize enough like so so often um I have past me has has been like you know using something from an open source project and I'll like be following whatever in the documentation and and the docs are wrong and and my first reaction is to be mad oh my God these idiots don't know what they're doing and you know what like turn that frown upside down because like what you should be doing if you if if you notice an issue in the documentation if there's a boo boo um something is wrong there's nothing that prevents you from filing a pull request to fix that documentation to prevent other people from getting frustrated from it being wrong so I mean that is the simplest thing that you can do and it makes such a huge impact so it doesn't have to necessarily be code it can be documentation and it can even it can be a minor correction so that's that's my little PSA there yeah that that was exactly our approach too so it was good like awesome we want some first issues let's go read the documentation is there anything that we can adjust or make more clear so exactly exactly yeah that's so great I I love hearing these stories and and I feel like these are the types of things that we just need to keep repeating over and over in in our community and and and tell people yes this this is a great way to contribute um you know we we talked about the benefits that you um that you you saw with um integrating open Telemetry um into your sdks what are some of the challenges um that you've seen um in open Telemetry especially like in in the in the mobile space like now now that you've you know U dipped your toes a little bit more more into that in in the community um I would say the the biggest challenge that we faced is really um you know we we this layer above the open Telemetry SDK and we are trying to streamline the process of of getting into observability and it's mostly because mobile developers just aren't familiar with this territory um they might what's interesting is a lot of developers are and there are a lot of existing tools out there that that will log events and things but um we're still trying to just flatten the learning curve uh for observability for people and so a lot of the challeng is just educating and figuring out okay what what clicks for people why is this um why is this valuable why is this useful um and then going from there and then you know educating them on how to you use our our product specifically um and so that's probably the one of the biggest non-technical challenges um the technically the the biggest challenge is that in the mobile space the environment is just chaotic it's it's something an app developer does not control um this is not a a server on in a box on some you know air conditioned shelf um that you have on Key and Lock uh it is in somebody's pocket hopefully um it could be on a nightstand you have no idea on the network status you have no idea how much battery is left on that device um or how angry that operating system is and whether or not it's just going to kill your app for who knows what reason and and so um the variability is is uh everpresent and and we need to be able to recover if any anything goes wrong so we have to handle things like okay we're trying to write Stu locally to dis so we can recover this data but there's no dis dis space What do we do um you know the battery is at 2% what do we do uh and it's just it's just a little more hectic than than you would expect or hope for if you're an app developer and and really the worst the worst thing is if if you're an app developer that's worried about performance um these things are out of your control but the user is going to blame you if if the app is not performing well and so uh if they are on 2% battery and they're you know rushing to to call a car to get to wherever they need to be um you need to make sure that they can complete that operation before the system just like okay I'm done time to sleep uh and so it's uh yeah it's just it's just crazy with with what we do yeah it's definitely a whole different ball game uh in in the mobile space compared to like the non-mobile space where where I guess it's it's a little less chaotic it's it's it feels like more unknowns in the in the mobile space that you're you're having to deal with right yeah and we're we're monitoring more than just the applications performance as well there's just how the user is interacting with with this um this application and what their behavior is is is something that you don't really have to deal with if it's you know if you are in a microservice architecture there's very consistent entry points very consistent uh exit points um in in any client side application not just mobile specific but even in a web browser um you know the user might be doing something on the page that that you just you don't know how they got there they came in with state that you didn't expect uh and so now you need to understand um you know how did you actually get to this issue I I can't reproduce it works on my machine but that's that's not an excuse can use so y yeah absolutely um now uh switching back to like you know um talking about contributing to open Telemetry um you know you you mentioned um getting getting started in in contributing um what you know now that that you've been doing it for for several months um how how has it been like you know one of the things that are um our Sig does is we want to understand what the contribution experience is for open Telemetry so we can provide that kind of feedback back to the sigs um to really um to really improve that right because if if we don't if we if if they're not aware of the feedback what what what can they do to improve so what are what are some of the some of the you know tell us the good the bad the ugly uh that's that you've experienced um with with uh you know becoming a a contributor um so it's it's good I I don't really have any bad or ugly um which uh maybe that's just me but um with joining the Swift Sig I think it was like a week or two in and these are weekly meetings so my second or third meeting um we were discussing an issue and and um nacho I think just said oh hey Austin can you take a stab and look at this um and uh I was like like o okay sure like you trust me to do that that that's that's awesome um and so that was kind of a a good like uh a sign of good faith and and confidence um and uh and then you kind of go from there it's like okay now what can I do to help out um and so some of the things that we're trying to do to help out are are push um proposals Upstream into the semantic conventions um make updates to the documentation to to try to flatten that learning curve so that you know you have specific examples on how to instrument uh you know on iOS what you know maybe what this view controller is doing or um you know making a network request with with URL session um these are very common patterns for iOS developers and um it just it makes it very easy if there's a snippet out there that you can just go grab and then and then tweak into in your application and so those are the the types of contributions that were um trying to make and and you know in terms of like feedback for the the sigs it's I don't know be be less friendly I I don't have much feedback it's um it's been great so that's awesome that that's really great to hear um and in terms of like I I know you said like it's been it's been a great experience so far do you have anything like that you'd want to uh improve that you you could see like maybe this could use a little tweak um I I don't know if it's an improvement but I It Like H finding that prior art is tough sometimes um and and mostly because you know some of these are proposals that are ongoing um a lot of things are just marked experimental and you don't know um how experimental that thing is um and and it's supposed to go I think experimental beta stable there there are two or three stages for for every proposal or or specification change and um and so making sure that those signals are are up to date um I think would be very useful and helpful as a proposal goes through the process but um you know that's just bookkeeping so it if you're joining and you're you're talking about things that's that's something that if you just ask a question someone would would help you out very quickly as well awesome awesome if I can add one thing there I think Austin Austin touches on something that took me a little bit to to kind of get used to as we started working with open Telemetry which is um it does take a bit longer than you would normally expect to get contributions accepted and move through that process like Austin is talking about but I think one thing to be aware of is that is by Design and I think when you're used to working like we're at a startup and so for us we try to make really quick decisions that concept of of oneway versus two-way doors and Implement them and see what happens and so working in a community like this it does take more consensus because we want that obviously the goal of this is to be a unified um solution that anyone can come use and and by design that means decisions take a little longer than you might be used to coming into this as someone that's new um and so I think that's just something to be aware of but but realize that that is part of what it means to create a standard that anyone can come in and use and really understand and is Thoroughly vetted so um that's something that people should probably be um aware of there yeah that's actually a really great call out and thanks for pointing that out um so we're we're finished now with the main questions uh portion so does anyone from uh from our audience have questions that they would like to ask yes Paige hello I am really curious what learning resources you would recommend if it was a particular blog or metaphor or tutorial that helped have that aha moment for you both um and then the kind of followup is any helpful metaphors is that you use for teaching your customers and kind of end users paig real quick before we answer that can can you tell us just who you are and and what your goal is yeah let me let me come on hello I'm Paige Cruz I work as a principal developer Advocate over at Kronos year um I used to be in Sr mostly focused on those beautiful backend systems the microservices running in beautiful air condition data centers and so I'm looking to learn more about mobile um I'm very interested in making sure observability is useful for everybody up and down the stack and mobile is a area that I need to brush up on and y'all are the experts so I was here to learn yeah thank you just wanted to um make sure I knew who I was talking to so appreciate that um actually it's funny that you mention blogs and books and resources it is required reading it embrace the learning open Telemetry book we literally have that oh you can't see it now but because my background but by uh Tedd young and Austin Parker um awesome we've started a book club around that at Embrace this is everyone in the company and I think it addresses particularly in mobile one of the other challenges that we face that's not technical which is um when you think about observability Concepts in open Telemetry it's been primarily focused on backend to date and so taking those Concepts and introducing them to front-end and mobile developers and mobile teams and why they're important and inand observability as a whole is a challenge so before we can even get them to start implementing our Solutions or understanding or uh or helping them understand why they're important they need to understand the fundamental concepts first and so before we can help them with that we need to understand those as well and so in our journey over the last nine months it's been various levels of Education through our entire org not just myself and Austin who are involved deeply in the sdks but our CSM teams our go to market teams our um sdrs all of them so we can all kind of speak the same language and understand the benefits of of open Telemetry not just for backend observability but for client side and mobile as well um Austin I don't know if you have other on oh and we intend a bunch of conferences too so that's great just being in the same space as people and and learning what they're going through has been super helpful yeah I I can't remember like when it flicked um but I think I'm a very visual person and a very Hands-On learner and so I think it probably the first time I saw a trace as a trace tree like a visualized out in in a timeline um I think that's when it was just like oh I this is this is my code running right now and I can see it and I can see the differences here you know if I'm comparing two that that should be identical but you know aren't because of stuff um you know that's when it was like oh okay this is this is interesting and useful awesome uh do anyone have any other questions yeah I was curious you were talk Austin I think you were talking about you know a lot of the challenges um with the mobile environment just being chaotic and you know a lot of things being out of control and um I think related to that you mentioned the data model for crash events and I was just kind of curious like what um what's going on like with all that like are you like yeah what's in development and kind of what are the current topics in that area um so I uh it was open to Elementry day this week so I didn't attend the client Sig this week uh because most people were were in person um but the latest I heard which would be as of last week um you know uh nine days ago um The Proposal is up in the GitHub repo um and uh the pr has been made I believe and we're maybe it's just in a Google doc but there's a document shared that describes a crash report as an event um the the big thing events are built on top of logs and log records Hanson's in chat he says proposal not up and PR not up yet but soon winky face so uh uh heads and nose um so yeah we we're discussing and there will be a proposal very soon um to use an event events are built on logs um but the key difference is there's event name um attribute that you can key off of and and so the value of that attribute attribute is is meant to represent kind of the schema of the log body and then I think also um the other attributes in in in that log record um and so if you start calling an event with a name you know this is a client crash um then the body can take a shape that is a little more formal um and uh what's interesting and what we're kind of working through is not all crash reports are the same um in a mobile environment um you know on on Android you have the the jvm which is you know the Java virtual machine and that runtime um then there's the native crashes below that um and those are mostly like C C++ um crash reports iOS you have Swift uh crash reports there's some C++ crash reports that are kind of weird and then we're started we we have these Hy apps react native and um uh the others that are are interesting because you actually you might get two crash reports and two stack traces uh for a single crash and and there's some weirdness there just because you know if a react native crashes it it might crash at the bridge layer um or in the JavaScript and and then it throws an exception down to the native code um and so you want to capture kind of all of this stuff and it's it's very different um and so what part of what's going into the proposal is okay how do we um how do we represent this in in a format that's consumable um but also understand that it can be very different um and when when a an otel collector receives this data um you know how should they start even trying to parse this out uh and so definitely recommend checking out the proposal as soon as it's up um sorry Hansen for putting the pressure on you but uh but uh uh we're looking forward to it because it it it's one of the biggest things that when we came into otel we we kind of asked ourselves well how are we going to represent crashes um and we we struggled with that um but uh we found a way we just kind of shove everything in as a log record right now and we're like this isn't good enough this could be better let's let's try to make this better interesting okay yeah I guess I didn't um think about you know these were things like low battery as being something that would affect my apps of course I just you know like you said stupid app and not yeah so well and low battery is one where where the systems will start throttling uh the processor so instead of a lot of mobile devices now there are these high efficiency processor cores and then the like you know the AI Crazy Fast processor cores uh and so when the system sees that the battery is getting low it's it starts throttling things and pushing things to those high efficiency uh CPUs and so um that has a downstream effect of your logic is now less performant um because it's it's either bottlenecked and waiting behind something or just the processor is slower it's just naturally not um not as quick as it would if it was you know charging or or had having a full battery interesting yeah thank you for sharing I'm sure I'll have more questions once I've processed this more but yeah this is this is good thanks for the question oh no problem well we prepare to wrap up we we will turn the tables and allow you to ask us questions okay I'm curious uh I think you know Paige Reese I think both of you are uh I think Reese your developer relations engineer page you work as an Sr to the other attendees as well I'm not sure exactly your roles but I'd be curious to what extent mobile observability has is part of your current observability practice um and how that fits in or how you marry that with your backend observability um that you have in place o yeah I mean I don't personally monitored any mobile applications um and you know I work with Adriana in the andies Sig and just you know anecdotally having heard a lot of end users being interested in this space like you know like Paige I want to learn more about it because I just don't really know much about mobile observability and I'm really interested yeah honestly I think it was um learning that Embrace existed kind of I'm like oh of course you'd want to do like observability for your mobile applications as well so that was you know honestly it wasn't until that was when I made the mental connection of like duh we should but why why w why aren't we talking about it so I I see this as an opportunity also to like really bring awareness to folks that of course you should be doing observability for your mobile apps yeah that triggers you said something earlier that I meant to that noted down for this question section what like what tactics have you found work best in that regard so for instance we have deep expertise in Mobile um we've just started working in open Telemetry just like over the last last like six to nine months and we want to bring that mobile expertise to the open Telemetry Community but I think there's a there's a thing around education from the mobile perspective that we also have to do that I think relate to a lot of the questions that we talked about today what have you guys found tactics that you guys have found to be particularly impactful in that regard whether it's something you guys have done or you've seen done um in terms of just um educating others on the kind of nuances of for us it'll be mobile but it can be whatever platform right or it could be open Telemetry or whatever um C would be interested to hear about that I would say like talks blogs um um yeah I mean apply to talk at conferences that'll bring bring awareness for sure blog posts YouTube videos get on people's podcasts um is another way to just bring that awareness yeah I don't know if uh any of the other folks have any additional thoughts for me it was really eye openening um at an org we sent all of our application data to the same observability platform and I was mostly focused on the kind of website website and I just for funsies looked at the mobile side just to see what's going on there just poking around and I saw oh my God like 90% of our requests came in through our IOS and Android apps and I was like I should just honestly ignore the website because to the business the impact was would be so much greater if we were to make these optimizations and that business observability is not always talked about inside org so really even just asking folks what's the split between um entry points from Mobile versus web that can really be a thread to start pulling on and then you also get your managers and other parts of the business like PMS um also interested and invited to the party and then last thing is at srecon this year I posted a link in the chat Isabelle gave a great talk called The Invisible front door reliability gaps in the front end and touched a bit on um mobile monitoring and how sres really need to turn I um towards this space so that's awesome thank you for sharing that that's great thanks Paige well we are almost out of time so um I thank you uh Austin and iLab for uh for joining us today this has been really great really educational and um thank you everyone who who joined us as well um see you for otel and practice diff --git a/video-transcripts/transcripts/2024-10-04T15:54:54Z-otel-q-a-with-steven-swartz.md b/video-transcripts/transcripts/2024-10-04T15:54:54Z-otel-q-a-with-steven-swartz.md index cdcb860..9536b26 100644 --- a/video-transcripts/transcripts/2024-10-04T15:54:54Z-otel-q-a-with-steven-swartz.md +++ b/video-transcripts/transcripts/2024-10-04T15:54:54Z-otel-q-a-with-steven-swartz.md @@ -10,120 +10,262 @@ URL: https://www.youtube.com/watch?v=3c9Bnldt128 ## Summary -In this YouTube Q&A session, Steven Schwarz, a Form Engineer specializing in Payment Processing, discussed his team's use of OpenTelemetry for observability in their Kubernetes architecture. The conversation covered the migration from a proprietary observability vendor to OpenTelemetry, highlighting the benefits such as cost control and improved support. Steven explained their setup, which includes using the OpenTelemetry operator for auto-instrumentation, sidecar deployments, and the integration of metrics with Prometheus and Grafana. He also shared challenges faced in scaling collectors, managing resource allocation, and educating teams on manual versus auto instrumentation. The session concluded with audience questions about sampling strategies, logging practices, and the overall experience with OpenTelemetry, emphasizing the importance of community engagement and shared learning. +In this YouTube Q&A session, Stephen Schwarz, a Form Engineering expert at a payment processing company, discusses his team's experience with observability using OpenTelemetry. The conversation, moderated by Dan, covers topics including the migration from a proprietary observability vendor to OpenTelemetry, the architecture and deployment strategies employed by Stephen's organization, and the challenges faced with scaling collectors. Stephen explains the use of the OpenTelemetry operator for automating instrumentation, the benefits of tail-based sampling, and the integration of Prometheus for monitoring metrics. He also shares insights on the learning curve for teams transitioning to OpenTelemetry and provides feedback for the OpenTelemetry maintainers regarding documentation and configuration examples. The session concludes with Stephen responding to audience questions, emphasizing the importance of community engagement in advancing observability practices. -# OpenTelemetry Q&A Session with Stephen Schwarz +## Chapters -**[Music]** +Here are the key moments from the livestream with their corresponding timestamps: -Thank you everyone for joining. We have quite a good audience for our OpenTelemetry Q&A. Today, we have Steven Schwarz with us. +00:00:00 Introductions and format explanation +00:01:30 Stephen Schwarz introduces himself and his role +00:05:00 Discussion on the architecture and programming languages used +00:11:30 Overview of the OpenTelemetry setup and Kubernetes usage +00:17:00 Explanation of tail sampling challenges and solutions +00:22:00 Discussion about migrating from a proprietary observability vendor +00:26:00 Challenges in scaling collectors and resource allocation +00:31:00 Experience with the OpenTelemetry operator and contribution to the project +00:37:00 Insights on manual vs. auto instrumentation in OpenTelemetry +00:42:00 Audience Q&A session begins with various questions about implementation and best practices -### Format Overview -Just a quick note on the format: I will be asking Stephen some questions, and afterwards, time permitting, you'll have the opportunity to post your questions for Stephen in the link that Dan provided. +# OpenTelemetry Q&A with Steven Schwarz -### Introduction -**Host:** Stephen, do you want to introduce yourself? +**Moderator:** Well, um, thank you everyone for joining. We have quite a good audience for our OpenTelemetry Q&A. We have Steven Schwarz with us today. -**Stephen:** Sure! My title at work is Form Engineering, and I work for a company that does payment processing. When you're working with payments and handling money, you need to be exact with your observability. My team manages all of the observability infrastructure, which includes about a hundred technical folks. We aim to be experts in observability, sharing best practices and helping solve problems. +**Format Note:** Just a quick note on the format which Dan actually posted in the chat. I’ll be asking Steven some questions, and then afterwards, time permitting, you’ll have the opportunity to post some questions for Steven as well using the link that Dan provided. -I joined this team almost a year ago. The project I started on was migrating from another observability vendor that used proprietary instrumentation to adopting OpenTelemetry and moving to a new vendor. I hope some of my learnings are useful to others going through that process. +--- -I also enjoy contributing to the OpenTelemetry collector contrib repository. I had to do this to unblock myself with a few bugs we faced, but it was also a great way to understand what the collectors are doing under the hood. I was completely new to Go, but the code is very approachable, so I recommend taking a look if you haven't already. It definitely made my job easier! +**Moderator:** So, first things first, Steven, do you want to introduce yourself? -### Architecture and Deployment Landscape -**Host:** Thanks for the intro! Can you tell us a little bit about the architectures you use in your organization? What programming languages and deployment landscape do you have? +**Steven:** Yeah, sure! My title at work is Form Engineering, and I work for a company that does payment processing. So, when you're working with payments, you're dealing with money, and you need to be exact with your observability. My team manages all of the observability infrastructure for about a hundred technical folks. We try to be the experts in observability, sharing best practices and helping solve problems. -**Stephen:** Sure! I have an architecture diagram that I shared; is it a good time to go over that? +I joined this team almost a year ago, and the project I started on was migrating from another observability vendor that used proprietary instrumentation. We adopted OpenTelemetry and moved to a new observability vendor. Hopefully, some of those learnings are useful to others going through a similar process. -**Host:** Yes, go for it! +I also really enjoyed contributing to the OpenTelemetry Collector contrib repo. I did that to unblock myself from a few bugs we faced, but it was also a great way to understand what the collectors are doing under the hood. I was completely new to Go, but the code is really approachable. I’d recommend taking a look if you haven't; it made my job a lot easier for sure. -**Stephen:** All right, I believe my screen is visible. Going from left to right on the diagram, we can dive into specific parts that seem interesting. +--- -We are running on Kubernetes with multiple production clusters. Within each cluster, we use the OpenTelemetry operator, which allows us to inject common SDK configurations into our teams' applications in their containers. It also enables us to automatically turn on auto-instrumentation, ensuring consistent spans are collected from all the applications. +**Moderator:** Thanks for the intro! Can you tell us a little bit about the architectures that you use in your organization? What programming languages are used and what’s the deployment landscape like? -A unique part of our setup is that we run OpenTelemetry as a sidecar alongside each application instance. The app sends all its telemetry over localhost to that sidecar. Currently, we send metrics and spans, but we aren't using logs through OpenTelemetry right now. For metrics, we use OpenTelemetry as a hop; we also use Prometheus to scrape the sidecar before it pushes to Grafana. +**Steven:** Sure! I have an architecture diagram I can share. Is it a good time to do that? -Another challenge we faced was getting tail sampling to work because we have spans coming from multiple Kubernetes clusters, and we only want to keep some of them. We have a separate cluster that collects all the spans, makes decisions on which to keep, and load balances based on trace ID to avoid piling up spans on a single collector. That’s a high-level overview of our setup. +**Moderator:** Yeah, go for it! -### OpenTelemetry Operator -**Host:** That's awesome! It's great to see this kind of setup in real life. What aspects of the OpenTelemetry operator are you currently using? Do you leverage auto-instrumentation or other capabilities? +**Steven:** Okay, I believe my screen is visible. Everyone can see the diagram? -**Stephen:** The main aspects we're using are injecting common SDK configurations and auto-instrumentation, which saves us a lot of time. We don't have to get teams to make code changes; we can manage that centrally and push it out to all the teams. We also use the sidecar feature of the OpenTelemetry operator. +**Moderator:** Yep, I can see it. Hopefully, everyone else can as well. -**Host:** I have to put in a plug for the operator's auto-instrumentation—it's impressive how little code teams need to write. +**Steven:** Great! Going from left to right, we can dive into specific parts that seem interesting. We are running on Kubernetes with multiple production clusters. Within each of those clusters, we are using the OpenTelemetry operator, which allows us to inject common SDK configuration into our teams' applications within their containers. It also lets us automatically turn on auto instrumentation, so we get consistent spans collected from all the apps. -**Stephen:** Absolutely! The less effort required from teams, the better. Even getting them to add an annotation can sometimes take time, so it definitely sped up the migration process. +Another unique part of our setup is that we are running OpenTelemetry as a sidecar. This means that alongside each instance of the application, we have a mini collector running, and the app can send all their telemetry over localhost to that sidecar. -### Migration from Proprietary Vendor -**Host:** You mentioned that you migrated from a previous vendor using their proprietary telemetry solution. What motivated that migration? +Currently, we are sending metrics and spans. For logs, it's a bit of a long story, but we aren't currently using logs through OpenTelemetry. For metrics, we use OpenTelemetry as sort of a hop; we also use Prometheus to scrape the sidecar before it pushes to Grafana. -**Stephen:** Cost was a significant factor. It was hard to control costs with the previous vendor, and the support wasn't great. A lot of stakeholders were unhappy, and we wanted more control over our observability solution, which the OpenTelemetry collector gives us. +Another interesting challenge we faced was getting tail sampling to work. We have spans coming from multiple Kubernetes clusters but only want to keep some of them. We don’t want to make the decision about which traces we keep until we have all the spans; maybe we wait to see if there’s an error. So, we have a separate cluster that all the spans go to in order to make that decision. We load balance to help scale that; we have collectors in that common cluster that load balance by Trace ID, so we don’t have a ton of spans piling up on one collector. That’s sort of a high-level overview of our setup. -### Scaling Collectors -**Host:** Have you encountered challenges in scaling your collectors with the sidecar deployment pattern? How many sidecars are you managing? +--- -**Stephen:** We run one sidecar for each pod or instance of the application, so it scales quite nicely. The tricky part is configuring resource allocation, like how much memory to request from Kubernetes. Every application is slightly different, and there's currently no way for specific applications to request a specific amount of memory. This creates some coordination overhead, but we’ve only had to tune it for one application so far. +**Moderator:** That's awesome! It’s really cool to see this kind of setup in real life, as I think that’s a burning question for a lot of practitioners: how do you set up your collectors? It’s nice to see that you’re using the OpenTelemetry operator. You're one of the few people I’ve spoken with recently who has been using the operator. What aspects of the operator are you currently using? Do you leverage the auto instrumentation capabilities? -### Load Balanced Collectors -**Host:** What about your load-balanced collectors? Did you face challenges setting that up? +**Steven:** The main aspects we’re using are injecting the common SDK configuration. That saves us a ton of time because we don’t have to get teams to make code changes; we can manage that centrally and push it out to all the teams. -**Stephen:** Yes, initially, we encountered a regression that caused a memory leak in one of the components. Our setup is memory-intensive because we cache metrics for Prometheus to scrape. We're currently using the Kubernetes Horizontal Pod Autoscaler to monitor memory usage and make scaling decisions based on that. +We also use the auto instrumentation, where they just add a line to their pod configuration to get the instrumentation. The other capabilities you mentioned, we aren’t using those, but we are using the sidecar, which is part of the OpenTelemetry operator. -### OpenTelemetry Operator Experience -**Host:** What was your experience running the OpenTelemetry operator? Was it straightforward? +--- -**Stephen:** Generally, it was smooth compared to the collector itself. The maintainers were very responsive in the Slack channels when we encountered minor issues. +**Moderator:** I have to put a plug in for the operator; auto instrumentation is so cool! It’s a minimal effort for teams. -### Custom Collector Distribution -**Host:** Are you building your own collector distribution? +**Steven:** Absolutely! The less effort you can make for teams, the better. Even getting them to add the annotation can sometimes take time, so I think it made the migration go much faster by minimizing the work they had to do. -**Stephen:** Yes, we are. When we faced bugs, we couldn't wait for the main distribution to push fixes, so we forked it. Now we have custom changes that are challenging to remove, but we want to move back to the main distribution eventually. +--- -### Contribution Experience -**Host:** How was the experience of contributing back to the collector? +**Moderator:** I wanted to ask: you mentioned that you migrated from a previous vendor using their proprietary telemetry solution. What made you decide to do that migration? -**Stephen:** It was good! People were helpful during the code review process, but it took a while to merge the code. There were also merge conflicts due to version upgrades. I think there's room for improvement in speeding up the merging process once someone approves the code. +**Steven:** Cost was a big factor; it was hard to control costs with the previous vendor. Additionally, the support wasn’t good, and a lot of stakeholders were unhappy with it. We wanted more control over costs, which the collector gives us. -### Manual Instrumentation -**Host:** Have you done any manual instrumentation as well? +--- -**Stephen:** Yes, we had to show teams how to convert their manual instrumentation from the proprietary format to the OpenTelemetry format. The process is ongoing, and I think they need more examples of the benefits of tracing to motivate them. +**Moderator:** Have you experienced any challenges in scaling your collectors? You mentioned that you have a sidecar deployment pattern. -### General Feedback -**Host:** Do you have any general feedback for the OpenTelemetry maintainers regarding your experience? +**Steven:** Yes, for the sidecars, we run one for each pod or instance of the application, so that scales on its own, which is quite nice. The tricky part with the sidecar is configuring how many resources to allocate, like how much memory to request from Kubernetes. Every application is slightly different in volume, and there’s currently no way for a specific application to request a specific amount of memory; we need to configure that in our centralized configuration. This creates a lot of coordination overhead if we want to tune that. We’ve only had to tune it for one application so far, so it's been quite easy to scale the sidecars. -**Stephen:** It felt like we were figuring a lot out ourselves, which took several iterations to get right. More example configurations for common problems would be helpful, especially for complex setups like tail sampling. Monitoring the collectors is another area where better documentation could improve the experience. +--- -### Audience Questions -**Host:** Now, let's check the Lean Coffee board for questions from the audience. +**Moderator:** What about your load-balanced collectors? Did you encounter any challenges in setting that up? -**Dan:** We have several questions! +**Steven:** Yes, there was a regression at the beginning that caused a memory leak in one of the components we were using for handling spans and metrics. That was an issue I fixed as part of my contribution to the open-source community. -1. **Do you only use tail-based sampling, or do you also use head-based sampling?** - - We only use tail sampling, as it's more effective for making sampling decisions. +Our setup is quite memory-intensive because we have to cache our metrics in memory for Prometheus to scrape them. So, it’s been about keeping memory at a reasonable level. We use the Kubernetes Horizontal Pod Autoscaler to watch the memory of each instance of the collector and make scaling decisions based on that. -2. **How do you handle container sizing on the collector sidecars?** - - We have a centralized configuration for sidecar resources, which creates some coordination overhead. +One thing I’ve noticed is that when our vendor has an outage, telemetry starts to pile up, causing a spike in CPU and memory. If the outage lasts a long time, the memory usage could become a problem for the collectors. -3. **Do you use serverless architectures? Any tips for instrumenting serverless workloads?** - - We don't use serverless, so I can't provide advice there. +--- -4. **Are you considering moving to the OpenTelemetry log file receiver?** - - We faced issues with durability in the past but may explore it again for logs. +**Moderator:** Going back to the OpenTelemetry operator, what made you decide to use it in the first place? -5. **How do you monitor the OpenTelemetry collectors?** - - We use Prometheus for metrics and FluentD for logs. +**Steven:** I heard about it from a colleague, who recommended it. It fit with the platform ethos of trying to get out of the way of developers and allowing them to do things without needing to be heavily involved. -6. **What are your thoughts on using no collector endpoint, sending data directly to the observability provider?** - - We opted for collectors to control costs and handle sampling. +--- -**Host:** Thank you, Stephen, for joining us today and answering all the questions. +**Moderator:** What was your experience running it? Did you find it straightforward? -**Stephen:** It was a lot of fun! Thank you for hosting. +**Steven:** Generally, it was very smooth compared to the collector itself. When we had minor issues, the maintainers were very responsive in the Slack channels. -**Host:** If anyone is interested in sharing their use cases or participating in future sessions, please reach out to us in the CNCF Slack. Thank you again, everyone! +--- + +**Moderator:** Are you building your own collector distribution? + +**Steven:** Yes, we are. When we faced some bugs, we couldn’t wait for them to be pushed to the main distribution, so we forked it to fix issues earlier. Once that happened, we started making other changes, so it’s been a bit hard to get off the fork, but it’s something we want to do. + +--- + +**Moderator:** Does that involve using the OpenTelemetry Collector Builder tool when you deploy your collectors? + +**Steven:** Yes, we are using that tool. We copied the list of components from the Dockerfile to build it. + +--- + +**Moderator:** You mentioned your own customizations, which led you to contribute back to the collector. How was that experience? + +**Steven:** It was a good experience. People were helpful during the code review. It took a while to get the code merged, though, and I had to deal with merge conflicts due to version upgrades. I think there’s room to merge things faster after approval to avoid those conflicts. + +--- + +**Moderator:** Have you done any manual instrumentation as well? + +**Steven:** Yes, we had to show teams how to convert their manual instrumentation from the proprietary format to the OpenTelemetry format. It’s still a slow process; I think the benefits of tracing aren’t obvious to everyone, so I need to provide more examples of why it’s useful to motivate teams to add more metadata to their spans. + +--- + +**Moderator:** Does your team ever get asked to instrument stuff for other teams? + +**Steven:** Not specifically for their applications, and I’m glad to hear that. + +--- + +**Moderator:** I noticed from your diagram that it looks like you're primarily using Java apps. Is that correct? + +**Steven:** Yes, mostly Java. We do have some Go as well. + +--- + +**Moderator:** How did you find the learning curve for instrumenting stuff in OpenTelemetry? + +**Steven:** Java has a good API, and the instrumentation process is pretty intuitive. We use Micrometer, a Java library for metrics, so the teams didn't have to change anything to get the metrics to work. With Go, it’s a bit more manual and harder to use than the Java one. I’m curious if anyone has used the auto instrumentation for Go; we aren't currently using it, but it would be nice to get that working. + +--- + +**Moderator:** Do you have any general feedback for the OpenTelemetry maintainers about your experience? + +**Steven:** I think we figured out a lot by ourselves, and it took several iterations to get to this point. More example configurations would be helpful for common problems, like tail sampling. Monitoring collectors is another area where I had to dig through the codebase to find what metrics are exposed, so documentation on useful metrics to monitor would be great. + +--- + +**Moderator:** Thank you for the feedback, Steven. Now, let’s check the questions from the audience. + +--- + +**Dan (Moderator):** We have many questions from the audience, which is great! Feel free to vote for the ones you find most interesting. I have marked one of them as answered already. + +Starting with the question about tail sampling: Do you only use tail-based sampling, or do you also use head-based sampling? + +**Steven:** We only use tail sampling. It’s more work to set up, but having the full trace to make sampling decisions allows us to make the best choices. + +--- + +**Dan:** How do you handle container sizing on the collector sidecars? + +**Steven:** We don’t have one size that fits all. It would be nice if pods could add an annotation for the resources they need. Currently, we have a centralized configuration that creates sidecar configurations for different resource combinations, which requires coordination. + +--- + +**Dan:** Do you use serverless? If so, do you have any tips for instrumenting serverless workloads with OpenTelemetry? + +**Steven:** We don’t use serverless, so I don’t have any advice there. + +--- + +**Dan:** You mentioned using FluentD for logs. Are you considering moving to the OpenTelemetry log file receiver? + +**Steven:** It's something I’d like to explore. We had issues with logs initially, so we pivoted to using FluentD, but I’m curious to see if others have found good solutions for durability in logs. + +--- + +**Dan:** Regarding monitoring your collectors, do you use OpenTelemetry to monitor your OpenTelemetry collectors? + +**Steven:** Yes, we use Prometheus to get metrics from the collectors and FluentD to collect logs, and that’s worked pretty well. + +--- + +**Dan:** You have a dedicated platform team managing the collectors. Do you have thoughts on not having a collector gateway and sending telemetry data directly to your observability provider? + +**Steven:** We decided to use collectors to control costs and provide durability if the vendor goes down. While it simplifies things, there are trade-offs. + +--- + +**Dan:** Do you find that developers are aware of auto instrumentation, or do they tend to do manual instrumentation unnecessarily? + +**Steven:** Yes, we have seen some redundant data from applications that instrument on their own. It comes down to education and tracking cardinality to help teams understand when they can disable unnecessary instrumentation. + +--- + +**Dan:** How do you manage sampling policies? Can teams dial that configuration themselves? + +**Steven:** We have a simple setup where we sample based on errors and trace lengths. Teams can force sampling by adding an attribute to their span, but we monitor its use to prevent issues. + +--- + +**Dan:** Do you currently use any configuration that may not be supported through environment variables, like metric views? + +**Steven:** We haven’t had to do that much. When we do, we usually show teams a merge request to copy paste from our configurations. + +--- + +**Dan:** How do you handle supply chain security for the OpenTelemetry SDKs? + +**Steven:** I created a wrapper dependency to manage OpenTelemetry library versions that work together, but it can cause dependency conflicts in some cases. + +--- + +**Dan:** Is the OpenTelemetry operator your preferred solution, or can it be used in parallel with SDKs? + +**Steven:** So far, we haven’t had issues with teams deviating from the injected configurations, but I’m curious how that will evolve over time. + +--- + +**Dan:** Are developers using the LGTM Docker container from Grafana for local testing and troubleshooting OpenTelemetry implementations? + +**Steven:** Not yet; they can only test by deploying to our clusters. We need a better local testing solution. + +--- + +**Dan:** If you had a magic wand, what would you love to add, remove, or change in OpenTelemetry and why? + +**Steven:** I would get rid of Prometheus in our architecture because we rely on it for aggregating data for cost savings. The collector doesn’t currently offer that functionality, so it would simplify our architecture. + +--- + +**Dan:** Are you using multiple OpenTelemetry backends for processing, or does everything go to one vendor? + +**Steven:** We have one vendor that we’re pushing to; we don’t push anything internally to anywhere else. + +--- + +**Dan:** Given the potential for bottlenecks in collectors, do you see disadvantages in having the SDK export data directly to the back end? + +**Steven:** It could be easier to handle smaller amounts of data directly from the pods, but it shifts the back pressure to the application, which could be impactful. + +--- + +**Moderator:** Thank you, Steven, for joining us today and for answering all these questions. It was great to see such engagement from the audience! + +**Steven:** Thank you, Dan, for hosting this and thanks to everyone for the great questions. It’s a topic that really interests people, and I appreciate the engagement. + +If anyone out there would like to participate in OpenTelemetry Q&A or OpenTelemetry in practice, we’re always looking for folks to share their use cases. You can DM either Dan Reyes or me or message us in the user channel in the CNCF Slack. + +We’ll post the recording of this session on YouTube, so let anyone know who may have missed it. Thank you again, everyone! ## Raw YouTube Transcript -[Music] well um thank you everyone for joining we have quite a uh a good audience for um for otel Q&A um we have uh Steven Schwarz uh today with us now um just a quick note on the format oh which Dan actually posted in the chat awesome um yeah so there I'll be asking Steph some questions and then afterwards um time permitting you'll have uh the opportunity to post some questions for Stephen as well in the uh in the link that Dan provided so um all right so first things first um Stephen do you want to introduce yourself yeah sure so uh my title at work is uh Form Engineering and I work for a company uh they do Payment Processing so you know when you're working with payments you're working with your money you need to be uh kind of exact with your your observability um what my team does is we manage all of the observability infrastructure we've got about a hundred uh technical folks that we're we're managing it for uh and we try and be sort of like the experts and observability and and share some of the best practices and you know help them solve their problems um so I recently join or I joined this team about almost a year ago and the project that I started on was actually migrating from uh another observability vendor where we're using proprietary instrumentation uh and we adopted open Telemetry and we moved to a new observability vendor uh so hopefully some of those learnings are you know useful to other uh folks going through that process um and also worth calling out is that I really enjoyed uh contributing to the open Telemetry collector contrib repo uh I had to do that to um unblock myself with a few bugs that we face but it was also just a really good way to uh you know understand what the collectors are doing under the hood uh I was completely new to goang but the the code is really approachable so yeah I I'd recommend taking a look if you haven't it's it's uh it made my job a lot easier for sure that's great thanks for the uh thanks for the intro now can you uh tell us a little bit about like uh the architectures that you use um in your organization like what programming languages are used and deployment landscape that kind of thing yeah so I have that architecture diagram I shared with you would it be a good time to share that yeah yeah go for it okay all right so believe my screen is visible everyone can see a diagram yep I can see it hopefully everyone else can as well well okay so go kind of left to right we can dive into uh specific parts that seem interesting um so we are running on kubernetes um we have multiple production clusters um and within each of those clusters we are using uh the open Telemetry operator uh and that gives us a way to inject like common configuration uh of the SDK into our uh you know our teams applications into their containers um and it also lets us to automatically turn on auto instrumentation uh so we get uh consistent spans collected uh from all of the all the apps and another uh sort of unique part is we we are using the open Telemetry uh we're running open tetes as a side car um meaning that uh alongside each instance of the application we we have a mini collector running and the app can send all their Telemetry over local hosts uh to that Sidecar um so we are currently sending metrics and spans uh logs we it's a bit of a long story but we we aren't currently using logs through open Telemetry um for metrics so we we use open Telemetry but it as sort of like a hop we we also use Prometheus um to scrape the side car before it pushes to uh grafana and another interesting uh challenge was um getting tail sampling to work um because we have spans coming from multiple kubernetes clusters and we only want to keep some of them um but we don't want to make the decision about which traces we keep until we have all the spans maybe we wait to see if there's an error um so uh we have a separate cluster that all the spans go to to make that decision um and we uh we load balance uh just to to help scale that we have these uh collectors in that common cluster they just uh load balance by Trace ID so we don't have a ton of spans piling up on on one collector and uh yeah that's sort of like a high level overview of our setup that's awesome um it's really cool to see like uh this kind of setup out in in real life because I think that's uh I do feel like that's kind of a burning question with with a lot of um our our practitioners is like how do you set up your collectors um so it's it's very nice to see um and and cool that you're using the open Telemetry operator um I think one of you're one of the few people that I've I've spoken with that's not to say people don't use the operator um but you're you're definitely one of the few people I've spoken with um recently who who has been using the operator uh what aspects of the operator are you currently using like do you do you leverage like the auto instrumentation do you um leverage the opamp capabilities um what's the other one uh for and due leverage like the The Collector uh deployment cap cap abilities like to you use all three or or combination uh so the main ones we're using are just you know injecting the common SDK configuration um that that saves us a ton of time we don't have to get teams to make code changes we can just manage that centrally and kind of push it out uh to all the teams um same with the auto instrumentation uh they just add you know a line to their configuration of their uh you know their pod and then they get the instrumentation um the other ones you mentioned we we aren't using but we are using the the side car which is the open Telemetry operator inject cool cool um yeah I I have to like put a plug for the operator um Auto instrumentation it is actually so cool it like you literally don't have to write code you just put an annotation in manifest and then well and and the uh instrumentation CR of course that you have to figure but like it's pretty cool I have to say um yeah yeah you know the least effort you can make for teams the better like even getting them to add The annotation you know can sometimes take time so uh it's it's yeah I think it made the migration go much faster being able to uh you know minimize the amount of work they have to do absolutely now um I wanted to ask because you you mentioned that you you had migrated from from a previous vendor using their own proprietary um Telemetry solution um what what what made you decide to do that migration yeah so um cost was a big one it was hard to control in previous vendor um and I guess support wasn't good so those are I think the main drivers I think just a lot of stakeholders were unhappy with it um and we wanted a bit more The Collector gives us a lot of control over uh cost um that this other vendor doesn't right right yeah that makes that makes a lot of sense now um another thing that I wanted to ask in terms of um scaling your collectors like have you experienced any uh any challenges in in scaling your your collectors because uh you know you mentioned that you have a sidecar deployment pattern um how many how many different side cars are you running at at a particular point in time like is it a lot that you're managing um yeah so the and for the side cards so we run one for each pod or instance of the application so that kind of almost scales on its own which is quite nice um so that that works um the tricky part with the sidecar is uh configuring the like how many resources to allocate like how much memory to ask from kubernetes uh just because every application you know is slightly different in their volume and um so and there's not really there currently there's not a way to uh like at the app for a specific application they can't say that they want a specific amount of memory we need to configure that in our centralized configuration so it creates a lot of coordination overhead if we want to tune that right um but we've only had to tune it I think for one one application so far um one of our monolist so that that has it's actually been quite easy to scale this uh the side C okay that's great that's great now what about uh what about your load balanced uh collectors um how have you did you encounter any like challenges in in setting that up yeah uh well I mean at the beginning we uh there's I think a regression that caused the memory leak um in one of the components we're using the span metrics one um so that that was a problem that was actually one of the things that I I fixed uh as like the open source contribution um and I guess our the way our setup is quite memory intensive because we have to Cache our metrics in memory uh for Prometheus to scrape them um so yeah it's really been a matter of keeping memory uh man I guess at a reasonable level um so right now we use uh the kubernetes horizontal pod a scaler to that watches the memory of the each of the instances of The Collector and it makes scaling decisions based on that yeah um it it works pretty well took some uh tinkering um one thing I've noticed and maybe other I'd be curious if other people have a good solution for this is when our vendor graa has because we're using when they have an outage uh I find the Telemetry starts to pile up um waiting for service to be restored and we see this big spike in CPU and memory so it's quite hard to uh uh I guess like I think if the outage were to go for quite a while probably the memory usage would just start to kill the kill the uh collectors or be very high uh so that's a bit of a problem um we still have right right um now another question that I had for you um just going back to the um Hotel operator um what made you decide to use the operator in the first place is it something that you had been made aware of or someone like did did you hear about it from from folks in the community uh so I I heard about it actually from a colleague um so it was their recommendation but it fit with the kind of platform I guess ethos of uh you know just getting try to get out of the way of the developers and yeah um you know have a way to uh I guess do things without the developers having to get involved right right yeah fair enough that that is one thing I I like about the operator now what was your experience in in in running it like was it something that you felt was straightforward um because I mean one of the things that's that we really value about these sessions is not only like learning how you're using these things in real life but also like learning like your user experience with using components of open Telemetry so um is it's something where felt like it was documented well enough did you have to do a little bit of digging poke around ask some questions in the operator slack um it was I'd say generally like compared to the collector itself it was very smooth but I guess it's a I mean anyways I'm not sure why that is but uh yeah it worked quite well um and the maintainers are when we did have sort of minor issues maintainers were very responsive in the slack channels awesome that that's good to hear i' I've had always really good experience also with the hotel operator maintainers so that's that's nice to hear as well great um now the other thing that I wanted to ask is um have you when you're using your collectors um are your are you building your own collector distribution um yeah we are um and that was because we uh when we fac some of those bugs um we couldn't wait until it was push to the main distribution um so we fored it just to to fix it earlier and then once that happened we kind of just had we started to make oh other changes so it's a bit hard to get off the fork now but it's something we do want to do now is that um does that involve using the otel collector Builder tool um when you're when you're deploying your your collectors um yeah it does uh I looked at sort of the the basically the docker file how they build the conty bro and just kind of copied uh the commands from there oh okay okay have you have you used the actual because there's um there's a a a tool where you basically specify like the components like which receivers which processors Etc that you want to include did you have you tried using that Tool uh yeah yes that's what we're using uh to build it uh we we copied like the list of components yeah oh okay okay got it got it sorry I misunderstood you um awesome um and you know you mentioned you you uh did your your own customizations and that um that kind of forced you to contribute uh back to the collector how is which is it sounds like you know it ended up being like a win-win for your team um how was the how was the experience of contributing back to the collector overall yeah it was um it was a good experience the you know people were helpful in uh you know during the code review it took a little while to get uh the code merged maybe this is a like I noticed um you know you'd have a separate person for approving it and then it would sit unmerged for a few weeks and then you get all these merge conflicts because like the the versions would be upgraded so I had to you know go back and update the go mod files a bunch of times so that that was I think maybe there's room to like once someone proves it to maybe merge it a bit faster but I don't know the sort of the rationale behind that yeah fair enough fair enough um and um the other thing I wanted to ask um you know what um you you mentioned that you um uh you mentioned that you use Auto instrumentation have you done any manual instrumentation as well yeah so the um yeah we we had to show teams because we had manual instrumentation using that proprietary format so we had to show teams how to convert it over to the otel format and then also educate them on how to um you know use otel manual instrumentation and how did that go um was did you find that teams were were um receptive to that um it's still I think they're slowly slowly doing it [Music] um it's I think tracing isn't maybe like the power of or the benefits of tracing isn't maybe as obvious so I think probably I just need more examples of like why it's useful to give them the motivation to to add more metadata to their spans for example right right right um now does does your team ever get asked to like instrument stuff for other teams um not uh not specifically for their applications yeah I'm very glad to hear one of my observability pet peeves well that's that's good to hear and so it looks like um now in terms of um the the instrumentation like it looks from your diagram that it's primarily like a Java app is that correct uh yeah yeah mostly Java we do have some some goang as well okay um now um have you um how did you find it in terms of like um you know all all the the stuff aside like the learning curve for for instrumenting stuff in open Telemetry um how did you find it like since you in a position to teach folks how to instrument their code with open Telemetry was it a big learning curve for your team as far as understanding how the uh how to how the manual instrumentation works worked for like both Java and go like did you notice any um anything that was like maybe easier in one language compared to the other or where you had to go back to the sigs to ask for clarification yeah the uh so Java it they've done a really good job with it I think the language just makes it easier um so yeah there because we use um like the kind of we use that micrometer which is like a Java library for mitting met metrics so they didn't actually have to change anything to get the metrics to work oh okay got it um and yeah the and the span API is pretty uh pretty intuitive as well U goaling I mean it's not my that's not my personal strength but I found it's a bit more manual uh it was it's definitely harder to use than the uh the Java one and right I'm curious if anyone's if anyone's used the auto instrumentation I haven't had a chance to play around with that but uh that would be we we're not currently using it for goaling but it would be nice if we could get that working yeah for folks uh for folks who are listening uh does anyone have have experience with that that would like to uh to share feel free to raise your hand if you do have any experience no takers all right um okay um the uh guess we're I'm just about done asking the questions um the only other thing is that I was going to ask is do you have any general feedback that you wanted to uh provide um to you know the open Telemetry maintainers around like your open to infmetry experience as far as like um using components like e of use uh you know you mentioned the the contribution experience you said it was a little bit challenging with with approving the PRS um anything else that you wanted to add to that um I think uh like it felt like we were kind of figuring out a lot of a lot of stuff out uh for ourselves uh like it took us quite a few iterations actually to get to this point um it might be hard to have a rest for every combination of like you know vendor and backend and and all that so that could be part of the problem um yeah I guess like more sort of uh example configuration or would be you know would be helpful like uh for example the these load balancing originally we actually had another set of uh collectors that all they did was load balancing yeah um but we realized oh we can actually load balance that you know th this deployment of collectors can low balance to itself um and that just is way easier to maintain so um yeah and I guess like sort of example configuration for common problems like tail sampling that we're doing right here right yeah that's really good piece of feedback I have heard from other folks as well that uh because you know we have the open Telemetry uh demo which is a great Showcase of I think it's really focused on on application instrumentation which is awesome um but I think there is definitely like a a desire from the community as far as um having some um like more collector specific use cases um and and specifically like stuff that's a little bit more complex than than what you would necessarily see in in the demo yeah for sure um another one is uh monitoring your collectors um I found I had to kind of dig through the code base to find like what metrics exposed like what are they doing um and I feel like that's probably there's probably some commment alerts or things that for most setups would be useful so maybe someone already has this but like uh you know some some we can document like hey these are useful things to monitor um the the actually the the dashboard that that for grafana they have that was pretty useful but uh I think on the alert side that would be uh that was I didn't find anything uh there right right cool well thank you for the feedback on that um now I guess we could um we can do one of two things uh we've got the uh the lean coffee board where folks can post questions for Stephen so I'm actually going to check the board oh we have some questions um yeah let's let's go through that and then if we have some time afterwards then Stephen you can ask some questions um to to our uh viewers as well okay okay um Dan do you wna do you want to take it on from here on uh the the questions from the board sure thing we've got many questions from the audience which is great um also feel free to go and vote for the ones that you think um are the most interesting there I have marked one of them as answered already because we talked about the collector um Builder and how you build your own dist for the collector on your side car uh starting from the the one that we talked about tail B uh sampling a little bit how you do the the how you use the obey sampling um but I wanted to ask this question here is like do you only use tail based sampling or do you also use head-based and any tips or uh thoughts on mixing these approaches uh yeah so we only use um tail sampling um I guess it if you like it's it's more work to get it set up but when you have the full Trace to make your sampling decision I find that you can make the the best decision um so yeah that's why we we just lean on tail sampling completely I guess we head B head Bas sampling there's always a risk and you're going to miss something important right because it's completely probabalistic so um yeah good point and moving on to the next one um how do you handle container sizing on the collector sidec cars you mentioned this already uh that you had to customize or tune one of the collector's uh collector sizes so do you have one size that fits all or anything you would like to improve in that setup or like resource allocation yeah so um I think what would be nice is if pods could add like an annotation or you know saying this is the resources I need uh instead what we have to do is kind of hacky um so for each side car it's like a separate res like kubernetes resource or configuration uh file so what we do is we have like a for Loop that goes through and creates uh those configurations for different resource uh combinations um so and so it's something we have to coordinate like hey you have to okay update the sidec card configuration that your pod wants to use to this otel sidecar 500 megabytes one CPU or you know so it's it's a bit of it's it's an extra coordination work there uh yeah cool um yeah okay next one is related to serverless um if you if you use serverless and um you know any tips or goas to look out for in instrumented serverless workloads with open Telemetry we don't use servus so don't have any uh advice there that's fine yeah I mean I I wanted to ask that in in case you you had some experience with that um also you mentioned that use fluent D for logs reading from standard out um are you considering moving to the otel lock file receiver um yeah it's something I'd like to explore we we sort of had a bit of a issue when we first tried using logs I'm curious if anyone has found a good solution for this um so what we wanted to do is uh like dur store our logs on disk so if there's like an outage we don't lose them um we ran into a bug with um the durable storage extension in The Collector uh and that kind of spooked us and we already had fluent D running so that's why we we kind of just pivoted to that but I don't know that was a while ago maybe and maybe people have had good success with with getting that durability uh for their logs yeah just to add on this uh with flu and D are you decorating your Logs with some of it semantic conventions or Trace ID span ID that you can so you can then correlate with your traces yeah we uh so we definitely have the logs and oh sorry the trace IDs and the span IDs and we're actually um adopting there's um a specification for the event uh I guess they call it like log events so it's sort of a structured format for logs uh in the opland Telemetry specification um so we're trying that out uh as well um yeah cool um do you use oel to monitor otel collectors and U we use um yeah it's like a big uh it's always speak Bol mud uh monitoring your monitoring but so these Prometheus to get the metrics from the collectors and fluent to collect the logs from the collectors and that that's worked pretty well um okay moving on to a bit more of your talking about your teams sounds like you uh you folks have DED a dedicated platform team uh that manages the oo collectors in your organization do you have any thoughts on the opposite of this where you have no collector Gateway or no collector um endpoint basically 100% of your Telemetry data is then sent to your observability provider uh yeah I guess in our case um we decided to use the collectors so that we could um do sampling uh to control costs of our traces um and also to try and um you know have some durability if uh I guess the vendor goes down we can kind of move that like retry logic and um off to the collector um yeah I'm curious if anyone has if anyone's doing that in in production it would definitely simplify things but I guess there's some tradeoffs as well there yeah I guess there's a trade-off as well of um authentication to a provider and so right is to sell in one place as well some of the benefits pros and cons I guess right um the next one it's related to um well we talked about Auto instrumentation the auto operator and how you inject those instrumentation packages and then we also talked about the fact that you are also using um like custom manual instrumentation um I guess the question is how do you find if your have your teams found that auto instrumentation is sufficient or is there a lot of custom instrumentation that's needed yeah so uh one thing I haven't talked about that I want to call it Les been great for us is um the SP metrics um component um so we we know we generate metrics so uh you know frequency latency eror error counts for every single span um and because we're using the automous instrumentation um every application emits the spans in the same format so basically out of the box without adding any uh instrumentation we get like I guess some vors call like APM or like red metrics so like all your API calls all your database queries um we have metrics on that automatically um so that that goes quite a long way and we can build like comment dashboards and commment alerts uh that all all these apps just get out of the box uh because of that standardization so it's a really good starting point um and then I guess so I like that is probably quite good but then if teams want to add metadata they they can um so yeah I would definitely recommend SP metrics uh if if that's like option for for people yeah I'm going to add a bit to this one as well do you find that you're as in like if you're moving from a from a from a world where like you know you had to instrument everything and now there is a lot of Auto instrumentation out there do you find that Engineers normally tend to not know about that auto instrumentation and they do it themselves so how do you manage that sort of like I guess learning or education piece saying that you may not need to instrument things is that is that has it been a challenge within your organization or everyone just uh yeah I i' say it is like we have some redundant uh like for example our API metrics like spring Boot and Java automatically emits some metrics so a lot of applications were getting some redundant data um so so yeah that that has been a problem uh but yeah so I guess it just comes down education and then having a way of tracking the cardinality if uh so when we've seen that like oh you're sending a ton of API metrics and we already have a capture the span metrics we can reach out to that team and say oh you can you can probably disable this you don't you don't really need this we don't even pay for it cool um regarding samplin um and then I'm assuming this is related to sampling policies can you advise on the is that the collector where you manage the policies for all traces or can teams Dial Dial that config up and down themselves uh so ours ours is pretty simple um so we sample you know errors we keep if the trace is longer than at something like 5 Seconds we keep but we do have an escape hatch if they add an attribute to their span to force sampling uh so far no one has like hasn't caused any problems um it might in the future you know teams overuse it but uh that that's work that's works so far and um related to some of that SDK conflict that you currently inject through the through the through environment variables um do you currently use any type of um other confli that may not be supported through environment variables like metric views for example is that do you have a way to apply defaults across the board for that type of uh config um I'm just reading the question oh I see um we haven't had to do that too much and when we have um we do have I mean our I guess our company's lucky and that we have sort of a tool that we can use to um it sort of generates uh applications from scratch um and then there's a way to sort of regenerate it the Regeneration doesn't work too well but um we know that most teams have a pretty similar format to their application so when we do need to make a change that we can't do with the operator um it's usually just boiler plate copy paste we can show them a merge request and they can copy it in but it does take you know can take a couple months for every team to adopt that cool okay um I guess one related to supply chain security um with the AEL sdks um so that do you how do you handle supply chain security so that otel sdks are safe to include them in application code or do you build them from sour yourself and then run your own security tests uh are you thinking this would be you know if there's like a c CB in one of the SDK versions how would we sort of manage patching that for everyone or I guess yeah or Downstream um or Upstream dependencies from from hotel packages yeah yeah um I mean for different reason I kind of created a wrapper dependency um that wraps like all the all the otel library versions that we use and we know they work together um just because we we found some versions were kind of incompatible um so from the app team's perspective they depend on one uh one dependency and then they get all the otel dependencies as transitive dependencies um so I suppose if we were to you know if there we need to upgrade it very quickly I guess it would just be one thing that they need to update um although that create like it works in theory but then it's also caused uh dependency conflicts just because at least how Maven Works in Java so I don't know if that's actually the best thing to do it might be more of a headache than it's worth cool um thank you okay so moving on to the next one uh related to um yeah so the auto operator and automatic instrumentation is that your preferred solution or can the oel operator used in parallel with sdks for example they not net in the case of the the person asking this question and you provide some balance of scalability and config configurability is there a recommendation here my immediate concern is that the manual instrumentation inside the application could make it difficult to configure and manage um at scale so it's this sort of like how how to balance um giving teams the flexibility to add manual sort of configuration and customization and then versus the sort of defaults that we inject for them um yeah so I mean so far we do I guess you know you can't override um the environment variables that are injected into um into the application automatically by the platform team uh um so far I guess it hasn't been uh too much of a problem um with teams teams haven't really wanted to deviate Too Much from the configuration that we've given them but yeah be curious longer term if that does cause problems thank you um that one this is an interesting one are your devs um using the lgtm docker container from grafana to develop Hotel implementation and then do local troubleshooting uh not yet right now they can only really test it and by deploying it to our clusters so we do need a better solution there um so they can test locally yeah I personally love there's like more and more solutions coming up that are open source that allow you to see that Telemetry in your own um computer I just testing uh so yeah a lot of uh really cool Solutions out there at the moment if you had a magic wand what's the thing that you would love to add remove or change in otel and why that is a logic question yeah I mean I think this is maybe short short cited but um if we could get rid of Prometheus in our architexture and the reasons we can't um right now is because we rely on Prometheus to um kind of aggregate our data together for cost savings um Source band metrics we we generate like 50 buckets uh histogram buckets for every span so it's it gets really expensive um and if each pod is you know has its own set of 50 buckets uh are cost scales with a number of PODS but we use Prometheus um to kind of like aggregate all the counts from all the pods and that like Cuts our cost significantly um I don't think the collector can do that today um so yeah that would I mean that would just make our architecture simpler makes sense um are you using multiple otel backends uh to process some of the things that internally uh compared to compared to your vendor or is it everything going to the vendor I guess um we have uh one one vendor if that that was the question that we're pushing to yeah so I guess but you're not um you're not pushing anything internally to anywhere else just everything goes to oh right yeah yeah we don't uh yeah okay and I think we've got time for one last one and that is given that the collector can quickly become a bottleneck due to back pressure and do you see any disadvantages having the SDK export that data directly to the back end except for the cost um con yeah I think that was similar to the other question about like why not just push directly to you know that your vendor from your your pod yeah it um it does make it easier to kind of scale you don't have to worry well yeah that's a good question I guess you're Shifting the back pressure then to your Apple um and then I'm then so yeah it might be though easier to handle if it's smaller amount of data but uh yeah that one I don't I don't I'm not too sure about Ben slots that' be interesting to discuss yeah that's um maybe easier to handle but also more impactful for the application so I guess there are pros and cons there as well right so if you're if you have your all your telemetry s of like shipped from your or out of your pod as soon as possible there are benefits for that as well right in terms of impacts so it's failed somewhere else in your infrastructure um because there will be at some point if your back end is down but um yeah pros and cons that's everything yeah and it seems like just looking at least the Java SDK they're pretty conservative about how much data piles up like I think it's like 1,000 24 spans is the max that will like buffer a memory before some low number that before we'll start dropping it because I guess they're really trying to lower the you know the footprint on the the application yeah exactly okay I think that's all we have time for um thank you very much Stephen for joining us we have we that was quite a lot of questions we went through from the audience uh thanks for everyone that joined and asked questions as well uh super nice to see uh engagement in as a session Q&A uh do you want to say a few words to um say goodbye yeah thanks uh thanks Dan and thanks for uh for doing the uh the questions yeah it's really nice to see the uh the engagement from from folks um in the uh in in the Q&A um I think this is a topic that really interests people so um we we love it when you engage also if anyone out there would like to participate in otel Q&A or otel in practice we are always looking for folks to share their use cases with us um you can just uh you can either DM either Dan reys or me um or if uh if you want you can also um just message Us in -- n- user our channel in the cncf slack um we are we we will be happy to hear your your use cases and have conversations and also as I mentioned recording for this will be up on YouTube so um anyone that you know that wishes they had attended but could not attend um let them know um we'll we usually post on the otel socials and also in the channel um once the video is up and thank you Stephen for for joining us um and and answering all the various questions we really appreciate your time yeah that was a lot of fun uh yeah thank you for hosting this +well um thank you everyone for joining we have quite a uh a good audience for um for otel Q&A um we have uh Steven Schwarz uh today with us now um just a quick note on the format oh which Dan actually posted in the chat awesome um yeah so there I'll be asking Steph some questions and then afterwards um time permitting you'll have uh the opportunity to post some questions for Stephen as well in the uh in the link that Dan provided so um all right so first things first um Stephen do you want to introduce yourself yeah sure so uh my title at work is uh Form Engineering and I work for a company uh they do Payment Processing so you know when you're working with payments you're working with your money you need to be uh kind of exact with your your observability um what my team does is we manage all of the observability infrastructure we've got about a hundred uh technical folks that we're we're managing it for uh and we try and be sort of like the experts and observability and and share some of the best practices and you know help them solve their problems um so I recently join or I joined this team about almost a year ago and the project that I started on was actually migrating from uh another observability vendor where we're using proprietary instrumentation uh and we adopted open Telemetry and we moved to a new observability vendor uh so hopefully some of those learnings are you know useful to other uh folks going through that process um and also worth calling out is that I really enjoyed uh contributing to the open Telemetry collector contrib repo uh I had to do that to um unblock myself with a few bugs that we face but it was also just a really good way to uh you know understand what the collectors are doing under the hood uh I was completely new to goang but the the code is really approachable so yeah I I'd recommend taking a look if you haven't it's it's uh it made my job a lot easier for sure that's great thanks for the uh thanks for the intro now can you uh tell us a little bit about like uh the architectures that you use um in your organization like what programming languages are used and deployment landscape that kind of thing yeah so I have that architecture diagram I shared with you would it be a good time to share that yeah yeah go for it okay all right so believe my screen is visible everyone can see a diagram yep I can see it hopefully everyone else can as well well okay so go kind of left to right we can dive into uh specific parts that seem interesting um so we are running on kubernetes um we have multiple production clusters um and within each of those clusters we are using uh the open Telemetry operator uh and that gives us a way to inject like common configuration uh of the SDK into our uh you know our teams applications into their containers um and it also lets us to automatically turn on auto instrumentation uh so we get uh consistent spans collected uh from all of the all the apps and another uh sort of unique part is we we are using the open Telemetry uh we're running open tetes as a side car um meaning that uh alongside each instance of the application we we have a mini collector running and the app can send all their Telemetry over local hosts uh to that Sidecar um so we are currently sending metrics and spans uh logs we it's a bit of a long story but we we aren't currently using logs through open Telemetry um for metrics so we we use open Telemetry but it as sort of like a hop we we also use Prometheus um to scrape the side car before it pushes to uh grafana and another interesting uh challenge was um getting tail sampling to work um because we have spans coming from multiple kubernetes clusters and we only want to keep some of them um but we don't want to make the decision about which traces we keep until we have all the spans maybe we wait to see if there's an error um so uh we have a separate cluster that all the spans go to to make that decision um and we uh we load balance uh just to to help scale that we have these uh collectors in that common cluster they just uh load balance by Trace ID so we don't have a ton of spans piling up on on one collector and uh yeah that's sort of like a high level overview of our setup that's awesome um it's really cool to see like uh this kind of setup out in in real life because I think that's uh I do feel like that's kind of a burning question with with a lot of um our our practitioners is like how do you set up your collectors um so it's it's very nice to see um and and cool that you're using the open Telemetry operator um I think one of you're one of the few people that I've I've spoken with that's not to say people don't use the operator um but you're you're definitely one of the few people I've spoken with um recently who who has been using the operator uh what aspects of the operator are you currently using like do you do you leverage like the auto instrumentation do you um leverage the opamp capabilities um what's the other one uh for and due leverage like the The Collector uh deployment cap cap abilities like to you use all three or or combination uh so the main ones we're using are just you know injecting the common SDK configuration um that that saves us a ton of time we don't have to get teams to make code changes we can just manage that centrally and kind of push it out uh to all the teams um same with the auto instrumentation uh they just add you know a line to their configuration of their uh you know their pod and then they get the instrumentation um the other ones you mentioned we we aren't using but we are using the the side car which is the open Telemetry operator inject cool cool um yeah I I have to like put a plug for the operator um Auto instrumentation it is actually so cool it like you literally don't have to write code you just put an annotation in manifest and then well and and the uh instrumentation CR of course that you have to figure but like it's pretty cool I have to say um yeah yeah you know the least effort you can make for teams the better like even getting them to add The annotation you know can sometimes take time so uh it's it's yeah I think it made the migration go much faster being able to uh you know minimize the amount of work they have to do absolutely now um I wanted to ask because you you mentioned that you you had migrated from from a previous vendor using their own proprietary um Telemetry solution um what what what made you decide to do that migration yeah so um cost was a big one it was hard to control in previous vendor um and I guess support wasn't good so those are I think the main drivers I think just a lot of stakeholders were unhappy with it um and we wanted a bit more The Collector gives us a lot of control over uh cost um that this other vendor doesn't right right yeah that makes that makes a lot of sense now um another thing that I wanted to ask in terms of um scaling your collectors like have you experienced any uh any challenges in in scaling your your collectors because uh you know you mentioned that you have a sidecar deployment pattern um how many how many different side cars are you running at at a particular point in time like is it a lot that you're managing um yeah so the and for the side cards so we run one for each pod or instance of the application so that kind of almost scales on its own which is quite nice um so that that works um the tricky part with the sidecar is uh configuring the like how many resources to allocate like how much memory to ask from kubernetes uh just because every application you know is slightly different in their volume and um so and there's not really there currently there's not a way to uh like at the app for a specific application they can't say that they want a specific amount of memory we need to configure that in our centralized configuration so it creates a lot of coordination overhead if we want to tune that right um but we've only had to tune it I think for one one application so far um one of our monolist so that that has it's actually been quite easy to scale this uh the side C okay that's great that's great now what about uh what about your load balanced uh collectors um how have you did you encounter any like challenges in in setting that up yeah uh well I mean at the beginning we uh there's I think a regression that caused the memory leak um in one of the components we're using the span metrics one um so that that was a problem that was actually one of the things that I I fixed uh as like the open source contribution um and I guess our the way our setup is quite memory intensive because we have to Cache our metrics in memory uh for Prometheus to scrape them um so yeah it's really been a matter of keeping memory uh man I guess at a reasonable level um so right now we use uh the kubernetes horizontal pod a scaler to that watches the memory of the each of the instances of The Collector and it makes scaling decisions based on that yeah um it it works pretty well took some uh tinkering um one thing I've noticed and maybe other I'd be curious if other people have a good solution for this is when our vendor graa has because we're using when they have an outage uh I find the Telemetry starts to pile up um waiting for service to be restored and we see this big spike in CPU and memory so it's quite hard to uh uh I guess like I think if the outage were to go for quite a while probably the memory usage would just start to kill the kill the uh collectors or be very high uh so that's a bit of a problem um we still have right right um now another question that I had for you um just going back to the um Hotel operator um what made you decide to use the operator in the first place is it something that you had been made aware of or someone like did did you hear about it from from folks in the community uh so I I heard about it actually from a colleague um so it was their recommendation but it fit with the kind of platform I guess ethos of uh you know just getting try to get out of the way of the developers and yeah um you know have a way to uh I guess do things without the developers having to get involved right right yeah fair enough that that is one thing I I like about the operator now what was your experience in in in running it like was it something that you felt was straightforward um because I mean one of the things that's that we really value about these sessions is not only like learning how you're using these things in real life but also like learning like your user experience with using components of open Telemetry so um is it's something where felt like it was documented well enough did you have to do a little bit of digging poke around ask some questions in the operator slack um it was I'd say generally like compared to the collector itself it was very smooth but I guess it's a I mean anyways I'm not sure why that is but uh yeah it worked quite well um and the maintainers are when we did have sort of minor issues maintainers were very responsive in the slack channels awesome that that's good to hear i' I've had always really good experience also with the hotel operator maintainers so that's that's nice to hear as well great um now the other thing that I wanted to ask is um have you when you're using your collectors um are your are you building your own collector distribution um yeah we are um and that was because we uh when we fac some of those bugs um we couldn't wait until it was push to the main distribution um so we fored it just to to fix it earlier and then once that happened we kind of just had we started to make oh other changes so it's a bit hard to get off the fork now but it's something we do want to do now is that um does that involve using the otel collector Builder tool um when you're when you're deploying your your collectors um yeah it does uh I looked at sort of the the basically the docker file how they build the conty bro and just kind of copied uh the commands from there oh okay okay have you have you used the actual because there's um there's a a a tool where you basically specify like the components like which receivers which processors Etc that you want to include did you have you tried using that Tool uh yeah yes that's what we're using uh to build it uh we we copied like the list of components yeah oh okay okay got it got it sorry I misunderstood you um awesome um and you know you mentioned you you uh did your your own customizations and that um that kind of forced you to contribute uh back to the collector how is which is it sounds like you know it ended up being like a win-win for your team um how was the how was the experience of contributing back to the collector overall yeah it was um it was a good experience the you know people were helpful in uh you know during the code review it took a little while to get uh the code merged maybe this is a like I noticed um you know you'd have a separate person for approving it and then it would sit unmerged for a few weeks and then you get all these merge conflicts because like the the versions would be upgraded so I had to you know go back and update the go mod files a bunch of times so that that was I think maybe there's room to like once someone proves it to maybe merge it a bit faster but I don't know the sort of the rationale behind that yeah fair enough fair enough um and um the other thing I wanted to ask um you know what um you you mentioned that you um uh you mentioned that you use Auto instrumentation have you done any manual instrumentation as well yeah so the um yeah we we had to show teams because we had manual instrumentation using that proprietary format so we had to show teams how to convert it over to the otel format and then also educate them on how to um you know use otel manual instrumentation and how did that go um was did you find that teams were were um receptive to that um it's still I think they're slowly slowly doing it um it's I think tracing isn't maybe like the power of or the benefits of tracing isn't maybe as obvious so I think probably I just need more examples of like why it's useful to give them the motivation to to add more metadata to their spans for example right right right um now does does your team ever get asked to like instrument stuff for other teams um not uh not specifically for their applications yeah I'm very glad to hear one of my observability pet peeves well that's that's good to hear and so it looks like um now in terms of um the the instrumentation like it looks from your diagram that it's primarily like a Java app is that correct uh yeah yeah mostly Java we do have some some goang as well okay um now um have you um how did you find it in terms of like um you know all all the the stuff aside like the learning curve for for instrumenting stuff in open Telemetry um how did you find it like since you in a position to teach folks how to instrument their code with open Telemetry was it a big learning curve for your team as far as understanding how the uh how to how the manual instrumentation works worked for like both Java and go like did you notice any um anything that was like maybe easier in one language compared to the other or where you had to go back to the sigs to ask for clarification yeah the uh so Java it they've done a really good job with it I think the language just makes it easier um so yeah there because we use um like the kind of we use that micrometer which is like a Java library for mitting met metrics so they didn't actually have to change anything to get the metrics to work oh okay got it um and yeah the and the span API is pretty uh pretty intuitive as well U goaling I mean it's not my that's not my personal strength but I found it's a bit more manual uh it was it's definitely harder to use than the uh the Java one and right I'm curious if anyone's if anyone's used the auto instrumentation I haven't had a chance to play around with that but uh that would be we we're not currently using it for goaling but it would be nice if we could get that working yeah for folks uh for folks who are listening uh does anyone have have experience with that that would like to uh to share feel free to raise your hand if you do have any experience no takers all right um okay um the uh guess we're I'm just about done asking the questions um the only other thing is that I was going to ask is do you have any general feedback that you wanted to uh provide um to you know the open Telemetry maintainers around like your open to infmetry experience as far as like um using components like e of use uh you know you mentioned the the contribution experience you said it was a little bit challenging with with approving the PRS um anything else that you wanted to add to that um I think uh like it felt like we were kind of figuring out a lot of a lot of stuff out uh for ourselves uh like it took us quite a few iterations actually to get to this point um it might be hard to have a rest for every combination of like you know vendor and backend and and all that so that could be part of the problem um yeah I guess like more sort of uh example configuration or would be you know would be helpful like uh for example the these load balancing originally we actually had another set of uh collectors that all they did was load balancing yeah um but we realized oh we can actually load balance that you know th this deployment of collectors can low balance to itself um and that just is way easier to maintain so um yeah and I guess like sort of example configuration for common problems like tail sampling that we're doing right here right yeah that's really good piece of feedback I have heard from other folks as well that uh because you know we have the open Telemetry uh demo which is a great Showcase of I think it's really focused on on application instrumentation which is awesome um but I think there is definitely like a a desire from the community as far as um having some um like more collector specific use cases um and and specifically like stuff that's a little bit more complex than than what you would necessarily see in in the demo yeah for sure um another one is uh monitoring your collectors um I found I had to kind of dig through the code base to find like what metrics exposed like what are they doing um and I feel like that's probably there's probably some commment alerts or things that for most setups would be useful so maybe someone already has this but like uh you know some some we can document like hey these are useful things to monitor um the the actually the the dashboard that that for grafana they have that was pretty useful but uh I think on the alert side that would be uh that was I didn't find anything uh there right right cool well thank you for the feedback on that um now I guess we could um we can do one of two things uh we've got the uh the lean coffee board where folks can post questions for Stephen so I'm actually going to check the board oh we have some questions um yeah let's let's go through that and then if we have some time afterwards then Stephen you can ask some questions um to to our uh viewers as well okay okay um Dan do you wna do you want to take it on from here on uh the the questions from the board sure thing we've got many questions from the audience which is great um also feel free to go and vote for the ones that you think um are the most interesting there I have marked one of them as answered already because we talked about the collector um Builder and how you build your own dist for the collector on your side car uh starting from the the one that we talked about tail B uh sampling a little bit how you do the the how you use the obey sampling um but I wanted to ask this question here is like do you only use tail based sampling or do you also use head-based and any tips or uh thoughts on mixing these approaches uh yeah so we only use um tail sampling um I guess it if you like it's it's more work to get it set up but when you have the full Trace to make your sampling decision I find that you can make the the best decision um so yeah that's why we we just lean on tail sampling completely I guess we head B head Bas sampling there's always a risk and you're going to miss something important right because it's completely probabalistic so um yeah good point and moving on to the next one um how do you handle container sizing on the collector sidec cars you mentioned this already uh that you had to customize or tune one of the collector's uh collector sizes so do you have one size that fits all or anything you would like to improve in that setup or like resource allocation yeah so um I think what would be nice is if pods could add like an annotation or you know saying this is the resources I need uh instead what we have to do is kind of hacky um so for each side car it's like a separate res like kubernetes resource or configuration uh file so what we do is we have like a for Loop that goes through and creates uh those configurations for different resource uh combinations um so and so it's something we have to coordinate like hey you have to okay update the sidec card configuration that your pod wants to use to this otel sidecar 500 megabytes one CPU or you know so it's it's a bit of it's it's an extra coordination work there uh yeah cool um yeah okay next one is related to serverless um if you if you use serverless and um you know any tips or goas to look out for in instrumented serverless workloads with open Telemetry we don't use servus so don't have any uh advice there that's fine yeah I mean I I wanted to ask that in in case you you had some experience with that um also you mentioned that use fluent D for logs reading from standard out um are you considering moving to the otel lock file receiver um yeah it's something I'd like to explore we we sort of had a bit of a issue when we first tried using logs I'm curious if anyone has found a good solution for this um so what we wanted to do is uh like dur store our logs on disk so if there's like an outage we don't lose them um we ran into a bug with um the durable storage extension in The Collector uh and that kind of spooked us and we already had fluent D running so that's why we we kind of just pivoted to that but I don't know that was a while ago maybe and maybe people have had good success with with getting that durability uh for their logs yeah just to add on this uh with flu and D are you decorating your Logs with some of it semantic conventions or Trace ID span ID that you can so you can then correlate with your traces yeah we uh so we definitely have the logs and oh sorry the trace IDs and the span IDs and we're actually um adopting there's um a specification for the event uh I guess they call it like log events so it's sort of a structured format for logs uh in the opland Telemetry specification um so we're trying that out uh as well um yeah cool um do you use oel to monitor otel collectors and U we use um yeah it's like a big uh it's always speak Bol mud uh monitoring your monitoring but so these Prometheus to get the metrics from the collectors and fluent to collect the logs from the collectors and that that's worked pretty well um okay moving on to a bit more of your talking about your teams sounds like you uh you folks have DED a dedicated platform team uh that manages the oo collectors in your organization do you have any thoughts on the opposite of this where you have no collector Gateway or no collector um endpoint basically 100% of your Telemetry data is then sent to your observability provider uh yeah I guess in our case um we decided to use the collectors so that we could um do sampling uh to control costs of our traces um and also to try and um you know have some durability if uh I guess the vendor goes down we can kind of move that like retry logic and um off to the collector um yeah I'm curious if anyone has if anyone's doing that in in production it would definitely simplify things but I guess there's some tradeoffs as well there yeah I guess there's a trade-off as well of um authentication to a provider and so right is to sell in one place as well some of the benefits pros and cons I guess right um the next one it's related to um well we talked about Auto instrumentation the auto operator and how you inject those instrumentation packages and then we also talked about the fact that you are also using um like custom manual instrumentation um I guess the question is how do you find if your have your teams found that auto instrumentation is sufficient or is there a lot of custom instrumentation that's needed yeah so uh one thing I haven't talked about that I want to call it Les been great for us is um the SP metrics um component um so we we know we generate metrics so uh you know frequency latency eror error counts for every single span um and because we're using the automous instrumentation um every application emits the spans in the same format so basically out of the box without adding any uh instrumentation we get like I guess some vors call like APM or like red metrics so like all your API calls all your database queries um we have metrics on that automatically um so that that goes quite a long way and we can build like comment dashboards and commment alerts uh that all all these apps just get out of the box uh because of that standardization so it's a really good starting point um and then I guess so I like that is probably quite good but then if teams want to add metadata they they can um so yeah I would definitely recommend SP metrics uh if if that's like option for for people yeah I'm going to add a bit to this one as well do you find that you're as in like if you're moving from a from a from a world where like you know you had to instrument everything and now there is a lot of Auto instrumentation out there do you find that Engineers normally tend to not know about that auto instrumentation and they do it themselves so how do you manage that sort of like I guess learning or education piece saying that you may not need to instrument things is that is that has it been a challenge within your organization or everyone just uh yeah I i' say it is like we have some redundant uh like for example our API metrics like spring Boot and Java automatically emits some metrics so a lot of applications were getting some redundant data um so so yeah that that has been a problem uh but yeah so I guess it just comes down education and then having a way of tracking the cardinality if uh so when we've seen that like oh you're sending a ton of API metrics and we already have a capture the span metrics we can reach out to that team and say oh you can you can probably disable this you don't you don't really need this we don't even pay for it cool um regarding samplin um and then I'm assuming this is related to sampling policies can you advise on the is that the collector where you manage the policies for all traces or can teams Dial Dial that config up and down themselves uh so ours ours is pretty simple um so we sample you know errors we keep if the trace is longer than at something like 5 Seconds we keep but we do have an escape hatch if they add an attribute to their span to force sampling uh so far no one has like hasn't caused any problems um it might in the future you know teams overuse it but uh that that's work that's works so far and um related to some of that SDK conflict that you currently inject through the through the through environment variables um do you currently use any type of um other confli that may not be supported through environment variables like metric views for example is that do you have a way to apply defaults across the board for that type of uh config um I'm just reading the question oh I see um we haven't had to do that too much and when we have um we do have I mean our I guess our company's lucky and that we have sort of a tool that we can use to um it sort of generates uh applications from scratch um and then there's a way to sort of regenerate it the Regeneration doesn't work too well but um we know that most teams have a pretty similar format to their application so when we do need to make a change that we can't do with the operator um it's usually just boiler plate copy paste we can show them a merge request and they can copy it in but it does take you know can take a couple months for every team to adopt that cool okay um I guess one related to supply chain security um with the AEL sdks um so that do you how do you handle supply chain security so that otel sdks are safe to include them in application code or do you build them from sour yourself and then run your own security tests uh are you thinking this would be you know if there's like a c CB in one of the SDK versions how would we sort of manage patching that for everyone or I guess yeah or Downstream um or Upstream dependencies from from hotel packages yeah yeah um I mean for different reason I kind of created a wrapper dependency um that wraps like all the all the otel library versions that we use and we know they work together um just because we we found some versions were kind of incompatible um so from the app team's perspective they depend on one uh one dependency and then they get all the otel dependencies as transitive dependencies um so I suppose if we were to you know if there we need to upgrade it very quickly I guess it would just be one thing that they need to update um although that create like it works in theory but then it's also caused uh dependency conflicts just because at least how Maven Works in Java so I don't know if that's actually the best thing to do it might be more of a headache than it's worth cool um thank you okay so moving on to the next one uh related to um yeah so the auto operator and automatic instrumentation is that your preferred solution or can the oel operator used in parallel with sdks for example they not net in the case of the the person asking this question and you provide some balance of scalability and config configurability is there a recommendation here my immediate concern is that the manual instrumentation inside the application could make it difficult to configure and manage um at scale so it's this sort of like how how to balance um giving teams the flexibility to add manual sort of configuration and customization and then versus the sort of defaults that we inject for them um yeah so I mean so far we do I guess you know you can't override um the environment variables that are injected into um into the application automatically by the platform team uh um so far I guess it hasn't been uh too much of a problem um with teams teams haven't really wanted to deviate Too Much from the configuration that we've given them but yeah be curious longer term if that does cause problems thank you um that one this is an interesting one are your devs um using the lgtm docker container from grafana to develop Hotel implementation and then do local troubleshooting uh not yet right now they can only really test it and by deploying it to our clusters so we do need a better solution there um so they can test locally yeah I personally love there's like more and more solutions coming up that are open source that allow you to see that Telemetry in your own um computer I just testing uh so yeah a lot of uh really cool Solutions out there at the moment if you had a magic wand what's the thing that you would love to add remove or change in otel and why that is a logic question yeah I mean I think this is maybe short short cited but um if we could get rid of Prometheus in our architexture and the reasons we can't um right now is because we rely on Prometheus to um kind of aggregate our data together for cost savings um Source band metrics we we generate like 50 buckets uh histogram buckets for every span so it's it gets really expensive um and if each pod is you know has its own set of 50 buckets uh are cost scales with a number of PODS but we use Prometheus um to kind of like aggregate all the counts from all the pods and that like Cuts our cost significantly um I don't think the collector can do that today um so yeah that would I mean that would just make our architecture simpler makes sense um are you using multiple otel backends uh to process some of the things that internally uh compared to compared to your vendor or is it everything going to the vendor I guess um we have uh one one vendor if that that was the question that we're pushing to yeah so I guess but you're not um you're not pushing anything internally to anywhere else just everything goes to oh right yeah yeah we don't uh yeah okay and I think we've got time for one last one and that is given that the collector can quickly become a bottleneck due to back pressure and do you see any disadvantages having the SDK export that data directly to the back end except for the cost um con yeah I think that was similar to the other question about like why not just push directly to you know that your vendor from your your pod yeah it um it does make it easier to kind of scale you don't have to worry well yeah that's a good question I guess you're Shifting the back pressure then to your Apple um and then I'm then so yeah it might be though easier to handle if it's smaller amount of data but uh yeah that one I don't I don't I'm not too sure about Ben slots that' be interesting to discuss yeah that's um maybe easier to handle but also more impactful for the application so I guess there are pros and cons there as well right so if you're if you have your all your telemetry s of like shipped from your or out of your pod as soon as possible there are benefits for that as well right in terms of impacts so it's failed somewhere else in your infrastructure um because there will be at some point if your back end is down but um yeah pros and cons that's everything yeah and it seems like just looking at least the Java SDK they're pretty conservative about how much data piles up like I think it's like 1,000 24 spans is the max that will like buffer a memory before some low number that before we'll start dropping it because I guess they're really trying to lower the you know the footprint on the the application yeah exactly okay I think that's all we have time for um thank you very much Stephen for joining us we have we that was quite a lot of questions we went through from the audience uh thanks for everyone that joined and asked questions as well uh super nice to see uh engagement in as a session Q&A uh do you want to say a few words to um say goodbye yeah thanks uh thanks Dan and thanks for uh for doing the uh the questions yeah it's really nice to see the uh the engagement from from folks um in the uh in in the Q&A um I think this is a topic that really interests people so um we we love it when you engage also if anyone out there would like to participate in otel Q&A or otel in practice we are always looking for folks to share their use cases with us um you can just uh you can either DM either Dan reys or me um or if uh if you want you can also um just message Us in -- n- user our channel in the cncf slack um we are we we will be happy to hear your your use cases and have conversations and also as I mentioned recording for this will be up on YouTube so um anyone that you know that wishes they had attended but could not attend um let them know um we'll we usually post on the otel socials and also in the channel um once the video is up and thank you Stephen for for joining us um and and answering all the various questions we really appreciate your time yeah that was a lot of fun uh yeah thank you for hosting this diff --git a/video-transcripts/transcripts/2024-11-04T21:03:07Z-otel-q-a-with-dan-ravenstone.md b/video-transcripts/transcripts/2024-11-04T21:03:07Z-otel-q-a-with-dan-ravenstone.md index caa221a..6af26fa 100644 --- a/video-transcripts/transcripts/2024-11-04T21:03:07Z-otel-q-a-with-dan-ravenstone.md +++ b/video-transcripts/transcripts/2024-11-04T21:03:07Z-otel-q-a-with-dan-ravenstone.md @@ -10,77 +10,102 @@ URL: https://www.youtube.com/watch?v=BMkckCQN8eg ## Summary -In this YouTube video, Dan Ravenstone, a staff engineer at Top Hat, discusses the evolution of monitoring to observability, particularly focusing on the use of OpenTelemetry. The conversation, hosted by Adrian, touches on the importance of understanding user experience and translating technical concepts into business value, especially for executives. Dan shares insights from his experience transitioning Top Hat away from vendor-specific monitoring tools to OpenTelemetry, highlighting the cultural and technical challenges faced during this shift. Key points include the need for effective communication, the significance of quality data over quantity, and how to encourage developers to take ownership of instrumentation in their code. The video also covers practical approaches to defining what to instrument and how to demonstrate the value of distributed tracing versus traditional logging practices. +In this YouTube video, Dan Ravenstone, a staff engineer at Top Hat, discusses the evolution of monitoring to observability, particularly in the context of using OpenTelemetry. The conversation includes insights about the challenges organizations face when adopting observability, the importance of user experience in guiding instrumentation choices, and the cultural shifts necessary for developers to embrace new practices. Dan reflects on his experiences transitioning Top Hat from a vendor-specific monitoring tool to OpenTelemetry, emphasizing the need for effective communication and patience while educating teams. He highlights the role of structured logging, the significance of service level objectives (SLOs), and the necessity of aligning tools with organizational needs. The discussion culminates with audience questions addressing telemetry quality, transitioning from logs to tracing, and best practices for ensuring user satisfaction. -# Otel Q&A with Dan Ravenstone +## Chapters -[Music] +Sure! Here are the key moments from the livestream along with their timestamps: -Welcome everyone to Otel Q&A! Thanks, Dan, for joining us on such short notice. We're glad to squeeze you in for October, especially with KubeCon coming up in November. For those going, a few of us from Top Hat will be there, including Dan and myself, so it will be busy! +00:00:00 Introductions and event overview +00:02:30 Dan Ravenstone introduces himself and shares his background +00:05:45 Discussion on the evolution from monitoring to observability +00:10:00 Key changes in observability and the challenges with alerting +00:15:15 The importance of understanding user experience in observability +00:20:00 Challenges organizations face in adopting observability +00:25:30 The significance of using open Telemetry versus vendor-specific libraries +00:30:45 Cultural and technical shifts required for observability adoption +00:35:00 The role of instrumentation in development and its impact on user experience +00:40:15 The current state of observability at Top Hat and future plans +00:45:00 Q&A session begins with audience questions -### Introduction +Let me know if you need any further assistance! + +# Otel Q&A with Dan Ravenstone -**Dan:** Hello! I am Dan Ravenstone, a staff engineer at Top Hat. I've been here for about two and a half years—actually, it will be three years this May. My background is primarily in monitoring and observability, which has been a significant part of my technical career. +Welcome everyone to the Otel Q&A! Thank you, Dan, for joining us on such short notice. I’m glad we were able to squeeze you in for October, especially with KubeCon coming up in November. For those who are going, a few of us will be there—Dan and I will represent from the team. -I got started in this field at a company called Affiliat in Toronto, where I managed the info.org domain registry. I initially worked in tech support, but after realizing how often our customers called to report outages, I took the initiative to learn more about monitoring tools. This led me to tools like Big Brother, Nagios, and Cacti, which I loved because they allowed me to look for patterns and solve problems proactively. +### Introduction -As technology evolved, I transitioned into the world of observability and found that being proactive is far better than being reactive—though it can take some time for people to understand this shift. +**Dan:** +Hello! I’m Dan Ravenstone, a staff engineer at Top Hat. I’ve been here for a bit over two and a half years, almost three by May. My background is mainly in monitoring and observability. I got my start at a company called Affiliat in Toronto, which managed the .info domain registry. -### Monitoring to Observability: Changes and Challenges +I started in tech support but quickly realized that I was tired of customers calling us to report downtime. So, I took the initiative to learn monitoring tools. Back in those days, we had tools like Big Brother, Nagios, and Cacti. I loved monitoring because it was like being on the front line, finding problems before they impacted customers. -When reflecting on the evolution from monitoring to observability, one thing remains unchanged: alerting. Alerting still presents challenges, but observability and OpenTelemetry have allowed us to view our services differently. Monitoring used to be reactive, focusing only on symptoms, but now we can look at the entire user experience, which is crucial. +As technology progressed, I fell in love with observability and how it provides a different perspective. Being proactive is far superior to reactive strategies, which many people still struggle to grasp. -It's essential to understand how users experience our services because that directly impacts revenue. If users are unhappy, they might stop using the service, which can lead to significant financial implications. Therefore, it’s vital to communicate the value of observability in terms that executives understand: dollars and cents. +### Changes in the Industry -### Challenges in Adoption +**Interviewer:** +You’ve been in this field for a while. What major changes have you seen in monitoring and observability? -One of the main challenges organizations face with observability is communication and culture. There's no technical reason to avoid adopting observability—it's often about resources and time. For instance, moving away from a vendor-specific library to OpenTelemetry is a significant undertaking. It requires careful planning and convincing stakeholders of the value of making that transition. +**Dan:** +One thing that hasn’t changed is alerting; it remains a problem. However, in observability and especially with OpenTelemetry, we’ve learned to look at our services differently. In the past, monitoring was reactive—we only had symptoms to deal with. For example, Nagios would alert us to issues based on CPU, memory, or disk space, but that’s not always indicative of a problem anymore. -Additionally, organizations often struggle with sampling and managing data volume, leading to increased costs without realizing the benefits. It's not just a lift-and-shift; it requires thoughtful implementation. +With the rise of Kubernetes and containers, we’ve had to adapt. We started aggregating logs, but even that posed challenges. Despite best practices, people would often log inconsistently. Observability has taught us to focus on overall user experience rather than getting lost in the minutiae. When in the middle of an incident, it’s crucial to decipher which data points truly matter. -### The Importance of User Experience +### Challenges in Adoption -In our experience at Top Hat, we emphasize the user journey to decide what to instrument. We encourage development teams to focus on user interactions and what they expect from the service. This approach helps identify key areas for instrumentation, ensuring that we capture the most relevant data to enhance user experience. +**Interviewer:** +What are some challenges organizations face today regarding observability? -### Instrumentation Culture +**Dan:** +Communication is a big issue, often stemming from cultural barriers. There’s no technical reason for organizations not to adopt observability other than resource constraints. For instance, our team is moving away from a vendor-specific library to OpenTelemetry. Transitioning takes time and can be a tough sell to executives, especially when discussing costs. -Currently, at Top Hat, instrumentation isn't fully ingrained in our culture, primarily because we've been migrating from a vendor to OpenTelemetry. Developers have relied on auto-instrumentation without fully understanding its importance. Education and patience are essential as we work to incorporate instrumentation into our daily practices. +Additionally, once organizations make the shift, they often struggle with implementation and proper usage. Many teams try to adopt OpenTelemetry but end up overloading their systems with data without understanding how to manage it effectively, leading to rising costs. -### Technology and Tools +### Tools and Cultural Shift -At Top Hat, we primarily use Python, and we've used auto-instrumentation for our code. We've built a wrapper library to simplify the process for developers, making it easier for them to implement without significant changes. Our goal is to gradually encourage teams to take ownership of their instrumentation and understand its impact on user experience. +**Interviewer:** +You mentioned that the adoption of observability tools is as much about culture as about technology. How do you approach changing the culture around instrumentation? -Regarding Kubernetes, we are not currently using it because it would add unnecessary complexity to our services. Instead, we're utilizing a cloud provider's container management service, which has served us well. +**Dan:** +It's essential to connect instrumentation to the user journey. When developers understand how their work affects the user experience, it becomes easier to convince them of the importance of proper instrumentation. We encourage teams to document their user interactions and understand the dependencies and prerequisites. -### Final Thoughts +Ultimately, the goal is to make instrumentation part of the development culture, not just a checkbox. We must support our developers in learning and adapting to new tools without overwhelming them. -In conclusion, when discussing observability and instrumentation, it’s crucial to focus on the user experience. We need to measure what matters most to our users and make decisions based on that data. While tools and technology are essential, they should be the last consideration. The priority should always be understanding user needs and how we can best support them. +### Observability in Practice -Thank you, Dan, for sharing your insights during this Otel Q&A session! +**Interviewer:** +What is the instrumentation culture like at Top Hat? ---- +**Dan:** +Currently, it’s still a work in progress. We have some auto instrumentation in place, but many developers are still getting used to it. There’s a tendency to view instrumentation as just a task to complete rather than a valuable part of their work. Our goal is to make it as seamless as possible and to show the value in real-time data. -### Questions from the Audience +### Final Thoughts -1. **What is your opinion on telemetry quality as a way to reduce spend? Is more data always better for observability?** +**Interviewer:** +What advice do you have for teams adopting observability practices? - - More data doesn't always mean better quality. It's about capturing meaningful data that reflects user experience. Quality data helps reduce spending because it allows for better sampling and focuses on what's valuable. +**Dan:** +Always focus on the user. If users are unhappy, it doesn’t matter how much data you collect; you need to know what frustrates them. Having clear metrics and objectives based on user experience is vital. The right tools can help, but they should be the last consideration. -2. **How do you approach the questions of what to instrument and how much?** +### Questions from the Audience - - This is linked to understanding the user journey. By identifying key user interactions and their expectations, we can determine what to instrument to ensure we capture relevant data. +1. **What is your opinion on telemetry quality as a way to reduce spending? Is more data always better for observability?** + - Quality is key. More data doesn’t necessarily mean better observability. It’s essential to capture data that reflects user experience. -3. **In an organization that has adopted distributed tracing, how do you encourage teams to leave behind outdated practices based on logs?** +2. **How do you approach the "what to instrument" question with your teams?** + - We start with understanding the user journey. Documenting user interactions helps pinpoint what needs to be instrumented. - - Communication is key. Understanding their reasons for preferring logs and showing the value of distributed tracing with concrete examples can help bridge the gap. +3. **In an organization that has adopted distributed tracing, how do you make teams leave behind outdated practices based on logs?** + - You need to understand their reluctance and restructure discussions around their viewpoints. Help them see the value of distributed tracing in understanding service interactions. 4. **What questions do you ask to understand what is most important to someone's organization?** + - Always focus on the user experience. If you understand their needs and frustrations, you can tailor your approach effectively. - - It's always about the user. We need to measure user satisfaction and understand what frustrates them. This insight allows us to make informed decisions about tooling and processes. - -Thank you all for attending this session! +Thank you, Dan, for sharing your insights today! It was a pleasure having you on the Otel Q&A. ## Raw YouTube Transcript -[Music] welcome everyone to otel Q&A and thanks Dan for joining us for otel Q&A especially on like we got you on such short notice um I'm glad we're able to squeeze you in for October um especially me too cucon is coming up in November so for folks who are going um a few of us will be there uh re Dan and I will be there from theel and user Sig so um be busy yeah it's going to be busy when is where is that happening by the way it's in Salt Lake City Salt Lake City nice I've never been there myself personally yeah me neither me neither oh oh so this be an experience for you too yeah yeah looking exciting exciting um start asking you questions I realize I'm the one here is supposed to be with the breaks folks it's gonna be a fun one oh yeah not sure about informative but it will be fun well here we go okay let's start uh let's start with the question so Dan why don't you introduce yourself and tell us what you do hello I am Dan Ravenstone I am a staff engineer at Top Hat uh I've been there for guess two and a half years maybe no a little bit longer than that come may I think it'll be three um a little of my background um I have been doing monitoring observability for the bulk of my technical career uh I started back in um where I actually this and I usually refer to as like where the bug bit the monitoring bug bit me was when I was at a company called affiliat which was at Young in 401 area in Toronto uh affiliat managed the info.org regist domain registry um in the back end of it uh for back in the day and I started there in texport but then I one day opened up my big mouth and said Hey listen I'm tired of our customers calling us up and telling us our stuff is down can we fix this and so they said well then you ask for it you get it so I went out and so I that's how I started learning about moning tools and that's back in the day of Big Brother nagios cacti all those fun things and um and I just loved it this kind of like I think one one of the things I liked about monitoring in those days was that like I like looking for patterns I like detective novels I like the all you all the fun things and what this do is like kind of allow for somebody to kind of like be in the front lines in the operations and dive in and find problems and report them back and then like the better more the better your monitoring tools were the more likely you were able to follow and find the problems a lot sooner before they become actually customer impact and I was always been my kind of like push when I did this and of course as things progressed and modern and changed uh we got into the wonderful world of observability and a whole different way of looking at it and I fell in love with with that obviously as well and tried to grow with the times and see how I you know how being proactive is a lot better than being reactive which is kind of like you would think is an obvious thing but it takes a lot of people time to kind of get their heads wrapped around um so yeah that is kind of uh how I got to the how I got to this part of the world world and a little bit about me there's tons of other things but we don't want to go into those because then we'll get lost in the weeds and we'll never get out and we'll be lost forever sounds good well thanks um and you know like you you've been in in in this for a while like when it's started out with monitoring evolved into observability what what kinds of like big changes have you seen um in in the course of that time uh let me start off the things I haven't I haven't seen change which is still a problem alert okay alerting still has not changed surprisingly um but what has changed and I think this is one of the things that um we're trying to get to I I so I focus on alerting because I was by myself time to think what has changed so now that you know my secret I just thought myself a couple more minutes so but what has changed though in observability and in the open Telemetry especially too is it allowed for us to actually look at our our services a lot differently where moning was kind of a very largely reactive kind of thing and we only had the symptoms of what was going on so if everybody has ever used nagios is well aware of the sort of out of thebox HST monitoring sort of plugin you usually incl include which is like your CPU your memory your disc SP and these things have been carried throughout time in you know to con be indicators of a problem but that's usually not quite the case anymore is it um especially way when we have kubernetes and containers and other tools now that these things shouldn't really play a role in us so it's like more like so what else is impacting the service and we never really had those that ability to look at it until we started sort of aggregating our logs and even when we aggregated our logs it was still strangely problematic um because you know we had guidelines on how to do structure logging and that kind of thing but still people would do their own thing and things were never and so even with all the good documentation in sort of best practice around it we still logs were still problematic and again you still have to still collect a bunch of data before you can actually understand oh wait a minute this is a problem before actually kind of knowing beforehand what observ abilia has done has taught us is how to sort of like set things up in a more of a a way that we're looking at it as an overall user experience and we don't get sort of sucked into minutia sometimes and there's because there's a lot of red herrings in operations when you're especially when you're in the middle of incident you're looking at all this all this data and you're like what is the actual problem we have high CPU over here and we have slow load times over here we have latency here we have some errors here but which one of these is the the smoking I would not I don't want to use a gun but for like better word Smoking Gun right and I see that that with observability has changed how we look at that however that's for this probably this group a lot of people are familiar with it there's a whole world of of Engineers out there who are still having are still trying to get their heads wrapped around this concept and this is why I'm clown and glad we have these conversations and I try to get those people this is like this is targeted towards them of why this is so important why you need to start looking at this because there's a number of good things one you know what the user is experiencing and that's all that matters if you know what the user is experiencing that means you can translate that to how much you're making like very very like very it's it's a cost thing and it's a money thing if you know if your users are happy that means you're making money because they're be spending money because they're going to keep using your service if they're not happy they're not going to be using your service and they're going to walk away and use something else or just not use it that often and try to avoid as much as possible that means you're not gonna be making the money anyway um and I'm gonna get in the weed so I'll you're gonna hear me say that a lot because I just do so you Adrian it's all up to you to rope me back in like you gota don't worries don't worries but I I do wanna um you said something really interesting um in in terms of translating it um in terms of money because you know like observability adoption and maybe I'm getting ahead of myself here but I I like way you said um in terms of observability adoption like it's as much a a top down thing as a bottom up thing and one way to speak to the benefits of observability is in dollars and cense and Executives speak in dollars and cense so um proving the the worth of of having an observable system I think becomes really really important and and being able to use that language um with Executives to be able to convince them I think is very valuable so and and that's what I've been trying to focus more on and like there are like there's quantif quantifiable yeah quantifiable reasons why to look at this we can do that why obsil excuse me uh provides value toward us the end of the day but I think we we' we're still I think we're still getting our we're still trying to get that figured out I feel like that um anyway you know what before we jump in let's let's keep moving forward because I I I could go into a whole other area and I don't know I don't want to sort of suck a lot of time in there but it is one of those things I think that does have is worth having conversations not with just those in the community but also talk like I mean if you're you know out there find out what your what the OB ability sort of platform or the concepts are your where you work what what are the drivers why why aren't you doing it why aren't your why aren't you using these things and ask those questions and then like you know that maybe we could that could help and feed that back to community because that could help us build out the reasons why you should be doing this because it does make better sense eventually um but yeah that's a sort of a general call to anyone who's out there who needs who wants to sort of get going on this and need help yeah I completely agree and I think this uh this takes us to our next question um which is uh you know like you've you've done this for a while now like first the monitoring space now moving into the observability space what do you think are some of the main challenges that most organizations are facing these days when it comes to observability uh there's a lot of challenges and I've been trying to get my head wrapped around some of this stuff one is communication I think so some of it I think it's mostly cultural really if you think about like I mean there's not a technical reason why people shouldn't go to this other than resourcing in time uh so and and I mean this might be jumping ahead of a little bit but where I am right now we we are using a current vendor uh which will remain nameless but we use a current vendor right now for our monitor oability purposes but we're getting away from them because of the library we're using is is is a vendor specific and that means that we're basically when it comes to doing any kind of of telemetry it has to use that that particular library and their's only getting and that's that that's one challenge because if you think about like okay so you're using this one vendor it's cost you x amount per month or per year it's exuberant however to get translate or get off of that one and go into another one where you have more options like with open Telemetry takes time it takes it's not like an like anybody who's ever gone on any kind of migration from one tool to another knows how long it takes and if you're and you have to have a few things in place to actually say why you should do this so one is like of course obviously the cost right well but you know it's hard for some Executives to say well we're spending this but it'll cost us this much to get over here and then we'll see the cost value so that there's this huge challenge of just trying to shift things over spend the time to do it and then do it properly and that's a whole other challenge is say okay yeah doing open Telemetry is one thing but doing it prop L like every other software or any other tool that you have out there you have to still use it for the right reasons and do it properly if you don't you'll still run into issues and whatever you're trying to sort of get away from over here will still might come up in a different way over here so there's those that's that's one of the challenges is just getting that just doing the whole Shi over uh I think that's a where a lot of people and they don't know how to do it and so and what happens too is then they do a shift they try it out and then they get burned for because you know they don't know how to do sampling or something like that and then they feed it off to some tool like let's say a tool that you get charged by event volume you're going to even though they're using otel they still became like oh I I saw one company who did that they start shifting you using otel but they started setting all this volume all this data to it they didn't do it right they were having like a 100,000 spans per Trace which is a huge amount and they weren't really getting a lot of value and their their costs are soaring and so it's like well you know then they kind of just drop and they well we'll figure it out another way we'll just use some other tools to figure out when there's problems with our service which is sort of like a defe kind of attitude but it does happen because they don't you know because there's a lot of work involved doing this is more than just a simple lift and shift it's a lot more involved to it um trying to think of other CH like those that to me is like some of one of the big some of the bigger challenges and I think a lot of people could probably emphasize with that one as well um there are others obviously but I mean I don't we don't have a whole lot of time and we could probably go on just you and I alone I bet you we both have experiences on some of those other challenges we have to experience but I mean uh I like to see like like how do I get how do we get past that is kind of like where I want to Target how do we get past these challenges and move forward which I don't have a proper answer to but I will want to talk about it you know that you you touched upon something that um I think is worth mentioning as well which is um moving away from like vendor libraries to open Telemetry um because you know even even though I I think one of the one of the wonderful things about open Telemetry is that it's vendor neutral um and so it makes it once you've moved all of your instrumentation to that super easy to switch vendors I would say Rel relatively easy right because they're still like yeah like at least on from the instrumentation standpoint it makes it super easy um but but it the challenge though is convincing folks to move away from those vendor libraries and I think that becomes um that becomes a bit of a sticking point because it's like they you need to move away from them because maybe that vendor is too expensive and you don't want to go with them and you want to try something fresh but there's so much time and effort and learning curve all that stuff associated with reinstrumentation and that what are your thoughts around that well and that's a very very powerful Point you're making because first off with vendor vendor libraries you get in the habit of doing things a certain way right and so again I do not want to there's no like but the thing is so some V some vendors may do things one way but it's not part of the industry standard or the best practice we see across the industry so they may like name things as certain way or do things a certain way and people get used to that so not only is it just just shifting a library over if it was just a matter of shifting a library to one or the other that that's in itself is a challenge but there's also all the behavior that gets attributed to using that older library that some for some things you got to go well listen you don't need to look at it like that anymore you need to look at it like this now and so it's not just a a technical shift it's also a cultural shift and how they even like Envision how their service is working with the new library with using open Telemetry um because open Telemetry is current it's trying to it's keep it's adding new things as we speak as every day uh it's you know it's you know we're seeing a lot more love now for mobile which is wonderful I'm so excited to see some of that because I feel that's largely been ignored for a long time but we're seeing these things getting pulled together and we're having this and it's kind of like keep following these sort of patterns and following these this this this these um standards and so make forces the developers to think a certain way there a lot of them are not used to that yet a lot of them still want to develop and I realizeed one of the things I noticed too was for developers perspective I don't I Adrian you you do you're a coder by Nature right like that's kind of yeah yeah that's when you yeah that's your back so it's not mine but I know that when I did do some coding though I would always print a line at certain parts in my code this okay where am I at so you're even like logging was is kind of part of your local development experience totally right now we're telling them don't worry so much about logging use instrumentation well you still have to have a mechanism for them to actually print that to screen and so it's just changing how they even develop in the beginning um and that and that's I think where we're facing some challenges there is that it's not just a it's it's um it's technical it's cultural it's it's about even how what you've been taught in the universities and how you even taught how to develop is changing a little bit and not everybody's quick to embrace change not and you know especially when you've got your CEO or your or your or your um your features team or product team saying we need this out now well you know you're not going to worry about trying to learn a new way of doing things and it's you know oh well now how my service is working from open Telemetry perspective it's like well I just need to get this out so I just slap these log lines in boom gone it passed you know and we're going now it's in production and we move on right so there's that that it's that kind of like also encouraging and supporting our developers to have the time to learn this and work with it so they can actually use it to their benefit and perhaps be able to do better code before even hits staging or development and then and then once it's a production you know how it should be based on all your stuff but that's it's getting that linking those people together getting them to work together in that thinking that's that's a that's a huge challenge yeah yeah absolutely and and getting getting them used to as you said like the nuances of you know instrumenting with open Telemetry which open Telemetry wants to standardize how you're doing like your your main signals in a certain way right which as you said maybe different from how you've been used to doing it whether it's like through your print statements or or vendor libraries or whatever it it's kind of like so I back in the day um this might be pre a lot of folks I don't know but I used to work with a lot with SNMP and what I liked about SNMP was it was it was basically a protocol that was agreed upon by the community and it was very basic but even then I saw this with loogy too but even with LMP I would still come across mibs I would still acoss across libraries that were not properly done so it required me even like because I knew it I would have to go in and kind of redo maybe a MIB to so I can translate the oid properly so when it goes into our or create a MIB because it wasn't available it just it just had these voids are available from an interface or something and you just have to guess what they were um even with standards in place like that something has been in our sort of part of our our industry for so long no longer really being used anymore um well actually still is by network devices but that's not hopefully we don't have to worry about that right now um is is that you know even with those things in place it still takes time for people that even then there's still mistakes and things happen so with a new thing like open Telemetry and and being setting standards it's taking a long time for people to understand how to use it to do it to set it up properly and we get their most value out of that instrumentation the most value out of using that from the beginning of the of the development cycle yeah absolutely um I do want to switch gears a little bit and talk um a little bit about specifically um the challenges that you're facing currently in in your role um when it comes to observability oh I'm really not my own mental health problems okay that's different um yeah so my my challenges have pretty much always been I feel like it's always been the same it's communication and not because I'm my challenge for me so I've been in this business for a while so what happens sometimes is as I make the mistake or the or make the assumption that folks know what I'm talking about and I have to be careful sometimes like for like I can have these conversations with many people in the same community is like within the monitoring obility community and we could go on for hours about little nuances of something whatever may people logging or or tracing anything like that and we can go on and talk about that as ad nauseum however when you when you're talking to other people and you you talk about basic things like um like the red method so rate of Errors um rate of requests durational requests which is a very basic kind of concept with the monitoring World um and which is helped kind of like we see a lot in even with notability too to a certain degree uh people don't understand that and you make the assumption that they actually kind of look at things in that light but they don't they still look at it from the old school and then like honestly like i' at some places I've been it's like they still see CPU as a indicator of a performance problem but it shouldn't be in my opinion it should because that's not CPU can is is cheap you can get more so that should be a scaling problem make sure you just you're scaling properly you should be good however I make the Assumption people know what I'm talking about and that's one thing is I got to con so I have to constantly be patient and communicate the same message over and over again to make sure they understand because people even if I tell them two or three times they still don't clue in I had one conversation with one engineer for about a month um and it took a month for them to actually understand what I was driving at and this is like because they were in different time zones so that's why it took a little bit longer because and so but it took a while before they actually oh that's what you meant and as soon as they were able to make that change everything was working better and like oh now I see where you're driving at I now see the point you're trying to make and it's just you have to be diligent you have to be patient you have to communicate and you have to change how you communicate sometimes too like you just can't say the same words over you have to rethink okay you're not getting what I'm saying what will help you understand and and then try to reward it so people understand um ad you met my friend Damian who who I work with he's actually the same place as I am but Damian who also works with me on a lot of stuff and we he's more of a he has a different background but he sometimes steps in and sort of translate what I'm saying because sometimes it helps to have a different because he puts a different sped on and all oh now I get it so that's those are some of the biggest things I find um to today are still the biggest sort of people get it it's just a matter of once but just getting them just lead you leave the horse to water type thing right but I am it's just leading them there once they're there they eventually do start to drink because they they kind of get oh I see this is good for me I will do it now yeah for sure but getting there there is sometimes you gotta kind of be patient you gota pause for a while and say okay yeah you want to look at the the pretty flowers for a moment that's cool you watch TR some grass great okay finally I'm equating developers and Engineers to horses so hopefully nobody will take offense to that but in the nicest possible way honestly I mean know it's not like I'm trying Hur I think you like you such a good point though with with like experimenting with different ways of of doing the messaging because as you said like you can't just keep repeating the same thing and then hoping that people are going to get it um you have to you have to come up with with different ways of saying it and and another thing that you and I have discussed before also is you can't tell them that they have an ugly baby you can't say your thing sucks my way is the better deal because people will get very offended rightfully so so you have to be very tactful in your messaging exactly no you're and that's right because they'll just take offense and then they'll walk and then everything you say afterwards it falls on deaf ears so you don't want to attack them you've got to draw them out and we we wear a lot of different hats in this role when it comes to observability I feel it's it's not just about knowing how to do it technically it's about how to assisting folks to see how they can learn from it and grow with it too and that is a huge challenge uh I that's why I've done like a lot of my videos I've done too like uh not too many of them been exposed publicly I've done them internally with at work but I will put on different characters like I would honestly like do different characters to help people understand just so they have something so it's not just another boring oh here we go get told off about monitoring obsil yes we know Dan logs are not monitoring okay like I but I try to build it up so they they go they it kind resonates a little bit because they're more engaged because oh Dan's put on a funny wig and he's making a funny voice but that point made actually does make sense what so then you know it starts to click eventually I hope maybe I'm wrong maybe they just look at me and laugh and just walk away nothing nothing six I like that though because you're you're you're also like putting on different personas right and I think people also res like different personas resonate with with different people they have to at the end of the day it's got to be how how does this benefit me right because otherwise if they don't see where they fit into all this then it's falling on exactly how does it benefit me that's that that's a good valid point and and it's getting them to see that there is a benefit to them it's just like all right well you just have to sort of showcase this several different times um I know somewhere along the way I heard somebody say that how when you're me creating a message for your organization you got to do it like seven times in like different formats to get it so people all kind of get it and think about it and understand it and I think that's got some truth to it to be honest because even to this day I'm still telling people by the way I've you know this is the reason why we're doing this and you kind of go through that process so um yeah no it's totally totally true yeah and I I think there's also the seeing is believing sort of thing to it too which is like yeah okay understand how like open Telemetry Works in theory right and but it's not until you put it in practice and instrument your code for the first time and even if it's like some you know I added a trace I added a log whatever then you see it go through the collector and Inter your observability back in and you're like what my life is transformed just even and you go like now what I do yeah right it's like you crave more right like give them taste it and it proofers in the pudding as the saying goes is a very very true too because I have seen a lot of success once you get the ball rolling they start to implement something and I and I I've seen it over the years in different tooling um I think only once have I seen it not work but that's only because I that was more my arrogance than anything else so I've dropped arrogance at the door after that and never did it again because I want to use a specific tool for the job and realize that that's not the way to approach it um and that's the other thing too as Engineers As Leaders technical leaders we should never push a tool in my opinion I think we should always look at what the organization needs what's most important thing to most value to the organization and then making sure we have the right practices in place the right standards in place and that will event then we can figure out what tooling we'll use after that tooling should be always the last thing you should decide on how you want to visualize or work with your data that's my just my two cents I don't know how the people that might cause problems but who knows now here here's a question for you along the lines of tooling because we see this a lot in in in Tech where you've got like a team they start using a tool but then the the corporate standard for tooling comes out how do you um how do you reconcile you know the the teams that are off doing their their own thing with the the corporate standard because sometimes it's it's really hard to get folks unstuck from that you you can you know try to force them but some folks are very adamant and and passionate about about the tooling that they use right they are and you you know what you got to I honestly personally that when it comes to that thing is like okay try to get them to as long as they're following best practices following industry standards then you can only push so far you know like you could say Hey listen you know stop using this particular tool because it's archaic it doesn't give you the the same value for what you have but you know and then they you're G to have push back I mean it doesn't matter where you are even when everybody's on board and they're you're going to get push back because people want to do things their own way right there's always that kind of like I I know better I know what I'm doing and you have to be delicate and you have to be patient and you have to be tactful um so it's better to just I would say just try to encourage them to use best practice and standards but don't try to push them off the tool if that's I mean if somebody if somebody higher up wants to make that decision and call them out on it let them do it but you want to I think it's more important to have the standards in place because if you can kind of tie everything together and you have a proper process flow and everybody can still can't get to the root of the cause of the issue then that's better than just saying you need to use x for your stuff moving forward tough NIS because that that kind of thing doesn't work either in my opinion yeah that makes a lot of sense especially like you know I I'd almost prefer to like okay you use whatever observability back end you want as long as we're all instrumenting with open Telemetry I feel like I feel like that is that is main thing yeah that is the bar exactly L now when you um when you joined your organization um were like was Top Hat doing observability at the time what was what was the landscape like at the time so at that time so we were at so we're we were not but we were um so this is one of those things where I kind of like I had to be very patient but they were like so when I first joined we were using a vender um uh everything was there was instrumentation in place there was logging in place there were um well I should say there was traces in place and there was logging in place they were linked but they weren't um it was done the initially was rolled out about five year ago years ago but no other real work was done just kind of building on what they already knew moving forward I was asked to come on and actually bring to take the company off the vendor and migrate to open Telemetry so that was two and a half years ago and we are almost there like this is uh I think uh we have one more service to migrate over to open telemeter this month and then we're done so it's taken to an almost about two and a half years to get to kind of get to this point amazing is take yeah uh it was hard obviously but it was also it was challenging um had to be a lot of patience involved um and what I did though in the beginning was actually kind of go back to remember the uh when we were at monorama together I did a talk on alerting and I did that talk because I kind of was that was a precursor to where I really wanted to go which is actually getting into slos right but that was sort of like here here's how you can first sort of understand your landscape when it comes to alerts and why so you've got and if you can categorize them in a certain way and realize a bulk of them were a lot of noise and doesn't have any value help people start seeing those things in that kind of light and then start showing the value of using service level objectives and how that can actually help with end user Journeys help actually bring the user experience home to the developer so they understand okay I don't need to alert on you know when the low balancer is hitting 5xx errors when I should be alerting on whether or not the service is being impacted by that and how it's being impacted and then yeah I mean those 5xr are important but where are they important how is that impacting the user experience if it's for some feature that we don't really care about yet we don't care about the 5xx because we don't need or we don't need to be alerted about that in the middle of the night but there's so that this was how we were trying to get that process going and then they start doing the migration okay now that we know how our Serv should look now we can do our instrumentation on that uh moving forward so we've we've done started off a lot of instrumentation but now we have those those user Journeys to reflect upon and go okay we're missing this this function in our Auto instrumentation let's make sure that's part of that because that impacts the users's experience so yeah so it's been a while uh but we are getting closer and closer to getting over over top and Telemetry we've set a lot of service level objectives two of them have already helped out like alerted us like two days before an incident actually happened so we're already seeing some value in this so it does again proof this is in the pudding right and we're already seeing some of that and we're you know it's it's great a lot of churn but in a lot of like turmoil a lot of folks were like oh no we're losing this but they're seeing that they're getting more value out of how we're doing things moving forward and they're able to query things better because they're able to have more richer data available at their fingertips so now in terms of uh vendor did you end up switching vendors from the one that you were using when you first came in or so we had the one vendor we were using we're still on that same vendor uh but as of the end of this month we're no longer going to be using that vendor we'll be on another vendor moving forward because we're pulling all the their vendors specific sdks out and we're just putting an open Telemetry in and we're doing that replacement so it's kind of like now we're getting to the point where the last in that in those open Telemetry sdks are pointing to a different vendor for our visualization of data cool awesome um all right moving on um yeah so the next question I had is um you know now now that you've gone through the exercise of um you know in of having of doing the instrumentation so actually around instrumentation is there um what what's the instrumentation culture like at Top Hat um is it because you know one one thing that I've spoken about in in the past is like it really instrumentation has to be the responsibility of the developers do you think that there is that attitude of developers instrumenting their own code where you're at not as of yet um and there's a the reason why and this is nothing against anybody so I hope anybody who's at top at listening don't take offense I love you all um no but it it's because we were kind of like once we started getting going on this whole transition we had a very short period of time so there's been a few instances where staff engineers and principal Engineers have stepped in and done some of the work laid laid the groundwork for the other team and they kind of just went flipped over things and did things the most effort they put in was actually in the user Journeys themselves so the auto instrumentation has been more largely being kind of like we've we moved this put this in and they've G of look done it so they haven't really it hasn't PARTA of been ingrained in part of their cultural thinking yet it's still largely kind of like a checkbox for them to sort of okay we've gotten we've we've moved migrated to Hotel job done move on to something else and little do they know that there's there's a lot more work involved but don't don't scare them off yet because it will help them but it is something that has to be I think and what I we've tried to do is try to make it so as easy as possible less not as impacting as far as their day-to-day life but try to slowly get them to start looking at things and so going over things and saying and and using the tooling that we have available saying here we know you did it in this particular tool this is how you should do it in this tool and providing that data and then doing training on that too so trying so they're slowly it's slowly becoming part but I would not say it's 100 like it's far from 100% yet it's still there's a lot of work to still to be done there just so that they start to in they all understand why it's it's part of the value right um it's only like I think I right now it's still a handful and I could be wrong maybe I might get in from snack later I'll get you know told no we all love it who knows in terms of instrumentation like what uh what languages um are part of like your your application landscape um and mostly python oh okay okay so mostly so we're what's that oh I was gonna say and you alluded to some Auto instrumentation there yeah so we did Auto instrumentation in our python code um and so we and we use a couple different Frameworks but I think we're just trying to work towards getting to one framework um and then we have and then yes it's fortunately there's not so when it comes to so we can you can actually almost kind of get away with like doing a rapper sort of library for people and have them just use that and then yeah to get and that's kind of what we did we have this little wrapper Library set up that uses oel with the you know all the different things that they need to pull in there and people just kind of just dumped it in and did Auto instrumentation the way they went so they kind of like so it's kind of like all done for them and they just kind of include this in a way they go um but I think as we progress it's going to be like okay let's start Target this is my next stage will be talking to first off the PMS talking to them okay this is what we have so far these are the gaps we have in our our Telemetry data these are the es we have set so far let's decide on how that should look because your team needs to own this your team needs to act on this so when your air budget gets exhausted your team needs to take that on in next Sprint and that's one of and like so that and again the technical side is not that difficult it's the the cultural change process change requires a lot of effort right because now you have to get people to say okay well instead of when our we burn through our error budget we need to take deal with this the next Sprint well that's usually well that are this future that we need to push out which which one do we do and so but they have to that that's what I said when I first started this is that you have to agree upon this process you have to think about you are making a contract with the company to say yeah we will act on these slos because they will impact the user Journey they will impact our users and that's what's most important to us today so um for those who are not aware topad is an educational platform so what our our our our you our um our users are teachers or professors and students so imagine you know if they're taking an exam and then everything goes sideways that's going to be really I mean anybody who has kids or has been to University knows that that could be a huge frustrating Factor when you're not you know if that's not working properly so yeah so it's important that um oh I did get a message somebody did tell me we all love it thanks [Laughter] Mark yeah so um yeah uh now I just kind completely thrown off there uh what was I talking about ADHD you were talking about the slos and and how frustrating it is uh if if the consumers of your platform um if it's not working as they expected yeah yeah so and that's this kind of fundamental shift is going to slos and helping them see that and get through that um yeah because I think now I just went off the rails I so but that's all good cool I have one more question that I wanted to ask and then we actually do have some questions from uh from folks see that our on our board which I'm very excited about um final final question um I guess there the two questions I guess um are you um if if are you using kubernetes and if so are you using the otel operator um to manage your collectors and to manage your auto instrumentation no no yeah so we um I'm glad we're not um just because and this is not because I think that kubernetes has its place in like all tools has its place and I think for what we're doing today it would not make sense it would add a layer of complexity that would completely destroy our service so glad we're not using KU I don't think it not saying that there's anything wrong with kubernetes I just think that you if you're going to run a kubernetes environment you need to know it yeah yeah AB right and so if you don't know what you're doing with kubernetes you can really really mess up your service and and I've been at places where we only had one or two people who knew kubernetes and we had a number of problems always with our clusters so I'm when we're ready though I think we'll probably make probably make that push but right we're now we're just in containers so it's all just through yeah just containers but not a kubernetes gotcha gotcha so you're you're using like another container Management Service like a cloud provided one kind of thing a cloud provider container service yes gotcha gotcha you're trying to avoid all names of vendors right now yeah yeah yeah let's be agnostic and and H final question before we get to the questions um collectors what's your collector landscape like so we're using a combination of alel collector um as a sar and then to a primary one that Aggregates it and then for our traces we use a with one company that we're work where where we're moving towards uh they have their own collector that we send our traces to that we can do additional sampling rules on and send off so um yeah yeah so we um so right now because and I I really really love the fact that you could choose you could do send from code itself but I love the hotel collector because simply because you have these processors you have these exporters and you can send to different places as you see fit I've had conversations with our data team saying Hey listen if you want stuff otel is collecting a lot of this right and we don't have to send all of it to our for production like as far as what we're looking at as far as like would give us an idea of the user experience but there may be things you are collecting that from R tell that we can send to an S3 bucket that you can pull in later for whatever you need however um you know we have we're still haven't gotten to that point but that's the beauty of the hotel collectors that you can kind of like just point to different things and I love that because then it there's no code change so if we decide we no longer want to use this or we want to add loging or we want to add set get uh we start using we don't use Prometheus now but let's say we start using Prometheus as a open to open um Source tool um as an example though uh you would we can easily just start getting that information right Off the Mark so it I I just if anything try not to use the the sampling and the old push from this code itself use a collector because I love the fact that because it makes life so much easier then you can just if you have to do little tiny changes of tweaks to your your sampling or whatever you don't have to go push it out to like the production environment again it's just a small change on your collectors themselves which makes life so much easier that's just how I think of it absolutely and and you're using all three like major signals traces logs metrics right now like for otel or mostly traces right now um so there was a decision to not do so metrics we still are capturing the infrastructure metrics from the cloud and the logs are primarily staying in our cloud provider so all of our services are are just logging by default from container to the logging um in our cloud provider and they are staying there for the time being um because that was another thing to to start tackling and I didn't I just I felt like that'd be too much for folks as they're transitioning from the one vendor to open Telemetry for them to like get their heads wrapped around would be like oh well by way you also need to have structured logs too and this is how you should do it no this is just gonna it'll make more confusion and so I figured just just focusing on one and then adding things as they go along as they as they prove useful so yeah that's how we kind of approach that whole process makes sense baby steps exactly cool um okay so we're going to turn to the board and if you want you can take a peek at the questions they're on the on the chat window there's a link to the board um take a look so um let's have a look at the questions so the first one is what is your opinion on Telemetry as a uh on Telemetry quality as a way to reduce spend is more data always better observability great question awesome question and that's a tough one um I don't think I can I hope I can answer that in my opinion um it's Bell quality is you need to actually because really with the end of because you can have a bunch of things going through but if you're sort of so let's just take a a standard Trace right if you have um a bunch of spans in there that are doing sort of some things that you really know it's just back and forth and you really like there's I've noticed some Tes where there it there's um I forget what it's called and I I I I wish I had a sample the code before it's been like a two weeks ago one of the one of our Engineers cleaned this up and what he did though is he looked and realized we're making these calls in the code that we're we're adding span events to and we don't need to because so you're looking at the Quality there and then you're not you're seeing what's invaluable and what's not so more data doesn't always mean better Telemetry quality data means better Telemetry so in my opinion you've got to look at what you're getting making sure does that reflect the user experience if it doesn't reflect the user experience or is not exposing things where you would say oh you know like that's kind of like the whole the concept of the unknowns right like you're exposing those unknowns you're not able to see those things and then you're not the qual it doesn't matter how much you have if you're not seeing that stuff then it doesn't mean you're it's the quality of the data that's most important and that will help you reduce spend because then you know what's good you can sample that partially out and you're going to make sure that those that that the air prone things Bubble Up and become more apparent when you're looking at your data hopefully that answered the question yeah that was a really great answer um yeah and I I I agree with you it's it's really like what's the point of capturing in Telemetry for your system that's useless to you like that's just and I think that's um that's a pitfall that a lot of people run into also with auto instrumentation initially right because there's a lot of stuff coming in and and and fortunately now I think there's a way to actually switch off auto instrumentation for certain libraries so that you can you can really zero in on the stuff that's uh super meaningful to you so yay yay cool okay next question um how do you approach the what to instrument and how much to instrument questions with your teams so that kind of is it linked to the previous question to in my opinion um because what instrument is again goes back to this is so I went through the whole process of working with the the teams the development teams and the the product teams to understand what the user Journey looks like so I went through and said okay what is the so we did the user Journey we looked at their servers and I asked this like some did more than others uh but more or less it was like I asked him what are the typical interactions a user would have your service and write it down and then write in in so in plain language and then in plain language describe the tech what's happening in the background and then we also talk about the prerequisites of what people expect when they interact with that system so they have to be logged in they have to have this Tech maybe this cookie or this cash or whatever um and then what are the dependencies and then we just kind of work through and go okay what are the things that where a user will potentially see problems based on this information and we went through that so that's how I kind of approach what the instrument because then once you kind of know okay so we know that user logs in so they know they have to do that this they have to interact with maybe uh let's hypothetically say they interact with a database they interact with an authorization service maybe they interact with something in the back end for something else uh another database and so all these queries are checked in there now that all those things all those functions are like talking to these different things if they don't work means they can't log in therefore we have key spots what they should be mostly looking at to make sure there are always going to be times where people miss things this is why we have the the slos this is why we have to have this constant part of our cycle to go back and make Ure that those things are identified so you know we're having problems with our servers but we don't know why that means maybe you're not instrumenting so what that means you have to go revisit some of the things you did because maybe you did some changes that no longer uses this kind of database uses maybe a cash service or uses a redice cluster instead instead of these things and so does your code or your user Journey reflect that change and does your instrumentation reflect that change and if not let's make sure they do so but it always boils down to the user journey and what the user is experiencing if as long as we're capturing that you're like 80% of the way in my opinion 80% of the way there uh because you at least have a general idea but you have to know you have to have that conversation awesome thank you okay next question the Q um in an org that has adopted distributed tracing how do you make individ idual teams leave behind their outdated practices based on logs how do you show value that was a spicy question that's a spicy one y slap them around and tell them what no you do not that's not the way to do things you have to be very exactly that's number one do not call her baby ugly um that's a tough one because you you really really have to be patient and there sometimes you do have to say okay listen this has been dictated by the AL hups that were doing this and you try to offer them as much help as possible but you have to you got to find out too why they're reticent about switching over uh having these conversations is important because then you understand what how they're looking at things and then you can restructure your discussions around that uh so oh I like using logging because you know I get to see immediately when I'm doing my development I can see it point out on the screen I know where things are broken so I can go back and fix and Blum Blum Blum and that's so you know that that's why they love logging as an example so how you have to then help them bridge that Gap and say okay well this is how you would do it with tracing and this is why you would probably want to do it because then with distributed tracing you're not just getting the value out of your service but you're seeing how service a interacted with your service service B and how service B is going to interact with service C and how all that kind of works together so this is how why you want to have these pieces in the puzzle so you can get that full picture but it takes time it takes patience it takes and again going back to which one of the things we talked about before restructuring how you approach the their their the uh the challenge of switching over it's not just repeating yourself the same words over and over again because eventually people are just going to tune you out like at my work I think they T me out half the time okay uh final question we can squeeze it in hopefully in two minutes um really love what you said about the tooling uh is the last thing you should think about and that you should lead with the question what is most important to your organization what are some of the questions you ask to understand what is most important to someone's organization it's always about the user in my opinion it's always about the user um because first because you're looking at if the user is not happy and you can't measure that or you don't know that then you're in trouble right uh and I always use streaming services like video streaming services as an example because and we everybody has tuned into one of one of many of these streaming servers we have now today so if you pull up a movie you push play it gets all pixelated for the first three seconds and then it clears up after that do you care no so if a person comes by but does no the the streaming service know that they probably are measuring that because they want to make sure that that doesn't become a bigger issue because they know that Runway of where you're okay to where you're not okay and you're G to get frustrated and go I'm gonna go switch to this now because they have better quality and so it's not it's always about what the user how we are we measuring the user experience do we know what the user experience is like do we know what frustrates them and if we don't how do we get there and that's how we get so that is usually with the tooling that we have available today today with the sdks and other things and the and the philosophies we have and eventually then you can go because you know what the tools comes out all the time there's so many tools out there that the landscape has gotten bigger and bigger as people become more and more involved but doesn't message mean they're all good and you know but you have to find the one that kind of works for you at the time and this is why we like open Telemetry is because you can switch from one to the other to the other without with very minimal change and that's why I like about getting people on open Telemetry but when it comes to the tooling that should be your last thing to think about don't build your solution around this tool in your spot Build Your solution about what your users are thinking and that's how I think a lot of people that resonates when you start talking in that light especially with the higher ups anyway so I think we got them all in yeah absolutely well we got through all the questions yay we filled up this hour quite nicely so thank you so much Dan uh for joining us for hotel Q&A stay tuned for our next uh Hotel Q&A and or hotel in practice and again thank you Dan for joining today I really appreciate it oh yeah that you know it's my my pleasure thank you for having me um you know how much I love talking about these things so you know I I think I actually did not go down a rabbit hole today I think I was I pretty good it's perfect it was great thank you again all right thank you everyone thank you for showing up see you +welcome everyone to otel Q&A and thanks Dan for joining us for otel Q&A especially on like we got you on such short notice um I'm glad we're able to squeeze you in for October um especially me too cucon is coming up in November so for folks who are going um a few of us will be there uh re Dan and I will be there from theel and user Sig so um be busy yeah it's going to be busy when is where is that happening by the way it's in Salt Lake City Salt Lake City nice I've never been there myself personally yeah me neither me neither oh oh so this be an experience for you too yeah yeah looking exciting exciting um start asking you questions I realize I'm the one here is supposed to be with the breaks folks it's gonna be a fun one oh yeah not sure about informative but it will be fun well here we go okay let's start uh let's start with the question so Dan why don't you introduce yourself and tell us what you do hello I am Dan Ravenstone I am a staff engineer at Top Hat uh I've been there for guess two and a half years maybe no a little bit longer than that come may I think it'll be three um a little of my background um I have been doing monitoring observability for the bulk of my technical career uh I started back in um where I actually this and I usually refer to as like where the bug bit the monitoring bug bit me was when I was at a company called affiliat which was at Young in 401 area in Toronto uh affiliat managed the info.org regist domain registry um in the back end of it uh for back in the day and I started there in texport but then I one day opened up my big mouth and said Hey listen I'm tired of our customers calling us up and telling us our stuff is down can we fix this and so they said well then you ask for it you get it so I went out and so I that's how I started learning about moning tools and that's back in the day of Big Brother nagios cacti all those fun things and um and I just loved it this kind of like I think one one of the things I liked about monitoring in those days was that like I like looking for patterns I like detective novels I like the all you all the fun things and what this do is like kind of allow for somebody to kind of like be in the front lines in the operations and dive in and find problems and report them back and then like the better more the better your monitoring tools were the more likely you were able to follow and find the problems a lot sooner before they become actually customer impact and I was always been my kind of like push when I did this and of course as things progressed and modern and changed uh we got into the wonderful world of observability and a whole different way of looking at it and I fell in love with with that obviously as well and tried to grow with the times and see how I you know how being proactive is a lot better than being reactive which is kind of like you would think is an obvious thing but it takes a lot of people time to kind of get their heads wrapped around um so yeah that is kind of uh how I got to the how I got to this part of the world world and a little bit about me there's tons of other things but we don't want to go into those because then we'll get lost in the weeds and we'll never get out and we'll be lost forever sounds good well thanks um and you know like you you've been in in in this for a while like when it's started out with monitoring evolved into observability what what kinds of like big changes have you seen um in in the course of that time uh let me start off the things I haven't I haven't seen change which is still a problem alert okay alerting still has not changed surprisingly um but what has changed and I think this is one of the things that um we're trying to get to I I so I focus on alerting because I was by myself time to think what has changed so now that you know my secret I just thought myself a couple more minutes so but what has changed though in observability and in the open Telemetry especially too is it allowed for us to actually look at our our services a lot differently where moning was kind of a very largely reactive kind of thing and we only had the symptoms of what was going on so if everybody has ever used nagios is well aware of the sort of out of thebox HST monitoring sort of plugin you usually incl include which is like your CPU your memory your disc SP and these things have been carried throughout time in you know to con be indicators of a problem but that's usually not quite the case anymore is it um especially way when we have kubernetes and containers and other tools now that these things shouldn't really play a role in us so it's like more like so what else is impacting the service and we never really had those that ability to look at it until we started sort of aggregating our logs and even when we aggregated our logs it was still strangely problematic um because you know we had guidelines on how to do structure logging and that kind of thing but still people would do their own thing and things were never and so even with all the good documentation in sort of best practice around it we still logs were still problematic and again you still have to still collect a bunch of data before you can actually understand oh wait a minute this is a problem before actually kind of knowing beforehand what observ abilia has done has taught us is how to sort of like set things up in a more of a a way that we're looking at it as an overall user experience and we don't get sort of sucked into minutia sometimes and there's because there's a lot of red herrings in operations when you're especially when you're in the middle of incident you're looking at all this all this data and you're like what is the actual problem we have high CPU over here and we have slow load times over here we have latency here we have some errors here but which one of these is the the smoking I would not I don't want to use a gun but for like better word Smoking Gun right and I see that that with observability has changed how we look at that however that's for this probably this group a lot of people are familiar with it there's a whole world of of Engineers out there who are still having are still trying to get their heads wrapped around this concept and this is why I'm clown and glad we have these conversations and I try to get those people this is like this is targeted towards them of why this is so important why you need to start looking at this because there's a number of good things one you know what the user is experiencing and that's all that matters if you know what the user is experiencing that means you can translate that to how much you're making like very very like very it's it's a cost thing and it's a money thing if you know if your users are happy that means you're making money because they're be spending money because they're going to keep using your service if they're not happy they're not going to be using your service and they're going to walk away and use something else or just not use it that often and try to avoid as much as possible that means you're not gonna be making the money anyway um and I'm gonna get in the weed so I'll you're gonna hear me say that a lot because I just do so you Adrian it's all up to you to rope me back in like you gota don't worries don't worries but I I do wanna um you said something really interesting um in in terms of translating it um in terms of money because you know like observability adoption and maybe I'm getting ahead of myself here but I I like way you said um in terms of observability adoption like it's as much a a top down thing as a bottom up thing and one way to speak to the benefits of observability is in dollars and cense and Executives speak in dollars and cense so um proving the the worth of of having an observable system I think becomes really really important and and being able to use that language um with Executives to be able to convince them I think is very valuable so and and that's what I've been trying to focus more on and like there are like there's quantif quantifiable yeah quantifiable reasons why to look at this we can do that why obsil excuse me uh provides value toward us the end of the day but I think we we' we're still I think we're still getting our we're still trying to get that figured out I feel like that um anyway you know what before we jump in let's let's keep moving forward because I I I could go into a whole other area and I don't know I don't want to sort of suck a lot of time in there but it is one of those things I think that does have is worth having conversations not with just those in the community but also talk like I mean if you're you know out there find out what your what the OB ability sort of platform or the concepts are your where you work what what are the drivers why why aren't you doing it why aren't your why aren't you using these things and ask those questions and then like you know that maybe we could that could help and feed that back to community because that could help us build out the reasons why you should be doing this because it does make better sense eventually um but yeah that's a sort of a general call to anyone who's out there who needs who wants to sort of get going on this and need help yeah I completely agree and I think this uh this takes us to our next question um which is uh you know like you've you've done this for a while now like first the monitoring space now moving into the observability space what do you think are some of the main challenges that most organizations are facing these days when it comes to observability uh there's a lot of challenges and I've been trying to get my head wrapped around some of this stuff one is communication I think so some of it I think it's mostly cultural really if you think about like I mean there's not a technical reason why people shouldn't go to this other than resourcing in time uh so and and I mean this might be jumping ahead of a little bit but where I am right now we we are using a current vendor uh which will remain nameless but we use a current vendor right now for our monitor oability purposes but we're getting away from them because of the library we're using is is is a vendor specific and that means that we're basically when it comes to doing any kind of of telemetry it has to use that that particular library and their's only getting and that's that that's one challenge because if you think about like okay so you're using this one vendor it's cost you x amount per month or per year it's exuberant however to get translate or get off of that one and go into another one where you have more options like with open Telemetry takes time it takes it's not like an like anybody who's ever gone on any kind of migration from one tool to another knows how long it takes and if you're and you have to have a few things in place to actually say why you should do this so one is like of course obviously the cost right well but you know it's hard for some Executives to say well we're spending this but it'll cost us this much to get over here and then we'll see the cost value so that there's this huge challenge of just trying to shift things over spend the time to do it and then do it properly and that's a whole other challenge is say okay yeah doing open Telemetry is one thing but doing it prop L like every other software or any other tool that you have out there you have to still use it for the right reasons and do it properly if you don't you'll still run into issues and whatever you're trying to sort of get away from over here will still might come up in a different way over here so there's those that's that's one of the challenges is just getting that just doing the whole Shi over uh I think that's a where a lot of people and they don't know how to do it and so and what happens too is then they do a shift they try it out and then they get burned for because you know they don't know how to do sampling or something like that and then they feed it off to some tool like let's say a tool that you get charged by event volume you're going to even though they're using otel they still became like oh I I saw one company who did that they start shifting you using otel but they started setting all this volume all this data to it they didn't do it right they were having like a 100,000 spans per Trace which is a huge amount and they weren't really getting a lot of value and their their costs are soaring and so it's like well you know then they kind of just drop and they well we'll figure it out another way we'll just use some other tools to figure out when there's problems with our service which is sort of like a defe kind of attitude but it does happen because they don't you know because there's a lot of work involved doing this is more than just a simple lift and shift it's a lot more involved to it um trying to think of other CH like those that to me is like some of one of the big some of the bigger challenges and I think a lot of people could probably emphasize with that one as well um there are others obviously but I mean I don't we don't have a whole lot of time and we could probably go on just you and I alone I bet you we both have experiences on some of those other challenges we have to experience but I mean uh I like to see like like how do I get how do we get past that is kind of like where I want to Target how do we get past these challenges and move forward which I don't have a proper answer to but I will want to talk about it you know that you you touched upon something that um I think is worth mentioning as well which is um moving away from like vendor libraries to open Telemetry um because you know even even though I I think one of the one of the wonderful things about open Telemetry is that it's vendor neutral um and so it makes it once you've moved all of your instrumentation to that super easy to switch vendors I would say Rel relatively easy right because they're still like yeah like at least on from the instrumentation standpoint it makes it super easy um but but it the challenge though is convincing folks to move away from those vendor libraries and I think that becomes um that becomes a bit of a sticking point because it's like they you need to move away from them because maybe that vendor is too expensive and you don't want to go with them and you want to try something fresh but there's so much time and effort and learning curve all that stuff associated with reinstrumentation and that what are your thoughts around that well and that's a very very powerful Point you're making because first off with vendor vendor libraries you get in the habit of doing things a certain way right and so again I do not want to there's no like but the thing is so some V some vendors may do things one way but it's not part of the industry standard or the best practice we see across the industry so they may like name things as certain way or do things a certain way and people get used to that so not only is it just just shifting a library over if it was just a matter of shifting a library to one or the other that that's in itself is a challenge but there's also all the behavior that gets attributed to using that older library that some for some things you got to go well listen you don't need to look at it like that anymore you need to look at it like this now and so it's not just a a technical shift it's also a cultural shift and how they even like Envision how their service is working with the new library with using open Telemetry um because open Telemetry is current it's trying to it's keep it's adding new things as we speak as every day uh it's you know it's you know we're seeing a lot more love now for mobile which is wonderful I'm so excited to see some of that because I feel that's largely been ignored for a long time but we're seeing these things getting pulled together and we're having this and it's kind of like keep following these sort of patterns and following these this this this these um standards and so make forces the developers to think a certain way there a lot of them are not used to that yet a lot of them still want to develop and I realizeed one of the things I noticed too was for developers perspective I don't I Adrian you you do you're a coder by Nature right like that's kind of yeah yeah that's when you yeah that's your back so it's not mine but I know that when I did do some coding though I would always print a line at certain parts in my code this okay where am I at so you're even like logging was is kind of part of your local development experience totally right now we're telling them don't worry so much about logging use instrumentation well you still have to have a mechanism for them to actually print that to screen and so it's just changing how they even develop in the beginning um and that and that's I think where we're facing some challenges there is that it's not just a it's it's um it's technical it's cultural it's it's about even how what you've been taught in the universities and how you even taught how to develop is changing a little bit and not everybody's quick to embrace change not and you know especially when you've got your CEO or your or your or your um your features team or product team saying we need this out now well you know you're not going to worry about trying to learn a new way of doing things and it's you know oh well now how my service is working from open Telemetry perspective it's like well I just need to get this out so I just slap these log lines in boom gone it passed you know and we're going now it's in production and we move on right so there's that that it's that kind of like also encouraging and supporting our developers to have the time to learn this and work with it so they can actually use it to their benefit and perhaps be able to do better code before even hits staging or development and then and then once it's a production you know how it should be based on all your stuff but that's it's getting that linking those people together getting them to work together in that thinking that's that's a that's a huge challenge yeah yeah absolutely and and getting getting them used to as you said like the nuances of you know instrumenting with open Telemetry which open Telemetry wants to standardize how you're doing like your your main signals in a certain way right which as you said maybe different from how you've been used to doing it whether it's like through your print statements or or vendor libraries or whatever it it's kind of like so I back in the day um this might be pre a lot of folks I don't know but I used to work with a lot with SNMP and what I liked about SNMP was it was it was basically a protocol that was agreed upon by the community and it was very basic but even then I saw this with loogy too but even with LMP I would still come across mibs I would still acoss across libraries that were not properly done so it required me even like because I knew it I would have to go in and kind of redo maybe a MIB to so I can translate the oid properly so when it goes into our or create a MIB because it wasn't available it just it just had these voids are available from an interface or something and you just have to guess what they were um even with standards in place like that something has been in our sort of part of our our industry for so long no longer really being used anymore um well actually still is by network devices but that's not hopefully we don't have to worry about that right now um is is that you know even with those things in place it still takes time for people that even then there's still mistakes and things happen so with a new thing like open Telemetry and and being setting standards it's taking a long time for people to understand how to use it to do it to set it up properly and we get their most value out of that instrumentation the most value out of using that from the beginning of the of the development cycle yeah absolutely um I do want to switch gears a little bit and talk um a little bit about specifically um the challenges that you're facing currently in in your role um when it comes to observability oh I'm really not my own mental health problems okay that's different um yeah so my my challenges have pretty much always been I feel like it's always been the same it's communication and not because I'm my challenge for me so I've been in this business for a while so what happens sometimes is as I make the mistake or the or make the assumption that folks know what I'm talking about and I have to be careful sometimes like for like I can have these conversations with many people in the same community is like within the monitoring obility community and we could go on for hours about little nuances of something whatever may people logging or or tracing anything like that and we can go on and talk about that as ad nauseum however when you when you're talking to other people and you you talk about basic things like um like the red method so rate of Errors um rate of requests durational requests which is a very basic kind of concept with the monitoring World um and which is helped kind of like we see a lot in even with notability too to a certain degree uh people don't understand that and you make the assumption that they actually kind of look at things in that light but they don't they still look at it from the old school and then like honestly like i' at some places I've been it's like they still see CPU as a indicator of a performance problem but it shouldn't be in my opinion it should because that's not CPU can is is cheap you can get more so that should be a scaling problem make sure you just you're scaling properly you should be good however I make the Assumption people know what I'm talking about and that's one thing is I got to con so I have to constantly be patient and communicate the same message over and over again to make sure they understand because people even if I tell them two or three times they still don't clue in I had one conversation with one engineer for about a month um and it took a month for them to actually understand what I was driving at and this is like because they were in different time zones so that's why it took a little bit longer because and so but it took a while before they actually oh that's what you meant and as soon as they were able to make that change everything was working better and like oh now I see where you're driving at I now see the point you're trying to make and it's just you have to be diligent you have to be patient you have to communicate and you have to change how you communicate sometimes too like you just can't say the same words over you have to rethink okay you're not getting what I'm saying what will help you understand and and then try to reward it so people understand um ad you met my friend Damian who who I work with he's actually the same place as I am but Damian who also works with me on a lot of stuff and we he's more of a he has a different background but he sometimes steps in and sort of translate what I'm saying because sometimes it helps to have a different because he puts a different sped on and all oh now I get it so that's those are some of the biggest things I find um to today are still the biggest sort of people get it it's just a matter of once but just getting them just lead you leave the horse to water type thing right but I am it's just leading them there once they're there they eventually do start to drink because they they kind of get oh I see this is good for me I will do it now yeah for sure but getting there there is sometimes you gotta kind of be patient you gota pause for a while and say okay yeah you want to look at the the pretty flowers for a moment that's cool you watch TR some grass great okay finally I'm equating developers and Engineers to horses so hopefully nobody will take offense to that but in the nicest possible way honestly I mean know it's not like I'm trying Hur I think you like you such a good point though with with like experimenting with different ways of of doing the messaging because as you said like you can't just keep repeating the same thing and then hoping that people are going to get it um you have to you have to come up with with different ways of saying it and and another thing that you and I have discussed before also is you can't tell them that they have an ugly baby you can't say your thing sucks my way is the better deal because people will get very offended rightfully so so you have to be very tactful in your messaging exactly no you're and that's right because they'll just take offense and then they'll walk and then everything you say afterwards it falls on deaf ears so you don't want to attack them you've got to draw them out and we we wear a lot of different hats in this role when it comes to observability I feel it's it's not just about knowing how to do it technically it's about how to assisting folks to see how they can learn from it and grow with it too and that is a huge challenge uh I that's why I've done like a lot of my videos I've done too like uh not too many of them been exposed publicly I've done them internally with at work but I will put on different characters like I would honestly like do different characters to help people understand just so they have something so it's not just another boring oh here we go get told off about monitoring obsil yes we know Dan logs are not monitoring okay like I but I try to build it up so they they go they it kind resonates a little bit because they're more engaged because oh Dan's put on a funny wig and he's making a funny voice but that point made actually does make sense what so then you know it starts to click eventually I hope maybe I'm wrong maybe they just look at me and laugh and just walk away nothing nothing six I like that though because you're you're you're also like putting on different personas right and I think people also res like different personas resonate with with different people they have to at the end of the day it's got to be how how does this benefit me right because otherwise if they don't see where they fit into all this then it's falling on exactly how does it benefit me that's that that's a good valid point and and it's getting them to see that there is a benefit to them it's just like all right well you just have to sort of showcase this several different times um I know somewhere along the way I heard somebody say that how when you're me creating a message for your organization you got to do it like seven times in like different formats to get it so people all kind of get it and think about it and understand it and I think that's got some truth to it to be honest because even to this day I'm still telling people by the way I've you know this is the reason why we're doing this and you kind of go through that process so um yeah no it's totally totally true yeah and I I think there's also the seeing is believing sort of thing to it too which is like yeah okay understand how like open Telemetry Works in theory right and but it's not until you put it in practice and instrument your code for the first time and even if it's like some you know I added a trace I added a log whatever then you see it go through the collector and Inter your observability back in and you're like what my life is transformed just even and you go like now what I do yeah right it's like you crave more right like give them taste it and it proofers in the pudding as the saying goes is a very very true too because I have seen a lot of success once you get the ball rolling they start to implement something and I and I I've seen it over the years in different tooling um I think only once have I seen it not work but that's only because I that was more my arrogance than anything else so I've dropped arrogance at the door after that and never did it again because I want to use a specific tool for the job and realize that that's not the way to approach it um and that's the other thing too as Engineers As Leaders technical leaders we should never push a tool in my opinion I think we should always look at what the organization needs what's most important thing to most value to the organization and then making sure we have the right practices in place the right standards in place and that will event then we can figure out what tooling we'll use after that tooling should be always the last thing you should decide on how you want to visualize or work with your data that's my just my two cents I don't know how the people that might cause problems but who knows now here here's a question for you along the lines of tooling because we see this a lot in in in Tech where you've got like a team they start using a tool but then the the corporate standard for tooling comes out how do you um how do you reconcile you know the the teams that are off doing their their own thing with the the corporate standard because sometimes it's it's really hard to get folks unstuck from that you you can you know try to force them but some folks are very adamant and and passionate about about the tooling that they use right they are and you you know what you got to I honestly personally that when it comes to that thing is like okay try to get them to as long as they're following best practices following industry standards then you can only push so far you know like you could say Hey listen you know stop using this particular tool because it's archaic it doesn't give you the the same value for what you have but you know and then they you're G to have push back I mean it doesn't matter where you are even when everybody's on board and they're you're going to get push back because people want to do things their own way right there's always that kind of like I I know better I know what I'm doing and you have to be delicate and you have to be patient and you have to be tactful um so it's better to just I would say just try to encourage them to use best practice and standards but don't try to push them off the tool if that's I mean if somebody if somebody higher up wants to make that decision and call them out on it let them do it but you want to I think it's more important to have the standards in place because if you can kind of tie everything together and you have a proper process flow and everybody can still can't get to the root of the cause of the issue then that's better than just saying you need to use x for your stuff moving forward tough NIS because that that kind of thing doesn't work either in my opinion yeah that makes a lot of sense especially like you know I I'd almost prefer to like okay you use whatever observability back end you want as long as we're all instrumenting with open Telemetry I feel like I feel like that is that is main thing yeah that is the bar exactly L now when you um when you joined your organization um were like was Top Hat doing observability at the time what was what was the landscape like at the time so at that time so we were at so we're we were not but we were um so this is one of those things where I kind of like I had to be very patient but they were like so when I first joined we were using a vender um uh everything was there was instrumentation in place there was logging in place there were um well I should say there was traces in place and there was logging in place they were linked but they weren't um it was done the initially was rolled out about five year ago years ago but no other real work was done just kind of building on what they already knew moving forward I was asked to come on and actually bring to take the company off the vendor and migrate to open Telemetry so that was two and a half years ago and we are almost there like this is uh I think uh we have one more service to migrate over to open telemeter this month and then we're done so it's taken to an almost about two and a half years to get to kind of get to this point amazing is take yeah uh it was hard obviously but it was also it was challenging um had to be a lot of patience involved um and what I did though in the beginning was actually kind of go back to remember the uh when we were at monorama together I did a talk on alerting and I did that talk because I kind of was that was a precursor to where I really wanted to go which is actually getting into slos right but that was sort of like here here's how you can first sort of understand your landscape when it comes to alerts and why so you've got and if you can categorize them in a certain way and realize a bulk of them were a lot of noise and doesn't have any value help people start seeing those things in that kind of light and then start showing the value of using service level objectives and how that can actually help with end user Journeys help actually bring the user experience home to the developer so they understand okay I don't need to alert on you know when the low balancer is hitting 5xx errors when I should be alerting on whether or not the service is being impacted by that and how it's being impacted and then yeah I mean those 5xr are important but where are they important how is that impacting the user experience if it's for some feature that we don't really care about yet we don't care about the 5xx because we don't need or we don't need to be alerted about that in the middle of the night but there's so that this was how we were trying to get that process going and then they start doing the migration okay now that we know how our Serv should look now we can do our instrumentation on that uh moving forward so we've we've done started off a lot of instrumentation but now we have those those user Journeys to reflect upon and go okay we're missing this this function in our Auto instrumentation let's make sure that's part of that because that impacts the users's experience so yeah so it's been a while uh but we are getting closer and closer to getting over over top and Telemetry we've set a lot of service level objectives two of them have already helped out like alerted us like two days before an incident actually happened so we're already seeing some value in this so it does again proof this is in the pudding right and we're already seeing some of that and we're you know it's it's great a lot of churn but in a lot of like turmoil a lot of folks were like oh no we're losing this but they're seeing that they're getting more value out of how we're doing things moving forward and they're able to query things better because they're able to have more richer data available at their fingertips so now in terms of uh vendor did you end up switching vendors from the one that you were using when you first came in or so we had the one vendor we were using we're still on that same vendor uh but as of the end of this month we're no longer going to be using that vendor we'll be on another vendor moving forward because we're pulling all the their vendors specific sdks out and we're just putting an open Telemetry in and we're doing that replacement so it's kind of like now we're getting to the point where the last in that in those open Telemetry sdks are pointing to a different vendor for our visualization of data cool awesome um all right moving on um yeah so the next question I had is um you know now now that you've gone through the exercise of um you know in of having of doing the instrumentation so actually around instrumentation is there um what what's the instrumentation culture like at Top Hat um is it because you know one one thing that I've spoken about in in the past is like it really instrumentation has to be the responsibility of the developers do you think that there is that attitude of developers instrumenting their own code where you're at not as of yet um and there's a the reason why and this is nothing against anybody so I hope anybody who's at top at listening don't take offense I love you all um no but it it's because we were kind of like once we started getting going on this whole transition we had a very short period of time so there's been a few instances where staff engineers and principal Engineers have stepped in and done some of the work laid laid the groundwork for the other team and they kind of just went flipped over things and did things the most effort they put in was actually in the user Journeys themselves so the auto instrumentation has been more largely being kind of like we've we moved this put this in and they've G of look done it so they haven't really it hasn't PARTA of been ingrained in part of their cultural thinking yet it's still largely kind of like a checkbox for them to sort of okay we've gotten we've we've moved migrated to Hotel job done move on to something else and little do they know that there's there's a lot more work involved but don't don't scare them off yet because it will help them but it is something that has to be I think and what I we've tried to do is try to make it so as easy as possible less not as impacting as far as their day-to-day life but try to slowly get them to start looking at things and so going over things and saying and and using the tooling that we have available saying here we know you did it in this particular tool this is how you should do it in this tool and providing that data and then doing training on that too so trying so they're slowly it's slowly becoming part but I would not say it's 100 like it's far from 100% yet it's still there's a lot of work to still to be done there just so that they start to in they all understand why it's it's part of the value right um it's only like I think I right now it's still a handful and I could be wrong maybe I might get in from snack later I'll get you know told no we all love it who knows in terms of instrumentation like what uh what languages um are part of like your your application landscape um and mostly python oh okay okay so mostly so we're what's that oh I was gonna say and you alluded to some Auto instrumentation there yeah so we did Auto instrumentation in our python code um and so we and we use a couple different Frameworks but I think we're just trying to work towards getting to one framework um and then we have and then yes it's fortunately there's not so when it comes to so we can you can actually almost kind of get away with like doing a rapper sort of library for people and have them just use that and then yeah to get and that's kind of what we did we have this little wrapper Library set up that uses oel with the you know all the different things that they need to pull in there and people just kind of just dumped it in and did Auto instrumentation the way they went so they kind of like so it's kind of like all done for them and they just kind of include this in a way they go um but I think as we progress it's going to be like okay let's start Target this is my next stage will be talking to first off the PMS talking to them okay this is what we have so far these are the gaps we have in our our Telemetry data these are the es we have set so far let's decide on how that should look because your team needs to own this your team needs to act on this so when your air budget gets exhausted your team needs to take that on in next Sprint and that's one of and like so that and again the technical side is not that difficult it's the the cultural change process change requires a lot of effort right because now you have to get people to say okay well instead of when our we burn through our error budget we need to take deal with this the next Sprint well that's usually well that are this future that we need to push out which which one do we do and so but they have to that that's what I said when I first started this is that you have to agree upon this process you have to think about you are making a contract with the company to say yeah we will act on these slos because they will impact the user Journey they will impact our users and that's what's most important to us today so um for those who are not aware topad is an educational platform so what our our our our you our um our users are teachers or professors and students so imagine you know if they're taking an exam and then everything goes sideways that's going to be really I mean anybody who has kids or has been to University knows that that could be a huge frustrating Factor when you're not you know if that's not working properly so yeah so it's important that um oh I did get a message somebody did tell me we all love it thanks Mark yeah so um yeah uh now I just kind completely thrown off there uh what was I talking about ADHD you were talking about the slos and and how frustrating it is uh if if the consumers of your platform um if it's not working as they expected yeah yeah so and that's this kind of fundamental shift is going to slos and helping them see that and get through that um yeah because I think now I just went off the rails I so but that's all good cool I have one more question that I wanted to ask and then we actually do have some questions from uh from folks see that our on our board which I'm very excited about um final final question um I guess there the two questions I guess um are you um if if are you using kubernetes and if so are you using the otel operator um to manage your collectors and to manage your auto instrumentation no no yeah so we um I'm glad we're not um just because and this is not because I think that kubernetes has its place in like all tools has its place and I think for what we're doing today it would not make sense it would add a layer of complexity that would completely destroy our service so glad we're not using KU I don't think it not saying that there's anything wrong with kubernetes I just think that you if you're going to run a kubernetes environment you need to know it yeah yeah AB right and so if you don't know what you're doing with kubernetes you can really really mess up your service and and I've been at places where we only had one or two people who knew kubernetes and we had a number of problems always with our clusters so I'm when we're ready though I think we'll probably make probably make that push but right we're now we're just in containers so it's all just through yeah just containers but not a kubernetes gotcha gotcha so you're you're using like another container Management Service like a cloud provided one kind of thing a cloud provider container service yes gotcha gotcha you're trying to avoid all names of vendors right now yeah yeah yeah let's be agnostic and and H final question before we get to the questions um collectors what's your collector landscape like so we're using a combination of alel collector um as a sar and then to a primary one that Aggregates it and then for our traces we use a with one company that we're work where where we're moving towards uh they have their own collector that we send our traces to that we can do additional sampling rules on and send off so um yeah yeah so we um so right now because and I I really really love the fact that you could choose you could do send from code itself but I love the hotel collector because simply because you have these processors you have these exporters and you can send to different places as you see fit I've had conversations with our data team saying Hey listen if you want stuff otel is collecting a lot of this right and we don't have to send all of it to our for production like as far as what we're looking at as far as like would give us an idea of the user experience but there may be things you are collecting that from R tell that we can send to an S3 bucket that you can pull in later for whatever you need however um you know we have we're still haven't gotten to that point but that's the beauty of the hotel collectors that you can kind of like just point to different things and I love that because then it there's no code change so if we decide we no longer want to use this or we want to add loging or we want to add set get uh we start using we don't use Prometheus now but let's say we start using Prometheus as a open to open um Source tool um as an example though uh you would we can easily just start getting that information right Off the Mark so it I I just if anything try not to use the the sampling and the old push from this code itself use a collector because I love the fact that because it makes life so much easier then you can just if you have to do little tiny changes of tweaks to your your sampling or whatever you don't have to go push it out to like the production environment again it's just a small change on your collectors themselves which makes life so much easier that's just how I think of it absolutely and and you're using all three like major signals traces logs metrics right now like for otel or mostly traces right now um so there was a decision to not do so metrics we still are capturing the infrastructure metrics from the cloud and the logs are primarily staying in our cloud provider so all of our services are are just logging by default from container to the logging um in our cloud provider and they are staying there for the time being um because that was another thing to to start tackling and I didn't I just I felt like that'd be too much for folks as they're transitioning from the one vendor to open Telemetry for them to like get their heads wrapped around would be like oh well by way you also need to have structured logs too and this is how you should do it no this is just gonna it'll make more confusion and so I figured just just focusing on one and then adding things as they go along as they as they prove useful so yeah that's how we kind of approach that whole process makes sense baby steps exactly cool um okay so we're going to turn to the board and if you want you can take a peek at the questions they're on the on the chat window there's a link to the board um take a look so um let's have a look at the questions so the first one is what is your opinion on Telemetry as a uh on Telemetry quality as a way to reduce spend is more data always better observability great question awesome question and that's a tough one um I don't think I can I hope I can answer that in my opinion um it's Bell quality is you need to actually because really with the end of because you can have a bunch of things going through but if you're sort of so let's just take a a standard Trace right if you have um a bunch of spans in there that are doing sort of some things that you really know it's just back and forth and you really like there's I've noticed some Tes where there it there's um I forget what it's called and I I I I wish I had a sample the code before it's been like a two weeks ago one of the one of our Engineers cleaned this up and what he did though is he looked and realized we're making these calls in the code that we're we're adding span events to and we don't need to because so you're looking at the Quality there and then you're not you're seeing what's invaluable and what's not so more data doesn't always mean better Telemetry quality data means better Telemetry so in my opinion you've got to look at what you're getting making sure does that reflect the user experience if it doesn't reflect the user experience or is not exposing things where you would say oh you know like that's kind of like the whole the concept of the unknowns right like you're exposing those unknowns you're not able to see those things and then you're not the qual it doesn't matter how much you have if you're not seeing that stuff then it doesn't mean you're it's the quality of the data that's most important and that will help you reduce spend because then you know what's good you can sample that partially out and you're going to make sure that those that that the air prone things Bubble Up and become more apparent when you're looking at your data hopefully that answered the question yeah that was a really great answer um yeah and I I I agree with you it's it's really like what's the point of capturing in Telemetry for your system that's useless to you like that's just and I think that's um that's a pitfall that a lot of people run into also with auto instrumentation initially right because there's a lot of stuff coming in and and and fortunately now I think there's a way to actually switch off auto instrumentation for certain libraries so that you can you can really zero in on the stuff that's uh super meaningful to you so yay yay cool okay next question um how do you approach the what to instrument and how much to instrument questions with your teams so that kind of is it linked to the previous question to in my opinion um because what instrument is again goes back to this is so I went through the whole process of working with the the teams the development teams and the the product teams to understand what the user Journey looks like so I went through and said okay what is the so we did the user Journey we looked at their servers and I asked this like some did more than others uh but more or less it was like I asked him what are the typical interactions a user would have your service and write it down and then write in in so in plain language and then in plain language describe the tech what's happening in the background and then we also talk about the prerequisites of what people expect when they interact with that system so they have to be logged in they have to have this Tech maybe this cookie or this cash or whatever um and then what are the dependencies and then we just kind of work through and go okay what are the things that where a user will potentially see problems based on this information and we went through that so that's how I kind of approach what the instrument because then once you kind of know okay so we know that user logs in so they know they have to do that this they have to interact with maybe uh let's hypothetically say they interact with a database they interact with an authorization service maybe they interact with something in the back end for something else uh another database and so all these queries are checked in there now that all those things all those functions are like talking to these different things if they don't work means they can't log in therefore we have key spots what they should be mostly looking at to make sure there are always going to be times where people miss things this is why we have the the slos this is why we have to have this constant part of our cycle to go back and make Ure that those things are identified so you know we're having problems with our servers but we don't know why that means maybe you're not instrumenting so what that means you have to go revisit some of the things you did because maybe you did some changes that no longer uses this kind of database uses maybe a cash service or uses a redice cluster instead instead of these things and so does your code or your user Journey reflect that change and does your instrumentation reflect that change and if not let's make sure they do so but it always boils down to the user journey and what the user is experiencing if as long as we're capturing that you're like 80% of the way in my opinion 80% of the way there uh because you at least have a general idea but you have to know you have to have that conversation awesome thank you okay next question the Q um in an org that has adopted distributed tracing how do you make individ idual teams leave behind their outdated practices based on logs how do you show value that was a spicy question that's a spicy one y slap them around and tell them what no you do not that's not the way to do things you have to be very exactly that's number one do not call her baby ugly um that's a tough one because you you really really have to be patient and there sometimes you do have to say okay listen this has been dictated by the AL hups that were doing this and you try to offer them as much help as possible but you have to you got to find out too why they're reticent about switching over uh having these conversations is important because then you understand what how they're looking at things and then you can restructure your discussions around that uh so oh I like using logging because you know I get to see immediately when I'm doing my development I can see it point out on the screen I know where things are broken so I can go back and fix and Blum Blum Blum and that's so you know that that's why they love logging as an example so how you have to then help them bridge that Gap and say okay well this is how you would do it with tracing and this is why you would probably want to do it because then with distributed tracing you're not just getting the value out of your service but you're seeing how service a interacted with your service service B and how service B is going to interact with service C and how all that kind of works together so this is how why you want to have these pieces in the puzzle so you can get that full picture but it takes time it takes patience it takes and again going back to which one of the things we talked about before restructuring how you approach the their their the uh the challenge of switching over it's not just repeating yourself the same words over and over again because eventually people are just going to tune you out like at my work I think they T me out half the time okay uh final question we can squeeze it in hopefully in two minutes um really love what you said about the tooling uh is the last thing you should think about and that you should lead with the question what is most important to your organization what are some of the questions you ask to understand what is most important to someone's organization it's always about the user in my opinion it's always about the user um because first because you're looking at if the user is not happy and you can't measure that or you don't know that then you're in trouble right uh and I always use streaming services like video streaming services as an example because and we everybody has tuned into one of one of many of these streaming servers we have now today so if you pull up a movie you push play it gets all pixelated for the first three seconds and then it clears up after that do you care no so if a person comes by but does no the the streaming service know that they probably are measuring that because they want to make sure that that doesn't become a bigger issue because they know that Runway of where you're okay to where you're not okay and you're G to get frustrated and go I'm gonna go switch to this now because they have better quality and so it's not it's always about what the user how we are we measuring the user experience do we know what the user experience is like do we know what frustrates them and if we don't how do we get there and that's how we get so that is usually with the tooling that we have available today today with the sdks and other things and the and the philosophies we have and eventually then you can go because you know what the tools comes out all the time there's so many tools out there that the landscape has gotten bigger and bigger as people become more and more involved but doesn't message mean they're all good and you know but you have to find the one that kind of works for you at the time and this is why we like open Telemetry is because you can switch from one to the other to the other without with very minimal change and that's why I like about getting people on open Telemetry but when it comes to the tooling that should be your last thing to think about don't build your solution around this tool in your spot Build Your solution about what your users are thinking and that's how I think a lot of people that resonates when you start talking in that light especially with the higher ups anyway so I think we got them all in yeah absolutely well we got through all the questions yay we filled up this hour quite nicely so thank you so much Dan uh for joining us for hotel Q&A stay tuned for our next uh Hotel Q&A and or hotel in practice and again thank you Dan for joining today I really appreciate it oh yeah that you know it's my my pleasure thank you for having me um you know how much I love talking about these things so you know I I think I actually did not go down a rabbit hole today I think I was I pretty good it's perfect it was great thank you again all right thank you everyone thank you for showing up see you diff --git a/video-transcripts/transcripts/2024-11-15T06:30:37Z-humans-of-otel-live-from-kubecon-na-2024.md b/video-transcripts/transcripts/2024-11-15T06:30:37Z-humans-of-otel-live-from-kubecon-na-2024.md index d95da08..a224ace 100644 --- a/video-transcripts/transcripts/2024-11-15T06:30:37Z-humans-of-otel-live-from-kubecon-na-2024.md +++ b/video-transcripts/transcripts/2024-11-15T06:30:37Z-humans-of-otel-live-from-kubecon-na-2024.md @@ -10,125 +10,156 @@ URL: https://www.youtube.com/watch?v=LjD454EPMgQ ## Summary -In this live session from KubeCon North America 2024 in Salt Lake City, host Rees and special guests Hazel and Ted Young engage in an insightful discussion about the OpenTelemetry community and the evolution of observability. Hazel shares her journey into observability, highlighting the importance of asking meaningful questions and the desire for better tools to empower others. She introduces the concept of "Observability 3.0," which emphasizes integrating business context with technology to enhance understanding across teams. Ted, a co-founder of OpenTelemetry, discusses the project's origins from the merging of OpenTracing and OpenCensus and details ongoing updates, including challenges with resource management and the need for better data integration. Both guests stress the importance of community, user involvement, and the collaborative nature of OpenTelemetry, underscoring the significance of vendor-neutrality in fostering trust and innovation within the ecosystem. +The video features a live discussion from CubeCon North America 2024, hosted by Rees, and includes special guests Hazel and Ted, both prominent figures in the OpenTelemetry community. The conversation delves into Hazel's journey into observability and OpenTelemetry, highlighting her frustrations with existing tools and her desire to improve the process of teaching others how to navigate these systems effectively. They discuss the importance of observability in platform engineering and the need for better integration of technology with business processes, coining the concept of "observability 3.0," which focuses on embedding tech understanding into non-tech stakeholders. Ted shares insights on his role as a co-founder of OpenTelemetry, the project's evolution from merging OpenTracing and OpenCensus, and the importance of community trust and involvement in maintaining a vendor-neutral approach. They emphasize the need for user feedback and engagement to refine the tools and processes within OpenTelemetry. The video concludes with a call to action for viewers to participate in surveys and community discussions to enhance the project. -# CubeCon North America 2024: OpenTelemetry Community Conversations +## Chapters -**Host Introduction:** -Hello, everyone! We are live from Salt Lake City. I'm Rees, and I'm here with some special guests at CubeCon North America 2024. Today, we’re going to talk with some incredible people in the OpenTelemetry community about their experiences and insights. Let’s dive in! +Sure! Here are the key moments from the YouTube livestream along with their timestamps: ---- +00:00:00 Introductions and welcome to the livestream from CubeCon North America 2024 +00:01:30 Guest introductions: Hazel and Adriana share their backgrounds +00:03:15 Hazel discusses how she got involved in observability +00:05:00 Transition to OpenTelemetry: Hazel's journey into the project +00:07:45 The importance of observability in platform engineering +00:10:30 Hazel shares her thoughts on the fragmentation within observability tools +00:12:15 Discussion on the challenges of data collection and operational context +00:15:00 Introduction to observability 3.0 and its significance +00:18:00 Overview of the OpenTelemetry Foundation and its origins +00:20:30 Ted Young joins the stream and introduces himself and his role +00:23:00 Ted discusses the merging of OpenTracing and OpenCensus into OpenTelemetry +00:28:00 Insights on maintaining community trust within the OpenTelemetry project +00:30:45 Ted shares updates on the project's current state and future direction +00:35:00 Discussion on the importance of user feedback for the SDK and instrumentation APIs +00:37:30 Ted talks about the vision for OpenTelemetry in the next five years +00:40:00 Closing remarks and encouraging audience participation in the project -**Guest Introductions:** -Good morning, everyone! I’m Hazel, and I have a lot of thoughts—thoughts that never stop. I’m thrilled to be here and hopefully make you laugh a little. +Feel free to reach out if you need more details or information! -And I’m Adriana Vela, working with Rees on OpenTelemetry and user experience. We’re excited to talk to our amazing OpenTelemetry guests today. +# CubeCon North America 2024 Live Stream Transcript ---- +**Reys:** Well, he hasn't done the countdown, so YouTube... Is this on the channel? Should I just shut up? Right, so I launched the counter for 10 minutes. That's probably too much. Let's drop it. No, we still have this. Oh, it's OpenTelemetry official. He's checking the OpenTelemetry channel to make sure that it’s queuing up on their live. Okay, cool. Perfect! Are you ready, M? -**Hazel’s Journey into Observability:** -Hazel: My journey into observability began quite similarly to many others. I didn’t start with the intention of getting into it. I became frustrated with various issues, which led me to ask better questions about my systems. The tools I had weren't sufficient, so I started searching for better options. +**Adriana:** Yeah, I will be in. -One of my superpowers is understanding entire systems intuitively. In college, I took an operating systems class where the goal was to learn to debug, but I found myself mentally tracing the code instead. While I could debug effectively, teaching others was a different story. I wanted to help people realize they already had the knowledge to find answers and learn how to ask meaningful questions. +**Reys:** Hello, live from Salt Lake City! I'm Reys, and I'm here with some special guests. We are live at CubeCon North America 2024, and we're here to talk with a couple of really cool people in the OpenTelemetry community about how they got involved and to learn a little bit more about them. Let's go! --- -**Getting Involved in OpenTelemetry:** -Interviewer: How did you get into OpenTelemetry? +**Reys:** Good morning, good morning, good morning! I'm really glad to be here. For those of you who don't know, I am Hazel Ha, and I have thoughts—lots of thoughts. They never stop thinking, and I am super glad to be here to talk with you all and hopefully make you laugh so hard you cry a little bit. Thank you! -Hazel: I was frustrated with the vendor-specific tools available. I wanted to find a way to unify everything, and that’s when I discovered OpenTelemetry. It felt like the core essence of observability—something we could build upon. +**Adriana:** And I'm Adriana Vela. I work with Reys on the OpenTelemetry and User SIG, so we're excited to be talking to our awesome OpenTelemetry guests today. Hazel, how did you get involved in OpenTelemetry? What is your story? ---- +**Hazel:** I think I got involved with observability in a very similar way that a lot of people do, which is to say I didn’t. What happened was I got really, really frustrated with things happening, and I needed to figure out how to ask better questions about my systems and understand what was going on. The tools I had weren't very good, so I started looking for better tools, better ways to do things, and more importantly, better ways to teach other people what we were doing and how it works. One of my superpowers is being able to take the entire system and hold it in my head, thinking about it intuitively. + +When I was in an operating systems class in college, the point of the class was to almost force you to learn how to debug, but I never did. I still don’t know how to debug because I just looked at the code and mentally traced it in my head. I can't teach that to people, though, so I always wanted to figure out how to take the vibes and turn them into a tool, an explanation, a technique, and a way to teach people that they already know the answers. They know how to find things; they’re already 80 or 90% of the way there. It’s not about a magical toolkit; it’s about asking meaningful questions, getting useful answers, and then doing something with them. + +After realizing this, I ended up getting into the observability space, where I met a lot of people who were also trying to do that. -**Contributions to OpenTelemetry:** -Interviewer: How long have you been using or contributing to OpenTelemetry? +--- -Hazel: I’ve been involved since 2019, focusing on empowering others and helping them navigate the various tools and integrations. +**Adriana:** And how did you get into OpenTelemetry itself? -Interviewer: That's impressive since 2019 was not too long ago! +**Hazel:** For me, it was actually looking at the shape of all the different vendor-specific things or the different instrumentation of logging, metrics... We didn’t really have traces when I first stepped into it. It was frustrating because I really hate doing things that are undifferentiated in different ways. I would much rather do the work to figure out how to think about this concept of optionality and put everything together. One of the best things about Linux is it turns out you don’t need to write a kernel to get something working; you can use one. One of the best things about Kubernetes is that you can write your own custom flavor on top of a core. -Hazel: I know, right? It feels just like two weeks ago! +So for observability, I started looking for that core essence, and I found that in OpenTelemetry. --- -**Observability and Platform Engineering:** -Interviewer: You’re heavily involved in platform engineering—how does observability fit into that? +**Adriana:** How long have you been using or contributing to the project? -Hazel: Observability is critical for effective platform engineering and Site Reliability Engineering (SRE). However, many people still treat it as a separate concern. The reality is that observability should be integrated into everything we do. +**Hazel:** I’ve been using the project for quite a while. I usually don’t get involved directly because I’m low enough on that platform side. What I’m almost always looking to do is enable people. I’ve looked at the OpenTelemetry Collector a lot, wanting to empower people and help them move from using one vendor here and another vendor on another team or a third-party integration to the OpenTelemetry Collector. -I see a lot of fragmentation in how teams manage observability, often leading to reimplementations of solutions. For instance, profiling discussions often overlook existing solutions we’ve developed in OpenTelemetry. +I want to say I really started digging deep into building those platforms for other people around 2019. --- -**Challenges in Observability:** -Interviewer: It sounds like you encounter many challenges in the industry. +**Adriana:** Wow, so basically from the beginning, really? + +**Hazel:** Yeah, that’s great! It's so weird because 2019 was not that long ago—like two weeks ago! -Hazel: Absolutely. The CNCF landscape is vast, and many teams end up reinventing the wheel rather than collaborating. We need to encourage communication between communities to avoid this. +**Adriana:** I know, right? It does feel like it! --- -**OpenTelemetry’s Potential:** -Interviewer: What aspects of OpenTelemetry intrigue you the most? +**Hazel:** I feel like observability is one of those things we often talk about as a separate thing, but you can’t have effective platform engineering without observability. You can't have effective SRE without observability. + +**Hazel:** It actually frustrates me a lot of the time because I understand why this happens. You have a massively wide platform and toolchain, and people keep adding more and more context into something. The context goes deep and wide, and you can’t possibly hold it all in your head unless you're weird like me. But you end up in a situation where OpenTelemetry can dive so deep into it that it becomes your whole thing. -Hazel: I find the disconnect between the end-user experience and the operational side fascinating. There’s a lot of potential to bridge that gap and create a more cohesive understanding of how to use the tools effectively. +A lot of companies with platform teams start off with an Ops team that gets rebranded, and then they split into two or three teams. One of those teams ends up being in charge of the Kubernetes part, labeled the platform team for some reason. This leads to complexity and fragmentation, much like the CNCF landscape, which is so massive that everything has its own corner, and nobody really talks to each other. + +I wish people would do that because they often end up re-implementing the same thing over and over. In profiling, for example, we have this Open Profiling aspect looking at a lot of OpenTelemetry stuff, and they’re reinventing the same discussions they need to have. --- -**Observability 3.0:** -Interviewer: You mentioned "Observability 3.0." Can you elaborate on that? +**Adriana:** That’s really fascinating, and I appreciate your perspective. + +**Hazel:** One of the things that’s intriguing about the OpenTelemetry project is that there’s this dichotomy between the end-user part and the operational side. The mental model for traces is often misleading—like there’s a little box that represents the start and end, but in reality, it's more complex. When you start sending data, it’s not a box; you’re reinventing distributed transactions. -Hazel: Sure! Observability 3.0 is about integrating non-technical perspectives into observability. It’s about making sure that not just technical teams, but also business analysts, marketing, and sales teams understand the systems they work with. We need to connect the technology with the broader business context for a more harmonious operation. +Sending everything is expensive, time-consuming, and often a waste of resources. You need to enrich it, correlate it, and finesse your understanding of the data in an operational context. This is often bundled into the OpenTelemetry Collector, but we don’t talk about the collector as a requirement, which hampers the SDK’s potential. --- -**The NLY Foundation:** -Interviewer: Can you share a bit about the NLY Foundation? +**Adriana:** Switching gears a bit, we talked about Observability 3.0. Can you elaborate on that? -Hazel: The NLY Foundation was started by Chris Nova and emerged from our experiences building a community on Mastodon. It focuses on empowering communities to own their architecture and engagement without relying on external platforms that may take control away from them. +**Hazel:** Sure! Observability 1.0 and 2.0 represent a natural progression in observability, focusing more on asking the right questions. Observability 3.0 is about embedding non-tech people and non-tech problems into your ability to reason about your system. It’s about making the tech and the people work together harmoniously. + +I’d love for businesses to understand how to better connect their operations with technology without it being a black hole where money goes in and nobody knows why. Can your business analysts, marketing teams, and customer success teams understand how to better serve the customer without tech people feeding them information? --- -**Closing Remarks:** -Rees: Thank you, Hazel, for sharing your insights and experiences! It’s been a pleasure. We’ll share your socials and how to get involved with OpenTelemetry at the end of this segment. +**Adriana:** That’s a fantastic vision. Finally, can you share the story behind the OpenTelemetry Foundation? + +**Hazel:** The OpenTelemetry Foundation was started by Chris Nova. It germinated from our experience with a project that became a massive thing, and we wanted to ensure that the community owned its architecture and engagement platform. We faced many challenges, particularly around legal liabilities and how to ensure that the community truly owned their project and didn’t lose control of it. -Now, let’s welcome our next guest, Ted! +The foundation's goal is to bridge the gap between a fun toy project and a community-driven initiative. We want to figure out how to navigate the cliffs of open source and ensure beautiful growth and knowledge sharing. --- -**Ted’s Introduction:** -Ted: Hi everyone! I’m Ted Young, one of the co-founders of OpenTelemetry. I come from the OpenTracing side, and I’m here to discuss my role in the project. +**Reys:** Thank you so much, Hazel. It was lovely talking with you. + +**Adriana:** We will share social links at the end and how you can get involved in future Humans of OpenTelemetry segments. I’m really excited to see the interactions that come from these conversations! --- -**Ted’s Role and Community Building:** -Ted: As a member of the governance committee, I help find consensus on design decisions. Building trust in the community is essential. Ultimately, everyone is working toward the same goal, and it's about finding common ground. +**Reys:** Now, let’s bring in our next guest! + +**Ted:** Hello! + +**Reys:** How's it going? + +**Ted:** Going great! It’s CubeCon—a big friend reunion. + +**Reys:** Can you tell us about your role in OpenTelemetry? + +**Ted:** Sure! I’m one of the co-founders of the project, coming from the OpenTracing side of things. OpenTelemetry is a merging of two prior projects: OpenTracing and OpenCensus. I work on the governance committee, focusing on various SIGs that require consensus and design solutions. --- -**Vendor Neutrality in OpenTelemetry:** -Interviewer: How does OpenTelemetry maintain a vendor-neutral stance? +**Reys:** That’s fantastic! What do you think helps foster the sense of community in OpenTelemetry? -Ted: It’s crucial. We structure the project to prevent any single company from dominating the conversation. This way, we can avoid conflicts and ensure that everyone has a voice. +**Ted:** Humans are generally good people. We all share a common interest in observability. When you’re involved long enough, you see people cycle through jobs, which makes it easier to trust one another. + +The second part is structuring the project in a way that discourages bad incentives. We made it clear how people are going to make money and how OpenTelemetry provides value for end-users and vendors, which has helped build a healthy community. --- -**Future of OpenTelemetry:** -Interviewer: Where do you see OpenTelemetry in five years? +**Adriana:** That’s insightful! What’s on the horizon for OpenTelemetry? -Ted: I believe we’ll see more advancements in analysis tools that leverage the data we’re collecting. The focus will be on connecting signals and providing actionable insights. +**Ted:** We’re at an interesting point where we’ve stabilized tracing, logs, and metrics. We’re looking at how to bring in profiling and other data sources. I think we’ll continue to see amazing new analysis tools that leverage all the data we’ve captured. --- -**Final Thoughts:** -Ted: If you’re an end user, please get involved in the developer experience SIG and provide feedback on your experiences with the SDK and APIs. We’re committed to improving the user experience. +**Reys:** Thank you, Ted! Before we wrap up, any last thoughts? + +**Ted:** I encourage end users to get involved in the Developer Experience SIG and provide feedback. We’re looking to clean up the SDK and instrumentation APIs. --- -Rees: Thank you, Ted! We appreciate your insights. Don’t forget to participate in the OpenTelemetry survey and get involved in our community discussions. +**Adriana:** We also do monthly events and end-user interviews to gather feedback. Don’t forget to take the doc survey that’s open until the end of the week. -Thank you all for joining us today, and we look forward to continuing these important conversations at CubeCon! +**Reys:** Thank you, everyone, for tuning in! We’re looking forward to more engaging discussions ahead. ## Raw YouTube Transcript -well he hasn't done the countdown so YouTube oh is this on the channel y should I just shut up right so I launch the counter of 10 minutes that's too much probably drop it no we still have this oh it's otel official Hotel Dash official Happ oh he's checking the otel channel to make sure that start it's queuing up on their live okay cool perfect are you ready M yeah and I will in hello live from Salt Lake City I'm reys and I'm here with with some special guests uh we are live at cubec con North America 2024 and we're here to talk with a couple of really cool people in the open slump Community um just about how they got involved and kind of learned a little bit more about them and let's go all right good morning good morning good morning I'm really glad to be here so for those of you who don't know I am ha a weekly and I have thoughts lots of thoughts they never stop thinking and they never stopped thinking and I am super glad to be here and talk with you all and hopefully make you laugh so hard to cry a little bit thank you and I'm Adriana Vela and I work with Rees on the otel and user Sig so we're excited to be uh talking to our awesome Hotel guests today so Hazel how did you get involved in open cemetry like what is your story how did you get involved we could start at the beginning how did you get involved in observability observability so I think I got involved with observability in a very similar way that a lot of people do which is to say I didn't and what happened was I got really really frustrated at things happening and I needed to figure out how to actually ask better questions about my systems and figure out what was going on and dig into them and the tools that I had weren't very good and so I duk around looking for better tools better ways to do things and more importantly better ways to teach other people what we're doing and how it works because one of my superpowers is being able to take the entire system and hold it into my head and be able to sort of intuitively think about it like when I was in an operating systems class in college the point of the class was to kind of almost force you to learn how to to do bunker but I never never did and I still don't know how to need to do bunker because I just looked at the code and mentally traced the colel in my head and debugged panic gloss in the schuer by just like staring at it but I can't teach that to people and I can't show them how to think about that so I've always set then want okay I can I can figure this out with just you know print F debuging or whatever I want to and Vibes but how do I actually how do I take the Vibes and turn into a tool and an explanation and a technique and a way to teach people that you have this you already know the answers you know how to find things you're already like 80 or 90% in the way there it's not this magical new tool kit it's this asking questions that are meaningful to you getting those answers that are useful and then doing something useful with that like learning and then after effectively on that and you don't do that you already know how to do that and just taking that and looking for really really nice things ended up getting me into that observability space where I met a lot of people that were also trying to do that and how did you get into open Telemetry itself so open Telemetry um so for me it was actually I looked at the shape of all the different vendor specific things or the different instrumentation of like locking or metric or we didn't really have traces when I first spped and it was actually really frustrating because I really really hate doing things that are undifferentiated in a different way I would much much rather do a bit work to figure out like how how can I SWA in down how can I think about this concept of optionality and putting everything together because one of the best things about linnux is it turns out you don't need to write a kernel in order to get something working you can use one and one of the best things about kubernetes is sure you write you know your own custom flavor on top but it's the thin veneer over this core and so for observability I started looking for that core the essence of it that you could build that thin ven or that layer on top and I found that in open Telemetry that's amazing so how long have you been using the project or um yeah contributing or using the project so I've been using the project actually ionically usually not directly because I'm low enough on that platform size what I'm almost always looking to do is enable people and so I have looked at the open tary collector a lot of being able to say how can I Empower people how can I enable them how can I take them from these different places of maybe they're using one vendor here and one vendor on another team and one vendor on another team or a third party integration or the open taryle code and so as far as being able to enable people and think about that I started I want to say really really digging deep in building those platforms for other people around in [Music] 209 okay wow so basically from the beginning really yeah yeah that's great aesome um which is so weird because like 2019 was not that long ago it's like 2 weeks ago I know right it does feel like it absolutely yeah I I was going to say um you're also like um you know heavily involved in in the platform engineering space and I feel like observability is one of those things like we often talk about it as like a separate thing um but really like you can't have effective platform engineering without observability you can't have effective SRE without observability um and I know you have many thoughts yeah so it's actually something that urts me a little bit a lot of the time because I get the I get why this happens because you have a massively wide platform and you have a massively wide tool chain and people keep adding more and more context into something and the context goes deep and it goes wide and it goes up when it goes in all the different places and you can't possibly hold on your head unless you know you're weird like me and you can hold way too many things in your head and but you do end up in this situation where yeah open tary you can dive so deep into it it can become your whole thing it can become like the only thing you really think about and as when you see and a lot of companies with the platform teams they start off with aici team that's really just rebranded Ops and then they take the rebranded Ops Team they split into you know two three teams and then one of those teams ends up being in charge of you know the kubernetes part and that you know gets labeled the platform team for some reason and then the platform team gets turned into two three different teams and then you keep going on and you know someone gets stuck with the the observability thing and it ends up being really complex because it there's a lot of moving parts so I see how things get split out and by at and sort of frally fragmented you know much in the same way as the CNC of landscape is so massive everything has it own the kind of corner nobody really talks to each other but I wish people would do that because they often end up reimplementing the same thing over and over so like in the profiling side of things we have this open profiling aspect and they're looking at a lot of the open lary stuff and they're Reinventing a lot of the same discussions that they need to have a lot of saying do you need standards do you need name Advent do you need plugins how does the collector of and I'm like we solve that 5 years ago like look at the preexisting stuff tweak it a bit and then you think about it or you have the like event streaming architecture people where you have like databas streaming you have lak House people and you have business and lits and business intelligence and all these different data science types of things in data science related stuff and it turns out that if you are trying to take a metric but load of data and derive useful actionable insights from it and then share that with people who doing data science that's kind of what we decided to call it you know 20 years ago and then we forgot about that and we reinvented open Telemetry instead of making a data Pipeline and then we reinvented profil instead of making a data Pipeline and now we're kind of looking at all these things and then we're going oh oh we should do this thing and then the you know the powerbi or the business analytics people in the corner are just crying a little bit because they've been doing this since the ' 80s and then way over here in the financial side uh did you know this is one of my favorite things actually so kdb is a financial analytic statem database and it is a colon database and so if you look at high frequency trading or you know data analytics people that are specialize in the financial World they actually predate the usage of all of these like oh no queries that rely on indices you can just grab the DAT s and they've been doing that for 40 years CU they had to and nobody else ever really thought to look it up and so it's it's it's fun to see all the communities reinvent things over and over bring their own favor and context into it and then hopefully we can with platform engineering start stirring all the people together talking to them and getting them to actually look outside the little window and go oh we solve the same problem well that's cool how can I learn from how you approached it and how can you learn from how I approached it and can we build kind of a more common thing that's like really awesome given that there's so many you know interesting points that you just brought up and like the way your brain works and how you can look at something and kind of intuitively understand what's happening what pieces of of uh the projects open projects are intriguing you so one of the things that's really intriguing to me about the open telary project is rather it's intriguing to me because I want more people to care about it and that is actually you have this dichotomy of open telary as the end user kind of Part F where you have this SDK and this API and there's how you use it and and you have this mental model and then you have the operational side of how you collect the data how you store it how you enrich it how you sample and how you do all these things and they're all deeply disconnected and they really don't need to be but the reason they're often deeply disconnected because they're all non intentionally or maliciously horrifically lying to each other so the mental model for example that you have with traces is oh there's like this little box and it's this start of it and the end of it and in the Box I put all my information stuff and I draw some information on the outside of it and then in the information stuff I had another box then I have another Bo and I have another box you I have so many boxes and when you actually use the SDC when you actually send things that's a complete lie you're not doing a box you are Reinventing the concept of distribut transactions really badly and without transactional santis because you're just firing stuff off like a stream of events but you don't have like this R ahead lock or any of the other stuff that the database people invented in the 60s and so there's that kind of Li of it and then when you get into the operation side increasingly it turns out sending everything is a expensive you know B really really timec consuming C actually a waste of effort and time and resources and you shouldn't do that so you need to take that enrich it corly other data look at better things figure out associations sample it and sort of figure out and finesse how you understand the data in this very operational context which is we often bundle that into the open telary collector but we never ever talk about the open open telary collector as like a required thing or even you know a thing and then so it hampers a lot of the potential of the SDK make because for example you can't update a span because a span is treated as a append only immutable log except it's a box but no it's a dependently immutable log it's a stream of events and but at the same time it turns out if you break this at any point because not are unreliable you completely M host the entire concept of what you're doing and EMB Brees all your ingestion pipelines and EMB Brees DUI of everything building everything and it's super annoying so I would love for people to think about what is it like if we made the concept of a collector or this concept of enriching rewriting the tree flattening the tree doing all these weird sorts of correlation Concepts or flattening or changing the shap of stuff to make it more malleable for you what if that was a more integral part of how we designed the SDK and if we design the Sate with more use cases in mind how do we do so in a way that gives people a simplement model but doesn't lie to them about what they can expect out in the platform so like there are client STIs Decay for example that don't send the data at all ever until they have received on the client the ending span which means that if your span is 10 seconds long and on second 9 and A2 the client process gets shut down and this not have time to send that span you lose all of that data but if you send it all to The Collector immediately you have so much traffic you can't really afford to do that but if you were able to send like a snapshot thing of a right ahead log of this isn't done yet but I'm going to I'm going to window it like databases do in 70s then okay you can work with it you can work with it and then you can patch it up and some vendors have actually started to work cleverly around hotel and work cleverly around the specification in the middle part to allow that capability in places where they need it like mobile clients and then patch it up and make it you know OTL compliant by the time you send to your back end so if we open up the Pandora b a little bit of okay maybe this is kind of necessary maybe we need to think about this in a weird way of we have to have all the things talk to each other and we need to make it more possible to do these things that make the mental model a bit more coherent when can we do that that's a lot he got thoughts um I wanted to just switch gears a little bit because um before we started uh the stream we're talking about you know you you have many thoughts on many things um and one of them was observability 3.0 now we've heard observability 2.0 um has kind of come into um our our um vernacular lately and you were talking now about observability 3.0 so can you tell us what you uh what you mean by that so what I mean by that is a really really fun thing where it gets into the heart of what I like about observability which is essentially the same thing that everybody ignores and I would like them to not ignore that and so if I brand it a little bit with like a cute little sticker maybe people will care so I'm servility 1.0 and the 2.0 is sort of a reframing of the natural progression that has happened and observability and the vendors and the capabilities and the needs of the platform and the what we need to do in order to ask the right questions and before it was sort of this is an observability and this is observability and then we're like well no it is all observability it's all about asking questions it's more what's the Fidelity of the questions what type of questions can you ask how much information is there how rich is it what are the properties of asking those types of questions and then so if we think of observability 2.0 as sort of being the ultimate inside the tech context of a company or a program or a platform can we ask essentially in question that's kind of the ultimate gole of observability 2.0 to me so for me observability 3.0 is defined by the idea that you need to take non-tech people and non-te problems and the larger context of the business and embed it into yourability to reason about your system and need to take the system and embed that into the rest of the company so the company can reason about that so observability 3.0 has a star difference in that it's not about tooling it's not about capabilities we it's not about all the sort of the technical side it's very much now we need to take the tech and we need to take the people and the processes and this you know massive budget that the you know it industry gets and stop spending it without accountability and stop spending it as this massive black hole where money goes in magic stuff comes out and nobody can explain why so can your business analysts can your marketing people can your sales people can your products people can your everybody else your customer success team can they all sit there and understand how to better serve the customer and how to better interoperate with the technology people without the technology people having screen feeding to them can you give them the capability to do a better job and can you take what they know and take all this cool stuff that they do and put it into context that lets platform engineers and product engineers and back end in and all these people do a really really good thing of being able to present technological options and be able to actually interoperate with what the company needs has a potential not just what the company has asked for you know in a guessing manner to me avability 3.0 they kind of sum it up is the business context comes in Tech the tech context goes into the business and they become one harmonious concept that's amazing and I'm sure this is going to spark a lot of conversations and I'm really excited to see you know other people's thoughts and kind of hopefully you know see the your vision um I also wanted to chat with you you know open Tre um is an open source project obviously we at cubec con um and you work you are at the nly foundation and I would love to you know have the audience learn more about kind of the story behind the foundation because um yeah she shared it earlier and I would love for you to be share that so the Le Foundation was started by Chris NOA and it was one of those last things that she did before she passed away and the nly foundation actually kind of started as an idea that germinated from hacker the Macedon instance that we all built together made into this massive thing and then did a huge migration it was live We Live streamed it and everybody learned from it it was awesome and the reason in haak that we chose the technology that we did is because you had a bunch of brains you had a bunch of people that are kind the world supp to doing a lot of things but we didn't want to rely on that because we wanted to experiment with the idea of what does it mean for a community to deeply own their own Community their own concept their own architecture their everything and this was right around the time when the community especially the Tech Community was first you know gr uping with the idea of we built this little thing and now it's kind of being taken away and we don't have control over our own you know engagement platform our own sort of communication WEA how do we never lose it again and then it turns out that as the massive like migration initially happened we almost closed registrations on haad not due to operational concerns but due to a massive legal liability and the course of how Federation works on Mason it ends up being a very very abusable Vector for putting illegal content somewhere because that content has to get syndicated onto your server and so if you run a Masson instance you are liable for anything that touches your computer but you don't really necessarily control what that is and so that is a very difficult concept and so Nova kind of ran into this and with her legal background and her legal brain she was like oh absolutely no no no no no no no and then so she ended up immediately finding lawyers sitting down and building like an LLC and we wrapped uh haad mount on L and very quietly spad this kind of information to a lot of other larger instances of you you need to now care about it you need immediately you have to immediately start caring about this because you are at risk of a huge vetor there's a huge surge of potential weight uh heat coming but it made us realize as we sat down and we kind of solved this problem when you go from a fun toy project and you take this toy project and it becomes a community there's this invisible Cliff of I have like my vision I have my dream my people and now there's everything like I I can't half ass my like licensing anymore I can't have ass my you know contributor like CLA agreement how do I get people from large SE companies or from regulated Industries to be able to contribute it turns out you can't just make their repository open shorts you have to actually make it so that their paperwork on there and is okay with it and then how about International people how about people with disabilities how about people with you know different needs how about people with different that happens okay you know we haven't even begun to solve the how you for open source how do you find open source no nobody knows the answer nobody's even gone close people are trying like Eva black as she says doing the Fant ftic job trying to create this sort of top down way and path and Avenue for the ability for companies to pay um to pay open source and for governments to pay and for all these things to happen it's going to take time but as a community we need to bottom up sort of figure out how do we pass this cliff and it's a massive cliff and nobody really knows it's coming and you don't have time to prepare for it because you don't get to choose when you become adopted and beloved by Community nobody sits down and says where Community now you just look around one day and you go these are all my friends and I love them and this is great and oh no and then there but it turns out that's only step two because it turns out that you can't really call the cncf community anymore it's an ecosystem and that's kind that third massive Cliff that very very few people talk about where you have this one project and this community and that's cool and the community maybe grows other things when it reaches this s of becoming a generation and incubation Hub of innovation of experimentation and it starts generating a fractal of different communities that all come together in this ecosystem and of this massive idea Exchange of community knowledge Shar and Community like growth when becomes this messy in articulate sort of ill defined but beautifully growing organic thing that takes on its own life that's an ecosystem and we don't even really know how to build those we definitely don't know how to fund them we definitely don't even know the differences between all the legal and the compliance how how do you facilitate that how do you how do you get there and so we looked at this problem Nova and I and a bunch of other people and we want someone need to start thinking about this it's going to take a while it's going to take several years maybe even decade or two to really deeply help the world understand how to to pass those Cliffs and turn them into this you know growing ramp of taking ideas and sharing with the world in a way that generates further ideas in a way that becomes this ever growing thing and the nly foundation which was originally sort of kind of starting as like a legal cover and legal ability to do this always was intended to be a vehicle that helps you understand that problem and helps figure out how do we make this beautiful growth and knowledge sharing possible that's so great um I think are we coming up on time so we are coming up on time um we are going to have our next guest on thank you so much Hazel it was lovely to talk with her and we will share um socials like how you can get in touch with her at the end and as well as like how you can get involved in the future humans of otel segment as well and I'm really excited if um to see you know what kind of interactions come up with uh from these conversations that we just had and yeah whatever questions comments thoughts you have definitely please let us know we would love to hear them and help connect you with hazel so we can learn even more and move forward to kind of her vision and also secretly jealous of her mental superpowers um I'm going to have to see if I can pick up any tips but um now we have our next guest on and I'm so excited hopefully you can see his cat pants because they're amazing um Ted hello hey hello how's it going going great how you all doing today not bad it's CU con thank awesome times it's like a big friend reunion mhm day two of the main cubec events which you know everyone is on fumes at this point y y MH yeah we had like we had a busy start cuz we had observability day which rejects re start with Rejects and then yep observability today and cucon day one yesterday I know it just feels like we've been just been here forever so we're so excited to have you um can you tell us a little bit about you know your role in open Telemetry and you know I'm sure a lot of our viewers who are more familiar with the project are already aware of who you are but would love an introduction yeah sure uh my name's Ted Young I'm one of the the co-founders of the project um and coming from the the open tracing side of the family uh people don't know open Telemetry is actually a merging of two prior projects open tracing and open census open sensus was mostly Google people and some Microsoft people and open tracing was everybody else and then we we merged to form open Telemetry and I continue to work on the project as a member of the governance committee uh mostly just focusing on the various um spec sigs implementation sigs uh that need um uh that that need a an extra helping hand when it comes to uh finding consensus figuring out what design is actually going to work there's some areas in otel where it's technically challenging enough that ironically it's easy to come to a a a design or at any rate is easy to determine which design is the correct one right because it's so technically challenging that the the requirements are very rigorous but then there's like other parts of open Telemetry where it's like a little more squishy where there's like one way that would be a really really good way and then there's some other ways that would like sort of work right you can't say they definitely would not work yeah right but they're not like great but when you have that situation it's so so so much harder to find consensus right because people will lock in to a particular idea yeah and then if it's possible to find some way to make it work cuz they're Engineers they will continue to promote but but there's this way it could work and uh trying to like find consensus there yeah of being like yes okay that would work but but this would work better and and getting everyone to be like well even though you prefer that one would you agree that um you're not going to convince everyone at this point to go that way like yes and like and would you be able to get everything done with this other one like yes and like okay well then would you be willing to come on board with this and help us move this forward because we all actually like need it to get done and so that's like a bit of just more like Community organizing that requires maybe kind of like an engineering design background yeah and I feel like that's where I provide the most value these days to the project that's so awesome cool so and and there's I I I feel like there's uh also you you have to like have some good diplomatic skills as well yeah uh build building trust in the community is important just absolutely um having people know that you don't have an agenda yeah yeah right and that you really are just there to like listen to everyone and help everyone really clarify the requirements um and help everyone really determine what what the best um what the best design solution would be for those requirements yeah yeah yeah and you know on on a similar vein it always brings me back too to like the whole thing with Hotel being such a like a lovely community and that it really takes the whole thing of being vendor neutral very seriously I mean we're all at three different places and we all get along um like we're all competitors but like not really because we're all moving towards the same goal and I was wondering if you could comment on like like what has to go on to like really continue to Foster that sense of of community and and not in US versus them yeah the trust yeah that's that is great so I really think there's there's two parts the simple part is that like humans generally speaking are good people um and also even though we might all work at different companies now the reality is for the most part there's just like many Tech domains there's like an observability scene right there's like the engine and product people whatever who's just like that's their bag and they're good at it and they just tend to kind of like Circle through the different orgs and companies that happen to be paying people to work on that stuff in that time so I think that's the thing that makes it easier right like it's like it's not like um we're from different planets that are in some Intergalactic war and we have like nothing in common or something we've got all this stuff to overcome it's just like like we all know each other uh um and when you're long around long enough it's like everyone cycled two or three jobs so you stop really looking at people as like representatives of a particular company it's so true yeah I think one of the things too just from like my you know very short experien is even when they are moving companies people want to stay involved in the project like they're actively seeking out roles where they can continue contributing to the project in their role so I think that speaks volumes honestly the the fact that um uh you can look at our maintainers and community members and and there's like quite a long lifespan at this point I I I actually haven't run the math but I would not be shocked if like the average age of a maintainer in otel was measured in years at this point like on average it's like two or three years I wouldn't be shocked if it's that high like people really do stick around it's so true so true and the second part of that is is structuring the project because companies there are very few like companies but there are a lot of bad incentives so when we started the project coming from other open- source projects where I had borne witness and had to deal with all the Fallout of these bad incentives or um muddy incentives causing various companies to like test the waters and like get into stuff with each other um so when we started this project we had a goal based off of our experiences of like how do we actually structure this project in a way that makes it obvious that these bad incentives are not present yeah where people can look at that and be like oh yeah that that won't work so we're not going to bother to try to we're we're not going to bother to see if we can just buy the whole TC right and take over the project by like what if just everyone who works on who's in a leadership position works for Splunk or something right like that you can't do that because once you get more than two people on the TC uh who work at one company that and you want a third person well one of those first two people has to drop out or remove companies or something yeah yeah and then when it comes to the actual shape of the project prior projects even these Big Industry things like kubernetes and whatever they tend to have this like veneer of Utopia like oh we're all just getting together for this alteristic reason of like making container scheduling work and it is like as an individual it's somewhat alteristic and that like I'm really interested in it I would love to like like push this domain forwards I want a real distributed operating system I don't even want kubernetes I want the next thing yeah um but sorry kubernetes uh but I also want the next I also want the next open Telemetry you know I want to keep pushing this stuff exactly exactly um so I'm motivated but like uh these are like billion dollar projects almost right the amount of engineering effort that goes into kubernetes over this like if you calculate the salaries that I'm sure that's at this point that's probably a billion dollars maybe maybe that's maybe that's crazy maybe it's like like a 100 million yeah it's I'm probably like way out on that one actually I can't math I mean chunk of change it's a chunk of change a lot of money when you think about developer salaries and how many people work on these projects otel on average like monthly average number of contributors is like a couple hundred Engineers I think wow on average is a couple hundred Engineers pushing some kind of PR or something at any given moment on otel it's a lot of people there A lot of money and the only reason people are getting paid to do that is because there is some incentive yeah yeah which is so nice to have that because otherwise like we're so busy already like so the fact that you have like all these like vendors who are backing otel I think speaks volumes for it cuz I I don't think we'd be where we are without that right but a problem kubernetes had when it first started and other projects that I worked on is it was there was all this interest and all this willingness to pay people to work on bootstrapping kubernetes but why that question was actually nebulous for maybe some vendors or Cloud providers it was clear how they were going to make money and thus why they should be paying people to work on this but for a lot of companies a lot of groups getting started especially the startups it was not clear how they were going to make money off of kubernetes so it was like step one spend all this money to bootstrap this thing step two figure out how to exploit it yeah and that just leaving that door open right just the fact that it wasn't really clearly defined how kubernetes was should be organized how everyone's expected to make money what part of this is going to be some shared open stack versus what part are we going to like sell things or whatever that was all just like a big question mark in the early days and should all the code live in a kubernetes GitHub or or should various startups and companies be allowed to have complete ownership of some component that lives within kubernetes and thus it lives within that company's repo right right all things like this led to just like a lot of problematic stuff right like what if one company kind of owns a component of open Telemetry they're going to build a lot of like startup brand around that component naturally right that would be the natural play would be to talk that component up and how it's kind of yours yeah right and then otel a project was like that was a cool component but now we're going in a different direction so we should deprecate that one and get a new thing yeah well that would be like the kiss of death to startup yeah for sure so you can start to see how like just leaving the door open starts to like create a situation where things like people's incentives can start to get at loggerheads yeah yeah and working on containerization stuff that led to these situations where you'd be in Spec meetings right people are making some technical proposals and everyone else is like I can't tell if this is a good idea or not this might be a good idea or this might be the beginning of some sneaky play to like insert some nasty thing so that this company can pull can can pull some crap on community later and why are they doing that they're evil no but because they need to make money off of all this money that they're spending so when we started otel we felt it needed to be very very very clear how people were going to make money yeah and how this project was going to provide value for end users for vendors for cloud providers for all the people are going to be involved that need to be Crystal Clear um cuz otherwise we were going to end up in that situation but because I we learned our lessons we did make it Crystal Clear yeah right all the code lives in open Telemetry do you want to donate something to open Telemetry that's fabulous you need to move it into otel you need to change the copyright and everything to be the otel authors you need to completely relinquish any individual ownership of this it's now collectively owned by otel and is now part of the cncf is no longer part of your startup yeah yeah um and also like uh what's the boundary of the project Telemetry we're going to standardize the data being emitted by all of these systems but storing and analyzing that data that's never going to be part of votel yeah because you wouldn't want to standardize that part you want to that's like Green Field that's the part where we're trying to figure out the Futures like what can we do with this information and we want everyone to compete on that and that's also where everyone's going to make money yeah exactly and making that very very clear yeah really did a lot in the early days as far as bringing in the second and third wave of contributing companies right like you have this first wave of people who are like we're just making this major bet because it's super clear this would be good for us and then this second wave of companies that are like well okay if those guys are all in that kind of changes the calculus for me I guess we should get involved um and then once those groups get involved there's like a third group it's like well now that all of those people are involved I guess that changes it for us and the fact that it was super clear what you would get out of this if you put something into it yeah uh made that that happened uh faster so you mentioned um you know open Telemetry is the merging of like open open sensus tracing and open senses thank you I was like open traces okay day three is okay sorry um and you know which was focused on tracing and metrics respectively and you know obviously you guys did the um project update yesterday we heard a lot of um new cool stuff coming on the pipeline did you was this part of your vision like at the beginning of the foundation of open Telemetry or is this kind of you mean like where the project is today yeah yeah I mean we've really like our original mandate were we're at kind of this interesting spot in open Telemetry where we had this original mandate which was to unify tracing metrics and logs those are the primary signals that we have available um but the problem in the past was these were siloed systems and what you're trying to do is get a completely connected coherent view of the system y um and those were the three major signals that we wanted to tackle first so that was like our original pitch we're going to merge these three signals into one graph of data that can then be walked by a computer system and we can dump all of this analysis off onto the computers instead of doing it in our brains when it comes to finding correlations across all of this information uh and that was a great pitch it's been part of actually transforming this whole industry right as the data completely changes the products of course have to completely change and if you're going from a bunch of siloed systems to like a unified platform you can see how that's just going to create a complete SE change within the industry because well if that's the future then what's going to happen well all these companies are going to consolidate right one company that was doing logging or whatever is going to acquire some companies that did the other signals and then they're going to try to merge all of that into a platform or if they're a new company they're right out of the gate they're going to be like are we a platform or how do we fit into this new new platform world where all the data is going into one spot so that's like a complete industry shift that probably would have happened anyways without otel but otel is like this massive accelerant so you incentivize basically the the the unification of the signals not not just supporting the three signals but actually right intertwining them so that you can really get that the most out of observability really right yeah and I I call you know the the old version of it I call the the three brows tabs of observability because calling it three pillars gives it too much credit like there was some intentional structure there there was that was like never an intentional design it's like a terrible design what should we do we should Silo all of the data and use our brains to try to figure out what how what connects to what like that's obviously dumb so obviously we didn't design that it just kind of acree over time yeah um and going from that to what I call the braid of data so it is all this data but it is being braided and connected in various ways yeah um and you can think of the value that comes out of a braid right you can think of like the individual strands in a rope and then you can think about a rope and how much more you can do with a rope than just uh pieces of straw yeah it's just so much stronger right exactly yeah yeah and um uh can you give folks uh just a a high level overview of what some of the updates uh that came out of the project update yesterday yeah yeah so i' would say the super high level and we didn't really cover this in the update but um exclusive exclusive but but getting back to this industry C change right being at this moment where we're finally stabilizing tracing logs and metrics right we had the kind of last piece of that um there's a little bit of work that needs to be done related to resources so if people don't know uh resources are in a way like the fourth signal right like when we there's the data there's the in the system and there's like the structure of the system and so the traces are kind of like the energy right that's the transactions that's the stuff moving through the system but then there's the actual like stuff that it's moving through and those are the resources the kubernetes Clusters the the virtual machines the um application binaries all all of that stuff um and when we started the project uh there was this idea that came from open census that these resources were immutable compared to the lifespan of a service when a service started it was associated with some resources and those resources did not change and this kind of made sense yeah uh uh because generally speaking the resources you care about with server side Computing they don't change now they do sometimes but this is just like Edge casy enough that at Google they kind of were just like B just going to ignore that um like you can freeze a virtual machine move it into a different Data Center and then like turn it back on right and now all of its resources changed it's running it's the same computer program and now it's running somewhere completely different but in practice we don't do that right so maybe you could just ignore this requirement and this was very convenient because have this obnoxious thing with resources where you want to figure all of them out before you start sending data you don't want to start sending partially indexed data which is what would happen if you started emitting Telemetry before you figured out your kubernetes pod ID or whatever but figuring out some of these resources can take time yeah you have to go query some API and like wait for the answer and you don't want to delay the booting up of your app necessarily CU what if one of these calls is failing right do you move forwards or is this gating turns out there's a lot of pernicious problems around actually Gathering all of these resources and associating them um and the biggest problem is dev's because it's annoying at startup to be like well whatever we we'll we'll do the dumb thing of we'll just tack these resources on as they come back from their various apis and then you end up with this situation where you have some partial data at the beginning how do you prevent devs from making this mistake well what if you made the resource API immutable where you could only set it once then they would try to do this right they would try to come back with some late binding resource and then the API would be like oh I can't oh this is annoying I guess I'm forced to do it the right way so it's not that resources are immutable by some intrinsic nature of what resources are resources are mutable because that was a convenient way to force the Google devs to not make this like stupid mistake oh and to to force them to do the extra effort figuring out how to resolve all of these resources uh at the beginning so to protect protect them from themselves to a certain extent this this is my claim yeah uh so um but then you get to clients right and all of this was fine this was just like fine all swans are black and then you get to client development which um we've been a little slow to get to in par because so almost all the engineers came from kind of a server side background it was just a little bit late before the client side people started showing up but when they did client side um Computing is very different right uh when the application starts and stops is actually like pretty arbitrary like it's pretty arbitrary when you hit quit on your web browser or whatever it doesn't really signify anything when program boots or ends but it goes through all these different states right like it might be foregrounded or backgrounded uh it might be on Wi-Fi or a cellular network right uh it might get put to sleep and then wake up in Hong Kong you know um and so all of these resources uh are changing over the lifespan of this application oops oops indeed damn it so that's kind of where we're at with the project where we finished our original mandate but there's like a couple of like squiggles around resources that are kind of like the last thing where it's like all right we have to do this really annoying work this is going to be really annoying for me personally because this is the exact kind of thing where it's like we need to fix this to make it work for the clients but we already told people it was immutable so can we like can we like take that back and say like some of these are immutable and some of these are not like would that screw anybody up or not like this is going to be very tricky and then we're going to have to come up with a solution and there's going to be a bunch of community people be like but we said it was immutable and then we're like but clients need to be like but I don't care cuz I'm a go developer you know like yeah so this this is this is what I do okay is try to try to to bring everyone together so we can move forward but if we can get past that yeah now we are in the Green Field of the future where we are looking at all of the data that we haven't captured yet and we want to figure out how to start bringing that data in how do we bring profiling in how do we bring um source code commits versioning all of that in yeah config files that's the future that's exciting yeah so we're you know at the like the five year five six year mark of uh the foundation what do you think is it's going to look like in another five years uh in another 5 years um pretty much the same but with more stuff tacked onto it sorry I'm going to be very boring with that answer because I think we move slow and I don't believe in 2.0's no I is just 1.0 forever uh and uh um and so I just think it'll be but there will be like a SE change that happens once you get enough of this data actually connected there's certain it unlocks certain things so I don't think you're going to see open Telemetry radically Chang in the way it looks but 5 years out from now is enough time for the analysis tools to start to catch up with where the data is so where I think you're going to see some amazing things is some amazing new analysis tools that are actually looking at all kinds of data that we can't look at today and are condensing that into to like like a helpful co-pilot at for uh an operator uh trying to figure out their system that's exciting right on and before you know we log off and get back into the real world I have to learn about your pants what is the story with these cat pants because they're fabulous they have cats on them uh you know these cats do I know them are they your cats they are not my cats I wish I wish I had personally uh fabricated these pants but I did not the internet provided these pants but I did know that these pants existed before I looked at them I was like I want pants that have kittens and space on them and I knew the internet would give this to me I had complete face and it took about 30 seconds wow internet came through it came it came through yeah you know they can make me get out of bed they can't make me get out of my pajamas that's kind of my attitude I love it all right yeah what else would you like to share before we head back into the actual event that we're all here for uh you know I would just encourage uh anyone watching um to if you're an end user please get involved in the developer experience Sig and give us feedback on your experience as an end user um in particular around uh installing the SDK and using the instrumentation apis because we want to we want to clean all that up now that things are stabilizing so yeah please get involved if that's what uh what you're interested in and speaking of end users Adriana and myself work in the end user Sig we do uh monthly events like open open Telemetry in practice um we do end user interviews to learn more about your adoption implementation process you know and get some feedback that we can share back with the sigs and we'll have a QR code at the end that you can scan to get in touch with us on the otel Sig user channel on CNC of slack and don't forget the survey that's open until the end of the week oh the doc survey the doc survey it ends Friday yes if you've used open tree. dos and any way shape or form and you have something to say about it whether good or bad or just comments or questions definitely please take the survey we will link that in the show notes I believe we can do that that is the thing we can do yeah and we should also if you go to the otel socials on I think every social platform we we're on Blu Sky X LinkedIn and Mastadon um there should some tweets also with links to the survey yes and we are available um on C slack at anytime and yeah reach out to us we'd love to hear from you and I think we're good [Music] [Applause] oh +well he hasn't done the countdown so YouTube oh is this on the channel y should I just shut up right so I launch the counter of 10 minutes that's too much probably drop it no we still have this oh it's otel official Hotel Dash official Happ oh he's checking the otel channel to make sure that start it's queuing up on their live okay cool perfect are you ready M yeah and I will in hello live from Salt Lake City I'm reys and I'm here with with some special guests uh we are live at cubec con North America 2024 and we're here to talk with a couple of really cool people in the open slump Community um just about how they got involved and kind of learned a little bit more about them and let's go all right good morning good morning good morning I'm really glad to be here so for those of you who don't know I am ha a weekly and I have thoughts lots of thoughts they never stop thinking and they never stopped thinking and I am super glad to be here and talk with you all and hopefully make you laugh so hard to cry a little bit thank you and I'm Adriana Vela and I work with Rees on the otel and user Sig so we're excited to be uh talking to our awesome Hotel guests today so Hazel how did you get involved in open cemetry like what is your story how did you get involved we could start at the beginning how did you get involved in observability observability so I think I got involved with observability in a very similar way that a lot of people do which is to say I didn't and what happened was I got really really frustrated at things happening and I needed to figure out how to actually ask better questions about my systems and figure out what was going on and dig into them and the tools that I had weren't very good and so I duk around looking for better tools better ways to do things and more importantly better ways to teach other people what we're doing and how it works because one of my superpowers is being able to take the entire system and hold it into my head and be able to sort of intuitively think about it like when I was in an operating systems class in college the point of the class was to kind of almost force you to learn how to to do bunker but I never never did and I still don't know how to need to do bunker because I just looked at the code and mentally traced the colel in my head and debugged panic gloss in the schuer by just like staring at it but I can't teach that to people and I can't show them how to think about that so I've always set then want okay I can I can figure this out with just you know print F debuging or whatever I want to and Vibes but how do I actually how do I take the Vibes and turn into a tool and an explanation and a technique and a way to teach people that you have this you already know the answers you know how to find things you're already like 80 or 90% in the way there it's not this magical new tool kit it's this asking questions that are meaningful to you getting those answers that are useful and then doing something useful with that like learning and then after effectively on that and you don't do that you already know how to do that and just taking that and looking for really really nice things ended up getting me into that observability space where I met a lot of people that were also trying to do that and how did you get into open Telemetry itself so open Telemetry um so for me it was actually I looked at the shape of all the different vendor specific things or the different instrumentation of like locking or metric or we didn't really have traces when I first spped and it was actually really frustrating because I really really hate doing things that are undifferentiated in a different way I would much much rather do a bit work to figure out like how how can I SWA in down how can I think about this concept of optionality and putting everything together because one of the best things about linnux is it turns out you don't need to write a kernel in order to get something working you can use one and one of the best things about kubernetes is sure you write you know your own custom flavor on top but it's the thin veneer over this core and so for observability I started looking for that core the essence of it that you could build that thin ven or that layer on top and I found that in open Telemetry that's amazing so how long have you been using the project or um yeah contributing or using the project so I've been using the project actually ionically usually not directly because I'm low enough on that platform size what I'm almost always looking to do is enable people and so I have looked at the open tary collector a lot of being able to say how can I Empower people how can I enable them how can I take them from these different places of maybe they're using one vendor here and one vendor on another team and one vendor on another team or a third party integration or the open taryle code and so as far as being able to enable people and think about that I started I want to say really really digging deep in building those platforms for other people around in 209 okay wow so basically from the beginning really yeah yeah that's great aesome um which is so weird because like 2019 was not that long ago it's like 2 weeks ago I know right it does feel like it absolutely yeah I I was going to say um you're also like um you know heavily involved in in the platform engineering space and I feel like observability is one of those things like we often talk about it as like a separate thing um but really like you can't have effective platform engineering without observability you can't have effective SRE without observability um and I know you have many thoughts yeah so it's actually something that urts me a little bit a lot of the time because I get the I get why this happens because you have a massively wide platform and you have a massively wide tool chain and people keep adding more and more context into something and the context goes deep and it goes wide and it goes up when it goes in all the different places and you can't possibly hold on your head unless you know you're weird like me and you can hold way too many things in your head and but you do end up in this situation where yeah open tary you can dive so deep into it it can become your whole thing it can become like the only thing you really think about and as when you see and a lot of companies with the platform teams they start off with aici team that's really just rebranded Ops and then they take the rebranded Ops Team they split into you know two three teams and then one of those teams ends up being in charge of you know the kubernetes part and that you know gets labeled the platform team for some reason and then the platform team gets turned into two three different teams and then you keep going on and you know someone gets stuck with the the observability thing and it ends up being really complex because it there's a lot of moving parts so I see how things get split out and by at and sort of frally fragmented you know much in the same way as the CNC of landscape is so massive everything has it own the kind of corner nobody really talks to each other but I wish people would do that because they often end up reimplementing the same thing over and over so like in the profiling side of things we have this open profiling aspect and they're looking at a lot of the open lary stuff and they're Reinventing a lot of the same discussions that they need to have a lot of saying do you need standards do you need name Advent do you need plugins how does the collector of and I'm like we solve that 5 years ago like look at the preexisting stuff tweak it a bit and then you think about it or you have the like event streaming architecture people where you have like databas streaming you have lak House people and you have business and lits and business intelligence and all these different data science types of things in data science related stuff and it turns out that if you are trying to take a metric but load of data and derive useful actionable insights from it and then share that with people who doing data science that's kind of what we decided to call it you know 20 years ago and then we forgot about that and we reinvented open Telemetry instead of making a data Pipeline and then we reinvented profil instead of making a data Pipeline and now we're kind of looking at all these things and then we're going oh oh we should do this thing and then the you know the powerbi or the business analytics people in the corner are just crying a little bit because they've been doing this since the ' 80s and then way over here in the financial side uh did you know this is one of my favorite things actually so kdb is a financial analytic statem database and it is a colon database and so if you look at high frequency trading or you know data analytics people that are specialize in the financial World they actually predate the usage of all of these like oh no queries that rely on indices you can just grab the DAT s and they've been doing that for 40 years CU they had to and nobody else ever really thought to look it up and so it's it's it's fun to see all the communities reinvent things over and over bring their own favor and context into it and then hopefully we can with platform engineering start stirring all the people together talking to them and getting them to actually look outside the little window and go oh we solve the same problem well that's cool how can I learn from how you approached it and how can you learn from how I approached it and can we build kind of a more common thing that's like really awesome given that there's so many you know interesting points that you just brought up and like the way your brain works and how you can look at something and kind of intuitively understand what's happening what pieces of of uh the projects open projects are intriguing you so one of the things that's really intriguing to me about the open telary project is rather it's intriguing to me because I want more people to care about it and that is actually you have this dichotomy of open telary as the end user kind of Part F where you have this SDK and this API and there's how you use it and and you have this mental model and then you have the operational side of how you collect the data how you store it how you enrich it how you sample and how you do all these things and they're all deeply disconnected and they really don't need to be but the reason they're often deeply disconnected because they're all non intentionally or maliciously horrifically lying to each other so the mental model for example that you have with traces is oh there's like this little box and it's this start of it and the end of it and in the Box I put all my information stuff and I draw some information on the outside of it and then in the information stuff I had another box then I have another Bo and I have another box you I have so many boxes and when you actually use the SDC when you actually send things that's a complete lie you're not doing a box you are Reinventing the concept of distribut transactions really badly and without transactional santis because you're just firing stuff off like a stream of events but you don't have like this R ahead lock or any of the other stuff that the database people invented in the 60s and so there's that kind of Li of it and then when you get into the operation side increasingly it turns out sending everything is a expensive you know B really really timec consuming C actually a waste of effort and time and resources and you shouldn't do that so you need to take that enrich it corly other data look at better things figure out associations sample it and sort of figure out and finesse how you understand the data in this very operational context which is we often bundle that into the open telary collector but we never ever talk about the open open telary collector as like a required thing or even you know a thing and then so it hampers a lot of the potential of the SDK make because for example you can't update a span because a span is treated as a append only immutable log except it's a box but no it's a dependently immutable log it's a stream of events and but at the same time it turns out if you break this at any point because not are unreliable you completely M host the entire concept of what you're doing and EMB Brees all your ingestion pipelines and EMB Brees DUI of everything building everything and it's super annoying so I would love for people to think about what is it like if we made the concept of a collector or this concept of enriching rewriting the tree flattening the tree doing all these weird sorts of correlation Concepts or flattening or changing the shap of stuff to make it more malleable for you what if that was a more integral part of how we designed the SDK and if we design the Sate with more use cases in mind how do we do so in a way that gives people a simplement model but doesn't lie to them about what they can expect out in the platform so like there are client STIs Decay for example that don't send the data at all ever until they have received on the client the ending span which means that if your span is 10 seconds long and on second 9 and A2 the client process gets shut down and this not have time to send that span you lose all of that data but if you send it all to The Collector immediately you have so much traffic you can't really afford to do that but if you were able to send like a snapshot thing of a right ahead log of this isn't done yet but I'm going to I'm going to window it like databases do in 70s then okay you can work with it you can work with it and then you can patch it up and some vendors have actually started to work cleverly around hotel and work cleverly around the specification in the middle part to allow that capability in places where they need it like mobile clients and then patch it up and make it you know OTL compliant by the time you send to your back end so if we open up the Pandora b a little bit of okay maybe this is kind of necessary maybe we need to think about this in a weird way of we have to have all the things talk to each other and we need to make it more possible to do these things that make the mental model a bit more coherent when can we do that that's a lot he got thoughts um I wanted to just switch gears a little bit because um before we started uh the stream we're talking about you know you you have many thoughts on many things um and one of them was observability 3.0 now we've heard observability 2.0 um has kind of come into um our our um vernacular lately and you were talking now about observability 3.0 so can you tell us what you uh what you mean by that so what I mean by that is a really really fun thing where it gets into the heart of what I like about observability which is essentially the same thing that everybody ignores and I would like them to not ignore that and so if I brand it a little bit with like a cute little sticker maybe people will care so I'm servility 1.0 and the 2.0 is sort of a reframing of the natural progression that has happened and observability and the vendors and the capabilities and the needs of the platform and the what we need to do in order to ask the right questions and before it was sort of this is an observability and this is observability and then we're like well no it is all observability it's all about asking questions it's more what's the Fidelity of the questions what type of questions can you ask how much information is there how rich is it what are the properties of asking those types of questions and then so if we think of observability 2.0 as sort of being the ultimate inside the tech context of a company or a program or a platform can we ask essentially in question that's kind of the ultimate gole of observability 2.0 to me so for me observability 3.0 is defined by the idea that you need to take non-tech people and non-te problems and the larger context of the business and embed it into yourability to reason about your system and need to take the system and embed that into the rest of the company so the company can reason about that so observability 3.0 has a star difference in that it's not about tooling it's not about capabilities we it's not about all the sort of the technical side it's very much now we need to take the tech and we need to take the people and the processes and this you know massive budget that the you know it industry gets and stop spending it without accountability and stop spending it as this massive black hole where money goes in magic stuff comes out and nobody can explain why so can your business analysts can your marketing people can your sales people can your products people can your everybody else your customer success team can they all sit there and understand how to better serve the customer and how to better interoperate with the technology people without the technology people having screen feeding to them can you give them the capability to do a better job and can you take what they know and take all this cool stuff that they do and put it into context that lets platform engineers and product engineers and back end in and all these people do a really really good thing of being able to present technological options and be able to actually interoperate with what the company needs has a potential not just what the company has asked for you know in a guessing manner to me avability 3.0 they kind of sum it up is the business context comes in Tech the tech context goes into the business and they become one harmonious concept that's amazing and I'm sure this is going to spark a lot of conversations and I'm really excited to see you know other people's thoughts and kind of hopefully you know see the your vision um I also wanted to chat with you you know open Tre um is an open source project obviously we at cubec con um and you work you are at the nly foundation and I would love to you know have the audience learn more about kind of the story behind the foundation because um yeah she shared it earlier and I would love for you to be share that so the Le Foundation was started by Chris NOA and it was one of those last things that she did before she passed away and the nly foundation actually kind of started as an idea that germinated from hacker the Macedon instance that we all built together made into this massive thing and then did a huge migration it was live We Live streamed it and everybody learned from it it was awesome and the reason in haak that we chose the technology that we did is because you had a bunch of brains you had a bunch of people that are kind the world supp to doing a lot of things but we didn't want to rely on that because we wanted to experiment with the idea of what does it mean for a community to deeply own their own Community their own concept their own architecture their everything and this was right around the time when the community especially the Tech Community was first you know gr uping with the idea of we built this little thing and now it's kind of being taken away and we don't have control over our own you know engagement platform our own sort of communication WEA how do we never lose it again and then it turns out that as the massive like migration initially happened we almost closed registrations on haad not due to operational concerns but due to a massive legal liability and the course of how Federation works on Mason it ends up being a very very abusable Vector for putting illegal content somewhere because that content has to get syndicated onto your server and so if you run a Masson instance you are liable for anything that touches your computer but you don't really necessarily control what that is and so that is a very difficult concept and so Nova kind of ran into this and with her legal background and her legal brain she was like oh absolutely no no no no no no no and then so she ended up immediately finding lawyers sitting down and building like an LLC and we wrapped uh haad mount on L and very quietly spad this kind of information to a lot of other larger instances of you you need to now care about it you need immediately you have to immediately start caring about this because you are at risk of a huge vetor there's a huge surge of potential weight uh heat coming but it made us realize as we sat down and we kind of solved this problem when you go from a fun toy project and you take this toy project and it becomes a community there's this invisible Cliff of I have like my vision I have my dream my people and now there's everything like I I can't half ass my like licensing anymore I can't have ass my you know contributor like CLA agreement how do I get people from large SE companies or from regulated Industries to be able to contribute it turns out you can't just make their repository open shorts you have to actually make it so that their paperwork on there and is okay with it and then how about International people how about people with disabilities how about people with you know different needs how about people with different that happens okay you know we haven't even begun to solve the how you for open source how do you find open source no nobody knows the answer nobody's even gone close people are trying like Eva black as she says doing the Fant ftic job trying to create this sort of top down way and path and Avenue for the ability for companies to pay um to pay open source and for governments to pay and for all these things to happen it's going to take time but as a community we need to bottom up sort of figure out how do we pass this cliff and it's a massive cliff and nobody really knows it's coming and you don't have time to prepare for it because you don't get to choose when you become adopted and beloved by Community nobody sits down and says where Community now you just look around one day and you go these are all my friends and I love them and this is great and oh no and then there but it turns out that's only step two because it turns out that you can't really call the cncf community anymore it's an ecosystem and that's kind that third massive Cliff that very very few people talk about where you have this one project and this community and that's cool and the community maybe grows other things when it reaches this s of becoming a generation and incubation Hub of innovation of experimentation and it starts generating a fractal of different communities that all come together in this ecosystem and of this massive idea Exchange of community knowledge Shar and Community like growth when becomes this messy in articulate sort of ill defined but beautifully growing organic thing that takes on its own life that's an ecosystem and we don't even really know how to build those we definitely don't know how to fund them we definitely don't even know the differences between all the legal and the compliance how how do you facilitate that how do you how do you get there and so we looked at this problem Nova and I and a bunch of other people and we want someone need to start thinking about this it's going to take a while it's going to take several years maybe even decade or two to really deeply help the world understand how to to pass those Cliffs and turn them into this you know growing ramp of taking ideas and sharing with the world in a way that generates further ideas in a way that becomes this ever growing thing and the nly foundation which was originally sort of kind of starting as like a legal cover and legal ability to do this always was intended to be a vehicle that helps you understand that problem and helps figure out how do we make this beautiful growth and knowledge sharing possible that's so great um I think are we coming up on time so we are coming up on time um we are going to have our next guest on thank you so much Hazel it was lovely to talk with her and we will share um socials like how you can get in touch with her at the end and as well as like how you can get involved in the future humans of otel segment as well and I'm really excited if um to see you know what kind of interactions come up with uh from these conversations that we just had and yeah whatever questions comments thoughts you have definitely please let us know we would love to hear them and help connect you with hazel so we can learn even more and move forward to kind of her vision and also secretly jealous of her mental superpowers um I'm going to have to see if I can pick up any tips but um now we have our next guest on and I'm so excited hopefully you can see his cat pants because they're amazing um Ted hello hey hello how's it going going great how you all doing today not bad it's CU con thank awesome times it's like a big friend reunion mhm day two of the main cubec events which you know everyone is on fumes at this point y y MH yeah we had like we had a busy start cuz we had observability day which rejects re start with Rejects and then yep observability today and cucon day one yesterday I know it just feels like we've been just been here forever so we're so excited to have you um can you tell us a little bit about you know your role in open Telemetry and you know I'm sure a lot of our viewers who are more familiar with the project are already aware of who you are but would love an introduction yeah sure uh my name's Ted Young I'm one of the the co-founders of the project um and coming from the the open tracing side of the family uh people don't know open Telemetry is actually a merging of two prior projects open tracing and open census open sensus was mostly Google people and some Microsoft people and open tracing was everybody else and then we we merged to form open Telemetry and I continue to work on the project as a member of the governance committee uh mostly just focusing on the various um spec sigs implementation sigs uh that need um uh that that need a an extra helping hand when it comes to uh finding consensus figuring out what design is actually going to work there's some areas in otel where it's technically challenging enough that ironically it's easy to come to a a a design or at any rate is easy to determine which design is the correct one right because it's so technically challenging that the the requirements are very rigorous but then there's like other parts of open Telemetry where it's like a little more squishy where there's like one way that would be a really really good way and then there's some other ways that would like sort of work right you can't say they definitely would not work yeah right but they're not like great but when you have that situation it's so so so much harder to find consensus right because people will lock in to a particular idea yeah and then if it's possible to find some way to make it work cuz they're Engineers they will continue to promote but but there's this way it could work and uh trying to like find consensus there yeah of being like yes okay that would work but but this would work better and and getting everyone to be like well even though you prefer that one would you agree that um you're not going to convince everyone at this point to go that way like yes and like and would you be able to get everything done with this other one like yes and like okay well then would you be willing to come on board with this and help us move this forward because we all actually like need it to get done and so that's like a bit of just more like Community organizing that requires maybe kind of like an engineering design background yeah and I feel like that's where I provide the most value these days to the project that's so awesome cool so and and there's I I I feel like there's uh also you you have to like have some good diplomatic skills as well yeah uh build building trust in the community is important just absolutely um having people know that you don't have an agenda yeah yeah right and that you really are just there to like listen to everyone and help everyone really clarify the requirements um and help everyone really determine what what the best um what the best design solution would be for those requirements yeah yeah yeah and you know on on a similar vein it always brings me back too to like the whole thing with Hotel being such a like a lovely community and that it really takes the whole thing of being vendor neutral very seriously I mean we're all at three different places and we all get along um like we're all competitors but like not really because we're all moving towards the same goal and I was wondering if you could comment on like like what has to go on to like really continue to Foster that sense of of community and and not in US versus them yeah the trust yeah that's that is great so I really think there's there's two parts the simple part is that like humans generally speaking are good people um and also even though we might all work at different companies now the reality is for the most part there's just like many Tech domains there's like an observability scene right there's like the engine and product people whatever who's just like that's their bag and they're good at it and they just tend to kind of like Circle through the different orgs and companies that happen to be paying people to work on that stuff in that time so I think that's the thing that makes it easier right like it's like it's not like um we're from different planets that are in some Intergalactic war and we have like nothing in common or something we've got all this stuff to overcome it's just like like we all know each other uh um and when you're long around long enough it's like everyone cycled two or three jobs so you stop really looking at people as like representatives of a particular company it's so true yeah I think one of the things too just from like my you know very short experien is even when they are moving companies people want to stay involved in the project like they're actively seeking out roles where they can continue contributing to the project in their role so I think that speaks volumes honestly the the fact that um uh you can look at our maintainers and community members and and there's like quite a long lifespan at this point I I I actually haven't run the math but I would not be shocked if like the average age of a maintainer in otel was measured in years at this point like on average it's like two or three years I wouldn't be shocked if it's that high like people really do stick around it's so true so true and the second part of that is is structuring the project because companies there are very few like companies but there are a lot of bad incentives so when we started the project coming from other open- source projects where I had borne witness and had to deal with all the Fallout of these bad incentives or um muddy incentives causing various companies to like test the waters and like get into stuff with each other um so when we started this project we had a goal based off of our experiences of like how do we actually structure this project in a way that makes it obvious that these bad incentives are not present yeah where people can look at that and be like oh yeah that that won't work so we're not going to bother to try to we're we're not going to bother to see if we can just buy the whole TC right and take over the project by like what if just everyone who works on who's in a leadership position works for Splunk or something right like that you can't do that because once you get more than two people on the TC uh who work at one company that and you want a third person well one of those first two people has to drop out or remove companies or something yeah yeah and then when it comes to the actual shape of the project prior projects even these Big Industry things like kubernetes and whatever they tend to have this like veneer of Utopia like oh we're all just getting together for this alteristic reason of like making container scheduling work and it is like as an individual it's somewhat alteristic and that like I'm really interested in it I would love to like like push this domain forwards I want a real distributed operating system I don't even want kubernetes I want the next thing yeah um but sorry kubernetes uh but I also want the next I also want the next open Telemetry you know I want to keep pushing this stuff exactly exactly um so I'm motivated but like uh these are like billion dollar projects almost right the amount of engineering effort that goes into kubernetes over this like if you calculate the salaries that I'm sure that's at this point that's probably a billion dollars maybe maybe that's maybe that's crazy maybe it's like like a 100 million yeah it's I'm probably like way out on that one actually I can't math I mean chunk of change it's a chunk of change a lot of money when you think about developer salaries and how many people work on these projects otel on average like monthly average number of contributors is like a couple hundred Engineers I think wow on average is a couple hundred Engineers pushing some kind of PR or something at any given moment on otel it's a lot of people there A lot of money and the only reason people are getting paid to do that is because there is some incentive yeah yeah which is so nice to have that because otherwise like we're so busy already like so the fact that you have like all these like vendors who are backing otel I think speaks volumes for it cuz I I don't think we'd be where we are without that right but a problem kubernetes had when it first started and other projects that I worked on is it was there was all this interest and all this willingness to pay people to work on bootstrapping kubernetes but why that question was actually nebulous for maybe some vendors or Cloud providers it was clear how they were going to make money and thus why they should be paying people to work on this but for a lot of companies a lot of groups getting started especially the startups it was not clear how they were going to make money off of kubernetes so it was like step one spend all this money to bootstrap this thing step two figure out how to exploit it yeah and that just leaving that door open right just the fact that it wasn't really clearly defined how kubernetes was should be organized how everyone's expected to make money what part of this is going to be some shared open stack versus what part are we going to like sell things or whatever that was all just like a big question mark in the early days and should all the code live in a kubernetes GitHub or or should various startups and companies be allowed to have complete ownership of some component that lives within kubernetes and thus it lives within that company's repo right right all things like this led to just like a lot of problematic stuff right like what if one company kind of owns a component of open Telemetry they're going to build a lot of like startup brand around that component naturally right that would be the natural play would be to talk that component up and how it's kind of yours yeah right and then otel a project was like that was a cool component but now we're going in a different direction so we should deprecate that one and get a new thing yeah well that would be like the kiss of death to startup yeah for sure so you can start to see how like just leaving the door open starts to like create a situation where things like people's incentives can start to get at loggerheads yeah yeah and working on containerization stuff that led to these situations where you'd be in Spec meetings right people are making some technical proposals and everyone else is like I can't tell if this is a good idea or not this might be a good idea or this might be the beginning of some sneaky play to like insert some nasty thing so that this company can pull can can pull some crap on community later and why are they doing that they're evil no but because they need to make money off of all this money that they're spending so when we started otel we felt it needed to be very very very clear how people were going to make money yeah and how this project was going to provide value for end users for vendors for cloud providers for all the people are going to be involved that need to be Crystal Clear um cuz otherwise we were going to end up in that situation but because I we learned our lessons we did make it Crystal Clear yeah right all the code lives in open Telemetry do you want to donate something to open Telemetry that's fabulous you need to move it into otel you need to change the copyright and everything to be the otel authors you need to completely relinquish any individual ownership of this it's now collectively owned by otel and is now part of the cncf is no longer part of your startup yeah yeah um and also like uh what's the boundary of the project Telemetry we're going to standardize the data being emitted by all of these systems but storing and analyzing that data that's never going to be part of votel yeah because you wouldn't want to standardize that part you want to that's like Green Field that's the part where we're trying to figure out the Futures like what can we do with this information and we want everyone to compete on that and that's also where everyone's going to make money yeah exactly and making that very very clear yeah really did a lot in the early days as far as bringing in the second and third wave of contributing companies right like you have this first wave of people who are like we're just making this major bet because it's super clear this would be good for us and then this second wave of companies that are like well okay if those guys are all in that kind of changes the calculus for me I guess we should get involved um and then once those groups get involved there's like a third group it's like well now that all of those people are involved I guess that changes it for us and the fact that it was super clear what you would get out of this if you put something into it yeah uh made that that happened uh faster so you mentioned um you know open Telemetry is the merging of like open open sensus tracing and open senses thank you I was like open traces okay day three is okay sorry um and you know which was focused on tracing and metrics respectively and you know obviously you guys did the um project update yesterday we heard a lot of um new cool stuff coming on the pipeline did you was this part of your vision like at the beginning of the foundation of open Telemetry or is this kind of you mean like where the project is today yeah yeah I mean we've really like our original mandate were we're at kind of this interesting spot in open Telemetry where we had this original mandate which was to unify tracing metrics and logs those are the primary signals that we have available um but the problem in the past was these were siloed systems and what you're trying to do is get a completely connected coherent view of the system y um and those were the three major signals that we wanted to tackle first so that was like our original pitch we're going to merge these three signals into one graph of data that can then be walked by a computer system and we can dump all of this analysis off onto the computers instead of doing it in our brains when it comes to finding correlations across all of this information uh and that was a great pitch it's been part of actually transforming this whole industry right as the data completely changes the products of course have to completely change and if you're going from a bunch of siloed systems to like a unified platform you can see how that's just going to create a complete SE change within the industry because well if that's the future then what's going to happen well all these companies are going to consolidate right one company that was doing logging or whatever is going to acquire some companies that did the other signals and then they're going to try to merge all of that into a platform or if they're a new company they're right out of the gate they're going to be like are we a platform or how do we fit into this new new platform world where all the data is going into one spot so that's like a complete industry shift that probably would have happened anyways without otel but otel is like this massive accelerant so you incentivize basically the the the unification of the signals not not just supporting the three signals but actually right intertwining them so that you can really get that the most out of observability really right yeah and I I call you know the the old version of it I call the the three brows tabs of observability because calling it three pillars gives it too much credit like there was some intentional structure there there was that was like never an intentional design it's like a terrible design what should we do we should Silo all of the data and use our brains to try to figure out what how what connects to what like that's obviously dumb so obviously we didn't design that it just kind of acree over time yeah um and going from that to what I call the braid of data so it is all this data but it is being braided and connected in various ways yeah um and you can think of the value that comes out of a braid right you can think of like the individual strands in a rope and then you can think about a rope and how much more you can do with a rope than just uh pieces of straw yeah it's just so much stronger right exactly yeah yeah and um uh can you give folks uh just a a high level overview of what some of the updates uh that came out of the project update yesterday yeah yeah so i' would say the super high level and we didn't really cover this in the update but um exclusive exclusive but but getting back to this industry C change right being at this moment where we're finally stabilizing tracing logs and metrics right we had the kind of last piece of that um there's a little bit of work that needs to be done related to resources so if people don't know uh resources are in a way like the fourth signal right like when we there's the data there's the in the system and there's like the structure of the system and so the traces are kind of like the energy right that's the transactions that's the stuff moving through the system but then there's the actual like stuff that it's moving through and those are the resources the kubernetes Clusters the the virtual machines the um application binaries all all of that stuff um and when we started the project uh there was this idea that came from open census that these resources were immutable compared to the lifespan of a service when a service started it was associated with some resources and those resources did not change and this kind of made sense yeah uh uh because generally speaking the resources you care about with server side Computing they don't change now they do sometimes but this is just like Edge casy enough that at Google they kind of were just like B just going to ignore that um like you can freeze a virtual machine move it into a different Data Center and then like turn it back on right and now all of its resources changed it's running it's the same computer program and now it's running somewhere completely different but in practice we don't do that right so maybe you could just ignore this requirement and this was very convenient because have this obnoxious thing with resources where you want to figure all of them out before you start sending data you don't want to start sending partially indexed data which is what would happen if you started emitting Telemetry before you figured out your kubernetes pod ID or whatever but figuring out some of these resources can take time yeah you have to go query some API and like wait for the answer and you don't want to delay the booting up of your app necessarily CU what if one of these calls is failing right do you move forwards or is this gating turns out there's a lot of pernicious problems around actually Gathering all of these resources and associating them um and the biggest problem is dev's because it's annoying at startup to be like well whatever we we'll we'll do the dumb thing of we'll just tack these resources on as they come back from their various apis and then you end up with this situation where you have some partial data at the beginning how do you prevent devs from making this mistake well what if you made the resource API immutable where you could only set it once then they would try to do this right they would try to come back with some late binding resource and then the API would be like oh I can't oh this is annoying I guess I'm forced to do it the right way so it's not that resources are immutable by some intrinsic nature of what resources are resources are mutable because that was a convenient way to force the Google devs to not make this like stupid mistake oh and to to force them to do the extra effort figuring out how to resolve all of these resources uh at the beginning so to protect protect them from themselves to a certain extent this this is my claim yeah uh so um but then you get to clients right and all of this was fine this was just like fine all swans are black and then you get to client development which um we've been a little slow to get to in par because so almost all the engineers came from kind of a server side background it was just a little bit late before the client side people started showing up but when they did client side um Computing is very different right uh when the application starts and stops is actually like pretty arbitrary like it's pretty arbitrary when you hit quit on your web browser or whatever it doesn't really signify anything when program boots or ends but it goes through all these different states right like it might be foregrounded or backgrounded uh it might be on Wi-Fi or a cellular network right uh it might get put to sleep and then wake up in Hong Kong you know um and so all of these resources uh are changing over the lifespan of this application oops oops indeed damn it so that's kind of where we're at with the project where we finished our original mandate but there's like a couple of like squiggles around resources that are kind of like the last thing where it's like all right we have to do this really annoying work this is going to be really annoying for me personally because this is the exact kind of thing where it's like we need to fix this to make it work for the clients but we already told people it was immutable so can we like can we like take that back and say like some of these are immutable and some of these are not like would that screw anybody up or not like this is going to be very tricky and then we're going to have to come up with a solution and there's going to be a bunch of community people be like but we said it was immutable and then we're like but clients need to be like but I don't care cuz I'm a go developer you know like yeah so this this is this is what I do okay is try to try to to bring everyone together so we can move forward but if we can get past that yeah now we are in the Green Field of the future where we are looking at all of the data that we haven't captured yet and we want to figure out how to start bringing that data in how do we bring profiling in how do we bring um source code commits versioning all of that in yeah config files that's the future that's exciting yeah so we're you know at the like the five year five six year mark of uh the foundation what do you think is it's going to look like in another five years uh in another 5 years um pretty much the same but with more stuff tacked onto it sorry I'm going to be very boring with that answer because I think we move slow and I don't believe in 2.0's no I is just 1.0 forever uh and uh um and so I just think it'll be but there will be like a SE change that happens once you get enough of this data actually connected there's certain it unlocks certain things so I don't think you're going to see open Telemetry radically Chang in the way it looks but 5 years out from now is enough time for the analysis tools to start to catch up with where the data is so where I think you're going to see some amazing things is some amazing new analysis tools that are actually looking at all kinds of data that we can't look at today and are condensing that into to like like a helpful co-pilot at for uh an operator uh trying to figure out their system that's exciting right on and before you know we log off and get back into the real world I have to learn about your pants what is the story with these cat pants because they're fabulous they have cats on them uh you know these cats do I know them are they your cats they are not my cats I wish I wish I had personally uh fabricated these pants but I did not the internet provided these pants but I did know that these pants existed before I looked at them I was like I want pants that have kittens and space on them and I knew the internet would give this to me I had complete face and it took about 30 seconds wow internet came through it came it came through yeah you know they can make me get out of bed they can't make me get out of my pajamas that's kind of my attitude I love it all right yeah what else would you like to share before we head back into the actual event that we're all here for uh you know I would just encourage uh anyone watching um to if you're an end user please get involved in the developer experience Sig and give us feedback on your experience as an end user um in particular around uh installing the SDK and using the instrumentation apis because we want to we want to clean all that up now that things are stabilizing so yeah please get involved if that's what uh what you're interested in and speaking of end users Adriana and myself work in the end user Sig we do uh monthly events like open open Telemetry in practice um we do end user interviews to learn more about your adoption implementation process you know and get some feedback that we can share back with the sigs and we'll have a QR code at the end that you can scan to get in touch with us on the otel Sig user channel on CNC of slack and don't forget the survey that's open until the end of the week oh the doc survey the doc survey it ends Friday yes if you've used open tree. dos and any way shape or form and you have something to say about it whether good or bad or just comments or questions definitely please take the survey we will link that in the show notes I believe we can do that that is the thing we can do yeah and we should also if you go to the otel socials on I think every social platform we we're on Blu Sky X LinkedIn and Mastadon um there should some tweets also with links to the survey yes and we are available um on C slack at anytime and yeah reach out to us we'd love to hear from you and I think we're good oh diff --git a/video-transcripts/transcripts/2024-12-17T22:14:29Z-humans-of-otel-kubecon-na-2024.md b/video-transcripts/transcripts/2024-12-17T22:14:29Z-humans-of-otel-kubecon-na-2024.md index 6d1e6da..9100d98 100644 --- a/video-transcripts/transcripts/2024-12-17T22:14:29Z-humans-of-otel-kubecon-na-2024.md +++ b/video-transcripts/transcripts/2024-12-17T22:14:29Z-humans-of-otel-kubecon-na-2024.md @@ -10,88 +10,106 @@ URL: https://www.youtube.com/watch?v=TIMgKXCeiyQ ## Summary -In this YouTube video, various professionals from the tech industry discuss their experiences and insights into OpenTelemetry, a framework designed to standardize observability for software systems. Participants include Hazel Weakly, Eromosele Akhigbe, Budha, Miguel Luna, Adriana Villela, David, Endre Sara, Braydon Kains, Christos, and Reese Lee, each sharing how they became involved with OpenTelemetry and its significance in their work. Key points include the importance of OpenTelemetry in enhancing troubleshooting, improving user experiences, and fostering collaboration across different organizations. The speakers emphasize the transformative potential of OpenTelemetry in standardizing data collection and facilitating better understanding of system behaviors, as well as the value of community engagement in open-source projects. Observability is defined through personal anecdotes and technical perspectives, highlighting its role in developing a deeper understanding of systems and ultimately improving software performance. +In this YouTube video, several professionals in the software development and observability space share their experiences and insights regarding OpenTelemetry, an open-source observability framework. Participants include Hazel Weakly, Eromosele Akhigbe, Budha, Miguel Luna, Adriana Villela, David, Endre Sara, Braydon Kains, Christos, and Reese Lee. They discuss how they became involved with OpenTelemetry, emphasizing its importance in standardizing observability practices across various organizations and improving the efficiency of troubleshooting and monitoring software. Topics explored include the significance of observability in understanding system behavior, the role of community collaboration in the OpenTelemetry ecosystem, and the various telemetry signals such as traces, metrics, and logs that aid in effective software monitoring. The conversation highlights the transformative impact of OpenTelemetry on both individual careers and organizational practices, reinforcing its potential as a future standard for observability in software engineering. -# OpenTelemetry Thoughts and Experiences +## Chapters -Hey there! My name is **Hazel Weakly** and I have thoughts—lots of thoughts. They never stop thinking. +Here are 10 key moments from the livestream along with their timestamps: -**Eromosele Akhigbe** here. I'm currently a software engineer at Sematext. +00:00:00 Introductions of speakers and their roles +00:05:30 Discussion on how individuals got involved with OpenTelemetry +00:10:15 Insights on the importance of open standards in technology +00:15:50 Explanation of the different signals in observability (traces, metrics, logs) +00:20:10 The role of OpenTelemetry in improving API management and troubleshooting +00:25:30 Personal experiences with OpenTelemetry and its impact on software engineering +00:30:45 Observability's definition and its significance in understanding system behavior +00:35:00 The collaborative nature of the OpenTelemetry community +00:40:20 Discussion on common standards and semantic conventions in OpenTelemetry +00:45:00 Final thoughts on the future of OpenTelemetry and its role in the industry -Hello everyone, I am **Budha**. I'm a developer advocate at Tyk. Apart from that, I have a very deep relationship with open standards as I am also the board chair for the OpenAPI Initiative and a board member for the GraphQL Foundation. +# OpenTelemetry Community Insights -My name is **Miguel Luna** and I'm a product manager at Elastic, where I'm the product lead for the OpenTelemetry efforts across the company and what we contribute to the community. +## Introductions + +Hello everyone! My name is **Hazel Weakly** and I have thoughts—lots of thoughts. They never stop thinking. + +I’m **Eromosele Akhigbe**, currently a software engineer at Sematext. + +Hi, I’m **Budha**, a developer advocate at Tyk. I have a deep relationship with open standards as I serve as the board chair for the OpenAPI Initiative and a board member for the GraphQL Foundation. + +My name is **Miguel Luna** and I’m a product manager at Elastic, where I lead the OpenTelemetry efforts across our company and contribute to the community. I’m **Adriana Villela**, a Principal Developer Advocate at Dynatrace. -My name is **David**, and I work at Monday.com as a software engineer on the CRM product. +My name is **David** and I work at Monday.com as a software engineer on the CRM product. -I’m **Endre Sara**, the co-founder of a company called Causely, which I started two years ago. +I’m **Endre Sara**, co-founder of Causely, which I started two years ago. -My name is **Braydon Kains**. I'm a software developer at Google in the Google Cloud Org, working for the Cloud Observability service. I mainly work on agents that customers install in their environments to collect telemetry signals and send them to Google Cloud. +My name is **Braydon Kains**. I’m a software developer at Google within the Google Cloud Org, working on the Cloud Observability service. -I'm **Christos**, a software engineer at Elastic. I've been working mainly in observability for the past five years, contributing mostly to the OpenTelemetry ecosystem since last year. +I’m **Christos**, a software engineer at Elastic. I’ve focused on observability for the past five years, contributing mostly to the OpenTelemetry ecosystem. -Hi, my name is **Reese Lee**, and I am a Senior Developer Relations Engineer at New Relic. +Hi, I’m **Reese Lee**, a Senior Developer Relations Engineer at New Relic. ## Getting Involved with OpenTelemetry -I got into OpenTelemetry almost accidentally. Initially, I was looking for answers to questions I had about teaching others how to find answers better and how to improve the learning processes in the teams I worked with. Eventually, I stumbled onto OpenTelemetry. +I got into the OpenTelemetry project almost accidentally. I was looking for answers to questions I had and how to teach others to find answers better. This journey led me to OpenTelemetry. + +In March, I participated in an internship called Outreachy, where I worked on building a logging bridge in Golang using OTel zerolog by the end of my internship. + +My involvement with OpenTelemetry came from the advocacy of our new group product manager who was a big proponent of observability and OpenTelemetry. Prior to that, I had experimented with OpenTracing and OpenCensus but hadn’t delved into OpenTelemetry. Her advocacy piqued my interest. -In March, I started an internship called Outreachy, where I worked on OpenTelemetry, building a logging bridge in Golang. By the end of the internship, I successfully created a logging bridge using OTel zerolog. +OpenTelemetry caught my attention primarily because of its status as an open standard. I believe investing time in open standards makes life easier and offers flexibility. -### Triggers to Get Involved +At Tyk, we started using OpenTelemetry both internally and externally, which significantly improved our troubleshooting efficiency across REST and GraphQL APIs. -There were several reasons OpenTelemetry caught my attention: +Initially, I began as a product manager, focusing on coordination rather than direct contributions. Recently, I’ve been involved in localizing documentation for Spanish speakers, translating the OpenTelemetry documentation into Spanish. -1. **Advocacy** - Our new group product manager was a strong proponent of observability and OpenTelemetry, which drew me in. -2. **Open Standards** - Anything that is an open standard is a no-brainer for me; I believe it simplifies life and adds flexibility. -3. **Internal Use** - At Tyk, we started using OpenTelemetry for internal and external purposes. This allowed us to see results in troubleshooting problems, not just with REST APIs, but also with GraphQL APIs. +My first experience with OpenTelemetry was at my previous role, where my manager encouraged me to attend community meetings. This sparked my first contribution to open source after over 20 years in tech. -Initially, I started as a product manager, focusing more on coordination rather than direct contribution. Recently, I’ve been involved in localizing documentation, specifically translating OpenTelemetry documentation into Spanish. +I started my career in embedded applications and was doing eBPF tracing before it was mainstream. After moving to Dropbox, where we handled all telemetry in-house, I continued to work with trace-based testing at Monday.com. -At my previous role, my manager encouraged me to join the OpenTelemetry community, marking my first experience contributing to open source. It was a gateway into OpenTelemetry, and I started my career in embedded applications, doing eBPF tracing before it was mainstream. +My team utilizes OpenTelemetry, specifically the OpenTelemetry Collector, to support customers in collecting data from their environments. Initially, we would report bugs and wait for fixes, but we aimed to change that approach by actively opening issues and contributing PRs. -### Observability and Its Importance +I was asked to contribute to OpenTelemetry by assisting with the Elastic Common Schema donation and focusing on the Collector and semantic conventions. -I’ve learned that OpenTelemetry presents a tremendous opportunity for the industry to standardize instrumentation and use common semantic conventions, making it easier for everyone to understand what's happening. I got excited about it and began working on it, initially with a few test applications, eventually demonstrating how to do instrumentation. +At New Relic, I began working with OpenTelemetry through support tickets from customers adopting it. I joined the dedicated OpenTelemetry team in November 2021 and integrated into the OpenTelemetry community quickly. -As my current company evolved, I emphasized the importance of proper instrumentation from day one for logs, metrics, and observability testing, which has greatly helped us. +OpenTelemetry has proven invaluable across organizations by providing a way to connect different vendors and tools. It allows for greater collaboration and efficiency and enhances internal communication among teams. -My team uses OpenTelemetry, particularly the OpenTelemetry Collector, to support our customers in collecting data from their environments. While we initially had light involvement in OpenTelemetry issues, we recognized the importance of opening issues with pull requests, which has encouraged our entire team to be more engaged. +Earlier this year, I organized a mini-conference called LEAP, which focused on API observability, gathering insights from various community members, including decision-makers. -I was asked to contribute to OpenTelemetry by assisting with the Elastic Common Schema donation, focusing on the specification and semantic conventions, and now I’m heavily involved in the Collector. +We at Elastic decided to donate the Elastic Common Schema to OpenTelemetry, aligning it with the goal of standardizing observability and improving efficiency in telemetry data collection. -Joining the dedicated OpenTelemetry team at New Relic in November 2021 allowed me to integrate into the community swiftly. The work my previous manager initiated to build the End User Working Group has now evolved into the End User SIG, significantly enhancing our ability to tie together different tools and vendors. +## Observability Insights -### Personal Growth Through OpenTelemetry +Observability is essential for understanding the user experience in increasingly complex software environments. OpenTelemetry provides the tools to answer challenging questions about user interactions with our products. -OpenTelemetry has become a transformative part of my work, facilitating cross-communication between engineering and non-engineering teams. It has improved our internal efficiency, allowing our SRE and DevOps teams to collaborate more effectively with the platform stack. +The primary purpose of observability is to understand and reason about system behavior. Merely collecting data is insufficient; the insights derived must drive actions and improvements in reliability and performance. -Earlier this year, I had the privilege of organizing a mini-conference called LEAP, the API Observability Conference, which brought together many community members to discuss various aspects of OpenTelemetry beyond just engineering. +To me, observability means having a clear understanding of your platform's pulse and recognizing what functions well and what does not. It’s about making informed decisions based on this understanding. -The initiation of the Elastic Common Schema donation to OpenTelemetry was a natural fit for standardizing observability and driving efficiency. +Monitoring answers known questions, while observability identifies unknown questions. Observability means gaining insights into your systems, transforming debugging processes into actionable insights. -Observability, to me, is understanding the pulse of the platform we’ve created. It involves knowing what functions and what doesn’t, enabling effective decision-making. Monitoring answers questions you know to ask, while observability helps you discover questions you didn’t even realize were important. +Observability engineers are akin to doctors for your systems. When something goes wrong, we pinpoint the issue to resolve it effectively. -### What Does OpenTelemetry Mean? +OpenTelemetry represents the future of observability. It fosters a collaborative community where vendors work together to innovate without competing over signal collection. -To me, OpenTelemetry represents a common language that unites various components of a platform stack. It simplifies the process for SRE and DevOps teams, allowing them to focus on evolving and maturing their systems. +## OpenTelemetry's Community and Future -It feels like home for me, as it has been transformative in my life and career, introducing me to open source and the CNCF community. It’s a lovely community where different vendors collaborate towards shared goals. +To me, OpenTelemetry is a home—a community that emphasizes collaboration among various vendors working towards the same goal of improving observability. -OpenTelemetry is a way to standardize efforts among engineers who seek to ask difficult questions about software. It enables collaboration among multiple vendors, allowing them to focus on delivering actionable insights rather than competing over instrumentation. +OpenTelemetry serves as a means to standardize efforts among engineers striving to tackle complex questions in software development. It enables collaboration, allowing companies to focus on adding value rather than competing over instrumentation. -Overall, OpenTelemetry means understanding the signals that help us differentiate between various aspects of our systems. It brings the observability community together, fostering learning and innovation across the industry. +In my experience, OpenTelemetry is a fantastic place to learn and collaborate with passionate individuals in the observability space. ## Favorite OpenTelemetry Signals -- My favorite OpenTelemetry signal is the one that provides the most useful answers to meaningful questions. -- **Traces** are my favorite signal because they provide a comprehensive view of everything happening in the system and help spot errors easily. -- **Logs** are essential for deep contextual insights, though I acknowledge the need for all signals—metrics, traces, and logs—to fully understand system issues. -- I have grown to appreciate **profiling** as it offers exciting opportunities to understand system behavior under different conditions, helping us to improve performance. +- **Traces**: They provide a comprehensive overview of system interactions, helping to pinpoint errors effectively. +- **Logs**: They offer deep contextual insights and are foundational for understanding system behavior. +- **Metrics**: Essential for identifying problems and understanding performance trends. -In conclusion, OpenTelemetry is a powerful tool for enhancing observability, encouraging collaboration, and driving effective decision-making in software development. +In conclusion, OpenTelemetry is not just a technical standard but a vibrant community that empowers engineers and organizations to enhance their observability practices and drive meaningful outcomes. ## Raw YouTube Transcript diff --git a/video-transcripts/transcripts/2024-12-20T07:03:14Z-otel-me-an-end-user-conversation-with-ariel-valentin.md b/video-transcripts/transcripts/2024-12-20T07:03:14Z-otel-me-an-end-user-conversation-with-ariel-valentin.md index 908d603..a608827 100644 --- a/video-transcripts/transcripts/2024-12-20T07:03:14Z-otel-me-an-end-user-conversation-with-ariel-valentin.md +++ b/video-transcripts/transcripts/2024-12-20T07:03:14Z-otel-me-an-end-user-conversation-with-ariel-valentin.md @@ -10,261 +10,156 @@ URL: https://www.youtube.com/watch?v=vB9_SiTV5CI ## Summary -In this episode of "Oh Tell Me," hosted by Reys, a Senior Developer Relations Engineer at New Relic, the format of the end-user Q&A has been revamped. Reys introduces Ariel Valentin, a Staff Software Engineer at GitHub, who specializes in observability. The discussion revolves around GitHub's adoption of OpenTelemetry, with Ariel sharing insights on his journey into observability, the challenges of transitioning from proprietary APM tools to OpenTelemetry, and the importance of distributed tracing. He highlights the growth in the number of services being instrumented for tracing since the adoption of OpenTelemetry and discusses technical aspects such as the architecture at GitHub and the use of probabilistic sampling. Audience engagement is encouraged throughout the conversation, with a focus on community contributions and feedback mechanisms in the OpenTelemetry ecosystem. Reys also provides information on how to get involved with the end-user Special Interest Group (SIG) and emphasizes the welcoming nature of the community. +In this episode of the "Oh Tell Me" end user Q&A, host Reys, a Senior Developer Relations Engineer at New Relic, introduces Ariel Valentin, a Staff Software Engineer at GitHub focused on observability. The discussion centers on GitHub's adoption of OpenTelemetry, with Ariel sharing insights into the challenges and processes involved in transitioning from proprietary SDKs to OpenTelemetry for distributed tracing. Key topics include the importance of consistent terminology in observability, the architecture landscape at GitHub, the benefits of using OpenTelemetry for tracing, and the collaborative spirit within the community for contributions. Ariel also emphasizes the need for better communication regarding updates in specifications and encourages more involvement from end users in the OpenTelemetry community. The episode concludes with a light-hearted exchange about holiday plans, showcasing the friendly dynamic between the host and guest. -# Oh Tell Me: End User Q&A Episode Transcript - -[Music] - -Hello everyone! Welcome to a brand new episode of *Oh Tell Me* and our End User Q&A. If you have attended one of these sessions in the past, you might notice that this looks a little different and has a different name. That's right! We have done some growing up since the last few times, and we're very excited to debut this new look. - -But don't worry, almost everything else is going to be the same. We have a great interview here for you today to learn from. Before I introduce myself and our guest, I would love to know where everyone is connecting from. I am online from Portland, Oregon today and would love to see where people are from. It looks like someone is here from Brooklyn, New York, so thank you so much for joining us out in New York! - -My name is Reys, and I am a Senior Developer Relations Engineer at New Relic. I also co-lead the Ner SIG, which is our cute little logo that you see up in the right corner. At the end of your SIG, we focus on connecting directly with users and are dedicated to helping the rest of the SIGs get feedback from end users so they can help improve the project. This is one of those events that we host to do that. - -Today, we are going to have Ariel on. Ariel, if you would like to join us and introduce yourself. - -**Ariel:** Hey, how's it going, everybody? I'm Ariel Valentin, a Staff Software Engineer at GitHub working on observability. Thanks so much, Reys, for inviting me here to chat with you today. It’s an absolute honor to be the first in this new format, so a big shoutout to everyone who has worked to get this new platform up and running. - -**Reys:** Oh no, thank you for being here! I'm really excited. To get you and everyone up to speed on the format, it's similar to the ones if you've been on one of these before, but we'll start with some warm-up questions to get Ariel comfortable. Then, we'll hit him with some meaty questions. We'll also get into questions about the OpenTelemetry community, where he'll have an opportunity to share feedback about his experiences contributing to the project. He'll also get a chance to ask us questions in a section we call "Turn the Tables." At the very end, if there are any audience questions—oh, actually scratch that—if you have questions that come up during our conversation, feel free to put them in the chat. If you're watching from YouTube, just put them in the live chat, and I think LinkedIn will also have its own chat, so put your questions there as well. We will get to them as we can throughout the conversation. At the end, if you have any questions, we will also get to those. - -Alright, I think we are good to get started. Ariel, you mentioned a little bit about your role at the company. How did you get started with observability and OpenTelemetry? - -**Ariel:** Oh, those are great questions! I should have mentioned that I'm here in Austin, Texas—Sunny Austin, Texas—so now you know where I am in the world. - -I think a lot of us started originally working with APM tools, which are generally proprietary tools that didn’t have distributed tracing in place at that time. As the community evolved and OpenTracing became a standard, that was my first experience with working with distributed tracing. I tried out different vendors and experiences with OpenTracing. By the time I got to GitHub, I was a champion for distributed tracing because I saw the power that was in there. Around that same time, folks were already moving towards developing and transitioning away from OpenTracing and OpenCensus to OpenTelemetry. I got very involved early on in trying to spread the word, learn more, and work with my observability team to start adopting tracing more and to make OpenTelemetry sort of our North Star for all of our telemetry signals. - -I feel like I keep saying the word "telemetry" over and over again. I'm going to have to find a better way to say that! - -**Reys:** We’re both going to stumble over "OpenTelemetry" or "distributed tracing" at some point; it's alright. This is a safe space! - -So, you mentioned you became a champion for distributed tracing. I think something that’s interesting is that we are so entrenched in the world of observability that at least I tend to presume everyone understands what that is. It’s always surprising to me when someone asks, "What’s distributed tracing?" What do you think is the main challenge that most organizations face regarding observability? Is it simply understanding the value of distributed tracing and these new concepts? - -**Ariel:** Yes, because these are all sort of new concepts to folks, right? People have different levels of understanding of the tools available to them, and there's all this vocabulary that you hear that’s a little hard to parse through. Sometimes it’s like, "Oh, when you say ‘trace,’ do you mean a trace log, or does it mean samples taken from a profiler?" There’s a lot of language that even though we’re converging on a lot of it and have these dictionaries defined and published everywhere, there’s still this hump we have to get over. There’s a challenge we face with trying to get everybody speaking the same language within the same context. - -It’s very similar to domain-driven design; we have these bounded contexts where the same term means a different thing. That flows over into our domain language when it comes to observability and SRE practices. I’m sure many folks have faced those challenges as well, and it’s like, let’s all get on the same page about what we mean. - -**Reys:** Absolutely! What are currently some of the most interesting problems that you are facing in your role? - -**Ariel:** One of those things is the transition. It’s really hard for an organization like us, which has been around for a long time—10 years or so—where the system has grown and evolved. There have been acquisitions that have been brought together, and there are disparate backends where we’re collecting this data. It’s trying to transition from one way of doing things to a new way of doing things. Learning something new is always going to be a big challenge. - -Folks are trying to do their job every day. They’re not trying to learn new SDKs or vocabulary; they’re trying to keep the system up and running and keep our customers happy. I think those are some of the challenges we all face. I know I face it, and I’m sure others do too, which is making these transitions with the fewest pain points possible. - -**Reys:** That’s a great segue into the next section. What was the process like for GitHub to adopt OpenTelemetry? - -**Ariel:** For us, it was the role of the advocate. I acted as an advocate and was a champion for OpenTelemetry at the company. I was specifically brought in to help advance the mission of tracing. There were other challenges we faced too, such as getting everybody to build a data dictionary and agree to the same language when it comes to our attributes. - -As you can imagine, as a system with many acquisitions, every team has their own log attributes or metric attributes. I said, "Look, here’s a North Star—here are semantic conventions from OpenTelemetry. Let’s anchor onto this and follow the rules around semantic conventions so that we can build our own internal dictionary." That comes with its challenges, like schema migrations and trying to keep up to date with the spec and what the instrumentations are doing. - -As an advocate, I was also a lead on the rollout. One of the things I wanted to do was sunset all our old SDKs and move over to our open-source SDKs. A few other members of my team and I were involved in OpenTelemetry Ruby, for example, because we’re a big Ruby shop. We got involved there to help the instrumentations get better and to test different releases of the SDK and get that rolled down into our GitHub monolith. We were very early adopters, ran into some challenges, and continue to give feedback to the community that way. - -I’m not only an end user; I’m also a maintainer, so I’m playing multiple roles there, trying to help the community along. - -**Reys:** That’s a really interesting position to be in—as an end user and contributor. I definitely want to circle back on that when we get to the OpenTelemetry community questions. Can you tell us about the architecture landscape at GitHub and the telemetry that you’re capturing? - -**Ariel:** Sure! I've got this little slide that I put together in Markdown, so it’s not anything official. We can bring that up now to take a look at it. I imagine that for a lot of folks out there who are working with virtual machines where they’re running systemd units or Kubernetes, we have different deployment styles here. Some of our main workloads are running in Kubernetes. - -The way we have everything set up in our Kubernetes cluster is that we run our own custom mesh. On every worker node, we have a deployment of the OpenTelemetry collector, which we build using OCB. We chose that route because we wanted to ensure that we had the most secure build possible with the minimum number of dependencies. We also wanted the ability to build our own custom processors so that we can address any issues specific to our needs. - -In each of these pods, we also have a mesh sidecar. Ingress traffic comes into the OpenTelemetry collector over HTTP. If you have another application running your service, it shoots over traces to the mesh and sends it to our OpenTelemetry collectors from which we generate span metrics, sample traces, and send those off to a SaaS provider where we aggregate all this data. - -On each individual worker, we’re running in a hybrid world right now. We’re only leveraging OpenTelemetry for traces and generating span metric data. We’re still living in a world where we’re using non-OpenTelemetry formats for collecting things like custom metrics, system metrics, and logs. On each of our worker nodes, we have a metrics agent that can speak various protocols, mostly using OpenMetrics or StatsD to collect data, aggregate it, and send it off to the SaaS provider. - -In addition to that, on all of our worker nodes, we have Fluent Bit running, and that’s what we’re using to collect our logs. All of our logs end up getting streamed out through Azure Event Hubs, processed by a bunch of consumers, and sent off to our log store and search system. - -In addition to these Kubernetes workloads, we have our own virtual machines where we run systemd units, such as Git file system services. Those are running on systemd, and we have the metrics agent and Fluent running there. We don’t yet have the OpenTelemetry collector deployed there, but that’s where we want to get to. We want our entire platform to run on open-source software—the OpenTelemetry collector—and we’re converging on OpenTelemetry for all these formats. +## Chapters -Some facts about our trace data: we’ve rolled out OpenTelemetry SDKs for different programming languages, including Ruby, Go, JavaScript, Node.js, and .NET. We had some experimental Rust usage but had to scale that back, and we had some Java experiments that didn’t pan out. +Here are the key moments from the livestream with timestamps: -A lot of what we do started before any of the automatic instrumentation was available for some of these languages, so we’re still deploying them through wrapper libraries. We maintain a set of wrapper libraries that give you the minimal defaults we require. We’ll do periodic updates of those through Dependabot, so as we roll out new versions, we keep everything updated. +00:00:00 Introductions and new format overview +00:01:30 Reys introduces himself and mentions the goal of the session +00:02:45 Ariel Valentin joins and introduces himself +00:04:15 Overview of the session format and audience interaction +00:06:00 Ariel discusses his journey into observability and open telemetry +00:09:00 Discussion on challenges organizations face with observability +00:12:00 GitHub's adoption process of open telemetry +00:15:30 Description of GitHub's architecture and telemetry capturing +00:20:00 Discussion on sampling techniques used at GitHub +00:25:00 Ariel shares GitHub's previous observability tools and changes since switching to open telemetry +00:30:15 Audience questions and interactive discussion on contributions to the project +00:35:00 Reys provides ways for new contributors to get involved in the end user Sig +00:40:00 Ariel asks about expectations for participation in the Sig +00:45:00 Conversation about holiday plans and cultural traditions -I feel like I’ve said a lot so far. Reys, do you have any follow-up questions for me? +Feel free to reach out if you need more information! -**Reys:** I actually have so many! You mentioned you’re leveraging OpenTelemetry for traces right now, and you talked a little bit about the plans and some of the things you tried. I was curious to find out more about that. Is that mainly because you’re primarily a Ruby shop, and maybe the signals for metrics and logs aren’t quite as mature yet? Is that the reason? - -**Ariel:** That’s one of the reasons. One of the other challenges I didn’t discuss regarding adoption is that even though OpenTelemetry says, “Hey, this is what the signals look like; this is what the semantic attributes are like,” there’s a lag between the time that something is published or declared in the OpenTelemetry spec and the time that vendors or open-source platforms can leverage those things. - -There’s a feedback loop as we go through OpenTelemetry and say, "Hey, we want to try this new thing out." We do a couple of things in experimental languages, but it was a big enough challenge for us to start rolling tracing out and adopting semantic conventions for logs that we didn’t want to take on yet another thing. - -As you said, some of the SDKs are ahead of others. With Ruby, we recently got help from our amazing maintainers—Schwan and AAA—who have been working diligently to get the metrics SDK up to speed for Ruby. So, that’s something that wasn’t available to us to use. - -One of the biggest advantages I see in tracing is the ability to generate span metrics from traces. The fewer things we impose on our users for them to try to figure out, the better for us, right? - -**Reys:** Are you using the span metrics connector? - -**Ariel:** We’re using a custom connector right now. I’d like to get us to the point where we’re using a span metrics connector, but we don’t have egress in OpenTelemetry right now. We’re still using vendor proprietary formats for export. What I want to get to is that one of the biggest strengths of OpenTelemetry is that it provides a standard format that we can make things portable for us. - -**Reys:** So it sounds like the plan is to migrate your other signals over once those reach GA in languages? - -**Ariel:** Absolutely! Once I have more time on the calendar, right? We have so many projects to do as engineers and companies, and it’s like, "Hey, where do we fit these in?" As you can imagine, GitHub is constantly growing every day. We just hit 150 million users, which is amazing growth for the company. So we’re here to support that volume growth and support our end users. - -**Reys:** I hope that answers your question! - -**Ariel:** Absolutely! I also want to mention real quick for those who joined us a little bit later: if you have questions that come up, feel free to pop them into the live chat of whatever platform you’re watching from, whether it’s YouTube or LinkedIn. Go ahead and pop the question in, and we will try to get to them throughout the show. - -I noticed on your second slide you mentioned probabilistic sampling, so it sounds like you’re not doing any kind of tail sampling right now. - -**Ariel:** No, not at the moment. One of the hard things about tail sampling is that the traces all have to go to the same collector to make that decision if you’re using the collector for tail sampling. Right now, we wanted to reduce that burden in complexity immediately, and when we started with probabilistic sampling, some of the things we want to do in the future is more advanced remote sampling. - -We want to have more fine-grained control over what we’re looking at and be able to leverage tail sampling rules, but as you know, that’s very difficult to implement and scale in our case. We have about 2,000 collectors supporting everything right now across our fleet of about 14,000 hosts. - -Right now, we’re doing about 26 million spans per second, and just yesterday during our peak times, we hit our all-time high of 32 million spans per second as we continue to grow our volume. This is one of the challenges we’ve had, and tail sampling is definitely on my list of things I want to do. - -**Reys:** At what point do you think you might get to the point where you could implement tail sampling? - -**Ariel:** That’s a tough question for me to answer because it’s on the pile of the list of things I want to do towards the bottom. So, it’s like, "When can we do it?" When I get to all my other wish list items! - -I’m also curious about the experiments you mentioned in Java and Rust that didn’t quite pan out. What was it that didn’t meet your expectations? - -**Ariel:** We’ve reduced the number of Java workloads we’re running, so those have been migrated over to different programming languages. That’s one reason we didn’t continue to roll out JVM workloads; we just didn’t have the return on investment we wanted. - -We also have a very limited set of applications that run Rust, and those ran into challenges with how it impacted their latency when they introduced the use of the SDK. I’m going to be honest with you—I’m not a Rust expert and couldn’t get in there to help address some of those problems and report them upstream. That was something we had to put on pause because, again, it was one program versus all these other services that are running in Go, Ruby, and JavaScript that we need to pay attention to. - -**Reys:** What was GitHub's observability tool before migrating to OpenTelemetry? - -**Ariel:** We were using proprietary SDKs for OpenTracing, which was how we collected traces before. Generally speaking, GitHub is a huge StatsD metrics user still to this day. Logs are a big part of what we utilize here, so folks are looking at exception stack traces a lot of the time to try to understand where errors are coming from. - -They’re looking at access log streams and trying to piece together where the request is going and where it’s slowing down. Then I showed up with my "magic tool" for distributed tracing and said, "Hey, look, here it is on the flame graph, or an ice cycle graph, or here's a waterfall view of this thing." People started looking into that as an additional tool in their toolbox to help debug things during an incident. - -**Reys:** How have things changed since GitHub switched to OpenTelemetry? - -**Ariel:** This year alone, we’ve seen a dramatic increase in the number of people using tracing and the number of services that have been instrumented. Part of that comes from the fact that I said, "Hey everybody, we’re all moving to this new SDK, so everyone install it," and let Dependabot do these installations on your apps. - -People started seeing the value of this investment, so they started to use it more. When we started this adventure, only about 80 services were instrumented, and now we’re closer to about 300 of those services, which effectively are Kubernetes deployment types or systemd types. - -The monolith itself has broken down into eight services, like web UI, API, GraphQL, and background workers. We saw quite an increase in the number of services that came in. We went from doing something like 5 million spans per second to now 32 million spans per second, which is just a huge increase in volume and usage. - -**Reys:** Along with that increase in usage and data volume, how has that impacted how your services are running and how quickly your team can debug issues as they arise? - -**Ariel:** Part of it is education. So I mentioned that I’m on the observability team, but really we’re two groups. One of the groups is called the Experience Team, which works directly with teams. They operate in a role sort of like developer relations and advocacy but also work on cost control and improving your experience. - -They help identify new workflows and introduce them into your incident command experience. Those teams set up education sessions to bring folks on board, but we try to do them not during an incident because that puts some stress on people. +# Oh Tell Me: End User Q&A Episode Transcript -When an incident is happening, we say, "Hey, here are some insights we’re gaining from the traces that you might not be able to see somewhere else." That has helped in many cases where we couldn’t pinpoint what was going on. There are also spots where not every system has been instrumented. We often rely on client metrics or client trace data and say, "Hey, look, this client is experiencing this problem. Can we take a deeper look at this other service that hasn’t yet been instrumented?" +**Host:** Hello everyone, welcome to a brand new episode of Oh Tell Me, an end user Q&A! If you have attended one of these sessions in the past, you might notice that this looks a bit different and has a new name. That's right, we’ve grown since the last few times, and we’re excited to debut this new look! But don’t worry, almost everything else is going to be the same. We have a great interview for you today to learn from. -It’s been sort of a mixed bag. Some teams have identified issues even before they go out to production, while other teams have leveraged it when it’s like, "Oh, we’re having an incident right now. Here’s the reason why this is failing." We’ve been able to identify bottlenecks and little coding mistakes, like forgetting to ack the message before pulling it out of Kafka, or the client timeout not matching the server timeout. +Before I introduce myself and our guest, I’d love to know where everyone is connecting from. I am online from Portland, Oregon today, and I’d love to see where others are from. It looks like someone is here from Brooklyn, New York. Thank you so much for joining us! -**Reys:** That’s fascinating. One more question from the main section: What would prevent you from implementing better telemetry? +My name is Reyes, and I am a Senior Developer Relations Engineer at New Relic. I also co-lead the Ner SIG, which is our cute little logo that you see in the right corner. At the end user SIG, we focus on connecting directly with users and are dedicated to helping the rest of the SIGs receive feedback from end users to improve the project. This event is one of those initiatives. -**Ariel:** I think it’s because there’s so much that’s in early stages. We were early adopters of a lot of things, but there’s a lot more that’s risky for us to roll out. For example, one of the things I’m really excited about since I came back from KubeCon is the continuous profiler. +So, we are going to have Ariel on. Ariel, if you would like to join us and introduce yourself. -If I could roll that out today, that’s what I would want to do. But because we’re still in the early stages of the profiler and the specification, and there’s still some churn with the data model, we don’t want to take that risk by going too early with those tools. +**Ariel:** Hey, how's it going, everybody? I’m Ariel Valentin, a Staff Software Engineer at GitHub, working on observability. Thank you so much, Reyes, for inviting me here to chat with you today. It's an absolute honor to be the first in this new format! Also, a big shout out to everyone who has worked to get this new platform up and running. So, thank you! -Also, there’s not a lot of support from our current vendors that will leverage that. It’s kind of like we would be experimenting with that to go nowhere. Once vendors start to support the OpenTelemetry profiling format data model, I’d love to jump in there and roll that out more widely. +**Reyes:** Thank you for being here! I’m really excited. Just to get you and everyone up to speed on the format, it’s similar to the ones if you’ve been on one of these before. We’ll start with some warm-up questions to help Ariel get comfortable, and then we’ll hit him with some meaty questions. We will also get into questions around the OpenTelemetry community, where he will have an opportunity to share feedback about his experiences contributing to the project. He’ll also get a chance to ask us questions in a section we call “Turn the Tables.” At the very end, we’ll address any audience questions. -Additionally, we have a bit of a bumpy road ahead in migrating semantic conventions from pre-1.0. We were early adopters of semantic conventions, so we’re at this pre-1.0 stage. Getting ourselves to migrate toward a 1.x version of semantic conventions is another big challenge because we’ve sent all this data out to our backends. We have to figure out a way to upgrade it or say, "Oh, this is version X." +If you have questions that come up during our conversation, feel free to put them in the chat. If you're watching from YouTube, just put them in the live chat, and LinkedIn will also have its own chat, so put your questions there. We’ll get to them as we can throughout the conversation. If you have any questions at the end, we’ll get to those as well! -It feels like a lot of progress was stalled there for the moment. +Alright, I think we are good to get started. Ariel, tell us a little bit about your role at GitHub. How did you get started with observability and OpenTelemetry? -**Reys:** I think that answers your question! +**Ariel:** Those are great questions! I should have mentioned that I’m here in sunny Austin, Texas, so now you know where I am in the world. I think a lot of us originally started working with APM tools, which are vendor-proprietary tools that didn’t have distributed tracing in place at that time. As the community evolved, and OpenTracing became a standard, that was my first experience with distributed tracing. -**Ariel:** Absolutely! I think this is another interesting topic—migrating some semantic conventions. I might jump back if we have time later, but I do want to get to some of the community questions! +When I got to GitHub, I was a champion for distributed tracing because I saw the power in it. Around the same time, folks were developing and transitioning away from OpenTracing and OpenCensus to OpenTelemetry. I got involved early on in trying to spread the word, learning more, and working with my observability team to adopt tracing more and embrace OpenTelemetry as our North Star for all of our telemetry signals. -Yes ma’am! One of the big things I think you’ve probably seen is that people want to contribute to the project but aren’t really sure where to get started. We have resources such as getting started in the documentation, various SIG channels, and the CNCF Slack. I think we all have the problem of how to reach these people because there are so many. +I feel like I keep saying the word telemetry over and over again, and I’m going to have to find a way to break that habit! -What was the contribution process for you and your team like for OpenTelemetry? +**Reyes:** No worries, we’re both going to stumble over OpenTelemetry or distributed tracing at some point. This is a safe space! You mentioned you became a champion for distributed tracing. What do you think is the main challenge most organizations face when it comes to observability? -**Ariel:** For us, it was a short process with a lot of observation. The things we looked for—myself and my teammates when we approached the community—were the code of conduct and the expectations of the code of conduct. We actually looked at previous PRs and the feedback that was in those PRs to see if the behaviors expected in the code of conduct were reflected in the PRs and issues. +**Ariel:** That’s a good question. There’s a lot of new language and concepts to parse through. When you say "trace," do you mean a trace log? When you say "trace," do you mean the samples taken from a profiler? There's a lot of language that even though we’re converging on it, there’s still a hump we have to get over to get everyone speaking the same language within the same context. It’s similar to domain-driven design, where the same term can mean different things. Many folks have faced these challenges as well; it’s about getting everyone on the same page. -There’s nothing worse than wanting to be part of this community and finding that those two things don’t match. When we saw that, we participated in the SIGs, which were very nice for us because the Ruby SIG was in US hours. We joined those meetings during the day and started to provide feedback as end users. +**Reyes:** Absolutely. What are currently some of the most interesting problems you are facing in your role? -If we identified something that was a challenge for us, we could contribute a PR. The maintainers were very generous with their time, did their reviews, and we were able to get a couple of custom propagators merged and some instrumentations merged. +**Ariel:** One of the significant challenges is that transition. It’s really hard for an organization like us, which has been around for about ten years, where the system has grown and evolved with various acquisitions. There are disparate backends where we collect data, and transitioning from one way of doing things to a new way is a challenge. People are trying to keep the system running and keep our customers happy, so making these transitions with the fewest pain points possible is a big challenge. -We identified other gaps for the instrumentations we use because one of the biggest challenges as a maintainer is that there’s so much to do. You’ve got the SDK, the API, documentation, and language-specific instrumentations. That’s daunting, especially with all the popular libraries out there. +**Reyes:** That’s a great segue into the meat of our discussion. What was the process like for GitHub to adopt OpenTelemetry? -We focused our energy and contributions back to the libraries we use heavily at GitHub, which are sort of "blessed" by their popularity. Anytime someone comes to me and says, "Hey, I have this challenge with this thing; it’s missing this attribute or I’d really like it to work like this," I say, "Let’s open up an issue and work on this together. I’d love to review your PR." +**Ariel:** For us, it was a lot of advocacy. I acted as an advocate and champion for OpenTelemetry at the company. It was something I pitched, and we also faced challenges like building a data dictionary and getting everyone to agree on the same language regarding our attributes. As you can imagine, as systems grow, every team has its own log attributes or metric attributes. We anchored onto the semantic conventions from OpenTelemetry to build our internal dictionary. -Creating an environment that lowers the barrier to entry for folks to collaborate and submit contributions is important. I want them to feel like this is ours, not mine, and I’m gatekeeping. We still have some standards for quality, obviously, and some expectations from you as a maintainer, but we try to make this a painless experience for you in the Ruby community. +This came with its own challenges, such as schema migrations and keeping up to date with the spec. I wanted to sunset all of our old SDKs and move over to our open-source SDKs. My team and I got involved in OpenTelemetry Ruby, for example, since we are a big Ruby shop. We helped make the instrumentations better, tested different releases, and got that rolled into our GitHub monolith. We were early adopters, ran into challenges, and continued to give feedback to the community. -That’s what the contribution process was like for me and getting involved in the community. +**Reyes:** That’s a really interesting position to be in as both an end user and contributor. I’d love to circle back on that when we get to the OpenTelemetry community questions. Tell us about the architecture landscape at GitHub and the telemetry you’re capturing. -**Reys:** That’s awesome! So you started by joining the SIG meetings as an end user and sharing feedback, then found information about how to open an issue, and from there built custom components and helped the community build out more of their components. +**Ariel:** Sure! I have a slide I put together, which we can bring up now. For many folks working with virtual machines or Kubernetes, we have different deployment styles. Some of our main workloads are running in Kubernetes, and we run our own custom meshing. On every worker node, we’re running a deployment of the OpenTelemetry collector that we build using OCB. We chose that route to ensure we had the most secure build possible with the minimum number of dependencies and the ability to build our custom processors. -Do you have any feedback for the SIG on ways to improve how things work based on your experiences with Ruby and with other SIGs? +In each pod, we have a mesh sidecar. Ingress traffic comes into the OpenTelemetry collector over HTTP. If you have another application running your service, it sends traces to the mesh and sends them to our OpenTelemetry collectors, which generate span metrics and sample traces before sending them to a SaaS provider for aggregation. -**Ariel:** Yes, I want to thank everybody for their amazing work contributing to the collector, in particular. We release our OCB build every time a new version of the collector package rolls out. Every week or two, Dependabot tells us, "Hey, it’s time to upgrade; a new release has rolled out." +We are currently leveraging OpenTelemetry for traces and generating span metric data, but we still collect custom metrics and logs using non-OpenTelemetry formats. On each worker node, we have a metrics agent that can speak various protocols. We have Fluent Bit running to collect logs, which get streamed out through Azure Event Hubs, processed by consumers, and sent off to our log store and search system. -We ran into some challenges where we identified performance issues in the collector. We worked with the team, provided profiles, and gave feedback. They were very responsive to our concerns, and they were happy to see we could contribute back just by providing feedback and actual data from production workloads. +For our trace data, we’ve rolled out OpenTelemetry SDKs for different programming languages, including Ruby, Go, JavaScript, Node.js, and .NET. We had some experimental Rust and Java usage but had to scale that back. We started before automatic instrumentation was available for some of these languages, so we’re deploying them through wrapper libraries, which we maintain and update periodically. -That’s very difficult to do as a maintainer; it’s hard for me to know what’s going on in a customer’s deployment or user deployment. So that spirit of collaboration was great, and that’s what I’d like to see happen in all the other SIGs. +**Reyes:** Wow, that’s a lot of information! I have many follow-up questions. You mentioned that you’re leveraging OpenTelemetry for traces right now. Is that mainly because you’re a Ruby shop and the signals for metrics and logs aren’t as mature yet? -I have limited experience with other SIGs. The other part was contributing to Semantic Conventions. I contributed to Semantic Conventions to introduce some new attributes, but there’s so much churn in that repository that it was hard for me to keep up with changes and deprecations. +**Ariel:** That’s one of the reasons. There’s also a lag between when something is published in the OpenTelemetry spec and when vendors or open-source platforms can leverage those things. We ran into challenges rolling out tracing that we didn’t want to take on yet, especially as some SDKs are ahead of others. With Ruby, we recently saw improvements to the metric SDK, which is something we wanted to use. -That was a little bit on me to keep up and find out that the stuff I submitted was deprecated in favor of something else. It’s those kinds of challenges that I’d like to find a better way for us all to keep up with each other. +**Reyes:** So, it sounds like the plan is to migrate your other signals over once they reach GA in languages? -One suggestion I might have is: when a change happens, here’s an OpenTelemetry change. Open issues for every maintainer repo and say, "Here’s a new thing that has changed; this is a new feature we expect the SDK to support." This way, we can track it because right now, the way it works is that we have someone who’s a representative at the SIG spec. They collect some data, come back to us, and say, "Hey, we need to implement this other thing." +**Ariel:** Certainly! Once we have more time on our calendar, we want to migrate everything over. GitHub is constantly growing. We just hit 150 million users, so it’s amazing growth, and we’re here to support that volume growth. -Sometimes we’re not diligent about project management; we’re individual contributors doing this in our spare time. We need to improve overall at being able to keep each other informed and track work. +**Reyes:** That absolutely answers my question! For those who joined us later, feel free to pop any questions into the live chat of whatever platform you're watching. -**Reys:** Got it! So, making sure everyone’s on the same page at the same time is a bit daunting. +I noticed on your second slide you mentioned probabilistic sampling. It sounds like you're not doing any kind of tail sampling. -What would you say are the things you interact with most right now? +**Ariel:** Not at the moment. One of the hard things about tail sampling is that traces have to go to the same collector to make that decision. We wanted to reduce the burden in complexity immediately, so we started with probabilistic sampling. More advanced remote sampling is something we want to pursue in the future. -**Ariel:** Right now, I almost exclusively participate in the Ruby SIG. I don’t have a lot of interaction with other SIGs at the moment, other than through an occasional issue or discussion that pops up. I’ll jump into a channel and say, "Hey, I’ve run into this problem," or "I have this question for you; can you help me?" +**Reyes:** At what point do you think you might get to the point where you could implement tail sampling? -I have limited interactions these days with other SIGs. Just recently, I met with the End User SIG at KubeCon, so I’ll probably get more involved now that I have this special invitation from you! +**Ariel:** That’s on the pile of things I want to do. It’s hard to say right now, but it’s something I’d love to explore. -**Reys:** I’m so excited! This brings us to our "Turn the Tables" section where you get to ask me or anyone on the call questions, and we’ll see if I can answer them or if anyone else can answer them. +**Reyes:** I’m curious about the experiments you mentioned in Java and Rust that didn’t pan out. What didn’t quite work out for those? -**Ariel:** For me, Reys, how do newcomers get involved, particularly in the End User SIG, to provide feedback? How do they get involved in other SIGs that you’ve been involved with? +**Ariel:** We reduced the number of Java workloads we’re running, migrating them over to different programming languages. We didn’t see the return on investment we wanted. Similarly, with Rust, we have a limited set of applications that run Rust, and they ran into performance issues that impacted their latency. I wasn’t able to get my hands dirty to help address those problems, so we had to put that on pause. -**Reys:** There are many different ways to contribute! I think a lot of people, when they think of contributing, think, "Oh, I have to open PRs; I have to write code." That’s not necessarily the case. There are so many different ways to provide feedback. +**Reyes:** What was GitHub’s observability tool before migrating to OpenTelemetry? -One way you mentioned is that you just started going to a SIG meeting and sharing things you’re seeing. That’s an absolutely great way to contribute to the project, and that’s what the End User SIG is all about. One of the things we’re all about is gathering feedback to give back to the appropriate SIG so we can help improve things. +**Ariel:** We were using proprietary SDKs for OpenTracing. GitHub is also a heavy StatsD metrics user, and logs are a big part of what we utilize. We were piecing together logs and metrics for debugging and started using distributed tracing as an additional tool in our toolbox. -We do things like surveys and open Q&A interview sessions where we dive deeper into the adoption and implementation process and find out your pain points and specific feedback. You can contribute blogs—maybe you love writing documentation! +**Reyes:** How have things changed since GitHub switched to OpenTelemetry? -If you’re not already part of the CNCF Slack instance, I would join. I just realized I did not include that link, but we will have it in the show notes. Once you’re a member of the CNCF Slack instance, you can scan the QR code to join the OpenTelemetry SIG End User channel, and we will be happy to direct you if you have questions about your implementation or if you just want to know, "Hey, where do I get started?" +**Ariel:** Just this year alone, we’ve seen a dramatic increase in the number of people using tracing and the number of services instrumented. When we started, around 80 services were instrumented, and now we’re closer to 300. We’ve gone from about 5 million spans per second to 32 million spans per second, which is a huge increase in usage. -We also have a survey going, which I also just realized I did not share the link to! I’ll see if I can get that right now. +**Reyes:** How has that impacted how your services are running and the speed at which your team can debug issues? -So yes, join the SIG meetings! Where can they access the calendar? +**Ariel:** It’s a mix of education. We have teams dedicated to improving the experience and helping identify new workflows. We try to provide insights during incidents and help teams identify issues even before they go to production. It’s been a mix of results—some teams have identified issues before they go live, while others have leveraged it during incidents to find the root cause. -**Ariel:** It’s through the community repository that has links to the calendar. +**Reyes:** What would prevent you from implementing better telemetry? -**Reys:** Perfect! If there’s a specific language you’re already working in, check out the calendar to find out when the SIG meets, and just hop on. +**Ariel:** There’s a lot in early stages. We were early adopters, but some tools are risky to roll out. For example, I’m excited about the continuous profiler, but it’s still in the early stages, and we don’t want to take that risk just yet. There’s also a challenge in migrating semantic conventions from pre-1.0 to 1.x, as we’ve already sent data out, and we need to figure out a way to upgrade it. -When I first started, I was like, "I’m just going to check out a few different SIGs." I went to the Python SIG, and everyone there was so welcoming and nice. I was so intimidated at first, but then I realized, "Wow, these people are so lovely!" That’s been my experience in almost every SIG meeting I’ve jumped into—everyone is so open to answering questions and wants your help. +**Reyes:** Let's shift to some community questions. What was the contribution process for you and your team like to OpenTelemetry? -It’s important that we make it clear that contributing doesn’t have to mean opening PRs. If that’s what you want to do and are able to, we would love that, but there are so many other ways to contribute, like providing feedback, donations, blogs, doing interviews, and surveys. +**Ariel:** It was a short process with a lot of observation. We looked at the code of conduct and previous PRs to ensure the behaviors matched our expectations. We participated in SIG meetings, provided feedback as end users, and were able to contribute some custom propagators and instrumentations. We focused our contributions on libraries that are popular at GitHub and aimed to create an environment that lowers the barrier to entry for collaboration. -Every little bit counts, right? Every little contribution can help us. If your contribution helps you, it’ll help the community. So, come if you’re having trouble with a specific problem. I guarantee you at least a dozen other people are having the same problem. If you don’t understand something, I guarantee at least a hundred other people don’t understand it either! +**Reyes:** Do you have any feedback for the SIG on ways to improve based on your experiences with Ruby and other SIGs? -**Ariel:** I really appreciate you answering my questions here. +**Ariel:** I’d like to see better communication about changes in specs. If a change happens, it should automatically open issues for every maintainer repo to track updates. Right now, it’s hard to keep up with changes and deprecations. -**Reys:** You are so welcome! I think we’re right about on time. +**Reyes:** What SIGs do you interact with most right now? -If anyone has any last-minute questions, we’ll hang out here for another minute and give people an opportunity to share if they would like. Otherwise, I’ll just ask about any fun Christmas or New Year plans? +**Ariel:** I mostly participate in the Ruby SIG and have limited interaction with other SIGs these days. I recently met the end user SIG at KubeCon, so I’m looking to get more involved there. -**Ariel:** We’re going to be spending it with family, so that will be really nice. We’ll be traveling, so that might be a little hectic. We are visiting my father, and we are traveling on Christmas Eve, so that should be fun! +**Reyes:** For our “Turn the Tables” section, what questions do you have for me or anyone on the call? -I’m going to show up there and we’ll be singing Puerto Rican Christmas carols at the top of our lungs, waking up the neighbors as soon as we arrive! +**Ariel:** What are the best ways for newcomers to get involved in the end user SIG? -**Reys:** Oh my gosh! How about you? What are your plans for the holiday season? +**Reyes:** There are many ways to contribute. Joining SIG meetings and sharing your experiences is a great way to start. We do surveys, open Q&A sessions, and even documentation. Joining the CNCF Slack instance is also essential. -**Ariel:** I’m going to be at home working on house projects that have been on my to-do list for three years! +**Ariel:** What kind of expectations of commitment do you have from folks who want to participate? -**Reys:** Nothing like it, right? You’re going to get to the tail sampling project, right? +**Reyes:** We welcome contributions in various forms. We hope people can join at least once a month, but if not, that’s okay! We understand everyone has commitments. We want to make it as welcoming as possible for everyone. -**Ariel:** Oh my gosh, yes! I’m really excited about it because I moved into this house about three years ago, and it’s like 85% done. +**Ariel:** Thank you for answering my questions, Reyes! -**Reys:** Just don’t look in the cabinets; that’s a project I need to organize! +**Reyes:** You’re welcome! If anyone has any last-minute questions, we’ll hang out for a minute. Otherwise, I’ll ask about your holiday plans! -**Ariel:** Exactly! And that’s the thing I tell people all the time: "Hey, don’t look inside the repositories! You may not like how things are arranged, but I know where things are!" +**Ariel:** We're spending it with family, traveling on Christmas Eve. It should be fun! -Thank you so much, Ariel! You were a fantastic guest. I’m really excited to learn more. As I said, I do have some follow-up questions, so I’ll be reaching out to learn more about your adoption processes, as well as some of the other components you mentioned you used, such as OCB. +**Reyes:** I’ll be working on house projects that have been on my to-do list for three years! -If anyone wants to reach out to either of us, you can find us in the CNCF Slack OpenTelemetry SIG End User channel. Again, the link is up here for you to scan, and I believe we will have the information in the show notes as well, which will be posted right after this. +**Ariel:** That sounds like a plan! -Thank you so much for joining us, and we look forward to seeing you all again next time! +**Reyes:** Thank you so much, Ariel! You were a fantastic guest. I’m excited to learn more from you. -**Ariel:** Thank you for having me, Reys! This was really awesome. Thanks, everybody, for watching wherever you are and whenever you are. I hope to see you all in a SIG room sometime! +If anyone wants to reach out, you can find us in the CNCF Slack's OpenTelemetry SIG end user channel. We’ll have the information in the show notes. Thank you all for joining us, and we look forward to seeing you again next time! Happy everything! -**Reys:** Yes, and happy everything to everyone! Adios! +**Ariel:** Thank you for having me, Reyes! This was awesome. Thanks, everyone, for watching, and I hope to see you in a SIG room sometime! -[Music] +**Reyes:** Adios! ## Raw YouTube Transcript -[Music] hello everyone welcome to a brand new episode of oh tell me and end user Q&A if you have been to one of these sessions in the past you might notice that this looks a little bit different and it has a bit of a different name that's right we have done some growing up since the last few times and we're very excited to debut this new look um but don't worry almost everything else is going to be the same we have a great interview here for you today to learn from um but before I introduce myself and our guest I would love to know where everyone is connecting from I am online from Portland Oregon today and would love to see where um people are from it looks like someone is here from Brooklyn New York so thank you so much for joining out in New York and so my name is reys I am a senior developer relations engineer at New Relic I also co-lead the ner Sig um which is our cute little logo that you see um up in the right corner and yeah um at the end of your Sig we really focus on connecting directly with users and we are dedicated to helping um the rest of the sigs um get feedback from end users so they can help improve the project um and to that end this is one of those uh events that we host to do that so we are going to have um Ariel on Ariel if you would like to join us and introduce yourself hey how's it going everybody um Ariel Valentin I'm a Staff software engineer at GitHub working on observability thanks so much ree for inviting me here to chat with you today as absolute honor to be the first in the new format also so um you know big shout out to you know everybody who's uh done work to try to get this um this uh new platform up and running so thank you oh no thank you for being here I'm really excited um so just to get you and everyone up to speed on the format um it's similar to the ones if you've been on one of these before but we're going to start with some warmup questions where we'll kind of massage Ariel into being comfortable and then we'll hit him with some meaty questions and then we will actually um also get into questions more around the open c community and where he'll have an opportunity to share um like feedback about um his experiences with contributing um using the project stuff like that and then he'll get the chance to ask us questions as well um in a little section we call turn the tables and the very end if there are any audience questions um oh actually scratch that if you have questions that come up during our conversation feel free to put them into the chat if you're watching from YouTube then just put them in the live chat and then I think LinkedIn will also have its own chat so put in your question there and we will get to them as we can throughout the conversation um and then at the end if you have questions at the end as well then we will get to those but yeah if you have any questions that pop up as Ariel as telling us his story we definitely want to get to those as they come in all right so I think we are good to get started so adiel ad told us a little bit about your role of the company um how did you get started with observability how did you get started with open Telemetry oh those are great questions I mean I I should have mentioned that I'm here in Austin Texas Sunny Austin Texas uh so you know where I am in the world um I think a lot of us started originally um uh and I don't think my experience is unique is working with sort of like APM tools uh which are vendor proprietary tools uh generally speaking that uh didn't have distributed tracing in place um at that time and as the community evolved and open tracing became a standard uh that was my first experience with working with distributed tracing was uh the open tracing uh um specification and uh working and trying out different vendors different experiences with uh open tracing and then um when by the time that I had gotten to GitHub um I was a champion for distributed tracing because I saw the power that was in there and around that same time folks were already moving towards developing and transitioning away from open tracing and open sensus to open Elementary so I got really uh involved very early on in trying to spread the word and learning more um and working with uh my obser my team which was the observability team to uh start to adopt um tracing more to embrace it more and to make otel sort of our North Star um for all of our Telemetry signals I feel like I keep saying the word Telemetry over and over again I'm going to have to find yeah me a lot of that we're both going to stumble over over open Telemetry at some point or distribute tracing at some point it's all right this is a safe [Laughter] space what um okay so you mentioned you became a champion for distribute tracing um and you know I think something that's interesting is you know we're so ins scon in the world of observability that we at least I you know tend to presume everyone understands what that is um and it's always surprising to me when someone's like what's distributed tracing and they working um what so kind of along with that what um do you think that's kind of a main Challenge and you think most organizations organizations face when it comes to observability is not just understanding the value of distribute tracing and yeah because these are all sort of new Concepts to folks right uh you know folks have their different levels of understanding of the tools that are available to them and so there's all this vocabul that you hear that's a little bit hard to to parse through so sometimes it's like oh when you say Trace do you mean like a trace log log level is at the trace level uh when you say Trace do you mean the samples taken from a profiler when you say so there's a lot of this sort of uh language that um even though we're we're uh converging on a lot of this language and we uh have these dictionaries that are defined and published everywhere everywhere there's still sort of like this uh there's a hump that we have to get over or like a challenge that we have with trying to get everybody speaking um um the same language within the same context right very similar to in domain driven design we have these bounded context where the same term means a different thing you know that that flows over into sort of our domain language when it comes to observability and Sr practices um and and I'm sure many folks have faced that the those challenges as well it's kind of like let's all get on the same page about what we mean right oh absolutely um what are currently some of the most interesting problems that you are facing in your role oh well I mean one of those things is uh is that transition right um it's really hard for an organization like us who's been around for a long time um and I say a long time but you know whatever it is 10 years um where the system has grown and involved there have been Acquisitions that have been brought together uh there's disparate kind of uh uh backends where we're collecting this data um it's it's uh trying to transition from one way of doing things to a new way of doing things right so it's learning something new that's always going to be a big Challenge and you know folks ever are are trying to do their job every day they're not trying to learn new sdks or trying to learn new vocabulary or trying to learn how to be um proficient in their back ends what they're trying to do is keep the system up and running and keep our customers happy right so I think you know those are some of the the the challenges that that we all you know I know I face it I'm sure others do which is um is is making these transitions with the fewest pain points as possible with like trying to avoid these paying points um and there's I mean there's so many more we can go on and on Reese but I I think right now that's kind of like one of the biggest challenges for adoption I think overall that's actually um that's a great segue into the mey section so what was the process like for GitHub to adopt open Telemetry so uh for us it was um the the role of the advocate right so I acted as an advocate and was a champion for otel at the company and I specifically was brought in to help Advance the mission of tracing and uh it was something that I had pitched and and also sort there there were uh other challenges that we faced too which was hey let's get everybody um uh building a data dictionary let's get everybody uh agreeing to the same language when it comes to what our attributes were going to be right because as you can imagine as a system of walls or Acquisitions or other teams are rolled in everybody has their own uh you know log attributes or their own metric attributes or whatever it is and and so we started to I said look here's a Northstar right here here's semantic conventions from otel let's anchor onto this and let's follow the rules around semantic convention so that we can all uh build up our own internal dictionary right and that that comes with its own challenges right scheme of migrations and trying to keep up to date with the spec and you know what the uh instrumentations are doing right and um and as an advocate you know was also uh a lead on the roll out so one of the things I wanted to do was sunset all of our old sdks and move over to our open source sdks um and you know myself and a few other members for my team we all were involved in um otel Ruby for example because you know U we're a big Ruby shop uh but uh we got involved in there to help the instrumentations get better and to help test different um releases of the SDK and get that rolled down into our GitHub monolith so um we you know were very like I said very early adopters ran into some challenges and um and and continue to give feedback um to the community that way as a and I am not only an end user but like I said I'm also a maintainer so um I'm playing multiple roles there you know trying to help the community along yeah that's a really interesting position to be in is as end user and contributor and I definitely want to Circle back on that when we get to the OA community questions um tell us about the architecture landscape at GitHub and the Telemetry that you're capturing sure so um I've got this little slide that I put together in mer marked down so it's not anything official we can bring that up now so we can take a look at it um and I imagine that for a lot of folks out there who are do working um with uh um virtual machines where they're running system D units or if they're running kubernetes we have this this uh uh this uh we have different deployment Styles here but some of our main workloads are uh running in kubernetes and the way uh that we've got everything set up you could see here is as part of our kubernetes cluster you know we run our own custom uh meshing grass um and on every worker node um or um um on the worker nodes we're running a deployment of the open telemetry collector which we uh build using OCB so we have our own version of The otel Collector that is specific to us uh we chose that route because we wanted to ensure that uh we had the most secure uh build possible with the minimum number of dependencies and we also wanted the ability to build our own custom processors so that um we can uh address any issues that we might have that are specific to our needs um in inside of each of these pods you know we also have a a mesh side car and so Ingress traffic is going to come into the hotel collector otel over H ltlp over HTP um and if you have another application that's running your service it's shooting over um OTP traces over to the over the the mesh and sending it to our oel collectors which we then from there generate span metrics and um sample traces and send those off to a SAS provider that has a um where we aggregate all this data um and then on each individual worker you know we're running in this hybrid world right now um we're only um leveraging oel for traces and for generating um a span metric data uh we're still living in a world where we're using nonel non-lp formats for collecting things like custom metrics system metrics and for collecting logs um we have uh on each one of our worker nodes a metrics agent um and that can speak various different um um protocols but um mostly it's either using open metrics or it's using statsd uh to collect uh data and aggregate it and send it off to the SAS provider um in addition to that on all of our worker nodes we have a fluent bit running and that's what we're using to collect our logs um and all of our logs are end up getting streamed out through Azure event hubs which are uh processed by a bunch of consumers and then sent off to our log Store and search system um in addition to that you know we in addition to these uh kubernetes workloads we have our own virtual machines where we run systemd units you have to think about you know like the git file system Services um those are all running on systemd and we have the metrics agent and fluent running there we don't yet have the otel collector deployed there but um that's where we want to get to we want to get to a world where essentially our entire platform is running um an open source software the open Telemetry collector and um we're converging on OTP for all these formats uh so those services are also using statsd open metrics uh flu bits pulling things out of Journal D because we you know um um that's where we're streaming all of our our logs of the system Journal so um and all that stuff again goes right through to the same channels and it all gets aggregated in these places where we do our work um a little bit about some facts about sort of our our uh Trace data is that um we've rolled out the otel sdks for different programming languages we have them for Ruby goang uh JavaScript nodejs net we had some experimental uh uh rust usage but um uh uh we've had to scale that back um and we had some Java experiments that didn't pan out we didn't roll those out uh to production um and a lot of the stuff that we do when we we started before any of the automatic instrumentation was available for some of these languages so we're still deploying them through rapper libraries so we maintain a set of rapper libraries you install those it gives you the sort of the what we consider to be the minimal defaults um that we require for for our needs and uh we'll do periodic updates of those through dependabot right so it's like as we roll out new vers versions of those things um I feel like I've said a lot so far and I don't know ree if you had any follow-up questions for me or I actually have so many but oh okay great um so you mentioned and um so you mentioned you're leveraging open for traces right now um and then you talked a little bit about you know um the plans and some of the stuff that you tried um and I was curious to find out more about um is that mainly because if you're primarily a ruby shop and you know the SK maybe the signals for metrics and logs aren't quite as mature yet is that the reason or that's one of the reasons so you know one of the other challenges that I didn't discuss when you asked about adoption challenges is even though oel says hey this is what the signals look like this is what the semantic attributes are like this is um there's a lag between the time that something is published or declared in the oel spec to the time that um vendors or open-source platforms are able to leverage those things and there's sort of like this feedback loop as we go through otps and say hey we want to try this new thing out we do a couple things in experimental languages we'll do this stuff and um you know it's it was a a big enough challenge for us to start to roll tracing out that we didn't you know and adopting semom for logs that we didn't want to take yet another thing on which was to say okay now we're going to switch over to Native OTP metrics as well um and like you said some of the sdks are ahead than others so with Ruby we're just recently you know with the help of our amazing maintainers um Schwan and um and AAA they've been working diligently to get the metric SDK uh up to speed for Ruby so that so that's something that wasn't available to us to use um and I think that one of the the biggest advantages that I that that I see in tracing is the ability to generate span metrics from traces it's like hey the the the less things that we have to impose on our users for them to try to figure out how to do the better for us right and are you using the spam metrics connector is that what you we're using a custom connector right now I'd like to be able to get us to the point where we're using a span metrics connector but we don't have egress in OTP right now we're we're still doing sort of like vendor proprietary formats for uh for export where I want to get to is you know that's the strength one of the biggest strengths of the of otel is OTP is that standard format that we can that makes things portable for us absolutely so that's definitely sort of high on my list of things that I want gotta okay so it sounds like the plan is to migrate your uh other signals over once those reach GA in languages oh certainly and you know once I have more time on the calendar too right we have so many projects that we have to do as engineers and companies and it's like hey where do we fit these one these in right um uh as you imagine you know GitHub is constantly growing every day um we just hit 150 million users uh you know so it's like just an amazing um amazing growth of the company so shout out to all the people at GitHub who've been working so diligently to make this happen you know um and so we're here to support we we're here to support that volume growth and support our end users so um um so I hope that answers your question absolutely um and also want to mention again real quick for those um who joined us a little bit later feel free if you have questions that come up um you want to learn more about something that Ariel has gone over feel free to pop it into the live chat of whatever platform you're watching from so whether it's YouTube LinkedIn um go ahead and pop the question in and we will try to get to them um throughout the show as you can um so I noticed on your second slide you mentioned probabilistic sampling so it sounds like you're not doing any kind of tail sampling no not at the moment so would you mind bringing that up again uh um L the second slide yeah that's awesome yeah so we started with probabilistic sampling one of the hard things about tail sampling um as you can imagine is that um the traces all have to go to the same collector in order to make that decision if you're using the collector for uh for for tail sampling so um right now we wanted to reduce that burden in complexity immediately um and we um when we started with probabilistic sampling some of the things that we want to do in the future is um more advanced remote sampling uh so that we can have um uh more Frank grain control of what we're looking at we want to be able to do leverage uh tail sampling rules but as you know that's very difficult to implement a little bit difficult to scale in our case because you know we have about 2,000 collectors which are supporting everything right now um across all of our our Fleet right of like 14,000 hosts or something like that and I'm making that number up off the top of my head I think that's the last number we had um and you know right now we're we're doing about 26 million spans per second and uh just yesterday during our peak times you know we hit our alltime high of 32 million spans per second as we continue to um uh to grow our volume so this is one of the the challenges that we've had and so on my list of all the things that I want to do tail sampling is definitely on there you know what I mean so okay at what point do you think you'll you might get to the point where you could Implement heill sampling would be more like side when the you know processor has been uh more develops or devs it's you know it's it's uh I don't know how to answer that question because it's on the pile of the list of things that I want to do towards the bottom so it's like when can we do it when I get to all my other wish list items got it got it um and although I am curious about the other but that's okay that's all right questions that I want to get to as well sure um also just going back uh you mentioned trying some experiments in like Java and Russ I didn't kind of pan out and I was just curious um what was it that didn't quite work out or that you know didn't meet your expectations well uh We've reduced the number I'm sorry um all the jvm people out there we've reduced the number of java workloads that we've uh that we're running um so those have been migrated over to different programming languages um so that's one of the reasons why we didn't go through and say oh let me continue to roll out uh jvm work like the we just didn't have the return on investment that that we wanted to you know uh try and um also like we don't have a lot of um uh the same thing was for rust is that we have like a very limited set of applications that run rust and um those are perform for performance reasons they ran into some challenges with um how it impacted their latency when they introduce the use of the SDK and I'm going to be honest with you me not being a rust expert and being able to get in there and get my hands dirty to try to help out with addressing some of those problems and Reporting them Upstream that was something that we had to put on pause cuz again it's like one program versus all of these other services that are running and go and Ruby and um and JavaScript that we need to pay attention to right and um so that that I think that's really where it was where sort of like a critical mass of application services that use these programming languages and was like hey we have to focus our attention ption on those that are going to get us a higher return on investment for this absolutely okay so what was oh yeah what was github's observability tool before migrating to I think you mentioned um you were using proprietary sdks uh yeah I mean we were use proprietary SDK for open tracing so you know open tracing was our uh how we collected traces before um but um generally speaking the I I I I think I covered this in the first slide GitHub is a huge uh sort of statsd metrics user still to this day um and um logs are a big part of uh of what we utilize here so it's like folks are looking at acception stack traces a lot of the times to try to understand where errors are coming from they're looking at um uh access log streams um and they're trying to and they were trying to piece together Hey where's the request going and where is it slowing down and this and that and I showed up with my you know magic Tool Distributor tracing I'm like hey look here's a here it is on the flame graph or an icycle graph or here's a waterfall view of this thing and it's like oh that's pretty cool and um people started looking into that as uh as a an additional tool in their tool box for them to help try to debug things during an incident um so I hope that answers your question there yes and kind of along those lines how have things changed since GitHub switch to open cemetry so uh We've what I'll tell you is that uh just this year alone we've seen a dramatic I wish I had these statistics off hand but we've seen a dramatic increase in the number of people using tracing and a number of services that have been instrumented part of that comes from the fact that I was like hey everybody we're all moving to this new SDK so everybody install it and let the pendot go ahead and do these installations on your on your apps and people started seeing the value of this uh of this investment as well so they started to use it more um but we um I want to say that like when we started this adventure uh only about uh 80 and I'm going to use the word services in air quotes only about 80 Services were uh instrumented and now uh we're closer to about 300 of those Services which are effectively like kubernetes deployment types or like system D types uh because as you imagine our monolith has th you know uh maybe a thousand services within it so there're like subservices in there um but uh the but the monolith itself has broken down into like eight Services as like you know web UI and API and graphql and background workers and stream processors and whatnot so um that's why I put them in air quotes as these are Services um but we saw that that that was quite an increase in the number of services that that came in we were doing something like uh uh like 5 million spans per second to now we're up to 32 million spans per second it's just a huge uh uh increase in volume in usage and along with that volume increase in usage and data volume um how would you say that's impacted how your services are running how um you know quickly your team is able to debug issues as they arise part of it is an education think so a lot of things that uh so I mentioned that I'm on the observability team but really we're two groups uh one of the groups is called The Experience team that works directly with teams and they're they operate in a role sort of like developer relations and advocacy but also working on um you know cost control and improving your experience uh helping you with identify new workflows and introduce them into uh your incident command experience um those those those teams set up sessions education sessions to bring folks on board uh but we try to do them not during an incident um because that that that puts some stress and so what we try to do is like you know we'll go into the if an incident is happening it's like hey here's some insights that we're gaining and we'll share with you that we're seeing uh from the traces that you might not be able to see you somewhere else and that has helped in a lot of cases uh uh where we couldn't exactly pinpoint what was going on uh but then um there's also these spots where not every system has been instrumented so we have to rely a lot on sort of like client metrics or client Trace data and say hey look this client is experiencing this problem can we take a look deeper at this other service that hasn't yet been instrumented um and so it's been sort of I'm going to say like this mixed results some teams have like identified issues even before they go out out to production and other teams have been able to leverage it when it's like oh this we're having an incident right now oh here's uh uh and and here's the here's the reason why this is failing um and and we've able to do things like identify bottlenecks and like mistakes that you know little little coding mistakes oh I forgot to act the message before I pulled it out of kavka oh I'm just retrying that same message millions of times or um or oh the client timeout doesn't match the server timeout so the client out and the server is continuing to turn along to try to do this request so we're identifying things like that that we couldn't identify before easily easily you know um and so those are some anecdotes oh no that's awesome I think I will probably be asking you more about that because I think this is really interesting um one more question from our mey section what would prevent you from implementing better Telemetry what would prevent me from doing that I think it's because there's so much of the stuff that's in early stages and you know we were early adopters on a lot of things but then there's a lot of stuff that's a lot more risky for us to try to roll out so for example one of the things I'm really excited about when I came back from cubec con is the continuous profiler I mean that is one one of the when you talk about one of the things on my wish list that we'd be able to use today if we could uh that's the thing that I would want to do but because we're still in the early stages of the of the profiler and the specification and there's still uh some turn with the data model um I think that there are when it comes to the resource intensive or sort of like introspective tools like that it's I we don't want to take that risk going a little bit too early to adopt those tools uh because also there's not a lot of support from our current U vends and vendors that will be able to leverage that it's kind of like we would be experimenting with that to go to nowhere and so that's kind of you know once vendors start to support the OTP profiling format um data model I should say and once the profiler is a little bit more stable I'd love to jump in there and and be able to roll that out a little more widely um and um I think that uh a little bit of a a a bumpy road for us to still is migrating um semom from pre 1.0 because we were early adopters of semom so we're at this pre 1.0 stage and sort and getting ourselves to migrate towards a a 1x version of of semom is uh another big challenge because we've already sent all of this data out it's our backends there we have to figure out a way to upgrade it or to say you know oh this is version X um and and I and I kind of it feels like a lot of the progress was stalled there for the moment um so that there I don't know that there's any real solutions in um available that would say okay we can start migrating these schemas over and keep our user experience very high quality um if that answers your question oh absolutely I think that is another interesting topic migrating some semantic conventions so I might I might jump back if we have time later but I do want to get to some of these Community questions yes ma'am um yeah so one of the big things I think um you know working worked in the community that uh probably you've seen you know I've seen sure um is people want to contribute to the project but they're not really sure where to get started and yeah we have like resources such as you know I think in the documentation we added you know getting started um and you have you know various uh Sig channels and cncm slack um and I think we all have the problem of how do we how are how do we reach these people because you know there's still some many I don't know so what was the contribution process for you and your team like to open cemetry for us it was um uh it was a a short process with a lot of observation so the things that we looked for uh myself and my teammates at the time when we approached the community we were looking at code of conduct what were the expectations of the code of conduct we went through and actually looked at issues previous PRS the feedback that was in those PRS to see if the behaviors expected in the code of conduct were reflected in the PRS and in the issues uh cuz there's nothing worse than you wanting to be part of this community and finding that those two things don't match and to see that that happened then we um participated in the sigs which was very nice for us to be able and convenient for us in the US because the Ruby Sig was in the US hours we joined those meetings during the day and started to provide some feedback as end users and say hey look we ran into this challenge if we identified something that was a challenge for us we would could contribute a PR um and the maintainers were very generous with their time um did their reviews and um and we were able to get a couple of Uh custom propagators uh merged we got some instrumentations merged we identified other uh gaps for the instrumentations that we use cuz one of the biggest challenges I think as a maintainer is there's so much that you have to do right you've got the SDK you've got the API you've got documentation to do and you have to have language specific instrumentations and uh that's daunting especially with all of the popular libraries that are out so for us it sort of we contributed back to the libraries that we use heavily right we have like a subset of libraries that are very popular at GitHub and um sort of blessed right so by by their popularity so that's where we focused our energy and our contributions um and then we reached out to people you know uh anytime somebody comes to me and says hey I have this challenge with this thing it's missing this attribute or I'd really like it to work like this or I've identified this bug the first thing I do I say is let's open up an instr and let's work on this together and I'd love to review your PR and creating in um an environment that makes it lowers a barrier to entry to have folks uh uh collaborate and submit contributions I want them to feel like this is yours this is ours this is not mine and I'm gatekeeping right we still have some standards for Quality obviously and some expectations from you as a maintainer but we try to make this an a painless experience for you um at least in the Ruby uh um um community so that's what the the contribution process was like for me and getting involved uh in the community that's awesome so you started basically joining the Sig meetings as an end user and sharing feedback and then finding information about like how to you can open uh you know an issue and then like from there building custom stuff and uh helping the community or sorry the Ruby s to build out more of their components okay yeah certainly and like you know with the home for open source right so uh for us um you know we open sourced uh for example new database drivers in Ruby for um MySQL right um and so we were the we had that instrumentation in house before all of that became public and as soon as we released that new driver I went right to the Sig and said Tada I have a donation for you and it's like oh yeah we able we were able to pull that in there so um it's this thing where um there's just this Spirit of collaboration right and this Spirit of supporting open source that um that's important to our mission right okay so it sounds like you had a pretty kind of positive experience it sounds like from for contributing to Ruby um I like to know do you have any feedback to the SS on ways to improve how things work based on your experiences with Ruby and then other sigs I know you mentioned also some other components that you've used but we'll come back to that yeah for sure and I again I I want to thank everybody for their amazing work that they've done um contributing to the collector in particular uh because you know we are uh we release our OCB build or sorry our our uh our custom build every time a new version of the cont package rolls out so um every week depend on bot is telling us or every two weeks it's telling us hey it's time to upgrade a new releases rolled out and uh we ran into uh some challenges where we identified some performance issues um in the in the collector and we uh worked with the team provided profiles provided feedback uh showed them our configurations and they were so responsive to um our concerns and uh they were happy to see that we were able to contribute back just in providing feedback just giving them actual data of production workloads which is very difficult to do right as a maintainer it's really hard for me to know hey what's going on in your um in this customer's deployment or this users uh a deployment we can't replicate it in our in our own environment so um uh that Spirit of collaboration was um was really great and that's the thing that I'd like to see uh happen in all the other SE I have limited experience uh with other sigs the other uh part was contributing to semom um and I contributed to simcom trying to uh introduce some new attributes uh but there's just so much turn in that repository that it was hard for me to keep up with uh changes and deprecations right um and so that was a little bit on me to be able to to keep up and find out that the stuff I had submitted was deprecated for example in favor of something else uh but it's it's those kinds of challenges that I like I I I have to find a better way for us to all be able to to to keep up with each other so one suggestion I might say is hey look a change happened here's an otop A Change happened in the spec that should automatically open issues for every maintainer repo and say Here's a new thing that has changed uh um this is a new uh feature we expect that the SDK should support and that way we're able to track it because right now the way it works is hey we have somebody who's a representative at the Sig spec they collect some data they come back to us and they tell us hey we need to implement this other thing and sometimes you know we're not diligent about project management right we're IC who are doing this in our spare time right um so it you know we don't um we need to improve I think as overall um at being able to keep each other informed and tracking work got so making sure everyone's on the same page at the same time which I can see obviously is a bit of a it's daunting yeah um so I know you mentioned some of the sigs that you have worked with in the past what would you say are the things you interact with most right now right now it's the I almost exclusively participate in the Ruby Sig um I don't have a lot of interaction with other sigs at the moment other than through an occasional issue or a discussion that'll pop up or in slack I'll jump into a channel and say hey I've run into this problem um or I have this question for y'all can you help me um and so um I've limited interactions now these days with with other s and uh just recently I'm meeting the end user Sig at uh cucon um so I'm going to probably get more involved in the end user Sig now that I have this special invitation from you so yes oh I'm so excited um and actually this brings us to our turn the table section where you get to ask me or anyone on the call questions and we'll see if I can answer them or if anyone else can answer them okay so uh for me re it's like that's how I got involved but maybe you know a special sort of formula or the best way to have someone who's new get involved in particular in the end user Sig to provide feedback or or how do they get involved in other s that that you've been involved in oh for sure so there's a lot of different ways to contribute right I think a lot of people for them when they think contribute they think it means oh I got to open PRS I got to be writing code um and that's not necessarily the case there's so many different ways it can provide feedback and you know like one of the ways you mentioned was um you just started out by going to a Sig meeting and sharing some um stuff that you're seeing that is an absolutely great way to contribute to the project and that's what the N Sig is all about well one of the things we're all about is gathering feedback to give back to the appropriate Sig so we can help improve things um so to that end we do things like surveys um open solary Q&A interview sessions um where we kind of dive deeper into the adoption implementation process and find out your pain points and specific feedback um you can contribute blogs maybe love writing documentation is a great way um and probably if you are not already a part of cncf slack instance I would join and I just realized that I did not include that link but um we will have it in the show notes and you can once you are a member of the CNC of slack instance um you can scan the secure code and join the otel Sig end user Channel and we will be happy to direct you um if you have questions about you know your implementation or you just kind of want to know hey where do I get started um I'm trying to do you know we are happy to help um direct you and we currently have a survey going which I also just realized I did not share the link I'm so sorry I'm gonna see if I can get that right now Henrik um and let's see so yes join the Sig meetings um where can they access the calendar it's through the the community Repository has links to the calendar perfect yes so if there's like a specific language that you are already working in um check out the calendar to find out where uh sorry when the Sig meets and just hop on and you know when I first started I was like I'm just GNA check out you know a few different sigs um I went to the python Sig and everyone there was so welcoming and so nice I just like I was so intimidated at first and then I was like oh wow these people are so lovely and that's been my experience with like almost yeah I mean not almost but every Sig meeting that I've jumped into is everyone is so open to answering questions and they want your help you know they they want to help you help them um so I think one of the things is just you know letting people know that it doesn't have to be contributions of course if that's what you want to do and are able to we would love that um but there's so many other different ways um feedback doation blogs doing interviews um surveys and yeah so you know similar related to that is like you know you're saying that hey you can contribute in all these other ways what kind of expectations of commitment do you have from folks who want to participate like um do they have to show up to every single meeting do they have to constantly be reviewing PRS do they have to review you know issues um if they miss a meeting are they like kicked out of the group never to be invited again you know got it I think so for the end s specifically we are a pretty small group right now um so we are always happy to you know if anyone wants to hop in and um if you're an end user and you're like yeah I I have some stories to share perfect please reach out to us in the Sig Channel um we would love to work with you to get something scheduled and talk to you a little bit further um and as for the expectation uh commitments that you asked about I I mean I would hope that you could you know join at least once one Sig meeting a month just so we can you know connect virtually um but even if you have a hard time making this thing meeting because of like other commitments or time zone slack is a great place to get in touch with us we are totally happy to um Converse async over slack um so don't worry about you know if you can't make the meetings um or every meeting and as far as what's needed I think it just depends on the priorities of each individual Sig um for us right now it's really just we're working out we're working on building out um our YouTube channel for instance um we are still working on building out some of the end user resources we have available which there is a link to here um you can scan to see to to learn more about some of the events that we do um but yeah time commitment wise you know we have some contributors that hop on as they can so we'll maybe see them like you know a few times a quarter and honestly that's fantastic you know we don't need someone to be of course it would be great but um if you have obviously your day job uh you know it's taking up most of your time like we completely understand uh you know we would just love to see you coming as you can so don't worry about having to yeah get a number certain number of PRS um each month or hit a certain number of meetings um yeah it's it's like every little bit counts right every little bit helps so if you find yourself interested in trying to contribute back or you don't know where to get started or you feel just like intimidated just know that all of us um we do our best to make the most welcoming communities possible in our sigs and I love that the end user Sig um creates these opportunities for folks to come and just provide feedback a space for them to say like hey I'm having trouble with this or I like this or this was a great experience for me um that's this this part has been very enjoyable for me to be able to share our my personal experience and our my team's experience with you so um uh I really hope that folks who are watching at home hey come on in the the water's fine don't worry and we have if you didn't host him we have booies we have floaties we got all kinds of gear yeah to to help you get started right so it's just a it's the little things every little contribution can help us um if it help if your contribution is there to help you it'll help out the community so um come if you're having trouble a specific problem I guarantee you at least the dozen other people are having the same problem if you don't understand something guarantee you probably at least a hundred other people don't understand it either so for sure for sure so I really appreciate you um answering my questions here oh you are so welcome um and I guess I kind of put this at right about time um in case anyone has any last minute questions we'll hang out here for another minute and just kind of chitchat um and give people an opportunity to share if they would like um otherwise I will just ask about any fun Christmas New Year plans yeah we're going to be just spending it with family so that's going to be really nice um and uh we're going to be traveling so that might be a little bit hectic uh we have to travel uh we're going to go visit my father but uh we are traveling on Christmas Eve so that should be fun right I'm going to show up there and we'll be seeing U poran Christmas carols um at the top of our lungs and waking up the neighbors as soon as we arrive so uh totally fine for me you know oh my gosh how about yourself whoa whoa whoa whoa whoa what are these Puerto Rican Christmas songs and where can I listen to them oh okay you can look at them on on YouTube on Tik Tok they're all over the place but um they're uh um in Puerto Rico you know folks and in other Caribbean island folks will join and make uh something called the trya and have a paranda which is a party basically and it's like a Katamari ball so you go from house to house knocking on doors and then you start singing these songs and the songs are like hey open up the door I'm here to say hello oh you turn the lights on I can see you in there let me in and then you know you have to feed the guests and give them um Beverages and stuff and then um what you do is you take those people whose home you invaded that we call it a we call an asto which is like we show up and um we uh I don't want to the direct translation is more like you've been Stu we're sticking you up we're holding you up but uh we we break into your home and then we bring you and add you to the Kari ball and we move to the next house and we come and you know we'll sing and stuff like that so uh it's really great um it's a lot of fun and so we're looking forward to doing that with our family that sounds so fun what is that um what did you say that was called uh the there's the it's a couple of words the trya the the aalos those are the songs that we will sing and the asalto like the we're we're showing up uninvited that's so that's lovely honestly that sounds fantastic it's a lot of fun it's a lot of fun I'm so excited you have to you have to share pictures I will thank you um and uh maybe videos too who knows what kind of trouble we can get ourselves into yes oh would love to have a video of you singing one of these songs uh there are videos of me on the internet singing songs but not not now uh ree what are you gonna be doing this uh this holiday season I am going to be at home busting out hopefully a bunch of house projects that have been on my to-do list for three years nothing like it right you're going to get to the tail sampling project right that want oh my gosh I have yeah but I'm I'm really excited about it because yeah I moved into this house uh about three years ago and it's like you know 85% there there's still are just don't look in the cabinets that's a project is I need to organize all the inside stuff yeah and that's the thing I tell people all the time it's like hey don't look on the inside of the of the repositories you know you may not know you may not like how the way things are arranged but you know I know where things are so that's totally fine well all right thank you so much ARA you were a fantastic guest um I'm really excited to learn more um as I said I did have some fla questions so I'm sure I'll be slacking you to learn more um even more about um some of your adoption processes as well as um some of the other components you mentioned you had used such as the OCB um but yeah if anyone wants to reach out to either of us you can find us on um in cncf slacks otel Sig and user Channel um again this link is up here um for you to scan and I believe we will have the information in the show notes as well which will be posted I think right after this so yeah thank you so much for joining us and we look forward to seeing you all again next time yeah thank you for having again Reese this was really awesome thanks everybody for watching uh wherever you are and whenever you are um and I hope to see yall in a Sig room sometime oh yes and happy everything to everyone adios [Music] +hello everyone welcome to a brand new episode of oh tell me and end user Q&A if you have been to one of these sessions in the past you might notice that this looks a little bit different and it has a bit of a different name that's right we have done some growing up since the last few times and we're very excited to debut this new look um but don't worry almost everything else is going to be the same we have a great interview here for you today to learn from um but before I introduce myself and our guest I would love to know where everyone is connecting from I am online from Portland Oregon today and would love to see where um people are from it looks like someone is here from Brooklyn New York so thank you so much for joining out in New York and so my name is reys I am a senior developer relations engineer at New Relic I also co-lead the ner Sig um which is our cute little logo that you see um up in the right corner and yeah um at the end of your Sig we really focus on connecting directly with users and we are dedicated to helping um the rest of the sigs um get feedback from end users so they can help improve the project um and to that end this is one of those uh events that we host to do that so we are going to have um Ariel on Ariel if you would like to join us and introduce yourself hey how's it going everybody um Ariel Valentin I'm a Staff software engineer at GitHub working on observability thanks so much ree for inviting me here to chat with you today as absolute honor to be the first in the new format also so um you know big shout out to you know everybody who's uh done work to try to get this um this uh new platform up and running so thank you oh no thank you for being here I'm really excited um so just to get you and everyone up to speed on the format um it's similar to the ones if you've been on one of these before but we're going to start with some warmup questions where we'll kind of massage Ariel into being comfortable and then we'll hit him with some meaty questions and then we will actually um also get into questions more around the open c community and where he'll have an opportunity to share um like feedback about um his experiences with contributing um using the project stuff like that and then he'll get the chance to ask us questions as well um in a little section we call turn the tables and the very end if there are any audience questions um oh actually scratch that if you have questions that come up during our conversation feel free to put them into the chat if you're watching from YouTube then just put them in the live chat and then I think LinkedIn will also have its own chat so put in your question there and we will get to them as we can throughout the conversation um and then at the end if you have questions at the end as well then we will get to those but yeah if you have any questions that pop up as Ariel as telling us his story we definitely want to get to those as they come in all right so I think we are good to get started so adiel ad told us a little bit about your role of the company um how did you get started with observability how did you get started with open Telemetry oh those are great questions I mean I I should have mentioned that I'm here in Austin Texas Sunny Austin Texas uh so you know where I am in the world um I think a lot of us started originally um uh and I don't think my experience is unique is working with sort of like APM tools uh which are vendor proprietary tools uh generally speaking that uh didn't have distributed tracing in place um at that time and as the community evolved and open tracing became a standard uh that was my first experience with working with distributed tracing was uh the open tracing uh um specification and uh working and trying out different vendors different experiences with uh open tracing and then um when by the time that I had gotten to GitHub um I was a champion for distributed tracing because I saw the power that was in there and around that same time folks were already moving towards developing and transitioning away from open tracing and open sensus to open Elementary so I got really uh involved very early on in trying to spread the word and learning more um and working with uh my obser my team which was the observability team to uh start to adopt um tracing more to embrace it more and to make otel sort of our North Star um for all of our Telemetry signals I feel like I keep saying the word Telemetry over and over again I'm going to have to find yeah me a lot of that we're both going to stumble over over open Telemetry at some point or distribute tracing at some point it's all right this is a safe space what um okay so you mentioned you became a champion for distribute tracing um and you know I think something that's interesting is you know we're so ins scon in the world of observability that we at least I you know tend to presume everyone understands what that is um and it's always surprising to me when someone's like what's distributed tracing and they working um what so kind of along with that what um do you think that's kind of a main Challenge and you think most organizations organizations face when it comes to observability is not just understanding the value of distribute tracing and yeah because these are all sort of new Concepts to folks right uh you know folks have their different levels of understanding of the tools that are available to them and so there's all this vocabul that you hear that's a little bit hard to to parse through so sometimes it's like oh when you say Trace do you mean like a trace log log level is at the trace level uh when you say Trace do you mean the samples taken from a profiler when you say so there's a lot of this sort of uh language that um even though we're we're uh converging on a lot of this language and we uh have these dictionaries that are defined and published everywhere everywhere there's still sort of like this uh there's a hump that we have to get over or like a challenge that we have with trying to get everybody speaking um um the same language within the same context right very similar to in domain driven design we have these bounded context where the same term means a different thing you know that that flows over into sort of our domain language when it comes to observability and Sr practices um and and I'm sure many folks have faced that the those challenges as well it's kind of like let's all get on the same page about what we mean right oh absolutely um what are currently some of the most interesting problems that you are facing in your role oh well I mean one of those things is uh is that transition right um it's really hard for an organization like us who's been around for a long time um and I say a long time but you know whatever it is 10 years um where the system has grown and involved there have been Acquisitions that have been brought together uh there's disparate kind of uh uh backends where we're collecting this data um it's it's uh trying to transition from one way of doing things to a new way of doing things right so it's learning something new that's always going to be a big Challenge and you know folks ever are are trying to do their job every day they're not trying to learn new sdks or trying to learn new vocabulary or trying to learn how to be um proficient in their back ends what they're trying to do is keep the system up and running and keep our customers happy right so I think you know those are some of the the the challenges that that we all you know I know I face it I'm sure others do which is um is is making these transitions with the fewest pain points as possible with like trying to avoid these paying points um and there's I mean there's so many more we can go on and on Reese but I I think right now that's kind of like one of the biggest challenges for adoption I think overall that's actually um that's a great segue into the mey section so what was the process like for GitHub to adopt open Telemetry so uh for us it was um the the role of the advocate right so I acted as an advocate and was a champion for otel at the company and I specifically was brought in to help Advance the mission of tracing and uh it was something that I had pitched and and also sort there there were uh other challenges that we faced too which was hey let's get everybody um uh building a data dictionary let's get everybody uh agreeing to the same language when it comes to what our attributes were going to be right because as you can imagine as a system of walls or Acquisitions or other teams are rolled in everybody has their own uh you know log attributes or their own metric attributes or whatever it is and and so we started to I said look here's a Northstar right here here's semantic conventions from otel let's anchor onto this and let's follow the rules around semantic convention so that we can all uh build up our own internal dictionary right and that that comes with its own challenges right scheme of migrations and trying to keep up to date with the spec and you know what the uh instrumentations are doing right and um and as an advocate you know was also uh a lead on the roll out so one of the things I wanted to do was sunset all of our old sdks and move over to our open source sdks um and you know myself and a few other members for my team we all were involved in um otel Ruby for example because you know U we're a big Ruby shop uh but uh we got involved in there to help the instrumentations get better and to help test different um releases of the SDK and get that rolled down into our GitHub monolith so um we you know were very like I said very early adopters ran into some challenges and um and and continue to give feedback um to the community that way as a and I am not only an end user but like I said I'm also a maintainer so um I'm playing multiple roles there you know trying to help the community along yeah that's a really interesting position to be in is as end user and contributor and I definitely want to Circle back on that when we get to the OA community questions um tell us about the architecture landscape at GitHub and the Telemetry that you're capturing sure so um I've got this little slide that I put together in mer marked down so it's not anything official we can bring that up now so we can take a look at it um and I imagine that for a lot of folks out there who are do working um with uh um virtual machines where they're running system D units or if they're running kubernetes we have this this uh uh this uh we have different deployment Styles here but some of our main workloads are uh running in kubernetes and the way uh that we've got everything set up you could see here is as part of our kubernetes cluster you know we run our own custom uh meshing grass um and on every worker node um or um um on the worker nodes we're running a deployment of the open telemetry collector which we uh build using OCB so we have our own version of The otel Collector that is specific to us uh we chose that route because we wanted to ensure that uh we had the most secure uh build possible with the minimum number of dependencies and we also wanted the ability to build our own custom processors so that um we can uh address any issues that we might have that are specific to our needs um in inside of each of these pods you know we also have a a mesh side car and so Ingress traffic is going to come into the hotel collector otel over H ltlp over HTP um and if you have another application that's running your service it's shooting over um OTP traces over to the over the the mesh and sending it to our oel collectors which we then from there generate span metrics and um sample traces and send those off to a SAS provider that has a um where we aggregate all this data um and then on each individual worker you know we're running in this hybrid world right now um we're only um leveraging oel for traces and for generating um a span metric data uh we're still living in a world where we're using nonel non-lp formats for collecting things like custom metrics system metrics and for collecting logs um we have uh on each one of our worker nodes a metrics agent um and that can speak various different um um protocols but um mostly it's either using open metrics or it's using statsd uh to collect uh data and aggregate it and send it off to the SAS provider um in addition to that on all of our worker nodes we have a fluent bit running and that's what we're using to collect our logs um and all of our logs are end up getting streamed out through Azure event hubs which are uh processed by a bunch of consumers and then sent off to our log Store and search system um in addition to that you know we in addition to these uh kubernetes workloads we have our own virtual machines where we run systemd units you have to think about you know like the git file system Services um those are all running on systemd and we have the metrics agent and fluent running there we don't yet have the otel collector deployed there but um that's where we want to get to we want to get to a world where essentially our entire platform is running um an open source software the open Telemetry collector and um we're converging on OTP for all these formats uh so those services are also using statsd open metrics uh flu bits pulling things out of Journal D because we you know um um that's where we're streaming all of our our logs of the system Journal so um and all that stuff again goes right through to the same channels and it all gets aggregated in these places where we do our work um a little bit about some facts about sort of our our uh Trace data is that um we've rolled out the otel sdks for different programming languages we have them for Ruby goang uh JavaScript nodejs net we had some experimental uh uh rust usage but um uh uh we've had to scale that back um and we had some Java experiments that didn't pan out we didn't roll those out uh to production um and a lot of the stuff that we do when we we started before any of the automatic instrumentation was available for some of these languages so we're still deploying them through rapper libraries so we maintain a set of rapper libraries you install those it gives you the sort of the what we consider to be the minimal defaults um that we require for for our needs and uh we'll do periodic updates of those through dependabot right so it's like as we roll out new vers versions of those things um I feel like I've said a lot so far and I don't know ree if you had any follow-up questions for me or I actually have so many but oh okay great um so you mentioned and um so you mentioned you're leveraging open for traces right now um and then you talked a little bit about you know um the plans and some of the stuff that you tried um and I was curious to find out more about um is that mainly because if you're primarily a ruby shop and you know the SK maybe the signals for metrics and logs aren't quite as mature yet is that the reason or that's one of the reasons so you know one of the other challenges that I didn't discuss when you asked about adoption challenges is even though oel says hey this is what the signals look like this is what the semantic attributes are like this is um there's a lag between the time that something is published or declared in the oel spec to the time that um vendors or open-source platforms are able to leverage those things and there's sort of like this feedback loop as we go through otps and say hey we want to try this new thing out we do a couple things in experimental languages we'll do this stuff and um you know it's it was a a big enough challenge for us to start to roll tracing out that we didn't you know and adopting semom for logs that we didn't want to take yet another thing on which was to say okay now we're going to switch over to Native OTP metrics as well um and like you said some of the sdks are ahead than others so with Ruby we're just recently you know with the help of our amazing maintainers um Schwan and um and AAA they've been working diligently to get the metric SDK uh up to speed for Ruby so that so that's something that wasn't available to us to use um and I think that one of the the biggest advantages that I that that I see in tracing is the ability to generate span metrics from traces it's like hey the the the less things that we have to impose on our users for them to try to figure out how to do the better for us right and are you using the spam metrics connector is that what you we're using a custom connector right now I'd like to be able to get us to the point where we're using a span metrics connector but we don't have egress in OTP right now we're we're still doing sort of like vendor proprietary formats for uh for export where I want to get to is you know that's the strength one of the biggest strengths of the of otel is OTP is that standard format that we can that makes things portable for us absolutely so that's definitely sort of high on my list of things that I want gotta okay so it sounds like the plan is to migrate your uh other signals over once those reach GA in languages oh certainly and you know once I have more time on the calendar too right we have so many projects that we have to do as engineers and companies and it's like hey where do we fit these one these in right um uh as you imagine you know GitHub is constantly growing every day um we just hit 150 million users uh you know so it's like just an amazing um amazing growth of the company so shout out to all the people at GitHub who've been working so diligently to make this happen you know um and so we're here to support we we're here to support that volume growth and support our end users so um um so I hope that answers your question absolutely um and also want to mention again real quick for those um who joined us a little bit later feel free if you have questions that come up um you want to learn more about something that Ariel has gone over feel free to pop it into the live chat of whatever platform you're watching from so whether it's YouTube LinkedIn um go ahead and pop the question in and we will try to get to them um throughout the show as you can um so I noticed on your second slide you mentioned probabilistic sampling so it sounds like you're not doing any kind of tail sampling no not at the moment so would you mind bringing that up again uh um L the second slide yeah that's awesome yeah so we started with probabilistic sampling one of the hard things about tail sampling um as you can imagine is that um the traces all have to go to the same collector in order to make that decision if you're using the collector for uh for for tail sampling so um right now we wanted to reduce that burden in complexity immediately um and we um when we started with probabilistic sampling some of the things that we want to do in the future is um more advanced remote sampling uh so that we can have um uh more Frank grain control of what we're looking at we want to be able to do leverage uh tail sampling rules but as you know that's very difficult to implement a little bit difficult to scale in our case because you know we have about 2,000 collectors which are supporting everything right now um across all of our our Fleet right of like 14,000 hosts or something like that and I'm making that number up off the top of my head I think that's the last number we had um and you know right now we're we're doing about 26 million spans per second and uh just yesterday during our peak times you know we hit our alltime high of 32 million spans per second as we continue to um uh to grow our volume so this is one of the the challenges that we've had and so on my list of all the things that I want to do tail sampling is definitely on there you know what I mean so okay at what point do you think you'll you might get to the point where you could Implement heill sampling would be more like side when the you know processor has been uh more develops or devs it's you know it's it's uh I don't know how to answer that question because it's on the pile of the list of things that I want to do towards the bottom so it's like when can we do it when I get to all my other wish list items got it got it um and although I am curious about the other but that's okay that's all right questions that I want to get to as well sure um also just going back uh you mentioned trying some experiments in like Java and Russ I didn't kind of pan out and I was just curious um what was it that didn't quite work out or that you know didn't meet your expectations well uh We've reduced the number I'm sorry um all the jvm people out there we've reduced the number of java workloads that we've uh that we're running um so those have been migrated over to different programming languages um so that's one of the reasons why we didn't go through and say oh let me continue to roll out uh jvm work like the we just didn't have the return on investment that that we wanted to you know uh try and um also like we don't have a lot of um uh the same thing was for rust is that we have like a very limited set of applications that run rust and um those are perform for performance reasons they ran into some challenges with um how it impacted their latency when they introduce the use of the SDK and I'm going to be honest with you me not being a rust expert and being able to get in there and get my hands dirty to try to help out with addressing some of those problems and Reporting them Upstream that was something that we had to put on pause cuz again it's like one program versus all of these other services that are running and go and Ruby and um and JavaScript that we need to pay attention to right and um so that that I think that's really where it was where sort of like a critical mass of application services that use these programming languages and was like hey we have to focus our attention ption on those that are going to get us a higher return on investment for this absolutely okay so what was oh yeah what was github's observability tool before migrating to I think you mentioned um you were using proprietary sdks uh yeah I mean we were use proprietary SDK for open tracing so you know open tracing was our uh how we collected traces before um but um generally speaking the I I I I think I covered this in the first slide GitHub is a huge uh sort of statsd metrics user still to this day um and um logs are a big part of uh of what we utilize here so it's like folks are looking at acception stack traces a lot of the times to try to understand where errors are coming from they're looking at um uh access log streams um and they're trying to and they were trying to piece together Hey where's the request going and where is it slowing down and this and that and I showed up with my you know magic Tool Distributor tracing I'm like hey look here's a here it is on the flame graph or an icycle graph or here's a waterfall view of this thing and it's like oh that's pretty cool and um people started looking into that as uh as a an additional tool in their tool box for them to help try to debug things during an incident um so I hope that answers your question there yes and kind of along those lines how have things changed since GitHub switch to open cemetry so uh We've what I'll tell you is that uh just this year alone we've seen a dramatic I wish I had these statistics off hand but we've seen a dramatic increase in the number of people using tracing and a number of services that have been instrumented part of that comes from the fact that I was like hey everybody we're all moving to this new SDK so everybody install it and let the pendot go ahead and do these installations on your on your apps and people started seeing the value of this uh of this investment as well so they started to use it more um but we um I want to say that like when we started this adventure uh only about uh 80 and I'm going to use the word services in air quotes only about 80 Services were uh instrumented and now uh we're closer to about 300 of those Services which are effectively like kubernetes deployment types or like system D types uh because as you imagine our monolith has th you know uh maybe a thousand services within it so there're like subservices in there um but uh the but the monolith itself has broken down into like eight Services as like you know web UI and API and graphql and background workers and stream processors and whatnot so um that's why I put them in air quotes as these are Services um but we saw that that that was quite an increase in the number of services that that came in we were doing something like uh uh like 5 million spans per second to now we're up to 32 million spans per second it's just a huge uh uh increase in volume in usage and along with that volume increase in usage and data volume um how would you say that's impacted how your services are running how um you know quickly your team is able to debug issues as they arise part of it is an education think so a lot of things that uh so I mentioned that I'm on the observability team but really we're two groups uh one of the groups is called The Experience team that works directly with teams and they're they operate in a role sort of like developer relations and advocacy but also working on um you know cost control and improving your experience uh helping you with identify new workflows and introduce them into uh your incident command experience um those those those teams set up sessions education sessions to bring folks on board uh but we try to do them not during an incident um because that that that puts some stress and so what we try to do is like you know we'll go into the if an incident is happening it's like hey here's some insights that we're gaining and we'll share with you that we're seeing uh from the traces that you might not be able to see you somewhere else and that has helped in a lot of cases uh uh where we couldn't exactly pinpoint what was going on uh but then um there's also these spots where not every system has been instrumented so we have to rely a lot on sort of like client metrics or client Trace data and say hey look this client is experiencing this problem can we take a look deeper at this other service that hasn't yet been instrumented um and so it's been sort of I'm going to say like this mixed results some teams have like identified issues even before they go out out to production and other teams have been able to leverage it when it's like oh this we're having an incident right now oh here's uh uh and and here's the here's the reason why this is failing um and and we've able to do things like identify bottlenecks and like mistakes that you know little little coding mistakes oh I forgot to act the message before I pulled it out of kavka oh I'm just retrying that same message millions of times or um or oh the client timeout doesn't match the server timeout so the client out and the server is continuing to turn along to try to do this request so we're identifying things like that that we couldn't identify before easily easily you know um and so those are some anecdotes oh no that's awesome I think I will probably be asking you more about that because I think this is really interesting um one more question from our mey section what would prevent you from implementing better Telemetry what would prevent me from doing that I think it's because there's so much of the stuff that's in early stages and you know we were early adopters on a lot of things but then there's a lot of stuff that's a lot more risky for us to try to roll out so for example one of the things I'm really excited about when I came back from cubec con is the continuous profiler I mean that is one one of the when you talk about one of the things on my wish list that we'd be able to use today if we could uh that's the thing that I would want to do but because we're still in the early stages of the of the profiler and the specification and there's still uh some turn with the data model um I think that there are when it comes to the resource intensive or sort of like introspective tools like that it's I we don't want to take that risk going a little bit too early to adopt those tools uh because also there's not a lot of support from our current U vends and vendors that will be able to leverage that it's kind of like we would be experimenting with that to go to nowhere and so that's kind of you know once vendors start to support the OTP profiling format um data model I should say and once the profiler is a little bit more stable I'd love to jump in there and and be able to roll that out a little more widely um and um I think that uh a little bit of a a a bumpy road for us to still is migrating um semom from pre 1.0 because we were early adopters of semom so we're at this pre 1.0 stage and sort and getting ourselves to migrate towards a a 1x version of of semom is uh another big challenge because we've already sent all of this data out it's our backends there we have to figure out a way to upgrade it or to say you know oh this is version X um and and I and I kind of it feels like a lot of the progress was stalled there for the moment um so that there I don't know that there's any real solutions in um available that would say okay we can start migrating these schemas over and keep our user experience very high quality um if that answers your question oh absolutely I think that is another interesting topic migrating some semantic conventions so I might I might jump back if we have time later but I do want to get to some of these Community questions yes ma'am um yeah so one of the big things I think um you know working worked in the community that uh probably you've seen you know I've seen sure um is people want to contribute to the project but they're not really sure where to get started and yeah we have like resources such as you know I think in the documentation we added you know getting started um and you have you know various uh Sig channels and cncm slack um and I think we all have the problem of how do we how are how do we reach these people because you know there's still some many I don't know so what was the contribution process for you and your team like to open cemetry for us it was um uh it was a a short process with a lot of observation so the things that we looked for uh myself and my teammates at the time when we approached the community we were looking at code of conduct what were the expectations of the code of conduct we went through and actually looked at issues previous PRS the feedback that was in those PRS to see if the behaviors expected in the code of conduct were reflected in the PRS and in the issues uh cuz there's nothing worse than you wanting to be part of this community and finding that those two things don't match and to see that that happened then we um participated in the sigs which was very nice for us to be able and convenient for us in the US because the Ruby Sig was in the US hours we joined those meetings during the day and started to provide some feedback as end users and say hey look we ran into this challenge if we identified something that was a challenge for us we would could contribute a PR um and the maintainers were very generous with their time um did their reviews and um and we were able to get a couple of Uh custom propagators uh merged we got some instrumentations merged we identified other uh gaps for the instrumentations that we use cuz one of the biggest challenges I think as a maintainer is there's so much that you have to do right you've got the SDK you've got the API you've got documentation to do and you have to have language specific instrumentations and uh that's daunting especially with all of the popular libraries that are out so for us it sort of we contributed back to the libraries that we use heavily right we have like a subset of libraries that are very popular at GitHub and um sort of blessed right so by by their popularity so that's where we focused our energy and our contributions um and then we reached out to people you know uh anytime somebody comes to me and says hey I have this challenge with this thing it's missing this attribute or I'd really like it to work like this or I've identified this bug the first thing I do I say is let's open up an instr and let's work on this together and I'd love to review your PR and creating in um an environment that makes it lowers a barrier to entry to have folks uh uh collaborate and submit contributions I want them to feel like this is yours this is ours this is not mine and I'm gatekeeping right we still have some standards for Quality obviously and some expectations from you as a maintainer but we try to make this an a painless experience for you um at least in the Ruby uh um um community so that's what the the contribution process was like for me and getting involved uh in the community that's awesome so you started basically joining the Sig meetings as an end user and sharing feedback and then finding information about like how to you can open uh you know an issue and then like from there building custom stuff and uh helping the community or sorry the Ruby s to build out more of their components okay yeah certainly and like you know with the home for open source right so uh for us um you know we open sourced uh for example new database drivers in Ruby for um MySQL right um and so we were the we had that instrumentation in house before all of that became public and as soon as we released that new driver I went right to the Sig and said Tada I have a donation for you and it's like oh yeah we able we were able to pull that in there so um it's this thing where um there's just this Spirit of collaboration right and this Spirit of supporting open source that um that's important to our mission right okay so it sounds like you had a pretty kind of positive experience it sounds like from for contributing to Ruby um I like to know do you have any feedback to the SS on ways to improve how things work based on your experiences with Ruby and then other sigs I know you mentioned also some other components that you've used but we'll come back to that yeah for sure and I again I I want to thank everybody for their amazing work that they've done um contributing to the collector in particular uh because you know we are uh we release our OCB build or sorry our our uh our custom build every time a new version of the cont package rolls out so um every week depend on bot is telling us or every two weeks it's telling us hey it's time to upgrade a new releases rolled out and uh we ran into uh some challenges where we identified some performance issues um in the in the collector and we uh worked with the team provided profiles provided feedback uh showed them our configurations and they were so responsive to um our concerns and uh they were happy to see that we were able to contribute back just in providing feedback just giving them actual data of production workloads which is very difficult to do right as a maintainer it's really hard for me to know hey what's going on in your um in this customer's deployment or this users uh a deployment we can't replicate it in our in our own environment so um uh that Spirit of collaboration was um was really great and that's the thing that I'd like to see uh happen in all the other SE I have limited experience uh with other sigs the other uh part was contributing to semom um and I contributed to simcom trying to uh introduce some new attributes uh but there's just so much turn in that repository that it was hard for me to keep up with uh changes and deprecations right um and so that was a little bit on me to be able to to keep up and find out that the stuff I had submitted was deprecated for example in favor of something else uh but it's it's those kinds of challenges that I like I I I have to find a better way for us to all be able to to to keep up with each other so one suggestion I might say is hey look a change happened here's an otop A Change happened in the spec that should automatically open issues for every maintainer repo and say Here's a new thing that has changed uh um this is a new uh feature we expect that the SDK should support and that way we're able to track it because right now the way it works is hey we have somebody who's a representative at the Sig spec they collect some data they come back to us and they tell us hey we need to implement this other thing and sometimes you know we're not diligent about project management right we're IC who are doing this in our spare time right um so it you know we don't um we need to improve I think as overall um at being able to keep each other informed and tracking work got so making sure everyone's on the same page at the same time which I can see obviously is a bit of a it's daunting yeah um so I know you mentioned some of the sigs that you have worked with in the past what would you say are the things you interact with most right now right now it's the I almost exclusively participate in the Ruby Sig um I don't have a lot of interaction with other sigs at the moment other than through an occasional issue or a discussion that'll pop up or in slack I'll jump into a channel and say hey I've run into this problem um or I have this question for y'all can you help me um and so um I've limited interactions now these days with with other s and uh just recently I'm meeting the end user Sig at uh cucon um so I'm going to probably get more involved in the end user Sig now that I have this special invitation from you so yes oh I'm so excited um and actually this brings us to our turn the table section where you get to ask me or anyone on the call questions and we'll see if I can answer them or if anyone else can answer them okay so uh for me re it's like that's how I got involved but maybe you know a special sort of formula or the best way to have someone who's new get involved in particular in the end user Sig to provide feedback or or how do they get involved in other s that that you've been involved in oh for sure so there's a lot of different ways to contribute right I think a lot of people for them when they think contribute they think it means oh I got to open PRS I got to be writing code um and that's not necessarily the case there's so many different ways it can provide feedback and you know like one of the ways you mentioned was um you just started out by going to a Sig meeting and sharing some um stuff that you're seeing that is an absolutely great way to contribute to the project and that's what the N Sig is all about well one of the things we're all about is gathering feedback to give back to the appropriate Sig so we can help improve things um so to that end we do things like surveys um open solary Q&A interview sessions um where we kind of dive deeper into the adoption implementation process and find out your pain points and specific feedback um you can contribute blogs maybe love writing documentation is a great way um and probably if you are not already a part of cncf slack instance I would join and I just realized that I did not include that link but um we will have it in the show notes and you can once you are a member of the CNC of slack instance um you can scan the secure code and join the otel Sig end user Channel and we will be happy to direct you um if you have questions about you know your implementation or you just kind of want to know hey where do I get started um I'm trying to do you know we are happy to help um direct you and we currently have a survey going which I also just realized I did not share the link I'm so sorry I'm gonna see if I can get that right now Henrik um and let's see so yes join the Sig meetings um where can they access the calendar it's through the the community Repository has links to the calendar perfect yes so if there's like a specific language that you are already working in um check out the calendar to find out where uh sorry when the Sig meets and just hop on and you know when I first started I was like I'm just GNA check out you know a few different sigs um I went to the python Sig and everyone there was so welcoming and so nice I just like I was so intimidated at first and then I was like oh wow these people are so lovely and that's been my experience with like almost yeah I mean not almost but every Sig meeting that I've jumped into is everyone is so open to answering questions and they want your help you know they they want to help you help them um so I think one of the things is just you know letting people know that it doesn't have to be contributions of course if that's what you want to do and are able to we would love that um but there's so many other different ways um feedback doation blogs doing interviews um surveys and yeah so you know similar related to that is like you know you're saying that hey you can contribute in all these other ways what kind of expectations of commitment do you have from folks who want to participate like um do they have to show up to every single meeting do they have to constantly be reviewing PRS do they have to review you know issues um if they miss a meeting are they like kicked out of the group never to be invited again you know got it I think so for the end s specifically we are a pretty small group right now um so we are always happy to you know if anyone wants to hop in and um if you're an end user and you're like yeah I I have some stories to share perfect please reach out to us in the Sig Channel um we would love to work with you to get something scheduled and talk to you a little bit further um and as for the expectation uh commitments that you asked about I I mean I would hope that you could you know join at least once one Sig meeting a month just so we can you know connect virtually um but even if you have a hard time making this thing meeting because of like other commitments or time zone slack is a great place to get in touch with us we are totally happy to um Converse async over slack um so don't worry about you know if you can't make the meetings um or every meeting and as far as what's needed I think it just depends on the priorities of each individual Sig um for us right now it's really just we're working out we're working on building out um our YouTube channel for instance um we are still working on building out some of the end user resources we have available which there is a link to here um you can scan to see to to learn more about some of the events that we do um but yeah time commitment wise you know we have some contributors that hop on as they can so we'll maybe see them like you know a few times a quarter and honestly that's fantastic you know we don't need someone to be of course it would be great but um if you have obviously your day job uh you know it's taking up most of your time like we completely understand uh you know we would just love to see you coming as you can so don't worry about having to yeah get a number certain number of PRS um each month or hit a certain number of meetings um yeah it's it's like every little bit counts right every little bit helps so if you find yourself interested in trying to contribute back or you don't know where to get started or you feel just like intimidated just know that all of us um we do our best to make the most welcoming communities possible in our sigs and I love that the end user Sig um creates these opportunities for folks to come and just provide feedback a space for them to say like hey I'm having trouble with this or I like this or this was a great experience for me um that's this this part has been very enjoyable for me to be able to share our my personal experience and our my team's experience with you so um uh I really hope that folks who are watching at home hey come on in the the water's fine don't worry and we have if you didn't host him we have booies we have floaties we got all kinds of gear yeah to to help you get started right so it's just a it's the little things every little contribution can help us um if it help if your contribution is there to help you it'll help out the community so um come if you're having trouble a specific problem I guarantee you at least the dozen other people are having the same problem if you don't understand something guarantee you probably at least a hundred other people don't understand it either so for sure for sure so I really appreciate you um answering my questions here oh you are so welcome um and I guess I kind of put this at right about time um in case anyone has any last minute questions we'll hang out here for another minute and just kind of chitchat um and give people an opportunity to share if they would like um otherwise I will just ask about any fun Christmas New Year plans yeah we're going to be just spending it with family so that's going to be really nice um and uh we're going to be traveling so that might be a little bit hectic uh we have to travel uh we're going to go visit my father but uh we are traveling on Christmas Eve so that should be fun right I'm going to show up there and we'll be seeing U poran Christmas carols um at the top of our lungs and waking up the neighbors as soon as we arrive so uh totally fine for me you know oh my gosh how about yourself whoa whoa whoa whoa whoa what are these Puerto Rican Christmas songs and where can I listen to them oh okay you can look at them on on YouTube on Tik Tok they're all over the place but um they're uh um in Puerto Rico you know folks and in other Caribbean island folks will join and make uh something called the trya and have a paranda which is a party basically and it's like a Katamari ball so you go from house to house knocking on doors and then you start singing these songs and the songs are like hey open up the door I'm here to say hello oh you turn the lights on I can see you in there let me in and then you know you have to feed the guests and give them um Beverages and stuff and then um what you do is you take those people whose home you invaded that we call it a we call an asto which is like we show up and um we uh I don't want to the direct translation is more like you've been Stu we're sticking you up we're holding you up but uh we we break into your home and then we bring you and add you to the Kari ball and we move to the next house and we come and you know we'll sing and stuff like that so uh it's really great um it's a lot of fun and so we're looking forward to doing that with our family that sounds so fun what is that um what did you say that was called uh the there's the it's a couple of words the trya the the aalos those are the songs that we will sing and the asalto like the we're we're showing up uninvited that's so that's lovely honestly that sounds fantastic it's a lot of fun it's a lot of fun I'm so excited you have to you have to share pictures I will thank you um and uh maybe videos too who knows what kind of trouble we can get ourselves into yes oh would love to have a video of you singing one of these songs uh there are videos of me on the internet singing songs but not not now uh ree what are you gonna be doing this uh this holiday season I am going to be at home busting out hopefully a bunch of house projects that have been on my to-do list for three years nothing like it right you're going to get to the tail sampling project right that want oh my gosh I have yeah but I'm I'm really excited about it because yeah I moved into this house uh about three years ago and it's like you know 85% there there's still are just don't look in the cabinets that's a project is I need to organize all the inside stuff yeah and that's the thing I tell people all the time it's like hey don't look on the inside of the of the repositories you know you may not know you may not like how the way things are arranged but you know I know where things are so that's totally fine well all right thank you so much ARA you were a fantastic guest um I'm really excited to learn more um as I said I did have some fla questions so I'm sure I'll be slacking you to learn more um even more about um some of your adoption processes as well as um some of the other components you mentioned you had used such as the OCB um but yeah if anyone wants to reach out to either of us you can find us on um in cncf slacks otel Sig and user Channel um again this link is up here um for you to scan and I believe we will have the information in the show notes as well which will be posted I think right after this so yeah thank you so much for joining us and we look forward to seeing you all again next time yeah thank you for having again Reese this was really awesome thanks everybody for watching uh wherever you are and whenever you are um and I hope to see yall in a Sig room sometime oh yes and happy everything to everyone adios diff --git a/video-transcripts/transcripts/2025-01-31T05:58:14Z-cfp-writing-q-a-livestream.md b/video-transcripts/transcripts/2025-01-31T05:58:14Z-cfp-writing-q-a-livestream.md index 50ee49f..6a66886 100644 --- a/video-transcripts/transcripts/2025-01-31T05:58:14Z-cfp-writing-q-a-livestream.md +++ b/video-transcripts/transcripts/2025-01-31T05:58:14Z-cfp-writing-q-a-livestream.md @@ -10,129 +10,170 @@ URL: https://www.youtube.com/watch?v=6SmO4yKjmCs ## Summary -In this panel discussion hosted by Dan as part of the OpenTelemetry End User SIG, experts in the tech conference arena shared valuable insights on how to effectively write and prepare for conference proposals, known as CFPs (Calls for Proposals). Panelists included Adriana Vela, Henrik Rexed, Josh, and Ree, who discussed various aspects of the speaking process, from choosing topics and overcoming imposter syndrome to the importance of unique storytelling and audience engagement. They emphasized the significance of networking at conferences, tips for finding suitable events, and the common experience of rejection, suggesting that aspiring speakers view it as part of the journey. The conversation also touched on the trends in topics such as AI and sustainability, with advice to be unique and relatable in presentations. The panel concluded with personal anecdotes about past speaking experiences, highlighting the challenges and lessons learned through live demos and audience engagement. +In this YouTube panel hosted by Dan, experts from the open Telemetry community discuss how to effectively apply to speak at tech conferences, sharing insights on crafting compelling call for proposals (CFPs). Panelists include Adriana Vela, Henrik Rexed, Josh, and Ree, who provide valuable tips on choosing topics, overcoming imposter syndrome, and preparing for presentations. They emphasize the importance of unique, engaging content and the value of end-user stories in tech talks. The conversation also touches on the significance of networking, the challenges of public speaking, and how to handle rejection after submitting proposals. The panelists collectively stress that while rejection is common, it can be a learning opportunity, and they encourage aspiring speakers to persist and refine their submissions. The session concludes with personal anecdotes from the panelists about their experiences at various conferences, underscoring the community aspect of speaking engagements. -# Panel Discussion on Writing Effective CFPs +## Chapters -[Applause] [Music] +Sure! Here are 10 key moments from the livestream transcript along with their timestamps: -Hello everybody! Good morning, good afternoon, or good evening, depending on where you are. I'm Dan, and as part of the OpenTelemetry End User SIG, I have the absolute pleasure of hosting this panel today. We typically focus on OpenTelemetry, as the name suggests, and we host end users discussing how they adopt observability best practices across the industry. If you want to know more, you can follow the QR code on the screen to get in touch. +00:00:00 Introductions to the panelists and the topic of discussion. +00:02:30 Explanation of what a CFP (Call for Papers) is by Adriana. +00:05:15 How to find conferences to speak at, with Josh sharing resources. +00:09:00 Ree discusses motivations for wanting to speak at conferences. +00:12:45 Overcoming imposter syndrome when applying to speak, with insights from Adriana. +00:16:30 Tips on selecting topics for CFPs, including the importance of uniqueness. +00:21:00 Discussion on knowing your audience and adjusting your talk accordingly. +00:25:00 The importance of catchy titles for CFPs and how they affect acceptance. +00:30:15 Advice on what to do after your CFP is accepted, including rehearsal tips. +00:35:00 Panelists share their horror stories from past speaking experiences. -This time, however, we're going to be gathering tips and tricks from true veterans in the tech conference arena. We’ll discuss how to write good CFPs (Call for Proposals), what topics to choose, how to prepare for a talk, and why going through the process of applying to speak at conferences is well worth it. If you have recently applied to speak at KubeCon, Observability Day in London, or any other conference, it doesn't matter if you made it or not; I think our experts will have advice that you can take home and use in your future talks. +Feel free to ask if you need any further information! -We want this to be an interactive panel, so if you're watching live on YouTube or LinkedIn and have questions related to the topics we're discussing, please drop them in the chat, and I will try to incorporate them into the discussion. +# Panel Discussion on Speaking at Conferences -## Meet the Panelists +**Dan:** +Hello everybody! Good morning, good afternoon, good evening. I'm Dan, and as part of the OpenTelemetry End User SIG, I have the absolute pleasure of hosting this panel today. We normally focus on OpenTelemetry, as it's in the name, but this time, we’re going to be getting some tips and tricks from true veterans in the tech conference arena. We'll discuss how to write good CFPs, what topics to choose, how to prepare for a talk, and why going through the whole process of applying to speak at a conference is worth it. -Let’s meet our panelists! If you’re watching live, I’d love to know where you’re connecting from, so drop a comment in the chat. I’m connecting from Edinburgh, Scotland, where it’s currently 5:02 PM. +If you've recently applied to speak at KubeCon, Observability Day in London, or another conference, it doesn't matter whether you got accepted or not. I believe our experts will have valuable advice that you can use for your future talks. -### Adriana -Hello, I’m Adriana Vela, connecting from Toronto, Canada. It’s noon here, and I work alongside Dan as one of the maintainers of the OpenTelemetry End User SIG. +We want this to be an interactive panel, so if you're watching live on YouTube or LinkedIn, please drop any questions related to our topics in the chat, and I'll do my best to incorporate them into the discussion. -### Henrik -Hi Dan, it’s a pleasure to be here! I'm Henrik Rexed, based in the sunny south of France. Even though it’s still 10-15 degrees Celsius at the moment, it’s 6:03 PM local time. I’m involved in a lot of observability topics and try to contribute to the End User SIG as well. +Now, let’s meet our panelists. If you’re watching live, I’d love to know where you’re tuning in from, so please drop a comment in the chat. I’m in Edinburgh, Scotland, where it's currently 5:02 PM. -### Josh -Hello, thanks for having me! I’m Josh, a Developer Advocate and Product Manager involved in OpenTelemetry, connecting from Brussels, where I’m here for FUM starting on Saturday. Looking forward to that! +**Adriana:** +Hey Dan, nice to see you! I'm Adriana Vela, connecting from Toronto, Canada. It’s noon here, and I work alongside Dan as one of the maintainers of the OpenTelemetry End User SIG. -### Ree -Hi everyone, I’m Ree, joining from Vancouver, Washington—not to be confused with Vancouver, BC—just 10 minutes north of Portland. I work in Developer Relations at New Relic and collaborate with these lovely folks in the OpenTelemetry Community, primarily as part of the End User SIG. +**Henrik:** +Hello! My name is Henrik Rexed, based in the beautiful sunny south of France. It's currently 6:03 PM here, and I'm involved in a lot of observability topics and I help out with the End User SIG as well. -## What is a CFP? +**Josh:** +Hello! Thanks for having me. I’m Josh, a Developer Advocate and Product Manager working in the realm of OpenTelemetry. I’m connecting from Brussels, where I’m here for FOSDEM, starting on Saturday. -Let’s dive into our first topic. Adriana, what exactly is a CFP? +**Ree:** +Hi everyone! I'm so excited to be here. I'm joining from Vancouver, Washington, not to be confused with Vancouver, BC, which is just 10 minutes north of Portland. I work in Developer Relations at New Relic and also engage with the OpenTelemetry community, primarily as part of the End User SIG. -A CFP, if I got this right, stands for Call for Proposals or Call for Papers, depending on the context. It’s essentially a request for a proposal for a talk at a conference. You have an idea for a talk, and you provide details about it. The specifics can vary depending on the conference. For example, applying for SREcon involves hashing out all details ahead of time compared to KubeCon, where you can be a bit more high-level. +--- -For me, a CFP is an opportunity to network and meet people. It’s a good excuse to reach out to a conference, and if I get accepted, I might even earn some air miles as an Air France member. +## What is a CFP? -## Finding Conferences to Speak At +**Dan:** +Let's kick things off with the first question directed to Adriana. What is a CFP? -So, if you're new to conferences, you might be wondering how to find the right ones to apply to. Josh, can you share your thoughts on this? +**Adriana:** +Great question! CFP stands for Call for Proposals or Call for Papers, depending on the context. It’s essentially a request for a proposal for a talk at a conference. You submit an idea along with the details, and the specifics can vary from conference to conference. For instance, applying to SREcon requires more detailed information compared to KubeCon, where you can be a bit more high-level. -Absolutely! A great way to find conferences is to use aggregator sites where many CFPS are posted. The big ones include SessionEyes, PaperCall, and PreTalks. You can also check GitHub repositories and Airtable databases that maintain updated lists of ongoing CFPS sorted by their due dates. +**Henrik:** +I’d add that for me, a CFP is an opportunity to network and connect with people at the conference. If I get accepted, it can also help me rack up some air miles! -When choosing conferences, I think the topic helps narrow down the initial list, but you also have to consider factors like travel distance and conference size. Speaking at a single-track conference as a first-time speaker is very different from a multi-track conference. +--- -## Why Speak at Conferences? +## Finding Conferences to Speak At -Ree, what makes you want to speak at conferences? +**Dan:** +If you're new to conferences, how do you find the right ones to speak at? Josh, can you share some insights? -One thing I discovered early on is that my job is so busy it can be hard to find time to learn something new. If I submit a topic and it gets accepted, it gives me the time to work on that topic. Another reason is the physical interactions and conversations that happen at conferences, which can be really enriching. Plus, I've made some great friends through this process. +**Josh:** +Absolutely! A great way to find conferences is to use aggregator sites that list CFPs, such as SessionEyes, PaperCall, and Pre-Talks. There are also GitHub repos and Airtable databases that maintain up-to-date lists of ongoing CFPs sorted by their due dates. -Adriana, you mentioned that for you, speaking is a bit of a challenge. Can you elaborate? +**Dan:** +Do you usually choose conferences based on the topic? -Yes, speaking is a way for me to push past my discomfort. Additionally, for those of us in underrepresented groups in tech, it's important to show visibility. The more we speak, the more we empower others in similar groups to do the same. +**Josh:** +Yes, the topic usually helps narrow down the list, but sometimes it also depends on the travel logistics and the size of the conference. Speaking at a single-track conference as a first-time speaker versus a multi-track conference can make a big difference. -## Overcoming Imposter Syndrome +--- + +## Motivation to Speak -Imposter syndrome is something many of us face. Adriana, how do you overcome that feeling of not being good enough to talk about a certain topic? +**Dan:** +Ree, what motivates you to speak at conferences? -You have to force yourself past it. If you don’t do it, someone else will, so why not you? You don’t have to be the expert; you can use this as an opportunity to learn. Sometimes having a newbie's perspective can be more relatable for the audience. +**Ree:** +I found that speaking at conferences forces me to take the time to learn something new, especially topics I want to dig into. Plus, the physical interactions and conversations are invaluable. The community aspect is incredible, and it helps me learn more while connecting with others. -Josh, do you have any thoughts on this? +**Adriana:** +For me, speaking is a challenge I enjoy, even though it can be terrifying. It also empowers those in underrepresented groups to share their experiences and show that we exist in tech. -Imposter syndrome never completely goes away, so you just have to push through it. Even if you're not the main contributor to a project, having a unique way of presenting your topic can still provide value to your audience. +**Josh:** +For me, it adds value to my role and gives me feedback from the community. It’s a great opportunity to gather insights from people outside my usual circle. -## Choosing Topics for Your CFP +--- -Let’s talk about how to choose topics to include in your CFP. Ree, how do you navigate this? +## Overcoming Imposter Syndrome -When I first submitted my CFP to KubeCon, OpenTelemetry was new, and I had success getting my topics accepted. However, as the space has grown, it’s become more challenging. I try to find niche aspects of OpenTelemetry that haven’t been covered extensively. +**Dan:** +Imposter syndrome can be a hurdle. Adriana, how do you get past that feeling of not being “good enough”? -Adriana, do you have anything to add? +**Adriana:** +You just have to force yourself to push through. Remember, if you don’t do it, someone else will. You don’t have to be the expert; you can use this as a chance to learn and share relatable experiences. -Yes, it’s essential to ensure that your topic is unique. AI, for instance, comes up often, and while there's nothing wrong with submitting a CFP on it, make sure your angle is different. Also, be cautious of project updates; they don’t typically get accepted unless they discuss a unique challenge. +**Henrik:** +I agree! It’s important to remember that you don’t need to be a perfect expert. Sharing your journey can be just as valuable. -## Audience Expectations and Depth +--- -Ree, how do you gauge the level of depth you want to present? +## Choosing Topics -Sometimes it’s tough. I consider how much background knowledge the audience might have. If the topic is newer, it may need more context, while established topics might require less. +**Dan:** +How do you decide what topics to present in your CFP? Ree, can you share your thoughts? -Henrik, any thoughts? +**Ree:** +It can be challenging. I try to find a niche, especially since OpenTelemetry is becoming more popular. Unique perspectives are essential to stand out among the many submissions. -Yes, knowing the audience is crucial. For technical conferences, you can usually expect a tech-savvy crowd, but adjusting complexity based on the specific audience is essential. +**Adriana:** +Yes, and it’s important to ensure your submission is unique. While AI topics are popular, they need to offer a fresh angle to be accepted. -## Post-Acceptance Preparation +--- -Now that your talk has been accepted, how do you prepare? Ree? +## Preparing Your Talk -I recommend rehearsing as much as possible. If you can, working with a public speaking coach can be beneficial. But even watching YouTube videos on public speaking can help you improve. +**Dan:** +Once accepted, how do you prepare for your talk? Ree, what’s your process? -Henrik, how do you prepare? +**Ree:** +I found that working with a public speaking coach helped me a lot. I also practice as much as possible to ensure I’m comfortable with the material. -I focus on ensuring my talk fits the duration and adjust content as necessary. I like to include graphics to make the experience smoother for the audience. +**Henrik:** +For me, I focus on timing and ensure I’m not trying to cram too much content into a limited timeframe. -## Handling Rejections +--- -It’s normal to get rejected. Josh, how do you manage expectations? +## Handling Rejection -It’s common to expect a 10% acceptance rate. If you want to speak at one conference, you might need to submit ten proposals. Rejections happen even to experienced speakers, so it’s vital to ask for feedback and resubmit your proposal. +**Dan:** +How do you manage expectations around rejection? Josh, what are your thoughts? -## Horror Stories +**Josh:** +It’s completely normal to get rejected. A good rule of thumb is to expect about a 10% acceptance rate. It’s crucial to keep submitting, and don’t be afraid to ask for feedback on your proposals. -Finally, let’s share a few horror stories from conferences. Ree, do you want to go first? +**Adriana:** +I agree. It’s okay to feel disappointed, but don’t let it deter you. Reflect on your submission and consider resubmitting to other conferences or tweaking your proposal for the next opportunity. -In my first talk about tail sampling with a live demo, everything worked before I went on stage, but it failed during the presentation. I still don’t know why! +--- -Henrik, any horror stories? +## Closing Thoughts and Horror Stories -Yes, once I experienced screen mirroring issues where my laptop wouldn’t connect to the projector. I had to skip my demo because the screen was blank, which was quite stressful. +**Dan:** +As we wrap up, let’s share some memorable conference experiences. Ree, do you have a horror story? -Josh, do you have a story? +**Ree:** +My first talk included a live demo that failed right in front of the audience. It was terrifying, but I managed to get it working later. -I had a talk where the audience could only see half of my slides while I could see everything. It was a lesson to never rely on notes again. +**Henrik:** +I had a demo where the screen mirroring didn’t work. I ended up scrambling to present without visuals, which was quite stressful! -## Conclusion +**Josh:** +Similar experience! The audience could only see half my slides, and I had to rely on my memory for the rest. -Thank you all for your insights and stories! This has been an enriching discussion, and I appreciate everyone’s participation. If you have further questions, feel free to reach out to us through the CNCF Slack or the resources provided. +**Adriana:** +I haven’t had any major mishaps, but I do prefer pre-recorded demos to avoid the stress of live presentations. -Thank you all for joining, and I look forward to seeing you at a conference! +--- -[Applause] [Music] +**Dan:** +Thank you all for sharing your insights and experiences! It’s been incredibly valuable. If you have more questions, feel free to reach out in the End User SIG channel. We’re always open to discussions about OpenTelemetry and beyond. Thank you, everyone, and see you at a conference! ## Raw YouTube Transcript -e [Applause] [Music] hello everybody good morning good afternoon good evening I'm Dan and as part of the as part of the open Telemetry and usus Sig today I have the absolute pleasure of Hosting this panel in the hotel end user s we normally focus on open Telemetry I mean it's in the name uh we host end user is telling us how they're adopting observability best practices across the industry if you want to know more you follow the QR code in the screen and you'll find how to get in touch however this time we're going to be getting some tips and tricks from True veterans in the Tech conference Arena we'll be talking about how to write good C cfps what topic what topics to choose how to prepare for a talk or even why going through the whole process of applying to speaker conference and how is well worth it so if you uh have recently applied maybe to speak at cubec con or observability day in London or another conference it doesn't matter if you made it or not I think these experts will have some advice that you can take home and use in your future talks we want this to be an interactive panel so if you're watching live on uh YouTube or LinkedIn and you have any questions related to the topics that we're discussing please please drop them in the in the chat and I will try to incorporate them into the discussion if best I can okay so it's time to meet our panelists um and uh if you're watching live on YouTube or LinkedIn I would love to know where you're watching from so drop a comment in the in the chat um and tell us where you're watching from myself I'm I'm in Edinburgh Scotland where is currently 5:02 p.m and so um let's meet our first panelist Adriana um hello uh can you tell us where you're connecting from and a little bit about yourself hey Dan nice to see you uh my name is Adriana Vela I am connecting from Toronto Canada it is noon here and um I work alongside Dan as one of the maintainers of the otel and user Sig thank you very much and uh Henrik hello Henrik hey pleasure to see here pleasure to be here uh so my name is Henrik rexed so look like it's written in the in the the bottom of my video uh so I'm based in south of France the beautiful sunny south of France even if it's still 10 15 degrees at the moment Celsius um and it's 603 local time in in France and uh otherwise I'm I uh I'm trying to be involved in a lot of observ topics so tag observ and I try to give a hands as well in the end user S as well thank you very much uh Josh are you there hello yes hello thanks for having me uh I'm Josh I've been a developer Advocate and product manager in all kinds of things around uh open Telemetry and uh I'm connecting from Brussels where I'm I'm here for fum uh starting on Saturday looking forward to that I like the European representation here reys and but not last but last but not least reys how's it going hi everyone I'm so excited to be here I am jining from Vancouver Washington not to be confused with Vancouver BC um right uh it's like 10 minutes north of Portland um I work in develop relations at New Relic and I also work with these lovely folks in the open solun Community um primarily as part of the Andy Sig right thanks thanks to everyone for being here right so let's start with uh the first topic the first question from from me um which is uh going to be directed to Adriana uh what is a an A cfp in the first place can you tell us more about it starting with the tough questions okay cfp if I I got this right stands for call for proposals or call for papers depending on on the context and so it's basically a request for a proposal for a talk in the context of um uh conferences where you you basically you know you have an idea for a talk and you give give the details for said talk and depending on what conference you're applying to um there the details will vary from conference to conference um for example uh if anyone's ever applied to sron um it is a song and a dance to apply to SRE con because I I feel like they they really want you to hash out all of the details of your proposal ahead of time compared to like say a cubec con where you can like be a little bit more high level so yeah that is cfp in a nutshell I don't know if anyone else wants to chime in uh for me uh cfp is the opportunity toh to be in a conference to meet people to network uh because at the end in my role to be able to reach out to that conference to excuse and the great excuse is to talk so for me a cfp is like oh maybe I will have some miles uh with with my air because I'm Air France member if I'm accepted that would be great yeah nice okay so uh we'll move to the next one I think uh you know if you're like new to conferences you're probably thinking okay so how do I find conferences to talk to uh to talk at so um Josh can you tell us a bit more about how you go about like finding what conferences you want to to apply to speak at yeah absolutely so um a really good way is that is to use the aggregator sites that a lot of the conferences will post their cfps on so that would be I think the two big ones would be session eyes and paper call and then um what's the third one anyone can help me out with the name of that third one that oh pre-t talks pre- talks is the third one um so yeah you can you can kind of check out um those or skedge right um those are not necessarily going to be specific to the the area that you want to speak at right of course for the cncf you can look at all the cncf events that's going to be relevant for open topics um there are also a lot of aggregators I think we can maybe add some of those to this to this resources list after this is is done but um yeah there there are some GitHub repos and some like air table databases that are maintained fairly well that just sort of keep an up-to-date list of all of the ongoing cfps sorted by their due dates nice um and then from those like do you normally choose by like the topic of the conference or or do you just I mean I I rather like have everyone talk about open Telemetry all the time everywhere in every conference like is that is that something that you do normally or like what how do you choose then you know what conference for a do you normally choose a specific topic of a conference um sometimes it's I think uh the topic helps narrow down the initial list and then from there uh oh maybe Henrik has something to say on this no no no I'm changing the the display that's uh um I forget what I was saying about that oh yeah so right so the topic I think Narrows down the list but especially if like if your topic is as broad as devops or even observability that's not going to narrow it down that much there are still a lot of conferences so then you have to pick like where am I okay to travel do I like to travel far like Henrik and rack up those miles or do I want to maybe get my start somewhere that I can drive to and I don't have to get a hotel um right and um how big will the conference be I think there's a really big difference between speaking at a single track conference as a first-time Speaker versus speaking at a multirack conference I think we'll maybe get into that and some of the other questions but yeah I think just the topic alone is not an enough narrow Town nice that's cool all right so we've thought we found the conference and we've got some resources there that people can go and check out to to find where to speak and I guess you know a question for re um what makes you want to speak what why do you why do you apply to speak at conferences you know um what so one thing that I kind of found out early on was um because my job is so busy that it sometimes can be hard to find time to learn something new or you know something maybe more Niche and so if I can submit a topic and it gets accepted on something that I want to learn more about then that automatically gives me the time and bandwidth to work on that topic and learn about it so that's one reason and now that I've been doing it a while while um just the benefits from the physical interactions of being able to be in the same space as people who want to learn about these things and having um conversations about it one I'm able to learn more and then also there's a lot of other people that also want to learn the same thing you know that I wanted to learn and so it just becomes like this um really cool Collective I guess and you get to meet a lot of cool people that way too and you know not just like for professional networking but also like some have become like personal friends it's it's just really great it's a great Community like exposure to community yeah is like um is that networking aspect as well not sure if Adriana for example do you wanna yeah I was gonna say um sometimes you know like uh for me personally like speaking is almost like a challenge I I like I love speaking in front of audiences but I also find it a little bit uh terrifying in so this is this kind of like forces me to get past um that discomfort um but another thing that I wanted to point out um on like why to speak um especially for those of us who are in an un under represented group in Tech I think the more of us in underrepresented groups that go out and do talks the more we can show those folks in underrepresented groups that we exist and we can Empower them to go out and speak as well and I think that's so so important 100% uh just to add as well I think um all of us um we we all working in in technical environments and sometimes we say we we think that what we are working on is quite normal and and nothing special but there's always some things to learn so if you if you like the community and and you want to share your work you want to share your experience you share your uh your journey to go through through being uh a completely beginners to and completely experts and the advice that you can share I think it's the best opportunity because I think sharing a talk I think it's it's May a lot of of sharing knowledge and educations to the communi I think it's it's it's an amaz amazing opportunity for for all one yeah and as well like for someone that's not like a Dev rail for example you know Josh that is working on the area like you know what how do you yeah what what motivates you for example yeah well no so I am a time devil so I get the same excuse as ree right like if I get a talk accepted that's brownie points for my boss and and it justifies my existence a little bit but for people who for people who don't have that motivation right um there's another thing that Devils actually do that's really really important right which is bring feedback back from the community so that's another thing if you're an engineer on an engineering team right like it can get a little bit insular in our bubbles and so going and talking at a conference as a chance to like kind of like kick the tires on some ideas with some people that are new people to you and and gather that feedback from the community as well yeah I think that's it as well sometimes you forget that what you're doing um it might be you know something that is ahead of a curve and you like you know you want to give back right that's that's great okay moving on let's they say that you know now basically I'm convinced that I want to talk but sometimes you know there's like I I think a lot of us suffer from imposter syndrome and you think that you're not good enough to to talk about a certain topic so I'll go back to to you Adriana how do you get how do you get past that imposter syndrome and that's s of like I don't know enough to talk about a topic I feel like you just have to force yourself past it like just force yourself to do it anyway because you know what if you don't do it someone else is going to do it so why not you right um and and I think um you know a lot of times I I think this is advice that someone gave me early on when I started applying for talks which is like you don't have to be the expert and I think ree touched on this too like you don't necessarily have to be the expert you can use this as an opportunity to force yourself to learn something about a topic or the other thing is like ah you know I have to like be the expert to be able to speak intelligently about it but like there's something to be said for having um kind of a newbie's point of view as well because you know you we are so much more relatable that way um when we talk about our experiences as newbies because there are so many people who are like you know new to things and and to to show people that you're human and you're not like some like perfect being that's up on this pedestal like I think it makes it um super relatable and and more fun too for for people to learn so yeah Adriana maybe I don't know you've gotten to this point I have not gotten to the point where I don't have a little bit of like impostor syndrome or or stage fright right like it just never goes away completely I think I'm with you yeah so yeah you really do just have to push through it because it's always there but I think I think it's uh it's the talk like you mentioned before it's the opportunity to learn and to be more expert on that side but even if you're not the uh the the maintainer the main contributor of the project or whatever topics you you decided to to present I think if you have a way of presenting that is super interesting brings a lot of entertainment and people will probably listen more to you talk to as someone who is very boring on stage uh you don't need to be the full experts it's just sharing a passion in a in a very funny way or in your way in fact and that brings value for the for for the audience that is such a good point I've seen talks where you know put subject subject matter experts who were really really knowledgeable about the subject but they weren't necessarily presenting it in the most absorbable way um for a larger audience and I've seen people who were newer to the subject you know do really creative things that for me I was able to like oh my gosh yes this makes complete sense and so I think that is a really good point you have your you know you can bring your own perspective into things and the way you like share your knowledge it will I guarantee you it will resonate with somebody in the audience yeah absolutely I will also say that this is why Ree and I have cats on our slides so you mean that you you you have uh you bought cats just for conferences or or you you had cats in the beginning oh uh so like re's cat Taco is like the most photogenic cat ever and so we just like feature her on a bunch of our slides whenever we do talks together she's a pretty short hair she's ridiculous real animal but anyways I I think we talked about go ahead yeah no go ahead I was just gonna add if the conference organizers chose your talk like right like don't be afraid to submit and then if you get chosen you got chosen they want you to talk about it more than anyone else so that is so true I think uh we talked about a bit about end users and like you know I think an open Telemetry and UPS ability for example you may think as an end user or you know people will want to know about the latest thing that maintainers have been working on in hotel or the latest thing that this particular vendor is is delivering but I think there's a lot of value from users like telling the stories as well like uh do you agree with that um Henrik for example I think you talked about that previously yeah I think uh um uh I mean having uh vendors presenting they will bring always their angle uh and I think having a user sharing their story they have always a different angle and a journey uh or an experience is always super interesting it's like a a book book where you have an adventure and you just follow the Adventure so I think it brings lots lots lots lots of value to the audience sometimes even more values to have someone like me like a devoll going on stage because I will I won't have necessarily this this experience and this journey I will I will bring uh the same topic in a different angle so I think a user has I in a real experience stock I think it's I think it's the best ones from my perspective anyone else that wants to chime in on the end user absolutely so we recently organized um the open source analytics conference at my company and um where we we participating in organizing it uh at the beginning of the process right choosing the talks we specifically separated out all the enduser talks because we didn't have enough of them and we wanted to make sure that they were well represented in the in the schedule um so we like we gave definitely gave preferential treatment to those and um you know the reason for that is for for everything that Henrik said right like it's like other people want to get these stories that they can follow along with and relate to um that don't have that that vendor spin necessarily yeah I think we got that basically from from one of our audience uh saying that yeah end users just give a different weight to that to like to what they're saying right because they are using it in production they're using it in their systems so yeah definitely okay I think we can move on to another topic which is uh the topic topic so we've got uh you know we've got an idea like we want speak to at a conference and then we how do you actually decide what topics to to go through um I guess you know like what what topics to put in that in that cfp uh and I will ask that to re so this is an interesting question um because so when I first when I submitted my very first cfp to cucon in uh it was was for cucon EU 2022 um I think open Telemetry was still pretty new as a topic at ccon at that time and um so I think for like the you know the first couple years um I was having pretty good success getting like topics my topics um selected and I've found you know um in the last year um it's been getting harder one because you know there's more people talking about it but also there's more and more things being covered um and so I think it's been interesting to kind of figure out like oh now it's time to get like really kind of Niche um I mean there's still you know a place for intro intro level talks um but I'm kind of in that space where I'm trying to figure out you know what are what are interesting um aspects and topics um related to open Telemetry and obser in general um to submit in or submit for and yeah I think Adriana you had a I did yeah I was gonna say on that same vein because yeah like as re said it's getting harder and harder especially in the observability space and then um it's interesting too because like I think some of the folks on this panel have like reviewed um cfps as well for cubec cons I've done a number of of um cfp reviews for cucon and it's interesting to see what topics come up over and over and over and I will tell y'all AI comes up a lot um and it's almost to the point where you're like oh my God not another freaking AIC cfp for the love of God and they're not unique because like here's the deal right there's nothing wrong with submitting a cfp on AI but make sure it's freaking unique at this point because a lot of the stuff that's out there it's like different permutations of the same thing and so it's really about like what what's your unique take on it and also like you know we we were talking about like end user stories I think those are still extremely useful and relevant but again after a while you start seeing a lot of the same end user stories so again when you're submitting an end user story what is special what makes you a snowflake um when when you're telling your end user story the other thing I would say is like um some some folks try to like um submit like project updates as part of like you know a cfp for for like cuon or whatever and it's like dude a project update does not automatically get you accepted like save that for you know the relevant um for the relevant uh venue uh because I think there's like special like project update sessions for example so that those are like a couple of my pet peeves I don't know if anyone anyone else wants to chime in no I agree with that right like how do you TR oh go ahead go ahead go go ahead I I I agree um how do you be trendy without being too trendy but I see um in the chat there's a thing about um people don't understand the basics of otel and Oli and REE when you were speaking about like trying to find your your Niche um does do other people call it olly or is it just me um I love it Ollie yeah yes um anyways uh maybe at cubec con right like that's true like you need to you got to have something unique and novel to bring to a cubec con conference um but outside of like our again getting outside of our bubble right like open Telemetry is still a very Niche topic and I've had my introduction to open Telemetry right like the smaller Regional conferences are clamoring for those introductory topics uh introductory talks on these you know Niche topics that are our Niche right so it doesn't feel that Niche to us um yeah and and in my case I I usually try to avoid those introduction talks because I think many people will do the same so it's just matter of uh be different or bring a different angle so then when the the people will review your talk then you get probably have an interesting angle uh that will bring value to the community in my my in my perspective I do talks that are very expensive expensive in terms of preparation because I do do love Benchmark because I think I like to have those studies where you show Numbers you show things it's like an experience and then you you show that to the to the stage because I think it's it brings another value uh because sometimes you don't show that uh in normal track and I think by having that difference uh you probably have more chance to be selected nice um I guess you know related to this I think the the introductory topics and the more in-depth topics sometimes you know when you're applying for or um to speak at a conference you don't know what level of um detail you want to go through either in the cfp or in the or in the the the Talk itself is it good to like how do you know your audience and how do you know what level of depth you you want to apply um re is that how do you normally go about it that's that I still find that tough sometimes um and and sometimes I'll you know start with like okay I kind of want to be more high level about this topic but in order for someone to have to come to this topic they might need you know XYZ knowledge and so even though it's more high level on that specific thing it might be considered like an intermediate if that makes sense um so it yeah it really depends and I'll also try to um you know consider like how much you know is this am I talking about something newer you know that was like more newly developed um so there's like less info about it or is it something that's been around for a while um so I don't I don't have like a really good answer um I I still find it tough honestly um I try to stick to like either beginner or intermediate level um in general because of the topics that I typically choose to do and the thing also it's a personal judgment as well sometimes you say oh I think it's it's beginner or I think it's it's I have no idea it's it's complicated yeah what what may seem basic to you might be very Advanced to others or the opposite right yeah yeah yeah do you have any any other tips there Josh to try to basically gauge the I have a quick anecdote so yeah it's it's hard to get it right um I gave one talk that was an introductory talk but I was trying to make it a little more advanced I thought oh this is you know this seems like a fairly technical audience I'd been having conversations with people at lunch everybody here's going to know what evf is so I didn't mention it and then the next speaker after me says raise your hand if you know what ebpf is and me and the one person I was talking to one raised our [Laughter] hand oh well awesome so yeah it's hard to get it right it's hard [Music] yeah right okay so Henrik I'm going to go back to you you um because I know that you're everywhere in open Telemetry as well so like I I know that there's a lot of Hot Topics in open Telemetry um but yes so what are the Hot Topics that you recommend people be writing about right now I know that you know doesn't have to be Hotel I know that we all love hotel but what are the what are the sort of like the Hot Topics that you would recommend thinking about writing about or talking about sorry uh I think uh what I would suggest is that but by the way AI I don't AI is not valid it is it is I think the in general the uh you you have to look at in previous conferences what has been presented and covered so far um so if you start to say I want to do open Telemetry on instrumentation there's plenty of them out there there's plenty of video out there and then there's no Hot Topic in general so usually I try to bring the the problem statement so for example how do I sample properly how do optimiz because the there is a big big in in case of observability there's a big concern about oh uh observ is going to be expensive so how can I control that cost uh so going through on that directions I think also there is another Trend that that I think is really important that people needs to be uh educated a bit more uh it's the sustainability so how can we make it green make it better so um talking about AI is not going to be green because you're going to consume more resources so maybe having another approach where you say oh you we we we we need to be good citizens of the worlds and and save energy I think that that's a pretty good one and also I think there is um anything that new projects comes in for example today uh as we speak open Telemetry profiling there is few talks out there in cucon and it's going to be more and more popular so there's a big chance that profiling will bring new problems new challenges so covering them that could be also interesting so try to follow what happens in the industry and and think about oh um if I start digging this I may have problems so how can I resolve them and maybe share that that uh that solution or that um this approach to to to a larger group to the community nice I just add Henrik Henrik you're really good at this right like you mentioned uh sampling and I saw that video right in my feed and I was like oh that's a great topic I'm gonna check that out later when I have time so uh yeah like if if henrik's talking about it he can only be one place at once right so maybe that's an interesting thing for you to put your own spin on somewhere else yeah I think so Absolut absolutely any other any other takes on you know what are the some of the some of the Hot Topics I would like to add something so uh you know I think piggybacking on what um Henrik was saying on on uh the topics of sustainability um there's some really cool like cncf projects out there um on on Tech sustainability and I I feel like this is a super hot topic um for for the year just because we're seeing a lot of stuff around like just wacky environmental things happening right like increase in forest fires bizarre temperature swings like and we're inherently in an industry that is contributing to the problem so um writing talks about um how we can use technology to less in the problem I think um can be really compelling and very timely and so there's there's a hot topic for yall for anyone considering it I would also mention you know like we talk a lot about cubec cons and I think Josh made a point earlier um about like there's a lot of conferences where like things like open Telemetry are still like kind of um you know not not super well known so um especially like a lot of these open- Source conferences like scale for example I think just added an observability track this year um fum I think would probably be another great one state of open con um another great one where you know we probably um there's probably not enough talks on observability so getting into those um sort of more nichy conferences um I think would be would be a good place to start especially if you're looking to do a talk on observability oh yeah all things open as well well Reese mentioned in in our yeah so right I'm going to take one question from the from the audience thank you so much for your questions and again you know if you've got any questions on what we're talking about drop them in the in the chat on YouTube on LinkedIn um and the question is should we go for a catchy title for a click baity title when you when you write your cfp or should you do more something more perhaps descriptive and accurate um Josh what do you think about about that do you normally go for something like that I almost always do a clickbaity title or at least I did that was always sort of my way um but I've only been doing this for a couple of years and someone who's been doing it for longer than me told me um actually it's a it's a it's a pendulum right it swings and what the conference organizers are looking for is going to swing back and forth almost like a cultural Zeitgeist and there are times where they want it to be super super specific and we're not in one of those times right now and be like maybe we're swinging in that direction but for me right now it does feel very much like the clickbaity titles are in and maybe we'll all get tired of them because of the AI topic and we'll be like no you need to tell me exactly what's in your talk so that I know that it's not a surprise ai ai talk and that'll be where we're at a year from now who knows I think um it if you can come up with a catchy title that also will give audience an idea of what to expect or like what your topic is about that is a great way to go I also do really like just clear straightforward titles as well um so I think they both have their place um and you know the content for sure is going to matter more I I say um because you could have a great catchy title and then the abstract is kind of you know maybe doesn't doesn't really fully flush out the idea um so I I I think the content is still king um but yeah title is is important as well um but as long as you have like a clear title and not necessarily like you know kind of click baity or whatever I think that's that's fine too in in when I preparing uh my talks usually I I try to bring the technical aspects and then try to find a funny angle or an analogy to a movie to video games to do whatever and then in the title I I try to bring that that fun uh friend angle because at the end uh U I think that that could be more attractive at the end because at the end when you have the schedule and you see a a title that is more I don't know sounds more fun then maybe more people will join your session for some some some reasons well I think the title is is is as as important as as the abstract from my perspective yeah and I think s go ahead go ahead oh I was going to say I I do generally agree with you on that the only thing I would caution again is a um make sure your titles don't sound like they were CH generated by chat GPT and B um make sure that your titles like aren't so pop cultur where like it ends up alienating potentially the cfp reviewers like I don't know if someone's like making a Game of Thrones reference in a talk title I'll be like I have no idea what you're talking about and I'm kind of like super put off by it um so I would just caution like it's it's a fine line to trade nice and I guess you know that's as well part of there are a couple of questions from from from YouTube which are related to to to that basically to knowing your audience right so who's going to be reading your your title your cfp so how do you know that how do you know like how to approach the the the audience in a way that they would understand or engage with and I'll ask that to Henrick it's it's a very very good question um uh usually goes to kcds and to cubec con and or open source friendly conferences so I know that the people that will be in that conference has at least tech technical background uh so then I I know that my talk would be not perceived as too technical um but yeah if you go to a conference that is more salesy uh then yes uh you may have to adjust and say okay um what are the type of persona that will be in this conference do I have the the normal audience that I'm talking to in general or should I uh reduce the complexity of my talk so then people can can follow uh the the idea behind behind this talk cool right I'm going to move on to uh to another topic which is like the post acceptance so like you got accepted hooray so what happens now what how do you train how do you rehearse what is like what can you give people to get ready for for the day I know was a very wide question so I'm going to start with h I'm going to start with ree how do you like prepare for it well so when I got my first talk so funny well funny story my very first cfp that I submitted ever was also my first one that was ever accepted which was huge and terrifying and my manager at the time she actually hired a well hired she got me a few lessons virtual lessons um with a public speaking coach and I found that that was super helpful it was just um three virtual um sessions and I came away with techniques that I still use but I also know people who you know are great speakers who don't who haven't gone through that profession training um but I say it's at least worth it to if you can if you have the resources to um I recommend it but if not there's also you know um YouTube's YouTubes YouTube videos you can watch um and of course like rehearse um as much as you can so that you feel comfortable and feels natural when you're like presenting but I'm sure other people will have their do any other topic any other like um tips or tricks to to basically get to the to the to the training rehearsing stage I uh I I usually don't I do just the rehearsal not a full fresh rehearsal I'm just checking that my talk is basically I'm respecting the duration because sometimes I put lot of content and then I realize oh maybe maybe too much here so I need to maybe figure out by timing yourself and then you know that when you will be on stage there's a big chance that you will take probably more time so I had a few minutes few minutes in a like a to to be always in time I think that's the best thing uh but what I usually do I'm trying to to put a lot of Graphics make it more nicer uh I put I mean that that's my my my style so I like to make a lot of graphics and make it more smoother experience for the audience so that's something that I prepare a lot in the background nice I'm going to take now a question from the audience and I think I'm um I think I'm GNA I'm going to choose I'm going to pick Josh to answer this question if a conference offers multiple formats for example lightning talk or a deep dive and you know I could make your talk fit more than one should I spam the organizers with multiple responses or is there a better way in my opinion that's something that will work if you know the organizer and can reach out to them directly right like or if they're having an office hours and you can talk to them about it and you can get that feedback that's fine but I do think that if you're a firsttime speaker at the conference or sort of unknown to the conference um you have to sell the talk you have to be a little bit more confident in the talk that you're proposing because they're going to be evaluating you based on the talk more than how they know you as a as a speaker see anybody else yeah I wanted to just mention something like especially um depending on the conference like some conferences don't have limits as to how many um talks you can submit um and so um sometimes you know especially if you're like a newbie speaker go for it and just see see how far you can take it um but other other conferences like Yukon where there is a limit on the number of sessions I would caution against um submitting like basically two versions of the session where it's like one's a lightning talk the other one the ones the other ones a longer form talk I think you have a you probably have a better chance of just submitting like two unique topics um because you never know also I will say um don't be afraid to recycle submissions at other conferences it doesn't always have to be a unique topic super super important it saves you a lot of like mental energy because putting together talk is a lot of work I would just add to the fact that um for the reviewers if you have a talk and usually you design your abstract for a 30 minute stock for example then the reviewer would expect some some details so then you say okay so he's GNA cover this and that makes sense in 30 minutes because it's going to be well covered and then if you then say uh if you do the same cfp in uh lightning talk then you say wow how is it going to cover that in five or 10 minutes so I think you need to adjust of course the way you're going to um present the abstracts I would definitely recommend to do like like Adana mentioned to have two different submission uh one lighter for the liting talks and one bigger for the the panel the the normal track cool okay I think it's time to start closing closing thoughts and I think one of the well not not quite there yet but one of the questions that that people normally get is I apply to you know many conferences and then you get you get rejected and uh how how normal is it to get rejected I think is it how do you manage your your expectations for getting your cfps accepted I corre yeah it's tough is it so Josh do you want to go uh give us give us your thoughts sure I think uh someone again you know getting advice from mentors is great someone uh told me on to expect a 10% acceptance rate so if you want to talk at one conference a year that means you need to submit 10 um and I I think that holds even for experienced speakers to some extent uh as well as new at least from what I've seen I don't know if you all how you feel about that ratio but you know that's interesting because I haven't actually done like a uh statistical analysis myself um but it is absolutely normal to get rejected you know you think about conferences some conferences get especially the bigger ones they get thousands sometimes um but usually like you know at least tens of dozens or hundreds of submissions um so and just because they rejected it you know this time doesn't mean that they won't next time I know people who've submitted the same one um a few times to the same conference before it got accepted so sometimes it's just timing sometimes it's just you know they just have a lot of topics that were just really really good um and sometimes too you can also ask for feedback from the from the um um conference um keepon for example they're pretty good about sharing anything um if they get feedback on your um proposal they'll they'll be happy to share it so that's something to think about too cool I I also wanted to add like it's okay to like mourn your rejection um because you know especially like when you're really invested in a topic and you you think like I've got such a great chance of getting this in and you get rejected you got like a great title and you're just yeah yeah it's like it's okay to mourn and and take that time to mourn and then as ree said like if possible ask for feedback um resubmit it um to that same conference or another conference or like in the case of cubec cons there's like so many across the globe submit it to a different cucon um you know like if didn't get in for na submit for EU there's open source Summit na there's open source Summit emia and also like take some time to reflect also on your cfp and see if there's like um areas where you see Improvement like sometimes I'll like write out a cfp and then I'll like you know it gets rejected I res submit it for another conference I'm like o yeah I can kind of see why they didn't like that and just make a few tweaks yeah agree well thank you so much I think now I've got one last question from you and I think as uh well if everyone's been following the the Golden Globes and the Academy Awards you'll see that we've got quite a lot of horror films coming up and the awards so I've got a question for you I think you've been to many conferences and I would like to hear some of the conferences that you've been to because I think we didn't cover that in the in the intro and I would like to know you know some of the some of the conferences that people could go and rewatch your videos but also like if you've got a quick anecdote or something that failed that was a bit of a Horror Story right so the mic doesn't work or you know something that doesn't work so um can you tell us about it and I'll go for re first because she's got her hands raised my very first talk that I did it was about tail sampling open Telemetry I had a live demo and everything was working I checked it right before my talk everything was working got up to the stage started the demo it didn't work um and I still don't know what it was because I got it to work a few minutes later and I know there's some people that I'm still remember that too so anyways that's my I have other ones too but that one is very um sticks close your recent uh conferences that You' talked at Ree I think so people can go and watch some of your of your talks uh um all things open I'm not sure if every one of those has recordings um but yeah those are the big ones that I I can remember off the top of my head co um Henrik do you want to go next uh has uh what was my worst experience I would say uh it happens once where I have to connect my laptop the screen mirroring didn't work and then I suddenly I did the demo and then you end up having nothing on your screen and you're like this and trying to type uh and looking at and it's like nightmare I was say okay the demo is going to be a nightmare impossible to achieve and so then I um I think the demo I I try to to skip it uh very fast uh but I was not very happy uh people didn't didn't see so much because I I I try to react as fast as possible to avoid having that blank effect where stress coming up up to you but um yeah at the end I was unhappy because I said ah my performance was just uh not normal and not acceptable so I had to improve that but yeah uh where can we see you next where can you so i w i was rejected for qcom so I've been crying uh so now I think uh I have no tears anymore so no it happens to the best it happens to the best it does uh so uh yeah the next I have a virtual conference that uh I don't know when it's it's going to happen in a few weeks I'm presenting talk and then I submitted lots of uh talks for the season so a lot of kcds and Cube con China and and Japan so we'll see didn't have the um the feedback yet so cross the fingers but otherwise if you want to watch any content from my end uh you can find it on cubec Con Europe cubec con North America and then I have plenty of other kcds where I have presented talks last year awesome Josh do you want to tell us your Horror Story yeah it's actually quite similar to henrik's it had to do with screen mirroring and not working and this is actually also like recent my very first uh talk um the audience could only see half of my slides I could see all of my slides but none of my notes um so my lesson that I took away from that is I never I never rely on my notes ever again um plus one plus one to that plus one to that yeah you have to have it all in your head you have to be able to just talk about the thing for 20 minutes with no notes and no slides if it comes down to it um I think not to scare anyone else off from doing this right like you can you can do that actually it's not as hard as it sounds um but yeah that was that was my worst Horror Story um although I would say like I feel like at least something small goes wrong pretty much every time uh and then where where you can find me right uh a lot of devop days I'm really big fan of those and I love that the the kcds are starting to sort of adopt that that Community format and sort of the open discussion format more um I really love those conferences and I love to get the combination of like the people who are are coming from all over um like you would see at the bigger conferences and then just also the local Representatives uh of the tech companies that are Employers in that area nice kcd and Edinburg yeah that is a kcd and Ed find out this this year and probably because London it's normally in London but like I guess London is getting cubec con so at least we're getting something I didn't see the cfp yet is it when is it open I it's in October so I'm not sure when the okay but yeah everyone in Edinburgh is really excited about it I mean I'm not sure the the normal person in the street would be but I am Adriana do you want to have some yeah tell us some of your horror stories um I think for we knock on wood I haven't had any like horrid things happen during talks um so I I will say that um I am not a fan of live demos and so I I do pre-recorded demos actually Ree and I have done a bunch of talks together um and and the ones that we've done together have had demos but they were pre-recorded but live narrated so that is my tip for you if you're scared of live demos like I am um I think for me the worst is like when you give a talk and you're like H that wasn't my best work and you feel like you're a little bit off like sometimes my voice is a little bit shaky or I just like kind of forget my talking points um I think that's uh you know just getting rattled by that and and and I it's interesting because um you know you come off a talk and people are like oh that was so great and you're like oh I sucked and so you kind of just have to like get over yourself um and just tell yourself like it happens um and it'll happen again and you just have to be okay with it and I would also suggest when you're practicing talks um practice like you know as you're practicing and you stumble just like power through it because that can happen in real life or you you stumble so if you if you practice that during your you know if you stumble during your practice don't like restart um see where it takes you um as far as conferences uh that I've spoken at and I'm just sending Henrik a link to my YouTube channel because I have a playlist of all my talks um but uh like I said recently I've done a bunch of talks we've spoken at cucon North America and EU a few times together we've spoken at observability day North America and EU together I've spoken at platform engineering day most recently in in North America we've done all things Reese and I have done all things open together we did umce open source Summit yeah open source North America and EU coming up and what we'll be at scale coming up yes we're going to scale in in March um we're speaking at cucon EU um yeah coming up in London um and I got in for observability day EU as well um and then I I did um I did a devop days in Montreal and most re and one of my favorites was a kcd Buu um where I got to give a keynote unfortunately there's no recording of of the keynote but I have a blog post version of the talk if anyone wants to check out my medium Channel awesome well thanks everybody so much for I mean you are really the the Dream Team so really like all the veterans from all the you know conference dos you can see that you know like from the amount of conference that you be so thank you so much for for yeah for your tips and tricks and your advice I think you know it's been really really useful um but I think that's all we had time for and uh I just want to say thanks for to the panelist thanks to the people that ask questions in the audience I think we didn't get to cover all the questions but um we are always open in the end user Sig you can come to the cncf slack you can go to the the resources that are here basically and get in touch with us I think we would love to hear more about your well in general anything about open Telemetry but also about topics like this one how do you go and talk about open Telemetry everywhere so um yeah with that that thank you very much and see you at a conference get the questions that were UN insed in the SL Channel yes right thank you bye bye bye [Music] [Applause] [Music] +e hello everybody good morning good afternoon good evening I'm Dan and as part of the as part of the open Telemetry and usus Sig today I have the absolute pleasure of Hosting this panel in the hotel end user s we normally focus on open Telemetry I mean it's in the name uh we host end user is telling us how they're adopting observability best practices across the industry if you want to know more you follow the QR code in the screen and you'll find how to get in touch however this time we're going to be getting some tips and tricks from True veterans in the Tech conference Arena we'll be talking about how to write good C cfps what topic what topics to choose how to prepare for a talk or even why going through the whole process of applying to speaker conference and how is well worth it so if you uh have recently applied maybe to speak at cubec con or observability day in London or another conference it doesn't matter if you made it or not I think these experts will have some advice that you can take home and use in your future talks we want this to be an interactive panel so if you're watching live on uh YouTube or LinkedIn and you have any questions related to the topics that we're discussing please please drop them in the in the chat and I will try to incorporate them into the discussion if best I can okay so it's time to meet our panelists um and uh if you're watching live on YouTube or LinkedIn I would love to know where you're watching from so drop a comment in the in the chat um and tell us where you're watching from myself I'm I'm in Edinburgh Scotland where is currently 5:02 p.m and so um let's meet our first panelist Adriana um hello uh can you tell us where you're connecting from and a little bit about yourself hey Dan nice to see you uh my name is Adriana Vela I am connecting from Toronto Canada it is noon here and um I work alongside Dan as one of the maintainers of the otel and user Sig thank you very much and uh Henrik hello Henrik hey pleasure to see here pleasure to be here uh so my name is Henrik rexed so look like it's written in the in the the bottom of my video uh so I'm based in south of France the beautiful sunny south of France even if it's still 10 15 degrees at the moment Celsius um and it's 603 local time in in France and uh otherwise I'm I uh I'm trying to be involved in a lot of observ topics so tag observ and I try to give a hands as well in the end user S as well thank you very much uh Josh are you there hello yes hello thanks for having me uh I'm Josh I've been a developer Advocate and product manager in all kinds of things around uh open Telemetry and uh I'm connecting from Brussels where I'm I'm here for fum uh starting on Saturday looking forward to that I like the European representation here reys and but not last but last but not least reys how's it going hi everyone I'm so excited to be here I am jining from Vancouver Washington not to be confused with Vancouver BC um right uh it's like 10 minutes north of Portland um I work in develop relations at New Relic and I also work with these lovely folks in the open solun Community um primarily as part of the Andy Sig right thanks thanks to everyone for being here right so let's start with uh the first topic the first question from from me um which is uh going to be directed to Adriana uh what is a an A cfp in the first place can you tell us more about it starting with the tough questions okay cfp if I I got this right stands for call for proposals or call for papers depending on on the context and so it's basically a request for a proposal for a talk in the context of um uh conferences where you you basically you know you have an idea for a talk and you give give the details for said talk and depending on what conference you're applying to um there the details will vary from conference to conference um for example uh if anyone's ever applied to sron um it is a song and a dance to apply to SRE con because I I feel like they they really want you to hash out all of the details of your proposal ahead of time compared to like say a cubec con where you can like be a little bit more high level so yeah that is cfp in a nutshell I don't know if anyone else wants to chime in uh for me uh cfp is the opportunity toh to be in a conference to meet people to network uh because at the end in my role to be able to reach out to that conference to excuse and the great excuse is to talk so for me a cfp is like oh maybe I will have some miles uh with with my air because I'm Air France member if I'm accepted that would be great yeah nice okay so uh we'll move to the next one I think uh you know if you're like new to conferences you're probably thinking okay so how do I find conferences to talk to uh to talk at so um Josh can you tell us a bit more about how you go about like finding what conferences you want to to apply to speak at yeah absolutely so um a really good way is that is to use the aggregator sites that a lot of the conferences will post their cfps on so that would be I think the two big ones would be session eyes and paper call and then um what's the third one anyone can help me out with the name of that third one that oh pre-t talks pre- talks is the third one um so yeah you can you can kind of check out um those or skedge right um those are not necessarily going to be specific to the the area that you want to speak at right of course for the cncf you can look at all the cncf events that's going to be relevant for open topics um there are also a lot of aggregators I think we can maybe add some of those to this to this resources list after this is is done but um yeah there there are some GitHub repos and some like air table databases that are maintained fairly well that just sort of keep an up-to-date list of all of the ongoing cfps sorted by their due dates nice um and then from those like do you normally choose by like the topic of the conference or or do you just I mean I I rather like have everyone talk about open Telemetry all the time everywhere in every conference like is that is that something that you do normally or like what how do you choose then you know what conference for a do you normally choose a specific topic of a conference um sometimes it's I think uh the topic helps narrow down the initial list and then from there uh oh maybe Henrik has something to say on this no no no I'm changing the the display that's uh um I forget what I was saying about that oh yeah so right so the topic I think Narrows down the list but especially if like if your topic is as broad as devops or even observability that's not going to narrow it down that much there are still a lot of conferences so then you have to pick like where am I okay to travel do I like to travel far like Henrik and rack up those miles or do I want to maybe get my start somewhere that I can drive to and I don't have to get a hotel um right and um how big will the conference be I think there's a really big difference between speaking at a single track conference as a first-time Speaker versus speaking at a multirack conference I think we'll maybe get into that and some of the other questions but yeah I think just the topic alone is not an enough narrow Town nice that's cool all right so we've thought we found the conference and we've got some resources there that people can go and check out to to find where to speak and I guess you know a question for re um what makes you want to speak what why do you why do you apply to speak at conferences you know um what so one thing that I kind of found out early on was um because my job is so busy that it sometimes can be hard to find time to learn something new or you know something maybe more Niche and so if I can submit a topic and it gets accepted on something that I want to learn more about then that automatically gives me the time and bandwidth to work on that topic and learn about it so that's one reason and now that I've been doing it a while while um just the benefits from the physical interactions of being able to be in the same space as people who want to learn about these things and having um conversations about it one I'm able to learn more and then also there's a lot of other people that also want to learn the same thing you know that I wanted to learn and so it just becomes like this um really cool Collective I guess and you get to meet a lot of cool people that way too and you know not just like for professional networking but also like some have become like personal friends it's it's just really great it's a great Community like exposure to community yeah is like um is that networking aspect as well not sure if Adriana for example do you wanna yeah I was gonna say um sometimes you know like uh for me personally like speaking is almost like a challenge I I like I love speaking in front of audiences but I also find it a little bit uh terrifying in so this is this kind of like forces me to get past um that discomfort um but another thing that I wanted to point out um on like why to speak um especially for those of us who are in an un under represented group in Tech I think the more of us in underrepresented groups that go out and do talks the more we can show those folks in underrepresented groups that we exist and we can Empower them to go out and speak as well and I think that's so so important 100% uh just to add as well I think um all of us um we we all working in in technical environments and sometimes we say we we think that what we are working on is quite normal and and nothing special but there's always some things to learn so if you if you like the community and and you want to share your work you want to share your experience you share your uh your journey to go through through being uh a completely beginners to and completely experts and the advice that you can share I think it's the best opportunity because I think sharing a talk I think it's it's May a lot of of sharing knowledge and educations to the communi I think it's it's it's an amaz amazing opportunity for for all one yeah and as well like for someone that's not like a Dev rail for example you know Josh that is working on the area like you know what how do you yeah what what motivates you for example yeah well no so I am a time devil so I get the same excuse as ree right like if I get a talk accepted that's brownie points for my boss and and it justifies my existence a little bit but for people who for people who don't have that motivation right um there's another thing that Devils actually do that's really really important right which is bring feedback back from the community so that's another thing if you're an engineer on an engineering team right like it can get a little bit insular in our bubbles and so going and talking at a conference as a chance to like kind of like kick the tires on some ideas with some people that are new people to you and and gather that feedback from the community as well yeah I think that's it as well sometimes you forget that what you're doing um it might be you know something that is ahead of a curve and you like you know you want to give back right that's that's great okay moving on let's they say that you know now basically I'm convinced that I want to talk but sometimes you know there's like I I think a lot of us suffer from imposter syndrome and you think that you're not good enough to to talk about a certain topic so I'll go back to to you Adriana how do you get how do you get past that imposter syndrome and that's s of like I don't know enough to talk about a topic I feel like you just have to force yourself past it like just force yourself to do it anyway because you know what if you don't do it someone else is going to do it so why not you right um and and I think um you know a lot of times I I think this is advice that someone gave me early on when I started applying for talks which is like you don't have to be the expert and I think ree touched on this too like you don't necessarily have to be the expert you can use this as an opportunity to force yourself to learn something about a topic or the other thing is like ah you know I have to like be the expert to be able to speak intelligently about it but like there's something to be said for having um kind of a newbie's point of view as well because you know you we are so much more relatable that way um when we talk about our experiences as newbies because there are so many people who are like you know new to things and and to to show people that you're human and you're not like some like perfect being that's up on this pedestal like I think it makes it um super relatable and and more fun too for for people to learn so yeah Adriana maybe I don't know you've gotten to this point I have not gotten to the point where I don't have a little bit of like impostor syndrome or or stage fright right like it just never goes away completely I think I'm with you yeah so yeah you really do just have to push through it because it's always there but I think I think it's uh it's the talk like you mentioned before it's the opportunity to learn and to be more expert on that side but even if you're not the uh the the maintainer the main contributor of the project or whatever topics you you decided to to present I think if you have a way of presenting that is super interesting brings a lot of entertainment and people will probably listen more to you talk to as someone who is very boring on stage uh you don't need to be the full experts it's just sharing a passion in a in a very funny way or in your way in fact and that brings value for the for for the audience that is such a good point I've seen talks where you know put subject subject matter experts who were really really knowledgeable about the subject but they weren't necessarily presenting it in the most absorbable way um for a larger audience and I've seen people who were newer to the subject you know do really creative things that for me I was able to like oh my gosh yes this makes complete sense and so I think that is a really good point you have your you know you can bring your own perspective into things and the way you like share your knowledge it will I guarantee you it will resonate with somebody in the audience yeah absolutely I will also say that this is why Ree and I have cats on our slides so you mean that you you you have uh you bought cats just for conferences or or you you had cats in the beginning oh uh so like re's cat Taco is like the most photogenic cat ever and so we just like feature her on a bunch of our slides whenever we do talks together she's a pretty short hair she's ridiculous real animal but anyways I I think we talked about go ahead yeah no go ahead I was just gonna add if the conference organizers chose your talk like right like don't be afraid to submit and then if you get chosen you got chosen they want you to talk about it more than anyone else so that is so true I think uh we talked about a bit about end users and like you know I think an open Telemetry and UPS ability for example you may think as an end user or you know people will want to know about the latest thing that maintainers have been working on in hotel or the latest thing that this particular vendor is is delivering but I think there's a lot of value from users like telling the stories as well like uh do you agree with that um Henrik for example I think you talked about that previously yeah I think uh um uh I mean having uh vendors presenting they will bring always their angle uh and I think having a user sharing their story they have always a different angle and a journey uh or an experience is always super interesting it's like a a book book where you have an adventure and you just follow the Adventure so I think it brings lots lots lots lots of value to the audience sometimes even more values to have someone like me like a devoll going on stage because I will I won't have necessarily this this experience and this journey I will I will bring uh the same topic in a different angle so I think a user has I in a real experience stock I think it's I think it's the best ones from my perspective anyone else that wants to chime in on the end user absolutely so we recently organized um the open source analytics conference at my company and um where we we participating in organizing it uh at the beginning of the process right choosing the talks we specifically separated out all the enduser talks because we didn't have enough of them and we wanted to make sure that they were well represented in the in the schedule um so we like we gave definitely gave preferential treatment to those and um you know the reason for that is for for everything that Henrik said right like it's like other people want to get these stories that they can follow along with and relate to um that don't have that that vendor spin necessarily yeah I think we got that basically from from one of our audience uh saying that yeah end users just give a different weight to that to like to what they're saying right because they are using it in production they're using it in their systems so yeah definitely okay I think we can move on to another topic which is uh the topic topic so we've got uh you know we've got an idea like we want speak to at a conference and then we how do you actually decide what topics to to go through um I guess you know like what what topics to put in that in that cfp uh and I will ask that to re so this is an interesting question um because so when I first when I submitted my very first cfp to cucon in uh it was was for cucon EU 2022 um I think open Telemetry was still pretty new as a topic at ccon at that time and um so I think for like the you know the first couple years um I was having pretty good success getting like topics my topics um selected and I've found you know um in the last year um it's been getting harder one because you know there's more people talking about it but also there's more and more things being covered um and so I think it's been interesting to kind of figure out like oh now it's time to get like really kind of Niche um I mean there's still you know a place for intro intro level talks um but I'm kind of in that space where I'm trying to figure out you know what are what are interesting um aspects and topics um related to open Telemetry and obser in general um to submit in or submit for and yeah I think Adriana you had a I did yeah I was gonna say on that same vein because yeah like as re said it's getting harder and harder especially in the observability space and then um it's interesting too because like I think some of the folks on this panel have like reviewed um cfps as well for cubec cons I've done a number of of um cfp reviews for cucon and it's interesting to see what topics come up over and over and over and I will tell y'all AI comes up a lot um and it's almost to the point where you're like oh my God not another freaking AIC cfp for the love of God and they're not unique because like here's the deal right there's nothing wrong with submitting a cfp on AI but make sure it's freaking unique at this point because a lot of the stuff that's out there it's like different permutations of the same thing and so it's really about like what what's your unique take on it and also like you know we we were talking about like end user stories I think those are still extremely useful and relevant but again after a while you start seeing a lot of the same end user stories so again when you're submitting an end user story what is special what makes you a snowflake um when when you're telling your end user story the other thing I would say is like um some some folks try to like um submit like project updates as part of like you know a cfp for for like cuon or whatever and it's like dude a project update does not automatically get you accepted like save that for you know the relevant um for the relevant uh venue uh because I think there's like special like project update sessions for example so that those are like a couple of my pet peeves I don't know if anyone anyone else wants to chime in no I agree with that right like how do you TR oh go ahead go ahead go go ahead I I I agree um how do you be trendy without being too trendy but I see um in the chat there's a thing about um people don't understand the basics of otel and Oli and REE when you were speaking about like trying to find your your Niche um does do other people call it olly or is it just me um I love it Ollie yeah yes um anyways uh maybe at cubec con right like that's true like you need to you got to have something unique and novel to bring to a cubec con conference um but outside of like our again getting outside of our bubble right like open Telemetry is still a very Niche topic and I've had my introduction to open Telemetry right like the smaller Regional conferences are clamoring for those introductory topics uh introductory talks on these you know Niche topics that are our Niche right so it doesn't feel that Niche to us um yeah and and in my case I I usually try to avoid those introduction talks because I think many people will do the same so it's just matter of uh be different or bring a different angle so then when the the people will review your talk then you get probably have an interesting angle uh that will bring value to the community in my my in my perspective I do talks that are very expensive expensive in terms of preparation because I do do love Benchmark because I think I like to have those studies where you show Numbers you show things it's like an experience and then you you show that to the to the stage because I think it's it brings another value uh because sometimes you don't show that uh in normal track and I think by having that difference uh you probably have more chance to be selected nice um I guess you know related to this I think the the introductory topics and the more in-depth topics sometimes you know when you're applying for or um to speak at a conference you don't know what level of um detail you want to go through either in the cfp or in the or in the the the Talk itself is it good to like how do you know your audience and how do you know what level of depth you you want to apply um re is that how do you normally go about it that's that I still find that tough sometimes um and and sometimes I'll you know start with like okay I kind of want to be more high level about this topic but in order for someone to have to come to this topic they might need you know XYZ knowledge and so even though it's more high level on that specific thing it might be considered like an intermediate if that makes sense um so it yeah it really depends and I'll also try to um you know consider like how much you know is this am I talking about something newer you know that was like more newly developed um so there's like less info about it or is it something that's been around for a while um so I don't I don't have like a really good answer um I I still find it tough honestly um I try to stick to like either beginner or intermediate level um in general because of the topics that I typically choose to do and the thing also it's a personal judgment as well sometimes you say oh I think it's it's beginner or I think it's it's I have no idea it's it's complicated yeah what what may seem basic to you might be very Advanced to others or the opposite right yeah yeah yeah do you have any any other tips there Josh to try to basically gauge the I have a quick anecdote so yeah it's it's hard to get it right um I gave one talk that was an introductory talk but I was trying to make it a little more advanced I thought oh this is you know this seems like a fairly technical audience I'd been having conversations with people at lunch everybody here's going to know what evf is so I didn't mention it and then the next speaker after me says raise your hand if you know what ebpf is and me and the one person I was talking to one raised our hand oh well awesome so yeah it's hard to get it right it's hard yeah right okay so Henrik I'm going to go back to you you um because I know that you're everywhere in open Telemetry as well so like I I know that there's a lot of Hot Topics in open Telemetry um but yes so what are the Hot Topics that you recommend people be writing about right now I know that you know doesn't have to be Hotel I know that we all love hotel but what are the what are the sort of like the Hot Topics that you would recommend thinking about writing about or talking about sorry uh I think uh what I would suggest is that but by the way AI I don't AI is not valid it is it is I think the in general the uh you you have to look at in previous conferences what has been presented and covered so far um so if you start to say I want to do open Telemetry on instrumentation there's plenty of them out there there's plenty of video out there and then there's no Hot Topic in general so usually I try to bring the the problem statement so for example how do I sample properly how do optimiz because the there is a big big in in case of observability there's a big concern about oh uh observ is going to be expensive so how can I control that cost uh so going through on that directions I think also there is another Trend that that I think is really important that people needs to be uh educated a bit more uh it's the sustainability so how can we make it green make it better so um talking about AI is not going to be green because you're going to consume more resources so maybe having another approach where you say oh you we we we we need to be good citizens of the worlds and and save energy I think that that's a pretty good one and also I think there is um anything that new projects comes in for example today uh as we speak open Telemetry profiling there is few talks out there in cucon and it's going to be more and more popular so there's a big chance that profiling will bring new problems new challenges so covering them that could be also interesting so try to follow what happens in the industry and and think about oh um if I start digging this I may have problems so how can I resolve them and maybe share that that uh that solution or that um this approach to to to a larger group to the community nice I just add Henrik Henrik you're really good at this right like you mentioned uh sampling and I saw that video right in my feed and I was like oh that's a great topic I'm gonna check that out later when I have time so uh yeah like if if henrik's talking about it he can only be one place at once right so maybe that's an interesting thing for you to put your own spin on somewhere else yeah I think so Absolut absolutely any other any other takes on you know what are the some of the some of the Hot Topics I would like to add something so uh you know I think piggybacking on what um Henrik was saying on on uh the topics of sustainability um there's some really cool like cncf projects out there um on on Tech sustainability and I I feel like this is a super hot topic um for for the year just because we're seeing a lot of stuff around like just wacky environmental things happening right like increase in forest fires bizarre temperature swings like and we're inherently in an industry that is contributing to the problem so um writing talks about um how we can use technology to less in the problem I think um can be really compelling and very timely and so there's there's a hot topic for yall for anyone considering it I would also mention you know like we talk a lot about cubec cons and I think Josh made a point earlier um about like there's a lot of conferences where like things like open Telemetry are still like kind of um you know not not super well known so um especially like a lot of these open- Source conferences like scale for example I think just added an observability track this year um fum I think would probably be another great one state of open con um another great one where you know we probably um there's probably not enough talks on observability so getting into those um sort of more nichy conferences um I think would be would be a good place to start especially if you're looking to do a talk on observability oh yeah all things open as well well Reese mentioned in in our yeah so right I'm going to take one question from the from the audience thank you so much for your questions and again you know if you've got any questions on what we're talking about drop them in the in the chat on YouTube on LinkedIn um and the question is should we go for a catchy title for a click baity title when you when you write your cfp or should you do more something more perhaps descriptive and accurate um Josh what do you think about about that do you normally go for something like that I almost always do a clickbaity title or at least I did that was always sort of my way um but I've only been doing this for a couple of years and someone who's been doing it for longer than me told me um actually it's a it's a it's a pendulum right it swings and what the conference organizers are looking for is going to swing back and forth almost like a cultural Zeitgeist and there are times where they want it to be super super specific and we're not in one of those times right now and be like maybe we're swinging in that direction but for me right now it does feel very much like the clickbaity titles are in and maybe we'll all get tired of them because of the AI topic and we'll be like no you need to tell me exactly what's in your talk so that I know that it's not a surprise ai ai talk and that'll be where we're at a year from now who knows I think um it if you can come up with a catchy title that also will give audience an idea of what to expect or like what your topic is about that is a great way to go I also do really like just clear straightforward titles as well um so I think they both have their place um and you know the content for sure is going to matter more I I say um because you could have a great catchy title and then the abstract is kind of you know maybe doesn't doesn't really fully flush out the idea um so I I I think the content is still king um but yeah title is is important as well um but as long as you have like a clear title and not necessarily like you know kind of click baity or whatever I think that's that's fine too in in when I preparing uh my talks usually I I try to bring the technical aspects and then try to find a funny angle or an analogy to a movie to video games to do whatever and then in the title I I try to bring that that fun uh friend angle because at the end uh U I think that that could be more attractive at the end because at the end when you have the schedule and you see a a title that is more I don't know sounds more fun then maybe more people will join your session for some some some reasons well I think the title is is is as as important as as the abstract from my perspective yeah and I think s go ahead go ahead oh I was going to say I I do generally agree with you on that the only thing I would caution again is a um make sure your titles don't sound like they were CH generated by chat GPT and B um make sure that your titles like aren't so pop cultur where like it ends up alienating potentially the cfp reviewers like I don't know if someone's like making a Game of Thrones reference in a talk title I'll be like I have no idea what you're talking about and I'm kind of like super put off by it um so I would just caution like it's it's a fine line to trade nice and I guess you know that's as well part of there are a couple of questions from from from YouTube which are related to to to that basically to knowing your audience right so who's going to be reading your your title your cfp so how do you know that how do you know like how to approach the the the audience in a way that they would understand or engage with and I'll ask that to Henrick it's it's a very very good question um uh usually goes to kcds and to cubec con and or open source friendly conferences so I know that the people that will be in that conference has at least tech technical background uh so then I I know that my talk would be not perceived as too technical um but yeah if you go to a conference that is more salesy uh then yes uh you may have to adjust and say okay um what are the type of persona that will be in this conference do I have the the normal audience that I'm talking to in general or should I uh reduce the complexity of my talk so then people can can follow uh the the idea behind behind this talk cool right I'm going to move on to uh to another topic which is like the post acceptance so like you got accepted hooray so what happens now what how do you train how do you rehearse what is like what can you give people to get ready for for the day I know was a very wide question so I'm going to start with h I'm going to start with ree how do you like prepare for it well so when I got my first talk so funny well funny story my very first cfp that I submitted ever was also my first one that was ever accepted which was huge and terrifying and my manager at the time she actually hired a well hired she got me a few lessons virtual lessons um with a public speaking coach and I found that that was super helpful it was just um three virtual um sessions and I came away with techniques that I still use but I also know people who you know are great speakers who don't who haven't gone through that profession training um but I say it's at least worth it to if you can if you have the resources to um I recommend it but if not there's also you know um YouTube's YouTubes YouTube videos you can watch um and of course like rehearse um as much as you can so that you feel comfortable and feels natural when you're like presenting but I'm sure other people will have their do any other topic any other like um tips or tricks to to basically get to the to the to the training rehearsing stage I uh I I usually don't I do just the rehearsal not a full fresh rehearsal I'm just checking that my talk is basically I'm respecting the duration because sometimes I put lot of content and then I realize oh maybe maybe too much here so I need to maybe figure out by timing yourself and then you know that when you will be on stage there's a big chance that you will take probably more time so I had a few minutes few minutes in a like a to to be always in time I think that's the best thing uh but what I usually do I'm trying to to put a lot of Graphics make it more nicer uh I put I mean that that's my my my style so I like to make a lot of graphics and make it more smoother experience for the audience so that's something that I prepare a lot in the background nice I'm going to take now a question from the audience and I think I'm um I think I'm GNA I'm going to choose I'm going to pick Josh to answer this question if a conference offers multiple formats for example lightning talk or a deep dive and you know I could make your talk fit more than one should I spam the organizers with multiple responses or is there a better way in my opinion that's something that will work if you know the organizer and can reach out to them directly right like or if they're having an office hours and you can talk to them about it and you can get that feedback that's fine but I do think that if you're a firsttime speaker at the conference or sort of unknown to the conference um you have to sell the talk you have to be a little bit more confident in the talk that you're proposing because they're going to be evaluating you based on the talk more than how they know you as a as a speaker see anybody else yeah I wanted to just mention something like especially um depending on the conference like some conferences don't have limits as to how many um talks you can submit um and so um sometimes you know especially if you're like a newbie speaker go for it and just see see how far you can take it um but other other conferences like Yukon where there is a limit on the number of sessions I would caution against um submitting like basically two versions of the session where it's like one's a lightning talk the other one the ones the other ones a longer form talk I think you have a you probably have a better chance of just submitting like two unique topics um because you never know also I will say um don't be afraid to recycle submissions at other conferences it doesn't always have to be a unique topic super super important it saves you a lot of like mental energy because putting together talk is a lot of work I would just add to the fact that um for the reviewers if you have a talk and usually you design your abstract for a 30 minute stock for example then the reviewer would expect some some details so then you say okay so he's GNA cover this and that makes sense in 30 minutes because it's going to be well covered and then if you then say uh if you do the same cfp in uh lightning talk then you say wow how is it going to cover that in five or 10 minutes so I think you need to adjust of course the way you're going to um present the abstracts I would definitely recommend to do like like Adana mentioned to have two different submission uh one lighter for the liting talks and one bigger for the the panel the the normal track cool okay I think it's time to start closing closing thoughts and I think one of the well not not quite there yet but one of the questions that that people normally get is I apply to you know many conferences and then you get you get rejected and uh how how normal is it to get rejected I think is it how do you manage your your expectations for getting your cfps accepted I corre yeah it's tough is it so Josh do you want to go uh give us give us your thoughts sure I think uh someone again you know getting advice from mentors is great someone uh told me on to expect a 10% acceptance rate so if you want to talk at one conference a year that means you need to submit 10 um and I I think that holds even for experienced speakers to some extent uh as well as new at least from what I've seen I don't know if you all how you feel about that ratio but you know that's interesting because I haven't actually done like a uh statistical analysis myself um but it is absolutely normal to get rejected you know you think about conferences some conferences get especially the bigger ones they get thousands sometimes um but usually like you know at least tens of dozens or hundreds of submissions um so and just because they rejected it you know this time doesn't mean that they won't next time I know people who've submitted the same one um a few times to the same conference before it got accepted so sometimes it's just timing sometimes it's just you know they just have a lot of topics that were just really really good um and sometimes too you can also ask for feedback from the from the um um conference um keepon for example they're pretty good about sharing anything um if they get feedback on your um proposal they'll they'll be happy to share it so that's something to think about too cool I I also wanted to add like it's okay to like mourn your rejection um because you know especially like when you're really invested in a topic and you you think like I've got such a great chance of getting this in and you get rejected you got like a great title and you're just yeah yeah it's like it's okay to mourn and and take that time to mourn and then as ree said like if possible ask for feedback um resubmit it um to that same conference or another conference or like in the case of cubec cons there's like so many across the globe submit it to a different cucon um you know like if didn't get in for na submit for EU there's open source Summit na there's open source Summit emia and also like take some time to reflect also on your cfp and see if there's like um areas where you see Improvement like sometimes I'll like write out a cfp and then I'll like you know it gets rejected I res submit it for another conference I'm like o yeah I can kind of see why they didn't like that and just make a few tweaks yeah agree well thank you so much I think now I've got one last question from you and I think as uh well if everyone's been following the the Golden Globes and the Academy Awards you'll see that we've got quite a lot of horror films coming up and the awards so I've got a question for you I think you've been to many conferences and I would like to hear some of the conferences that you've been to because I think we didn't cover that in the in the intro and I would like to know you know some of the some of the conferences that people could go and rewatch your videos but also like if you've got a quick anecdote or something that failed that was a bit of a Horror Story right so the mic doesn't work or you know something that doesn't work so um can you tell us about it and I'll go for re first because she's got her hands raised my very first talk that I did it was about tail sampling open Telemetry I had a live demo and everything was working I checked it right before my talk everything was working got up to the stage started the demo it didn't work um and I still don't know what it was because I got it to work a few minutes later and I know there's some people that I'm still remember that too so anyways that's my I have other ones too but that one is very um sticks close your recent uh conferences that You' talked at Ree I think so people can go and watch some of your of your talks uh um all things open I'm not sure if every one of those has recordings um but yeah those are the big ones that I I can remember off the top of my head co um Henrik do you want to go next uh has uh what was my worst experience I would say uh it happens once where I have to connect my laptop the screen mirroring didn't work and then I suddenly I did the demo and then you end up having nothing on your screen and you're like this and trying to type uh and looking at and it's like nightmare I was say okay the demo is going to be a nightmare impossible to achieve and so then I um I think the demo I I try to to skip it uh very fast uh but I was not very happy uh people didn't didn't see so much because I I I try to react as fast as possible to avoid having that blank effect where stress coming up up to you but um yeah at the end I was unhappy because I said ah my performance was just uh not normal and not acceptable so I had to improve that but yeah uh where can we see you next where can you so i w i was rejected for qcom so I've been crying uh so now I think uh I have no tears anymore so no it happens to the best it happens to the best it does uh so uh yeah the next I have a virtual conference that uh I don't know when it's it's going to happen in a few weeks I'm presenting talk and then I submitted lots of uh talks for the season so a lot of kcds and Cube con China and and Japan so we'll see didn't have the um the feedback yet so cross the fingers but otherwise if you want to watch any content from my end uh you can find it on cubec Con Europe cubec con North America and then I have plenty of other kcds where I have presented talks last year awesome Josh do you want to tell us your Horror Story yeah it's actually quite similar to henrik's it had to do with screen mirroring and not working and this is actually also like recent my very first uh talk um the audience could only see half of my slides I could see all of my slides but none of my notes um so my lesson that I took away from that is I never I never rely on my notes ever again um plus one plus one to that plus one to that yeah you have to have it all in your head you have to be able to just talk about the thing for 20 minutes with no notes and no slides if it comes down to it um I think not to scare anyone else off from doing this right like you can you can do that actually it's not as hard as it sounds um but yeah that was that was my worst Horror Story um although I would say like I feel like at least something small goes wrong pretty much every time uh and then where where you can find me right uh a lot of devop days I'm really big fan of those and I love that the the kcds are starting to sort of adopt that that Community format and sort of the open discussion format more um I really love those conferences and I love to get the combination of like the people who are are coming from all over um like you would see at the bigger conferences and then just also the local Representatives uh of the tech companies that are Employers in that area nice kcd and Edinburg yeah that is a kcd and Ed find out this this year and probably because London it's normally in London but like I guess London is getting cubec con so at least we're getting something I didn't see the cfp yet is it when is it open I it's in October so I'm not sure when the okay but yeah everyone in Edinburgh is really excited about it I mean I'm not sure the the normal person in the street would be but I am Adriana do you want to have some yeah tell us some of your horror stories um I think for we knock on wood I haven't had any like horrid things happen during talks um so I I will say that um I am not a fan of live demos and so I I do pre-recorded demos actually Ree and I have done a bunch of talks together um and and the ones that we've done together have had demos but they were pre-recorded but live narrated so that is my tip for you if you're scared of live demos like I am um I think for me the worst is like when you give a talk and you're like H that wasn't my best work and you feel like you're a little bit off like sometimes my voice is a little bit shaky or I just like kind of forget my talking points um I think that's uh you know just getting rattled by that and and and I it's interesting because um you know you come off a talk and people are like oh that was so great and you're like oh I sucked and so you kind of just have to like get over yourself um and just tell yourself like it happens um and it'll happen again and you just have to be okay with it and I would also suggest when you're practicing talks um practice like you know as you're practicing and you stumble just like power through it because that can happen in real life or you you stumble so if you if you practice that during your you know if you stumble during your practice don't like restart um see where it takes you um as far as conferences uh that I've spoken at and I'm just sending Henrik a link to my YouTube channel because I have a playlist of all my talks um but uh like I said recently I've done a bunch of talks we've spoken at cucon North America and EU a few times together we've spoken at observability day North America and EU together I've spoken at platform engineering day most recently in in North America we've done all things Reese and I have done all things open together we did umce open source Summit yeah open source North America and EU coming up and what we'll be at scale coming up yes we're going to scale in in March um we're speaking at cucon EU um yeah coming up in London um and I got in for observability day EU as well um and then I I did um I did a devop days in Montreal and most re and one of my favorites was a kcd Buu um where I got to give a keynote unfortunately there's no recording of of the keynote but I have a blog post version of the talk if anyone wants to check out my medium Channel awesome well thanks everybody so much for I mean you are really the the Dream Team so really like all the veterans from all the you know conference dos you can see that you know like from the amount of conference that you be so thank you so much for for yeah for your tips and tricks and your advice I think you know it's been really really useful um but I think that's all we had time for and uh I just want to say thanks for to the panelist thanks to the people that ask questions in the audience I think we didn't get to cover all the questions but um we are always open in the end user Sig you can come to the cncf slack you can go to the the resources that are here basically and get in touch with us I think we would love to hear more about your well in general anything about open Telemetry but also about topics like this one how do you go and talk about open Telemetry everywhere so um yeah with that that thank you very much and see you at a conference get the questions that were UN insed in the SL Channel yes right thank you bye bye bye diff --git a/video-transcripts/transcripts/2025-02-26T15:40:58Z-what-is-otel-otel-for-beginners-the-javascript-journey.md b/video-transcripts/transcripts/2025-02-26T15:40:58Z-what-is-otel-otel-for-beginners-the-javascript-journey.md index 642c8ab..9938318 100644 --- a/video-transcripts/transcripts/2025-02-26T15:40:58Z-what-is-otel-otel-for-beginners-the-javascript-journey.md +++ b/video-transcripts/transcripts/2025-02-26T15:40:58Z-what-is-otel-otel-for-beginners-the-javascript-journey.md @@ -10,93 +10,191 @@ URL: https://www.youtube.com/watch?v=iEEIabOha8U ## Summary -In this video titled "Otel for Beginners," Lisa Jung introduces OpenTelemetry (otel) as a crucial framework for observability in modern software systems. She emphasizes the importance of turning complex, opaque systems into transparent ones by effectively collecting and managing telemetry data—metrics, logs, and traces. Lisa explains how otel mitigates vendor lock-in by establishing an open standard for instrumentation, allowing developers to easily switch between observability backends without the need to learn new proprietary systems. The video also outlines resources for beginners, including official documentation, a YouTube channel, and a community on Slack for support. The next episode promises to focus on starting the JavaScript journey with otel. +In this introductory video titled "OTel for Beginners," Lisa Jung, a member of the OpenTelemetry (OTel) Communication and End User Special Interest Groups, shares her personal journey of learning OTel and aims to guide viewers through the process. The video explains the importance of observability in modern software systems and how OTel serves as an open-source framework to generate, collect, manage, and export telemetry data, thereby preventing vendor lock-in. Lisa emphasizes the advantages of using OTel over proprietary solutions, including reduced switching costs and ease of use across different observability backends. She also highlights the resources available for getting started with OTel, including official documentation, YouTube tutorials, and community support on Slack. The next episode will focus on beginning the JavaScript journey with OTel. -# OpenTelemetry for Beginners +## Chapters -Hi, welcome to OpenTelemetry (Otel) for Beginners! My name is Lisa Jung, and I'm a member of the OpenTelemetry Communication SIG and End User SIG. I recently began my journey with OpenTelemetry, and as a new user, I had a tough time figuring out how to get started. I want you to have a different experience, so I'll learn alongside you and share what I'm discovering through this series. +Sure! Here are the key moments from the livestream along with their timestamps: -Depending on which programming language you're working with, you'll follow a language-specific journey. In this series, I'll focus on the JavaScript journey to get you started with OpenTelemetry. +00:00:00 Introductions to OTel for Beginners +00:01:30 What is OpenTelemetry (OTel)? +00:02:45 Importance of Observability +00:04:00 Turning a Black Box into a Glass Box +00:05:30 Types of Telemetry Data: Metrics, Logs, and Traces +00:07:15 Instrumentation and Data Collection in OTel +00:09:00 The Problem of Vendor Lock-In +00:11:30 Comparison of OTel to USB-C Standard +00:14:00 Benefits of Using OTel for Observability +00:16:15 Resources to Get Started with OTel -## What is OpenTelemetry? +Feel free to ask if you need more details on any specific section! -Before we get our hands dirty, let’s discuss what OpenTelemetry is, why you should consider using it, and the resources to get started. +# OTel for Beginners -OpenTelemetry (Otel) stands for **OpenTelemetry** and plays an important role in observing your systems. First, let's talk about observability and how OpenTelemetry fits into this concept. +Hi! Welcome to OTel for Beginners. My name is Lisa Jung, and I'm a member of the OTel Communication SIG and the End User SIG. I recently began my journey with OTel, and as a newbie, I had a tough time figuring out how to get started. I want you to have a different experience, so I'll learn alongside you and share what I'm discovering through this series. -### Observability +Depending on which programming language you're working with, you'll follow a language-specific journey. In this series, I'll guide you through the JavaScript journey to help you get started with OTel. -Modern software systems can consist of complex, multi-layered, and distributed architectures with many interdependencies. A system without observability is like a black box; we have no idea what’s going on inside. If something goes wrong, it becomes more difficult and time-consuming to solve the problem. +## What is OTel? -With observability, we can turn this black box into a glass box. Observability helps you collect data necessary to visualize and understand what’s happening in your system. +Before we get our hands dirty, let's talk about what OTel is, why you should consider using it, and the resources available to get started. OTel stands for **OpenTelemetry**, and it plays an important role in observing your system. -### How Does It Work? +Let’s discuss observability first, then delve into how OTel fits into this concept. Our modern software systems can be complex, multi-layered, and distributed, with many interdependencies. A system without observability is like a black box; we have no idea what's going on inside. If something goes wrong, it becomes more difficult and time-consuming to solve the problem. -To make this possible, you first have your infrastructure or applications that you want to observe. You’ll collect data from it and send that data to the observability backend of your choice. Then, you connect the backend to a visualization frontend where you can query and use the data that interests you. +With observability, we turn this black box into a glass box. It helps you collect the data necessary to visualize and understand what's happening in your system. -The most common types of data collected for observability are **metrics, logs,** and **traces**—these are known as telemetry data. Getting the telemetry data into the backend is crucial for understanding your infrastructure or applications, and this is where OpenTelemetry comes in. +### Making Observability Possible -### OpenTelemetry's Role +How do we make this possible? First, you have your infrastructure or applications that you want to observe. You'll collect data from it and send that data to the observability backend of your choosing. Then, you connect the backend to a visualization frontend where you can query and use the data that interests you. -OpenTelemetry is an open-source framework that allows you to add software to your applications or systems to generate telemetry data. This process is known as **instrumentation**. OpenTelemetry collects, manages, and exports telemetry data to an observability backend for storage. +The most common types of data collected for observability are **metrics, logs,** and **traces**. These are known as telemetry data. Getting this telemetry data into the backend is crucial for understanding your infrastructure or applications, and this is where OTel comes in. -Different aspects of OpenTelemetry make this process possible, and we will discuss them in detail as we go through the JavaScript journey. For now, remember that OpenTelemetry focuses on the generation, collection, management, and export of telemetry data. You can easily instrument your applications or systems, regardless of their language, infrastructure, or runtime environment. The storage and visualization of telemetry are intentionally left to other tools. +OTel is an open-source framework. Using OTel, you can add software to your applications or systems to generate telemetry data. This process is known as **instrumentation**. OTel collects, manages, and exports telemetry data to an observability backend and the database for storage. -## Why Use OpenTelemetry? +Different aspects of OTel make this process possible, and we'll discuss these in more detail as we go through the JavaScript journey. For now, remember that OTel focuses on the **generation, collection, management,** and **export** of telemetry data. You can easily instrument your applications or systems, regardless of their language, infrastructure, or runtime environment, with the storage and visualization of telemetry intentionally left to other tools. -Now, why should we consider using OpenTelemetry? +## Why Should We Use OTel? -Let's first discuss something that many of us have experienced. If we dig through our drawers at home, we could probably find a bunch of cables of different types. Why? Because the devices we own come from different vendors, and each vendor has a specific type of cable and port to charge or connect their gadgets. +Now, why should we consider using OTel? Before we answer this question, let's talk about something many of us have experienced: the issue of **vendor lock-in**. -We are all familiar with buying multiple products from the same vendor and investing in additional accessories and apps specifically designed for that product. Before we know it, we get so used to using a line of products that switching to another vendor can become quite difficult. The product from another vendor may operate differently, which will take time to learn and adapt to, costing us more money because our additional accessories or apps won’t work with the new product. +If we dig through our drawers at home, we could probably find a bunch of cables of different types. Why? Because the devices we own come from different vendors, and each vendor has a specific type of cable and port to charge or connect your gadget. -This issue is known as **vendor lock-in,** where the costs of switching vendors are so high that customers feel stuck with what they have. +We're all familiar with buying multiple products from the same vendor and investing in additional accessories and apps specifically designed for that product. Before we know it, we become so accustomed to using a line of products that switching to another vendor can become quite difficult. -### The Shift Towards Standards +A product from another vendor may operate differently, requiring time to learn and adapt. It would also cost us more money because the additional accessories or apps we've invested in don't work with a product from a new vendor. This challenge is known as **vendor lock-in**, where the costs of switching vendors are so high that customers feel stuck with what they have. -Things are slowly changing with USB-C ports becoming the universal standard. For example, it doesn’t matter if you’re using an iPhone or an Android phone; you can charge or connect many of these phones with a USB-C cable. Because a universal standard has been set, users can use the same cable regardless of the brand of phone they have. +### The Shift Towards Universal Standards -OpenTelemetry has similar implications in observability as USB-C does for phones. +However, things are slowly changing with **USB-C ports** becoming the universal standard. For example, with the newest model of phones—regardless of whether you're using an iPhone or an Android—you can charge or connect many of these phones with a USB-C cable because a universal standard has been established. This allows users to use the same cable, no matter how many times they charge their phones. -### Vendor Flexibility with OpenTelemetry +OTel has similar implications in observability as USB-C does for phones. -To summarize, we have our infrastructure or applications that we want to observe. We want to collect telemetry data and send it to an observability backend for storage, so this data can be queried and visualized with an observability frontend. +### A Quick Review -Imagine you have three observability backends to choose from: vendors A, B, and C. Each vendor often has proprietary instrumentation agents or collectors. Let’s say you picked vendor A. You're using their proprietary instrumentation agents or collectors and sending the data to its backend. But what if your needs change later, and you want to switch to vendor B? +Let’s review briefly. We have our infrastructure or applications that we want to observe. We want to collect telemetry data and send it to an observability backend for storage, enabling the data to be queried and visualized with an observability frontend. -Switching vendors is not as simple as it might seem. You can’t just send the existing data from vendor A to B because vendor B requires its own instrumentation agents or collectors, which means you cannot accept data from vendor A's proprietary instrumentation. Your development team would need to change their instrumentation, possibly to a new proprietary solution from vendor B. +Imagine you have three observability backends to choose from: vendors A, B, and C. Each vendor often has proprietary instrumentation, agents, and/or collectors. -Now, imagine doing this for thousands of Linux machines or dozens of applications. Is the cost of money, time, and effort worth the benefits of changing vendors? As you can see, with this setup, the cost of switching vendors can be so high that customers can feel effectively locked in by their choices. +Let’s say you choose vendor A and start using their proprietary instrumentation, agents, or collectors to send data to their backend. But what if your needs change down the road, and you want to switch your backend to vendor B? Switching vendors isn't as simple as it seems. You can't just send the existing data from vendor A to B because vendor B requires its own instrumentation, agents, or collectors that cannot accept data from vendor A's proprietary instrumentation. -### The OpenTelemetry Advantage +Now your development team has to change their instrumentation, possibly to a new proprietary solution from vendor B. Imagine doing this for thousands of Linux machines or dozens of applications. Is the cost in money, time, and effort worth the benefits of changing vendors? -Now, imagine if all vendors accepted the same standard for sending or receiving telemetry data, similar to many phone companies accepting USB-C as a standard. When you switch a vendor, you wouldn’t need to learn proprietary instrumentation agents or collectors each time. You would only need to learn one technology and a single set of APIs and conventions associated with it. Whatever data you generate with this technology is yours, and you could send it to any observability backend that accepts the standard. This would make switching vendors substantially easier and reduce costs, time, and effort. +As you can see, the cost of switching vendors can become so high that customers can be effectively locked in by their choices. -This is exactly what OpenTelemetry does for you. It creates an open standard—a set of guidelines, rules, or specifications for sending and receiving data. +### The Solution: OpenTelemetry -There’s a significant incentive for vendors to accept OpenTelemetry. Many customers prefer OpenTelemetry to avoid vendor lock-in; they want to learn a single set of APIs and conventions instead of having to learn new ones every time they change a vendor. They also want to own their data and send it to any observability backend that accepts these standards. +Now, imagine if all vendors accepted the same standard for sending or receiving telemetry data, similar to how many phone companies accept USB-C as a standard. When you switch vendors, you wouldn't need to learn proprietary instrumentation, agents, or collectors each time. You just need one technology and a single set of APIs and conventions associated with it. -Vendors benefit from accepting OpenTelemetry as well. Customers may already be familiar with it, and there is a vibrant OpenTelemetry community that serves as a great resource. By accepting OpenTelemetry, vendors can reduce their support and implementation costs. Moreover, there is more demand for innovation since vendors are receiving the same data, prompting them to innovate to stand out from competitors. +Whatever data you generate with this technology is yours. You could send the data to any observability backend that accepts the standard, significantly reducing the cost, time, and effort involved in switching vendors. This is exactly what OTel offers. -As you can see, there are many benefits to accepting open standards, and you can take advantage of these benefits by using OpenTelemetry. +OTel creates an **open standard**, a set of guidelines, rules, or specifications for sending and receiving data. There’s a strong incentive for vendors to accept OTel. Many customers now prefer OTel to avoid vendor lock-in, wanting to learn a single set of APIs and conventions instead of a new one every time they change vendors. They also want to own their data and send it to any observability backend that accepts these standards. -## Getting Started with OpenTelemetry +Vendors benefit from accepting OTel as well. Since customers may already be familiar with using OTel, there's a vibrant OTel community that serves as a great resource. Accepting OTel helps vendors reduce their support and implementation costs. Moreover, it drives innovation because vendors receive the same data, compelling them to innovate to stand out from competitors. -Now that we've covered what OpenTelemetry is and why you should use it, let’s go over the resources to get started. +## Getting Started with OTel -The links to all the resources are included in the description of the video. The best place to begin is the **OpenTelemetry documentation**. The documentation is continuously being improved, so the page may look different by the time you watch this video. Be sure to use the link in the description to check out the latest page. +Now that we've covered what OTel is and why you should use it, let's go over the resources to get started. The links to all the resources are included in the description of the video. -The first place in the documentation you should start with is the **language APIs and SDKs**. As I mentioned earlier, your OpenTelemetry journey will differ depending on the programming language you’re working with. Select the language of your choice, and you should end up on a page that lists all the resources to get started. +1. **OTel Documentation**: The best place to start is the OTel documentation, which is continuously being improved. The page may look different by the time you watch this video, so use the link in the description to check out the latest version. Start with the **Language APIs and SDKs** section. Your OTel journey will differ depending on the programming language you're using, so select your language of choice to find all the necessary resources. -Next, we have the **official OpenTelemetry YouTube channel**, where you'll find helpful videos along with the OpenTelemetry for Beginners series. +2. **Official OTel YouTube Channel**: You'll find helpful videos, including the OTel for Beginners series on this channel. -Lastly, as you start your OpenTelemetry journey, you may have many questions. We have a huge community of OpenTelemetry users on Slack. Join the CNCF Slack Channel, post your questions in the OpenTelemetry Channel, and connect with other community members. Again, check the description of this video to access all these resources. +3. **Community Support**: As you embark on your OTel journey, you may have many questions. We have a huge community of OTel users on Slack. Join the **CNCF Slack Channel**, post your questions in the **OpenTelemetry channel**, and connect with other community members. -In the next episode, we’ll talk about how to get started with the JavaScript journey, so stay tuned for that! +Again, check the description of this video to access all these resources. -Thank you for watching, and I’ll see you in the next episode. +In the next episode, we'll talk about how to get started with the JavaScript journey, so stay tuned for that. Thank you for watching, and I'll see you in the next episode! ## Raw YouTube Transcript -hi welcome to otel for beginners my name is Lisa Jung and I'm a member of the otel communication Sig and end user Sig I recently began my journey with otel and as a new V I had a tough time figuring out how to get started I want you to have a different experience so I'll learn alongside you and share what I'm learning through the series depending on which programming language you're working with you'll follow a language specific journey in this series I'll go through the JavaScript journey to get you started with otel before we get our hand dirty let's talk about what otel is why you should consider using it and the resources to get started otel stands for open Telemetry and it plays an important role in observing your system so let's talk about observability first then delve into how otel fits into all of this our modern software systems can consist of complex multi-layered and distributed systems with many interdependent icies a system without observability is like a black box we have no idea what's going on inside so if something goes wrong it's going to be more difficult and more timec consuming to solve the problem with observability we turn this black box into a glass box as a matter of fact it helps you collect the data necessary to visualize and understand what's going on in your system how do we make this possible first you have your infr structure or applications that you want to observe you'll collect data from it and send the data to the observability backend of your choosing then connect the back end to a visualization front end where you can query and use the data that you're interested in the most common types of data collected for observability are metrics logs and traces these are known as Telemetry data getting the Telemetry data into the back end is an important part of understanding your infrastructure or applications and this is where otel comes in otel is an open-source framework using otel you can add software to your applications or systems to generate Telemetry data this process is known as instrumentation then it collects manages and exports Telemetry data to an observability backend the database for storage different aspects of otel makes this process possible and we're going to talk about that more in detail as we go through the JavaScript Journey for now remember that otel is focused on the generation collection management and export of telemetry data you can easily instrument your applications or systems no matter their language infrastructure or runtime environment and the storage and visualization of telemetry are intentionally left to other tools so why should we consider using otel before we answer this question let's talk about something that almost all of us have experienced now if we dig through our doors at home we could probably find a bunch of cables of different types why because the devices we own come from different vendors and each vendor has a specific type of cable and port to charge or connect your Gadget we're all familiar with buying multiple products from the same vendor and investing in additional accessories and apps specifically designed for that product before we know it we get so used to using the line of products that switching over to another vendor could get pretty difficult the product from another vendor May operate differently which will take some time to learn and get used to it would also cost us more money because the additional accessories or apps we invested in don't work with a product from a new vendor this is a problem known as vendor lockin where the costs of switching vendors are so high that customers feel stuck with what they have then things are slowly changing with USBC ports becoming the universal standard let's take the newest model of phones as an example it doesn't matter if you're using an iPhone or an Android you could charge or connect many of these phones with a USBC cable because a universal standard has been set users can use the same cable regardless of how many times they charge their phones otel has similar implications in observability as a USBC does for phones let's do a quick review we have our infrastructure or applications we want to observe we want to collect Telemetry data and send it to an observability backend for storage so this data could be queried and visualized with an observability front end say you have three observability backends to choose from vendors a b and c each vendor often has proprietary instrumentations agents and or collectors now let's say you picked vendor a you're using their proprietary instrumentations agents or collectors and sending the data to its back end but your needs change down the road you want to switch your back end to vendor B but switching vendors is not as simple as you might think you can't just send the existing data from vendor A to B Because vendor B requires its own instrumentation agents or collectors so you can't accept data from vendor A's proprietary instrumentation now your development team has to change their instrumentation possibly to a new proprietary instrumentation of vendor B now imagine doing this for thousands of Linux machines or dozens of applications is a cost of money time and effort worth the benefits of changing vendors as you could see with this setup the cost of switching vendors can get so high that customers can be effectively Locked In by their choices now imagine if all vendors accepted the same standard to send or receive Telemetry data similar to many phone companies accepting USBC as a standard when you switch a vendor you don't need to learn proprietary instrumentations agents or collectors each time you just need one technology and learn a single set of apis and conventions associated with it whatever data you generate with this technology is yours you could send the data to any obser durability back end that accepts the standard so you could easily switch vendors with substantially reduce cost time and effort this is exactly what otel does for you it creates an open standard a set of guidelines rules or specifications to send and receive data there's quite an incentive for the vendors to accept otel many customers now prefer otel to avoid vendor lock in they want to learn a single set of apis and conventions rather than having to learn new on once every time they change a vendor they also want to own their data and send the data to any observability backend that accepts these standards now the vendors benefit from accepting hotel as well customers may be already familiar with using otel there's a Vibrant otel Community who serves as a great resource because of that accepting otel helps the vendors reduce their support and implementation costs on top of that there is even more d for Innovation vendors are receiving the same data so they got to innovate to stand out from the competitors as you can see there are many benefits to accepting Open Standards and you can take advantage of these benefits by using otel now that we covered what otel is and why you should use it let's go over the resources to get started the links to all the resources are included in the description of the video the best place to get started is the otel documentation the the documentation is continuously being improved so the page may look different by the time you watch this video so use a link in the description to check out the latest page the first place in the dog you should start with is the language apis and sdks as I mentioned earlier your otel journey will differ depending on the programming language you're working with select the language of your choice and you should end up on a page that lists all the resources to get started next we have the official otel YouTube channel you'll find helpful videos along with the otel for beginners series on this channel last but not least as you start your otel journey you'll come across a lot of questions we have a huge community of otel users on slack so join the cncf slack Channel post your questions on the open Telemetry Channel and connect with other community members again check the description of this video to access all these resources in the next episode we'll talk about how to get started started with a JavaScript Journey so stay tuned for that thank you for watching and I'll see you in the next episode +Hi! Welcome to OTel for Beginners. My name  +is Lisa Jung and I'm a member of the OTel Communication SIG and end User SIG. I recently  +began my journey with OTel and as a newbie, I had a tough time figuring out how to get started.  +I want you to have a different experience so I'll learn alongside you and share what I'm learning  +through the series. Depending on which programming language you're working with, you'll follow  +a language specific journey. In this series, I'll go through the JavaScript journey to get you  +started with OTel. Before we get our hand dirty, let's talk about what OTel is and why you  +should consider using it and the resources to get started. OTel stands for OpenTelemetry and it  +plays an important role in observing your system. So let's talk about observability first, then  +delve into how OTel fits into all of this. Our modern software systems can consist of complex,  +multi-layered, and distributed systems with many interdependencies. A system without observability  +is like a black box we have no idea what's going on inside so if something goes wrong it's going to  +be more difficult and more time consuming to solve the problem. With observability, we turn this  +black box into a glass box. As a matter of fact, it helps you collect the data necessary to  +visualize and understand what's going on in your system. How do we make this possible? First,  +you have your infrastructure or applications that you want to observe. You'll collect data from it  +and send the data to the observability backend of your choosing. Then connect the back end to a  +visualization front end where you can query and use the data that you're interested in. The most  +common types of data collected for observability are metrics, logs, and traces. These are known as  +telemetry data. Getting the telemetry data into the backend is an important part of understanding  +your infrastructure or applications, and this is where OTel comes in. OTel is an open-source  +framework. Using OTel, you can add software to your applications or systems to generate telemetry  +data. This process is known as instrumentation. Then it collects, manages, and exports telemetry  +data to an observability backend and the database for storage. Different aspects of OTel make  +this process possible, and we're going to talk about that in more detail as we go through  +the JavaScript Journey. For now, remember that OTel is focused on the generation, collection,  +management, and export of telemetry data. You can easily instrument your applications or systems, no  +matter their language, infrastructure, or runtime environment, and the storage and visualization of  +telemetry are intentionally left to other tools. So why should we consider using OTel? Before we  +answer this question, let's talk about something that almost all of us have experienced. Now, if we  +dig through our drawers at home, we could probably find a bunch of cables of different types. Why?  +Because the devices we own come from different vendors, and each vendor has a specific type of  +cable and port to charge or connect your gadget. We're all familiar with buying multiple products  +from the same vendor and investing in additional accessories and apps specifically designed for  +that product. Before we know it, we get so used to using the line of products that switching over  +to another vendor could get pretty difficult. The product from another vendor may operate  +differently, which will take some time to learn and get used to. It would also cost us more money  +because the additional accessories or apps we invested in don't work with a product from a new  +vendor. This is a problem known as vendor lock-in, where the costs of switching vendors are so high  +that customers feel stuck with what they have. Then things are slowly changing with USB-C  +ports becoming the universal standard. Let's take the newest model of phones as an example.  +It doesn't matter if you're using an iPhone or an Android. You could charge or connect many  +of these phones with a USB-C cable because a universal standard has been set. Users can  +use the same cable regardless of how many times they charge their phones. OTel has similar  +implications in observability as a USB-C does for phones. Let's do a quick review. We have  +our infrastructure or applications we want to observe. We want to collect telemetry data and  +send it to an observability backend for storage, so this data can be queried and visualized with  +an observability frontend. Say you have three observability backends to choose from: vendors  +A, B, and C. Each vendor often has proprietary instrumentations, agents, and or collectors. Now,  +let's say you picked vendor A. You're using their proprietary instrumentations, agents, or  +collectors and sending the data to their backend. But your needs change down the road,  +you want to switch your backend to vendor B. But switching vendors is not as simple as you might  +think. You can't just send the existing data from vendor A to B, because vendor B requires its own  +instrumentation, agents, or collectors, so you can't accept data from vendor A's proprietary  +instrumentation. Now your development team has to change their instrumentation, possibly to a  +new proprietary instrumentation of vendor B. Now imagine doing this for thousands of Linux machines  +or dozens of applications. Is the cost of money, time, and effort worth the benefits of changing  +vendors? As you could see with this setup, the cost of switching vendors can get so high that  +customers can be effectively locked in by their choices. Now, imagine if all vendors accepted the  +same standard to send or receive telemetry data, similar to many phone companies accepting USB-C  +as a standard. When you switch vendors, you don't need to learn proprietary instrumentations,  +agents, or collectors each time. You just need one technology and learn a single set of APIs and  +conventions associated with it. Whatever data you generate with this technology is yours. You could  +send the data to any observability backend that accepts the standard, so you could easily switch  +vendors with substantially reduced cost, time, and effort. This is exactly what OTel does for you.  +It creates an open standard, a set of guidelines, rules, or specifications to send and receive  +data. There's quite an incentive for the vendors to accept OTel. Many customers now prefer OTel to  +avoid vendor lock-in. They want to learn a single set of APIs and conventions rather than having to  +learn a new one every time they change a vendor. They also want to own their data and send the data  +to any observability backend that accepts these standards. Now, the vendors benefit from accepting  +OTel as well. Customers may already be familiar with using OTel. There's a vibrant OTel community  +that serves as a great resource because of that. Accepting OTel helps the vendors reduce their  +support and implementation costs. On top of that, there is even more drive for innovation. Vendors  +are receiving the same data, so they have to innovate to stand out from the competitors. As you  +can see, there are many benefits to accepting open standards, and you can take advantage of these  +benefits by using OTel. Now that we covered what OTel is and why you should use it let's go over  +the resources to get started. The links to all the resources are included in the description  +of the video. The best place to get started is the OTel documentation. The documentation is  +continuously being improved, so the page may look different by the time you watch this video.  +So use a link in the description to check out the latest page. The first place in the doc you  +should start with is the Language APIs and SDKs. As I mentioned earlier, your OTel journey  +will differ depending on the programming language you're working with. Select the language of your  +choice and you should end up on a page that lists all the resources to get started. Next, we have  +the official OTel YouTube channel. You'll find helpful videos along with the OTel for Beginners  +series on this channel. Last but not least, as you start your OTel journey, you'll come across a lot  +of questions. We have a huge community of OTell users on Slack. So join the CNCF Slack Channel,  +post your questions on the OpenTelemetry channel and connect with other community members. Again,  +check the description of this video to access all these resources. In the next episode, we'll  +talk about how to get started with a JavaScript Journey, so stay tuned for that. Thank you for  +watching, and I'll see you in the next episode. diff --git a/video-transcripts/transcripts/2025-03-21T05:53:37Z-otel-me-with-jerome-johnson-cal-loomis.md b/video-transcripts/transcripts/2025-03-21T05:53:37Z-otel-me-with-jerome-johnson-cal-loomis.md index 0a47fc2..44a9779 100644 --- a/video-transcripts/transcripts/2025-03-21T05:53:37Z-otel-me-with-jerome-johnson-cal-loomis.md +++ b/video-transcripts/transcripts/2025-03-21T05:53:37Z-otel-me-with-jerome-johnson-cal-loomis.md @@ -10,183 +10,132 @@ URL: https://www.youtube.com/watch?v=DrD35XxTDsY ## Summary -In this episode of "Hotel Me," hosts Ree and Adriana, joining from Vancouver and Toronto respectively, discuss open telemetry with guests Cal and Jerome from Relativity, a company that automates legal e-discovery processes. The conversation focuses on Relativity's tech stack, their early adoption of open telemetry, and the challenges and successes they faced in implementing it. They detail their complex observability platform, which serves over 1500 services globally and processes around 10-15 terabytes of data daily. The guests emphasize the cultural shift towards observability within their organization, noting increased acceptance and benefits of open telemetry over time. They also provide feedback on the project's documentation, expressing a desire for improved standardization of telemetry data and greater support for downstream consumers. The episode concludes with mentions of upcoming events and resources related to open telemetry. +In this episode of "Hotel Me," hosts Ree and Adriana welcome guests Cal and Jerome from Relativity, who discuss their journey with open telemetry and their complex tech stack. The conversation highlights Relativity's early adoption of open telemetry and how it has significantly improved their observability practices. They delve into their architecture, which includes a network of over 200 to 500 collectors managed via Kubernetes and Helm charts, and share insights into the challenges they faced with legacy systems. The guests emphasize the importance of standardizing telemetry data and the cultural shift within their organization towards embracing observability. They also address audience questions regarding data consumption, collector management, and the benefits of auto instrumentation. Overall, the discussion showcases Relativity's commitment to leveraging open telemetry for enhanced operational insights while providing constructive feedback for further improvements in the open telemetry project. -# Hotel Me Episode Transcript - -[Music] - -Hello! Welcome everyone to a new episode of **Hotel Me**, formerly known as the End User Q&A. I'm very excited to be here. If you have been here before, I am Ree, and I have Adriana here with me. I am coming to you live from Vancouver, Washington, not BC. Adriana, where are you coming from? - -I am coming to you live from Toronto, Canada. So, we are at opposite ends of the coast, which is pretty neat. - -For those of you who are familiar with the flow, hang tight. We are going to bring our guests on pretty soon. For those of you who are new, welcome! Here’s how this is going to go: we will bring on our two special guests from an end user organization shortly. We will have a quick introduction, and they will introduce us to their tech stack and how long they’ve been using OpenTelemetry. This will be interesting because they were actually one of the first guests we talked to when we started this segment back in 2022. I’m pretty excited to follow up with them and see how far they’ve come in their journey, what struggles they’ve had along the way, and what they’ve figured out. - -Then we’ll get into some time for them to give us feedback about the project. We should also have some time at the end for any audience questions. If you have questions that come up as we go, feel free to pop them in the chat, whether you're on LinkedIn or YouTube, and we will get to them as appropriate during the conversation. If not, we will wait until the end. - -And with that, I believe we can bring our guests on. Oh, and Adriana, do you want to talk about our resources just a little bit? - -Oh, yeah! So, our resources... I believe this will take us to [OpenTelemetry.io](https://opentelemetry.io). We have a page in the Hotel site all about our End User SIG. If you want to learn more about the End User SIG, we welcome you to join. I believe there's a link on there for joining our Slack channel on CNCF Slack. If you're already on CNCF Slack, please come find us; we welcome everyone. People come along to ask questions. We might not have all the answers, but we can definitely direct you to where you need to be. - -We would also love to know where you are listening from, so feel free to put that in the chat as well. We just like to know where people are tuning in from. - -The guests we have for you today are Jerome Johnson and Calumnus from Relativity. Hello! - -Hi, everyone! - -Howdy, howdy! - -We would love to learn about you. Can you tell us a little bit about your role in the company and what Relativity is focused on? You can go ahead, Cal. Alphabetical order and whatnot. - -Oh, okay! Excellent. Very democratic. - -So, a little bit about Relativity: Relativity is a company that predominantly has a SaaS product providing services to the legal community. We automate essentially all the processes around legal e-discovery, including the exchange of documents and all of that. It is a SaaS platform, and we have a lot of companies around the world that actually use it. As part of the observability team, we are really tuned into how our SaaS project is actually working at any particular point in time. - -As for me, I started with Relativity about four almost five years ago now. I was brought in basically to upgrade the way we do telemetry. As Ree mentioned, we were one of the first ones to have a chat with you folks and one of the first adopters of OpenTelemetry in production. So, we were very early adopters of OpenTelemetry. I’m an architect here. Now, I’ll let Jerome introduce himself. +## Chapters -Hello! As Cal said, we’re on the observability team. My name is Jerome. I’ve been at the company for about 10 years or so now, but I’ve only been on the observability team, specifically the one dealing with OpenTelemetry, for a little over a year now. Hopefully, I can bring some perspective on how easy it is to use from the internal side, and I’m looking forward to chatting with you guys. +Here are the key moments from the livestream along with their timestamps: -Cool! That's so exciting. This is a great opportunity for us to start digging in. Can either of you or both of you give us some insights into your tech stack and architecture? +00:00:00 Introductions and welcome to the stream +00:02:20 Introduction of guests from Relativity +00:05:30 Overview of Relativity's tech stack and use of OpenTelemetry +00:12:00 Discussion on the decision to adopt OpenTelemetry +00:18:00 Insights into the complexity of the observability platform +00:25:00 Explanation of the collector setup and management +00:32:00 Discussion on maintaining the collector pipeline and configurations +00:39:00 Evolution of the OpenTelemetry implementation over the years +00:45:00 Discussion on data volume and budget management for telemetry +00:52:00 Sharing feedback on the OpenTelemetry project and its documentation -Yeah, I could talk about that a bit. Like I said, we were an early adopter of OpenTelemetry. Before that, we had a completely bespoke system for collecting telemetry from a lot of different sources. In addition to that, we did use New Relic at the time, but we also had a lot of other vendors in there too. So, it was kind of a nightmare from an operational standpoint to troubleshoot and debug because you had to go to lots of different sources to sort of put all that telemetry together in your head. +Feel free to reach out if you need further assistance! -At the time when I was hired, the company made a decision to try to unify everything. We looked around to see what would be the best platform for doing that, and the one that checked all the boxes, even though it was really immature, was OpenTelemetry. So, that was the one we sort of pointed to do that. - -Now, internally we have a very complicated platform. We have over 1,500 different services running on our platform, which is a huge mix of things. We are predominantly a Windows and .NET shop, so most of that is actually Windows and .NET. But we have a mix of basically every other language in the world, so we have a very complicated tech stack and need to pull in the telemetry from all those different things. - -Most of the complexity in our observability platform, which I’ll probably mention at various points, is just being able to pull those signals in from lots of different sources. We really prefer that people use the OpenTelemetry SDKs now, and we push that as hard as we can. However, we still have people using our old platform, which is Relativity APM, and we translate that into OpenTelemetry now. We pull logging from lots of different places, and that’s probably the part which is most difficult and most heterogeneous in terms of the platform itself. - -We have a whole fleet of OpenTelemetry collectors that we run in 20 different regions around the world. Each one of those is load-balanced, so we’re running anywhere between 200 and probably 800 individual collectors around the world to pull all this telemetry in. We then send that to various data stores for operations. This is really New Relic at this point, which is where most of our engineers go to see what the telemetry is doing. We have a subset of people who use LaunchDarkly for future releases, so we send a set of telemetry there. We also archive all of our telemetry for compliance reasons for a year and for forensic analysis. - -We have a reporting platform that uses Azure Blob Storage as its backend to make reporting for managers and things like that. This makes it look simpler than it is; it’s a lot more gnarly than this diagram lets on. But we do have a large scale: 20+ regions around the world and 1,500 services all feeding into this is a fairly complicated platform. - -Dang, that was really cool! I have so many questions. I do actually have a lot of follow-up questions, but I also want to get into why OpenTelemetry. I know you mentioned that your team was early adopters back in 2022. How did you come upon OpenTelemetry, and why did you decide that it made sense for you? - -I think the thing that ticked all the boxes for us was that we came from a place where we were having to maintain our own observability libraries. Given that we were .NET and then trying to diversify into lots of different languages, it meant that we were having to translate that thing again and again. On top of that, we had a mix of our own libraries for some stuff and a mix of things from New Relic. - -One of the complicated parts was that every time we wanted to move something or change something, it meant code changes. That’s something we wanted to avoid going forward. So, we wanted to standardize on something that was ideally open source, that we could include in our services one time and maintain that without having to change the code every time we needed to modify where we put things. OpenTelemetry was the only one that ticked those boxes for us at the time. - -The only concern we had was that it was a new project, so there was a lot of risk in going that direction. It wasn’t an easy decision; we discussed this a long time internally. But we decided that it was worth the risk to go ahead and adopt it because it looked like it was going to meet our needs. Basically, we wanted to isolate our codebase from what we do with the telemetry afterwards, which was the selling point for us. - -That is very awesome! I don’t know if Jerome has anything to add. - -Oh, yeah! I can give some perspective on what it was like before we had OpenTelemetry. I was serving on the performance team for several years, and it was very challenging to analyze specific applications because, as Cal mentioned, it was kind of the wild west on how we handled metrics. - -Every time we had to analyze a specific application, we would have to do a deep dive into what telemetry they were putting out. Usually, it would take half the time just trying to understand what the product was, and then the latter half would be coming up with a solution. Since joining the observability team, I've been amazed at how much cleaner it is to understand what Relativity is doing as a whole. - -As a follow-up question, we are always curious about collector setup in the SIG. What kind of setup do you use for your fleet of collectors? Can you give us an idea of how many collectors you have and how they are managed? - -In terms of how things are usually deployed, we use a gateway setup. All of our collectors are basically in a gateway setup, so people send their telemetry to us, we process it, and send it on. Over the years, we’ve had more or less complicated pipelines, but in general, we have a single set of collectors with a common configuration that handles all of that telemetry. - -We are running in over 20 regions, and each region has between 10 and 100 instances, depending on the load. So, we’re running at any point in time somewhere between 200 and 500 individual collectors, all running in Kubernetes. We are very focused on compute and Kubernetes, so that’s where almost all these things are running. - -In terms of deployment, we handle all of our deployments via Helm charts. We use Helm charts to deploy everything to our Kubernetes clusters. We take care to ensure that all of our configurations are isolated. Our collectors, except in very few cases, do not reach out to external services. We manage the configurations locally to ensure they are as robust as possible. +# Hotel Me Episode Transcript -One of the reasons we went with OpenTelemetry is that we have some internal attribute enrichments to do specifically for our product, and the way we use things. We implement those inside the collector itself, so we have custom processors that enhance the data in various ways. We validate it and drop data that contains things that shouldn’t be there, for instance. We do a lot of validation on the fly as things go through the collectors before we write it out to our various data stores. +**Hello and welcome everyone to a new episode of Hotel Me**, formerly known as the End User Q&A. I'm very excited to be here. If you've been here before, I am Ree, and I have Adriana here with me. I am coming to you live from Vancouver, Washington, not BC. Adriana, where are you coming from? -I think that covers most of your questions. Did I miss anything? +**Adriana:** I am coming to you live from Toronto, Canada. So, we're at opposite ends of the coast, which is pretty cool! -I think you covered it well. A follow-up question: since you deploy your collectors in Kubernetes via Helm charts, have you used the OpenTelemetry operator to manage any of your collectors? Is that something you’ve played with? +**Ree:** Yes, it is! For those of you who are familiar with the flow, hang tight. We will bring our guests on pretty soon. For those of you who are new, welcome! Here's how this will go: we will bring on our two special guests from an end-user organization shortly. We’ll have a quick introduction, and they’ll introduce us to their tech stack and how long they’ve been using OpenTelemetry. This segment is particularly interesting because they were actually one of the first guests we spoke with when we started this segment back in 2022. I'm excited to follow up with them and see how far they've come in their journey, what struggles they've faced along the way, and what they've figured out. -We have not. The OpenTelemetry operator sort of came out after we were already doing things. We don’t have any particular reason for not using it; we consider it tech debt. We had the Helm chart beforehand, and we’ve just continued doing that. We’re kind of used to managing things that way. +Then, we'll have some time for them to give us feedback about the project, and we should also have time at the end for any audience members to ask questions. If you have questions that come up as we go, feel free to pop them in the chat, whether you’re on LinkedIn or YouTube, and we will address them as appropriate during the conversation. If not, we will wait until the end. -We will have an internal shift in our deployment tooling coming up, so that might be an opportunity to switch how we manage things. But at the moment, we don’t have any experience with the OpenTelemetry operator. +With that, I believe we can bring our guests on. Oh, Adriana, do you want to talk about our resources just a little bit? -Got it. Another similar question: since you manage a fleet of collectors, have you experimented with the OPAMP protocol for managing them? +**Adriana:** Sure! Our resources will take us to [OpenTelemetry.io](https://opentelemetry.io). We have a page on the Hotel site dedicated to our End User SIG. If you want to learn more about the End User SIG, we welcome you to join. There’s a link on there for joining our Slack channel on CNCF Slack. If you’re already on CNCF Slack, please come find us. We welcome everyone. People come along to ask questions. We might not have all the answers, but we can definitely direct you to where you need to be. -We haven’t. We’ve tried very hard to make our fleet essentially standalone, so even for configurations and things like that, we don’t reach out to anything else. We deployed static config maps for that. It was a choice we made because we have such a large, distributed deployment; we wanted to be as resilient against networking problems as possible. Even for configuration changes, we push out new configs rather than using a centralized method. +We would also love to know where you are listening from, so feel free to put that in the chat as well. Now, the guests we have for you today are Jerome Johnson and Calumnus from Relativity. -Do you have a dedicated team responsible for maintaining the collector pipeline, or is it just whoever is on the observability team? +**Jerome:** Hello! -Yes, we do have an observability team split between Poland and here. Our other major engineering office is in Krakow, Poland. Each team has around five people at the moment, and collectively we are responsible for maintaining those pipelines and configurations. +**Calumnus:** Hi everyone! -We try to avoid shared responsibility because shared responsibility often means no one is responsible. +**Ree:** We would love to learn about you. Can you tell us a little bit about your role in the company and what Relativity is focused on? -As the project has matured, it sounds like you have a pretty good setup working well for you now. How has the implementation evolved over the years? Have you added additional instrumentation signals? +**Calumnus:** Sure! A little bit about Relativity: we predominantly have a SaaS product that provides services to the legal community. We automate essentially all the processes around legal e-discovery, such as the exchange of documents. It’s a SaaS platform used by many companies around the world. Our observability is tuned towards how our SaaS product is actually working at any given time. -It’s interesting in that the deployment hasn’t really changed in terms of architecture since the beginning. It’s been the same architecture the entire time. We’ve made tweaks here and there and simplified things as new features have come out, but by and large, it has stayed the same. +As for me, I started with Relativity about four, almost five years ago now. I was brought in to upgrade our telemetry. As Ree mentioned, we were one of the first to adopt OpenTelemetry in production. I’m an architect here. Jerome, would you like to introduce yourself? -We’ve gone through the alpha and beta phases, and things are starting to become stable now. We haven’t noticed any significant changes that we needed to make in the platform. That being said, we have added new signals. Initially, it was just metrics and traces; logs came in afterward. We had logs coming into our system even earlier than that, so we’ve folded those changes into the platform as they come along, and it hasn’t been terribly painful. +**Jerome:** Sure! As Cal mentioned, we’re on the observability team. I’ve been with the company for about 10 years now, but I’ve only been on the observability team, specifically dealing with OpenTelemetry, for a little over a year. I’m looking forward to chatting with you all! -In terms of the community, we haven’t seen any negative consequences from the alpha and beta phases. We’ve had to update occasionally, but those updates are usually pretty easy. Jerome has been handling a lot of the following for the collector. We have our own build of the collector, which we try to keep in sync with the external one, and he can probably provide some feedback on that process. +**Ree:** That’s exciting! I think this is a great opportunity for us to start digging in. Can either of you, or both of you, give us some insights into your tech stack and architecture? -Oh yeah! To be honest, updating the collectors has been a relatively painless process. There have been a few changes that we’ve had to discuss as a team, but overall, pushing out changes has been simple, especially with how detailed the change logs have been. They clearly communicate any breaking changes or deprecations. +**Calumnus:** Certainly! As I mentioned, we were an early adopter of OpenTelemetry. Before that, we had a completely bespoke system for collecting telemetry from various sources. We were using New Relic at the time, but we also had a lot of other vendors, which made troubleshooting and debugging quite complicated. -I have a follow-up on building your own collectors. I think you might be the ones who have built your own collector. What was your experience doing that, especially with the documentation provided on the OpenTelemetry site? +When I was hired, the company decided to unify everything. We looked for the best platform for that, and even though OpenTelemetry was immature at the time, it checked all the boxes for us. Internally, we have a very complicated platform with over 1,500 different services running, predominantly on Windows and .NET, but also incorporating many other languages. -We started with our own build of the OpenTelemetry collector because we needed custom receivers, processors, and exporters. Initially, it was very painful; we had to clone the entire repository and make our changes. Once the collector builder came along, it made the process much easier. +We prefer that people use the OpenTelemetry SDKs, but we still have some people using our old platform. We run a fleet of OpenTelemetry collectors in over 20 different regions worldwide, with load balancing. Depending on the time of day, we run between 200 to 800 individual collectors globally to pull all this telemetry in. We send that telemetry to various data stores for operations, primarily New Relic, which is where most of our engineers go to see what the telemetry is doing. -Now we have our own repository with a single config file for the collector builder that specifies what we pull in from contrib and the main OpenTelemetry collector. The only thing we have in our repo is our custom stuff. The builder has streamlined the process significantly, and once it was available, everything became easier. +We also archive all our telemetry for compliance reasons and have a reporting platform that uses Azure Blob Storage as its backend. Overall, we have a large-scale system supporting 1,500 services across the globe. -We did that right at the beginning, so I can’t comment on the current documentation. However, the builder works seamlessly for us now. +**Ree:** That sounds really cool! I have so many questions. I’d like to learn more about why you chose OpenTelemetry. When did you first come upon it, and why did you decide it made sense for you? -Regarding building your own components, how was that experience? Was it tricky? +**Calumnus:** The main reason for us was the need to maintain our own observability libraries while trying to diversify into various languages, which required constant translations. We were also using New Relic, so we had a mixed setup that complicated things further. -I think the biggest hurdle was learning Go. We weren’t initially familiar with it, but now we have good experience with Go, so it’s not an obstacle anymore. In terms of building our own components, we started by copying something that worked and modifying it, which was fairly straightforward. +We were looking for something open source that we could include in our services once and maintain without changing the code every time we needed to move something. OpenTelemetry was the only option that ticked those boxes. There was a lot of risk in going in that direction, but we decided it was worth it. -The documentation has improved since then, making it easier to start building. However, there are still areas where I would like better documentation, like internal telemetry and how to hook into it better. We had to reverse engineer some aspects, but overall, it was manageable. +The primary selling point was to isolate our code base from telemetry processing, allowing us to adapt without constant code changes. -We have an audience question from Sumit: How much data size do you consume, and have you found the need to reduce it via tail sampling or other methods? +**Jerome:** To add to that, before OpenTelemetry, we had a very challenging time analyzing specific applications. We had to deep dive into each application to understand what telemetry they were emitting, which was often very inconsistent. Since adopting OpenTelemetry, it’s been much cleaner and easier to understand what Relativity is doing as a whole. -We typically write between 10 and 15 terabytes a day. The split between logs, traces, and metrics is not even, but we take all of those in. We’ve never had issues with scale; we can handle whatever is thrown at us due to autoscaling on all our collectors. +**Ree:** That’s fantastic! Now, regarding your collector setup, how do you manage your fleet of collectors? -The place where we’ve had to reduce data is mainly for budget reasons since we push our data into New Relic and pay for what we push there. We manage the volume of data we send to New Relic mainly for budget reasons rather than technical ones. In those cases, we do sample down the traces and manage logs by log levels. Metrics, however, we tend to leave untouched, although we do look into cases of abuse. +**Calumnus:** We have a gateway setup for our collectors. They receive telemetry from users, process it, and send it on. We have a common configuration for our collectors, which handles all the telemetry. -Since your organization is supportive of the observability journey and OpenTelemetry adoption, can you talk more about the culture of observability? +We run over 20 regions, each with 10 to 100 instances, so we usually have between 200 to 500 individual collectors running in Kubernetes. We deploy everything via Helm charts and make sure all configurations are isolated. The collectors typically don’t reach out to external services. -When I started, I was brought in to change how we do observability, and a major part of that has been OpenTelemetry. It’s been nice to see that the technical aspects have been the easiest parts. The tech has just worked well. +We also implement custom processors to enhance the data, validate it, and drop any invalid data on the fly before writing it to our data stores. -The harder part has been the cultural changes. There has been some resistance to migration because we have a large code base. However, as people gain experience with OpenTelemetry and see the benefits, there has been less resistance over time. People are starting to lean into OpenTelemetry as a platform. +**Ree:** That’s an impressive setup! Have you used the OpenTelemetry operator to manage any of your collectors? -One thing that has enabled people to do nice new interesting things is that we can write to different places easily. For example, we write telemetry to LaunchDarkly, which allows us to use their release guardian feature. This feature automatically rolls back flags based on telemetry data, which is powerful because it integrates another platform seamlessly into our telemetry without changing our code fundamentally. +**Calumnus:** We have not, mainly because the operator came out after we had already set things up. We have our Helm chart and are used to managing things that way. However, we are considering an internal shift in our deployment tooling, which might be an opportunity to switch over how we manage things. -Thank you for sharing that! In the interest of time, I want to give you space to share additional feedback that we could share with the project maintainers. You mentioned the issue with conventions for internal data schemas; can you provide more detail on that? +**Jerome:** To add to that, we have focused on keeping our fleet standalone to avoid networking issues, so we push out new configurations rather than relying on external management. -I think it’s important to note that I am fully in support of OpenTelemetry. The maintainers are doing a fantastic job both on the documentation and the code level. This feedback isn’t necessarily about the collector implementations but about our own journey. +**Ree:** That makes sense. Who maintains the collector pipeline? Is there a dedicated team? -As we generate more telemetry, it becomes increasingly useful for various downstream consumers, such as our AI division and managers who need reporting. Standardizing data is becoming more important. We enforce a global schema for the attributes coming into the system, but how this fits with semantic conventions and data contracts is something we are thinking hard about. +**Calumnus:** Yes, we have an observability team split between our offices in Poland and here. Each team has about five people who are responsible for maintaining our pipelines and configurations, avoiding shared responsibility to ensure accountability. -Discussions around semantic conventions, the Elastic Common Schema, and how to create data contracts with OpenTelemetry data will become increasingly important. It would be great to have common themes and tools in the community regarding this. +**Ree:** As the project has matured, how has your implementation evolved? Have you added additional instrumentation signals? -The tech stack is really solid, so I don’t have much feedback on that. Looking forward, it’s crucial to standardize the data produced and make it more useful for downstream consumers. +**Calumnus:** Interestingly, the overall architecture of our deployment hasn’t changed much since we started. We’ve made tweaks and simplified things as new features have come out, but the core architecture has remained stable. -Thank you! Now, let’s catch up on some audience questions. Doug on YouTube wants to know if cloud providers will offer OpenTelemetry as a boring text, similar to SMTP or SFTP. Is that a crazy idea? +We initially started with metrics and traces, then added logs. We’ve folded in changes as they come along, and updates have been relatively easy to implement. Jerome has been handling a lot of the collector updates, and it’s been a smooth process. -The first question we ask any vendor now is whether they support OTLP. We treat that as our standard protocol, and it’s a negative mark against them if they don’t support it. So, I don’t think that’s a pipe dream; pushing vendors in that direction is a good thing. We have consistently given feedback to Azure that we want Azure Monitor to produce OpenTelemetry signals. +**Jerome:** Yes, updating the collectors has been relatively painless. The detailed change logs from OpenTelemetry have been very helpful in navigating updates and breaking changes. -Manas has a question about OpenTelemetry tutorials. We will respond directly with some resources for you in the LinkedIn chat. +**Ree:** That’s great to hear! Can you tell us about your experience building your own collector and components? -Another question: Did you ever feel there were classes of telemetry data that OpenTelemetry didn’t collect well, requiring you to add another collection method like eBPF? +**Calumnus:** We had to build our own collector because we needed custom receivers, processors, and exporters. Initially, it was quite painful, but once the collector builder came along, the process became much easier. Now we have our own repository with a single config file that tells us what we need and pulls in from the OpenTelemetry collector and contrib. -We had to develop some custom receivers and things that weren’t covered beforehand. One early need was for a webhook into OpenTelemetry, which now exists. The biggest gap we’ve had internally is front-end telemetry. We are currently working on a project to collect more front-end telemetry using the OpenTelemetry JavaScript SDK. +**Jerome:** In terms of building our own components, the biggest hurdle was learning Go, but now it’s not an issue. The process of copying existing components and modifying them worked well for us, and the documentation has improved over time. -In terms of other telemetry types, we haven’t had significant issues. Most things we needed to collect were either OTLP or could be captured with tools like Splunk, Fluent Bit, etc. +**Ree:** That’s insightful! Regarding data volume, how much data do you consume, and have you found the need to reduce it via methods like tail sampling? -Perfect! Thank you. Our time is just about up. Any last parting words? +**Calumnus:** We typically write between 10 to 15 terabytes a day. We haven’t had scaling issues with our collectors, but we do manage the data volume we push to New Relic for budget reasons. We sample down traces and manage log levels dynamically to control what we send. -Just thanks for inviting us! If anyone wants to reach out, I am available on Slack, so feel free to contact me directly if you have questions or if discussing what we’ve done would be helpful. +**Ree:** It sounds like your organization is very supportive of observability and OpenTelemetry adoption. Can you talk about the culture of observability within your organization? -Thank you, Cal and Jerome, for being on and sharing your journey with us. We will link to our OpenTelemetry SIG and user channel so you can find us there. You can also reach out to Cal directly via CNCF Slack. +**Calumnus:** When I started, I was tasked with changing how we approached observability. The technical aspects have been relatively easy, but the cultural changes have been more challenging. There was resistance to migration at first, but as people see the benefits of OpenTelemetry, that resistance has lessened. -We have a link for you, Manas, that will provide resources for OpenTelemetry documentation. +The ability to write telemetry to various platforms has enabled teams to do interesting things, which has encouraged adoption. -Before we wrap up, I want to mention that Hotel Community Day is coming up on June 25th, and we are currently accepting talk proposals. If you would like to submit a talk, we encourage you to submit a topic for OpenTelemetry Day in Colorado, which is part of Open Source Summit North America. +**Jerome:** To add to that, I’ve seen a significant shift in attitudes toward telemetry. Teams are now more responsive to adopting telemetry practices and seeing the benefits, which is a positive change. -Also, if you are going to or are already in Europe and planning to attend KubeCon EU in London in less than two weeks, we would love to see you there! Adriana and I will be speaking at the event, and we have a blog post about the event that outlines all the OpenTelemetry-specific talks. +**Ree:** Before we wrap up, do you have any feedback for the project maintainers? -The recordings will be available on the CNCF YouTube channel about one to two weeks after the event. So, if you can make it in person, that’s great, but if not, you can catch the recordings. +**Calumnus:** Absolutely! I think OpenTelemetry is a fantastic platform, and the maintainers are doing a great job. However, as we generate more telemetry, standardizing data for downstream consumers is becoming increasingly important. Discussions around semantic conventions and data contracts will be vital for organizations moving forward. -If you are at KubeCon EU, we will be hanging out at the Hotel Observatory off and on, so come say hi. The observatory is always fun, with many OpenTelemetry contributors and maintainers around. +Overall, the tech stack is solid, and I’m excited to see how it evolves. -There will also be project updates at KubeCon about OpenTelemetry, so I strongly encourage folks to join that session to learn what's new and exciting in OpenTelemetry. +**Ree:** Thank you so much for sharing your insights! We appreciate you both being here today. -If you enjoyed this recording but have a friend who couldn't make it, you can replay it on our LinkedIn or share the link to this live recording. It will also be available on our OpenTelemetry YouTube channel. +Before we go, I want to remind everyone about the upcoming Hotel Community Day on June 25th, where we are accepting talk proposals. Also, if you’re attending KubeCon EU in London soon, Adriana and I will be speaking there. -Thank you all so much, and we will see you next time! +If you missed this recording but have friends who couldn’t make it, you can replay it on our LinkedIn or find it on our YouTube channel. -[Music] +Thank you all so much, and we will see you next time! Goodbye! ## Raw YouTube Transcript -[Music] Hello. Welcome everyone to a new episode of Hotel Me, formerly known as the end user Q&A. I'm very excited to be here. Um, if you have been here before, I am Ree and I have Adriana here with me. I am coming to you live from Vancouver, Washington, not BC. Um, Adriana, where are you coming from? I am coming to you live from Toronto, Canada. So, we're at opposite ends of the coast, which is pretty We are. Yeah. And so, for those of you who are familiar with the flow, um, hang tight. We are going to bring our guests on pretty soon. Um, for those of you who are new, welcome. So, how this is going to go is we are going to bring on our two special guests from a an end user organization on shortly. We're going to have a quick introduction. Um, they'll introduce us to their tech stack and how long they've been using open telemetry. This one's gonna be a really interesting one because they were actually one of the first guests we talked to when we started this segment um back in 2022. So, I'm pretty excited to follow up with them and see, you know, how far they've come in their journey, what, you know, what kind of struggles they've had along the way and um what they've figured out. Um and then we'll get into some time for them to give us some feedback about the project. And we should also have some time at the end for any audience members to ask questions. Um, if you do have questions that come up as we go, feel free to pop them in the chat, whether you're on LinkedIn or YouTube, and we will get to them as appropriate um during the conversation. If not, we will wait until the end. And with that, I believe we can bring our guests on. Oh, and Adrian, do you want to talk about our resources just a little bit? Oh, yeah. So, uh, our resources, um, I believe this will I'm going to scan the QR code myself because I don't remember what this does. Um, this will take us to this takes us to open telemetry.io. Um, and we have a page in the hotel uh, site all about our um, all about our end user SIG. If you want to learn more about the end user SIG, we welcome you to join. I believe there's a a link on there also for joining our uh Slack channel on CNCF Slack. So, if you're already on CNCF Slack, um please come find us. We welcome everyone. Um people come along to ask questions. We might not have all the answers, but we can definitely direct you to where you need to be. Oh, and also we would love to know where you are listening from. So feel free to put that in the chat as well so we can we just like to know where people are listening from. And so the guests we have for you today are Jerome Johnson and columnist from relativity. Hello. Hi everyone. Howdy howdy. Hello. Would you we would love to um learn about you and can you tell us a little bit about your role in the company and kind of what relativity is focused on? You can go ahead, Cal. Alphabetical order and whatnot. Oh, okay. All right. Excellent. Very democratic. Yeah. So, a little bit about relativity. So, Relativity is a company that has a predominantly a SAS product that uh provides services to the legal community. So, we automate essentially all the uh processes around legal eiscocovery. So, exchange of documents and all of that. Um it is a SAS platform. We have a lot of companies around the world that actually use it and uh we as observability are really tuned toward how how the cloud our our SAS project is actually working at any particular point in time. Um as far as me I um I started with relativity about four almost five years ago now. I was brought in basically to upgrade uh the way we do telemetry and as Ree mentioned we were one of the probably the first first ones to have a chat with with you folks but also one of the first adopters of open telemetry in production. So we were very very early adopters of of open telemetry. Um I'm an architect here. Um and then I'll let Jerome introduce himself. Hello. As Cal said, we're on the observability team. My name is Jerome. Um, I've been at the company for about 10 years or so now. Uh, but, uh, I've only been on the observability team, specifically the one that's dealing with open telemetry for a little over a year now. So hopefully I can bring some perspective on like how easy it is to use from an on the internal side and how nice it is to so looking forward to chatting it up with you guys. Cool. That's so exciting. Well, um I think this is a great opportunity for us to uh start digging in and can um can either of you or both of you give us some insights into your tech stack and architecture? Yeah, so I could talk about that a bit and uh you know, so like I said, we were an early adopter of open telemetry. Uh what we had before that was an entirely bespoke I would say uh system for collecting telemetry from a lot of different sources. Um, in addition to that, um, we we did use New Relic at the time, but we also had a lot of other vendors in there, too. So, it was kind of a nightmare from an operational standpoint to actually do any uh troubleshooting and debugging because you had to go to lots of different sources to to sort of put all that telemetry together in your head. Um, at the time when I was hired, we we uh the company made sort of a decision to try to unify everything. We looked around to see what would be the best platform for doing that. Um and the one that sort of checked all the boxes even though it was really immature was open telemetry. So that was the one which we sort of pointed to to do that. Now internally we have a very complicated platform. Um and maybe this is time to bring up the diagram on on what we actually do now with um um with open telemetry. We have 1500 plus different services running on our platform and that is a huge mix of things. We are predominantly a Windows andnet shop. So most of that is actually Windows and .NET. Um but we have a mix of basically every other language in the world. Uh so we have a very very complicated tech stack and we need to pull in the telemetry from all those different things. So most of the complexity in our observability platform and I'll probably mention at various points. So if I say RELI, that is our observability platform that's built over open telemetry. Um most of the complexity is just being able to pull those signals in from lots of different sources and you see most of them here. We really prefer that people use the open telemetry SDKs now and we push that as hard as we can. But we still have people using our old platform which is relatively APM which we translate into open telemetry now. We have things like SNMP. uh we pull logging from lots and lots of different places and that's probably the part which is most most difficult and most uh heterogeneous in terms of the platform itself. We have a whole fleet of uh open telemetry collectors that we run in 20 20 different regions around the world. Each one of those is load balanced. So we're running we're running anywhere between uh depending on the time of day and all that between 200 and probably 800 individual collectors around the world to to pull all this telemetry in. Um and then we we send that to various uh data stores for operations. This is really new relic at this point. Um so that's where most of our uh engineers go to actually see what the what the telemetry is doing. We have a subset of people who use launch darkly and use that for future releases. So we send some set of telemetry there. Um and then we actually archive all of our telemetry for compliance reasons for a year and for forensic analysis and stuff like that. Uh we also have a reporting platform that uses the Azure blob storage basically as as its back end to to make that reporting for managers and things like that. Um so this makes it look simpler than it is. It's a lot more uh gnarly than than this diagram uh uh lets on. But we do have a large scale. I mean 20 plus regions around the world, 1500 services all feeding into this is is a fairly complicated platform. Dang, that was really cool actually. Um questions. So many questions. I do actually have a lot of um followup questions, but I also want to um get into why open telemetry like how like when exactly I know you mentioned, you know, kind of back in 2022 you you know your team was early adopters. Um how did you come upon open telemetry and like why did you decide like oh this makes sense for us? Yeah. So you know I think the thing which which sort of ticked all the boxes for us we we came from a place where we were having to maintain our own observability libraries um and given the fact that we werenet and then trying to diversify into lots of different languages uh meant that we were having to translate that thing again and again and again. Um and on top of that we also had uh well we were using New Relic at the time and we had a lot of the New Relic agents in the mix as well. So we had this this weird mix of our own libraries for some stuff. We had uh a mix of of things from open from New Relic uh with the uh with the agents and all of that kind of stuff. Um and we had a lot of people who were writing to New Relic directly from from their code as well uh as well as to other sources. So one of one of the things which was really complicated on our side was every time we wanted to move something or we wanted to change something it meant code changes. Um and that's something we wanted to avoid going forward. So we wanted to standardize on something which was ideally open source that we could include in our uh in our services one time and maintain that and then change what where we put things but didn't didn't have to change the code every time. So we're really looking for um you know something that would give us stability in terms of the codebase but allow us the flexibility to send things to other places as we needed to do that and open telemetry was the only one which sort of ticked those boxes for us at the time and the only worry we really had at the point was that it was it was a new project. Um there was a lot of risk in going that direction. uh we decided in the end it it wasn't an easy decision. We we actually discussed this a long time internally but we decided that it was worth the risk to to go ahead and adopt that because it looked like it was going to take tick the boxes for what we wanted to do there. Um you know so basically it's it's to isolate basically our code base from from the from what we do with the telemetry afterwards which was sort of the selling point for what we wanted to do there. That is very awesome. Um I don't know I don't know if Jerome has anything. Uh oh yeah I yeah just to add on to that um I can give somewhat of a perspective to what it was like before we had open telemetry like um I was serving on the performance team for a good number of years and um it was very challenging to do uh analysis of specific applications just because like Cal had mentioned things were it was kind of the wild west on how we handled metrics like performance metrics and monitoring are are somewhat look similar, a little different, but enough to where like every time we had to analyze a specific application, we would have to do a deep dive on a set application like what telemetry are you guys putting out? Oh, you're not putting out any at all. And then we'd have to like dig into the code and that would really muck up like how much time we could actually like drill down into what the core problem of said application performance uh perspective was. Usually it would take half the time we would do an analysis it would be like trying to understand what the product was and then actually coming up with a solution would be the the la latter 50 or 40% they have. So um since joining uh observability like and even like even though Cal showed a simplified version of um our reli stack and uh it's a like you said it's a bit more complicated once you drill down into it. I was amazed just how like how much cleaner it is to just understand like what our what relativity is doing as a whole. So, uh take that with what you want. That's very cool. Um now as a as a follow-up question, um one of the things that we are always curious about in the SIG is collector setup. Um what kind of you you mentioned I I believe um Cal mentioned that you guys manage a fleet of collectors. Um how do you manage that fleet of collectors? How many can you give us an idea of how many collectors? Is there a particular setup that you use? Like we we would love to know. Yeah. So, you know, I think um in terms of how things are usually deployed, I think you call it a gateway setup. So, all of our collectors are basically in a gateway setup. Um so, people send their telemetry to us, we process it and send it on. Um over the years, we've had more or less complicated pipelines, but in general, we have essentially a single set of collectors with a common configuration that handles all of that uh all of that uh telemetry. We um so in terms of the numbers of things so we are running over 20 plus regions each uh each region has somewhere between depending on the load between 10 and probably 50 to 100 uh instances themselves. So we're running at any point in time somewhere between 200 and 500 individual collectors. These are all running in Kubernetes. Um we are uh very focused on compute and Kubernetes. So that's where almost all these things are running. In terms of the deployment itself, we actually handle all of our deployments um uh via Helm charts. So we use Helm charts basically to deploy these all out to the um to the um uh Kubernetes clusters. Uh we do take pains to make sure that essentially all of our configurations are isolated. So our collectors except in a very few cases do not reach out to external services. We manage the configurations all locally to make sure that they are as available and uh as as available and robust as possible, right? Um so that's kind of how we do all of that. In terms of the pipelines themselves, um one of the other reasons why we went with open telemetry is we have some internal uh I would say attribute enrichments that we do specifically to our product and the way we use things. We actually implement those inside of the uh collector itself. Um so we have some custom processors which en enhance the data in various ways. We actually validate it. We actually drop data which is not uh which contains things which shouldn't be there for instance. Um so we do a lot of validation on the fly as things go through the the collectors themselves before we write it out to our various data stores. Um I think I I think that hit most of the points you're asking for. Did I miss any of the uh the things you're asking? I think I think you got most of them. And and yeah, actually one one follow-up question I had was uh also since you you mentioned you deploy your your collectors in Kubernetes via Helmcharts, have you used um the hotel operator to manage any of your collectors? Is that something you played with? We have not. The the hotel operator sort of came out after we were already doing stuff. So in fact, we uh we don't uh but not for any particularly good reason. No. Okay. Okay. Fair enough. So, it's it's that we had uh you consider it sort of tech debt. You know, we had the Helm chart and all of that beforehand. Um so, we've just continued doing that and we're kind of used to managing things that way. Um we actually will have an internal shift in in our deployment uh tooling uh coming up for the organization. So, that might be an that might be an opportunity to switch over how we're actually managing things. But at the moment, yeah, we're not we don't have any experience with the uh open telemetry operator at the moment. Got it. Got it. And I I uh another similar question. Um since you do meet manage a fleet of collectors, um have you played around with the opamp protocol for for doing that? We've not again again we've tried very hard to make our our fleet essentially uh standalone. So even for configurations and things like that, we don't actually reach out to anything else. We really deployed uh sort of static config maps for that. Um that that was basically a choice that we made um just to because we have such a large deployment and because it's distributed so widely, we wanted to make sure we were as um resilient against networking problems as possible. So even for things like configuration changes, we we push out new new configs for those things rather than than doing something that Got it. Got it. Do you have a It's not to not to say that's a bad idea. It's just it's a choice. It's a choice that we made. Of course. Of course. That makes sense. That makes sense. What about um maintaining the collector pipeline? Who is there a dedicated team? Is that just whoever is on the observability team or how do you maintain the local configurations and the pipelines? Yeah, so we do have an observability team. Um, so we are uh split between Poland and here um our other big engineering office is actually in Krakow, Poland. Um, so we have two uh two observability teams split between the two regions. Uh, each one has I think around five people at the moment. Um but collectively we are responsible for maintaining those pipelines and that configuration. Um almost I would say almost entirely we are responsible for that. There are a few areas where we share responsibility with certain places where we pull in configurations. One of them is our Kubernetes team. um they actually define part of our configuration for which metrics we actually pull into the global system rather than just keeping it local to their Prometheus instances. But by and large we are responsible for that entire configuration. Um and we try to avoid shared responsibility there just because we want to make sure that uh you know shared responsibility is always means no one's responsible. So we try to really avoid that. So, as you know, the project has matured um and it sounds like you've got a pretty good setup that work is working well for you guys right now. Um how has the implementation evolved over the years, you know, as um things have become GA? Have you added additional instrumentation signals? Um how has that looked as you've kind of grown alongside the project? Yeah. So, um yeah, it's actually an interesting question um in the sense that the thing that surprised me overall is that the deployment that we have hasn't really changed in terms of its architecture since the beginning. It's been the same architecture the entire time. Now, we've made tweaks here and there and we've simplified things, you know, as new features have come out and that kind of stuff, but by and large, it's stayed the same. and and that I think is amazing given the fact that it was really sort of pre-alpha when we started. Um so we've gone through you know we sort of gone through the uh the alpha beta you know and things are starting to become stable now and we've not really noticed uh any really heavy changes that we've needed needed to make in the platform. Now that being said I mean we have added new signals. So at the beginning of course it was metrics and traces that were were there at the beginning. Logs came in afterwards. Um we had logs coming into our system even earlier than that. So we we had some of our own receivers to do some of that before it was an official signal. So we've we've sort of folded those changes into the platform as they as they come along and it hasn't really been terribly painful in doing that. Um, so you know, as far as the community goes, the, you know, the the alpha beta stable kind of thing, you know, we've not seen really any negative consequences from that. We've had to update on occasion. Um, but by and large, those updates are pretty easy. and and Jerome I mean his he's been doing a lot of the following for the collect we have our own uh build of the collector uh which of course we try to keep in sync with the external one and Jerome has been doing quite a lot of that so you can probably give some feedback on how difficult that process is and and where we've run into issues. Oh yeah, I'm sure. I mean, for the most part, um, uh, it's, and I don't know if this speaks to open telemetry or the automation we put around it, but as far as just updating the collectors, it's been a relatively painless process. I mean, sure, um, there have been a few changes that we've had to uh, kind of team uh, just file down on as a team and like, hey, what path do we want to go down? But um as far as just like pushing out those changes, they've been pretty simple, especially with how elaborate and um detailed the white papers or the change logs have been out of open telemetry just explicitly telling hey these libraries or modules you've been using uh they here's an uh we're introducing a breaking change or an API is uh going to be deprecated. Uh but overall it's been uh quite e uh use of ease experience and um uh getting more people in our team to actually utilize that process has been uh fairly easy as well. Um, I I have a followup actually on on um on on building your your collectors because I think uh of I think out of all the people we've talked to, I think you guys might be the ones that have done uh have built your own collector, which makes me very excited. Um did you um in in terms of building like your your own dro um did you have like what was your experience around that like with as far as like the documentation provided on the hotel site because that that's another thing that we we want to hear from from um practitioners of OTEL is how how usable is is the documentation and and have you noticed an improvement in the documentation? So even um building like building your your own collector, how useful was that documentation? Did you have to do a little extra like Sherlock Holmesing to figure stuff out? And and also um were you able to uh build like a nice streamlined process for building the collector that uh uh like did you I guess have to go outside of the confines of of what was already documented to do that? Yeah. So um I'm not sure I'm going to be able to provide necessarily documenta you know feedback on the documentation but the process as a whole I mean we one of the things we started with was we had to start with our own uh our own build of the open open telemetry collector mainly because we do have custom custom receivers we have custom processors we have custom exporters as well and we kind of needed that at the beginning to cover some of our some of our use case so we were forced basically at the beginning to to build our own collector, right? Um, at the beginning it was very painful. It was basically us cloning the entire repository and then making the changes we needed to build it. Um, once once the collector builder came along um that made that whole process so much easier and it was it was a great addition to that uh that process. Um right now basically we have moved to a situation where we have our own repository. Um and basically we have the single config file for the uh for the collector builder that just tells us what we pull in from from contrib and from the uh the main open telemetry collector and the only thing we have in our repo at this point is our own custom stuff. Um so once the once the builder was there everything since then has been really really easy and smooth to to build our own stuff. Um we did that right at the beginning. So before there was documentation so um and it's worked since then. So I I can't say you know whether the current documentation is good there or not but we use the builder and it it works seamlessly for us. And uh just as a followup because you mentioned you you built your own components. Um, how how was that experience of of building your own components? Was that um was that tricky? Like what was the um uh because I that that's definitely something that um I I would say personally feels a little bit lacking in the hotel documentation for from building like your own receivers, processors, exporters kind of thing. How how is that for you? Yeah. So I think in terms of the team itself um I think the biggest hurdle there was was learning go we we weren't uh we weren't um so I think from the team standpoint that was sort of the biggest uh initial obstacle for that um all of us I think have some good experience with go uh now so I don't think that's an obstacle anymore um in terms of how to do it um we again were doing this very early. So it came came at a time when it was basically well let's take one which we know works copy it over and then make the changes make the changes you know that we need to do to make our own stuff right. Um since then since then the documentation has gotten much better and it's much easier to sort of start with with those things and get them done. Um, but you know the I must say the the process of just copying something that works and moving it over and making the changes you need that also worked pretty easily as well. So you know I don't think there's a huge for an experienced programmer I don't think it's a huge barrier to do those things. Um now there are some things which I would love to have better documentation on like in terms of internal telemetry how to you know how to hook in better to the internal telemetry and and that kind of stuff. Um we sort of worked it out uh you know by uh reverse engineering what was there but some some of the things like that would be much more helpful if uh they were better documented for instance. Thanks. Oh, I was just going to say um yes, I remember um that when we were catching up um a bit yesterday and we definitely want to get into more of like the feedback that you have for the project. Um there was a an audience question that I think is interesting from Sumit. uh how much data size do you consume and have you found the need to reduce it for example via tail sampling or other methods and from application logs and security logs perspective. Yeah. So uh in terms of the data that we push through the system we are somewhere around uh we write typically somewhere between 10 and 15 terabytes a day. So that's sort of the the uh the volume of the data that we we consume. Um the in terms of the split between logs and and and traces and metrics, it's not quite an even slip between them, but you know, it's we we we uh take all of those things in, right? Um so it's it's not so different between the between the three of them. Um in terms of the platform itself, for the stuff that flows through the collectors, we've never had an issue with scale. uh we've been able to take whatever people have thrown at us uh because we have autoscaling on all of our collectors and everything else. Um we've not run into any issues at that scale and there are times when that scale goes up by a factor or two because someone's doing something stupid. So you know uh so in terms of of tech and all of that things have just scaled correctly. The place where we've had to reduce data is mainly because we do push this data into New Relic and we pay for what we push into New Relic and we do have budget constraints. So we do manage essentially the volume of data we push into New Relic mainly for budget reasons rather than technical reasons. Um and in those cases yes we do sample down the traces. That's what gets sampled down most heavily in in the things that we we pull in. We obviously manage our logs by log levels. So we really do have dynamic ways of changing the log levels we use for that. Um typically coming from the Kubernetes platform as well. Um and metrics we tend to leave untouched. Um but there are cases where people abuse that and and we and we go after people. It tends to be rather than a proactive thing. It tends to be reactive. uh we see that uh we do look at who's who's pushing out data um and then go after them if we think that they're doing something unreasonable. Um but I will say in terms of tech, we've not run into any issues with scaling which I think is really amazing uh for for this for open telemetry in general. Um and most of the most of the things have been really uh based on budget rather than rather than the actual technical stack. Um, I wanted to uh ask a follow-up question to one of the comments that you made because actually overall um because the the vibe I get from from speaking with both of you guys is that your organization is very much into like very much supportive of of like the observability journey um and and open telemetry adoption. Can you talk a little bit more about like this culture of observability? Uh yeah, so you know the I was brought in basically when I when I started to to change the way we do observability at the at the organization you know and and a major part of that has been open telemetry and it's been really nice to see that the technical aspects of that have been have been actually the easiest parts uh you know the tech has just worked. uh you know not to say that there aren't hiccups and all of that but um you know the the tech stack is actually working really well and the data flows that we have in place I think have been uh have been really good. The harder part of this has been the cultural changes which you mentioned a bit. um and changing the way the organization deals with observability. Um so there's the you know resistant to migration. So you know there there are migrations there and people are resistant to changing code and we have a large code base. So that's one of the reasons why we still have some of our old libraries still producing some telemetry. Um so there there are res there has been resistance there. Um, and the the nice thing I've seen over the last few years is that as people get more and more experienced with the open telemetry and see what it can do for them and how it, you know, sort of simplifies the way they generate the telemetry, there's been less and less resistance going forward, which is which is a nice change to see, right? Um, so people are starting to see the benefits of this and they're starting to really, you know, uh, lean into open telemetry as a platform for that. Um the other thing I've seen on the other side of things is because we can write to various different places very easily. Um being able to do that has uh has sort of enabled people to to do nice new interesting things. And you know one of the things which is you know on the diagram I showed at the beginning was one of the places we write telemetry is launch darkly. Um, and they have a really cool feature where you can say, um, uh, it's called release guardian, and it's a way of actually, uh, changing a flag and having launch darkly ch make those changes automatically and progressively roll it out based on the telemetry it's seeing, right? So, we send spans there and if the spans look like we're getting a lot more errors or that the latency is getting larger or things like that, it will automatically roll back. And the fact that we could actually roll that out by just a configuration change on the open on the observability side is really cool because we've you know incorporated an entire another platform with our telemetry without having to change anything fundamental in our in our code and that uh for me is really really powerful and I think as people see things like that I think they're they're more convinced that this is a good choice. That's great. Um oh sorry go ahead. I don't know just going to give another before story as well. Um like like I mentioned uh I was on the performance team initially and we did have uh like Cal said like uh people are very uh hesitant for change. Uh like to give you some perspective like we did uh we had an initiative uh in the performance sphere to try to get more people to start using more performance telemetry and that dragged out for like almost a year and a half with teams like moaning and complaining all the way through uh just because of how non-industry standard it was and even at the end of it people weren't 100% sure like how much benefit it was bringing but Um right but to bring that to back to today we have another initiative uh along with open telemetry and we've gotten feedback basically just to get more telemetry uh maturity and coverage throughout all of our systems and services and we've gotten feedback from teams like almost immediately like how much they're benefiting from it as they mature their own uh application telemetry. uh and uh we we've seen like just like uh like response times to like issues that have come up uh that uh have brought about this because of open telemetry. So it's at least from someone who's been on both halves of this, it's like night and day. That's really exciting and you actually answered the question I was about to ask you. So that's great. Um Reese, do you have anything else that you wanted to touch upon? Um, I'll say quite a bit, but um, in the interest of time, um, you know, I did want to give some space for you to share additional feedback that we could share with the project maintainers. Um, I know you talked a little bit about, um, the issue with, um, conventions for internal data schemas. Um, so if you want to go into a little bit more about that and anything else that you would like to see improvements in or feature requests, things like that. Yeah. So, um, yeah. So, first let me preface this by, you know, I I think probably from what I've said already, you know, I am 100% in with open telemetry and I think it's a great platform and what the maintainers are doing is fantastic both on the documentation level and the code level, right? So, Um, that's all great. Um, and and this isn't feedback necessarily uh in terms of the actual uh collector implementations or open telemetry project as a whole, but I think it's sort of feedback on I think where we're going in our own journey and probably that has some bearing on other people as well as as we're generating more and more telemetry and it's becoming more and more useful for people. We have a lot of downstream consumers now. So we have an entire AI division who's now looking at our telemetry. We have people managers who are trying to do reporting off of this and getting standard data is becoming more and more important. Um one of the one of the things we built initially and that one of the reasons why we had to have a custom processor is we do have standards for the attributes which come into the system. So we enforce a sort of a global schema on everyone even though that's extensible. Um now how that fits in with the semantic conventions, how that fits in with you know the elastic common uh uh elastic common schema um in general how this fits in with sort of data contracts and how you have contracts between the producers of the data and the consumers of it. Um all those things are things we're thinking very hard about right now and we don't have good ideas on how to you know apply that across the board um and good ways and the level at which we need to do that. So you know I think discussions about uh the semantic conventions and the ECS and you know how to do things like data contracts with with open telemetry data I think is going to become more and more important. uh I see that in our own organization and it'd be good to come up with sort of common common themes there and common tooling and common ideas. Um so a little more convergence I think within the conver within the community there I think would go a long way. Um in terms of other feedback the tech stack I think uh in general is really really solid and you know the way things are managed I think is great. So I don't have a whole lot of feedback there. Um yeah so I think it's mainly looking forward and seeing how we can sort of use the technological base that we have and and come up with better ways of standardizing the data which is produced uh making it more useful for downstream consumers I think is the the main thing I would like to see discussed more right on thank you and I think we have a bit of time to catch up on some audience questions um so let's see one of them is from Doug Doug on YouTube and Doug wants to know with regards to collectors, their dream is that cloud providers will offer as a boring text similar to an SMTP or SFTP. Is Doug crazy to want that? Um we um the first question we ask any vendor now is do they support OTLP? Um, so you know, we treat that as our standard protocol and uh it's a negative uh a negative check mark against someone if they're not supporting it already. So we we're pushing all of our vendors and and various tech products and whatever to support that as much as possible. Um so I don't think that that's um I don't think that's a pipe dream. I think uh pushing people in that direction is is a good thing. Um as I said we are uh very heavily a Microsoft shop so we use Azure uh very much and the feedback we've consistently given them is uh we want you know Azure monitor and those types of things to produce open telemetry signals because we don't want to have to do that translation ourselves. Thank you. And Manas I see you have a question about OPM tutorials. we will respond directly with some resources for you um in the LinkedIn chat. Um in the meantime, there was another question. So, did you ever feel that there were classes of telemetry data that hotel didn't collect well and you needed to add another collection method such as eBPF? we've had to uh we've had to come up with some um some custom receivers and things like that that uh that weren't covered beforehand. One of the early things we did was there was no way to sort of uh provide a web hook into open telemetry at the beginning. There is now. Um so that's one of our one of our legacy receivers that we've had. Um and that was something that was not difficult to write but something that was sort of lacking. Um in terms of other classes of telemetry uh I would say that the one of the biggest gaps we've had internally as an organization is front-end telemetry. Um and that's something that we're actually closing now. We actually have a pro project in uh in currently going on basically to collect more of our front end telemetry uh with the open telemetry SDK the JavaScript one. Um so that was something that we internally was difficult to do for for many reasons and we're now sort of closing that gap. In terms of other types of cle telemetry that's been difficult. I don't think we've had any real issues. most of the things that we've had to collect if it's not OTLP or things like Splunk Hack or Fluent Bit and things like that where we've been able to get things into into our collectors fairly easily. Perfect. Thank you. And I think our time is just about up. Um were there any last well not last parting words? No just uh thanks thanks for inviting us I think is the is the the last thing from our side. Um and you know if anyone wants to reach out or whatever I am in the uh in the slack so feel free to reach out to me directly if you have questions or whatever happy to discuss what uh what we've done in detail if it's if it's helpful. Thank you so much for offering that and thank you so much Kyle and Jerome for being on and sharing your journey with us. Um we will link to our um hotel sig and user channel as well. Um so you can find all of us in there. Um you can also reach out to Cal directly via the CNCF Slack as well. Um, and so we do have a link manus for you um that we'll put up here in a second um that links to OPE documentation and oh well we might have time for one more quick question. This is kind of what do you think about auto instrumentation? Uh me personally I think it's a great thing uh you know and we use it where we can uh and in fact for the front end stuff we are planning to use some of that auto instrumentation there. Um the downside we've seen uh we have tried to use it and we do use it on the net side in the net SDK for uh HTTP requests and things like that. The the problem we've run into there is volume. So it it it tends to be difficult to sort of scale down to exactly which ones you want. So the auto instrumentation tends to be overly broad sometimes, but that's the only sort of downside we've seen with uh with it. Uh so in general, we you know, I think it's a great thing and if it works for you, then yeah, definitely use it. Awesome. Thank you. Well, uh, Adriana and I would like to thank you again so much for being on here. This has been really great and I'm sure I'll have um myself and other people will have more questions to learn more about open s implementation since you guys were early adopters. Um, we do have a couple things we want to share real quick before we head out. Um, hotel community day is coming up on June 25th, I believe, and we are currently accepting uh talk proposals. So, if you would like to submit a talk, we definitely encourage you to submit a topic for open telemetry day. It's going to be in Colorado as part of open source in Denver. It it's a collocca I think it's colllocated event of opensource summit North America in Denver, Colorado. Yes. So if you want to come out get um well I guess that's too late to ski but the beautiful mountains I guess. Yes. Um, also if you are going to or if you are already in Europe and you're planning to um head out to CubeCon EU in London in two weeks, less than two weeks. Yeah. Um, we would love to see you there. Adrienne and I will be speaking um at the event and we have a blog post. Yes. and we have a blog post um about the event that outlines all the open specific talks. Um the recordings if you're not able to make it will be available on the CNCF YouTube channel I think about one to two weeks after I think um depending. So yeah, they're they're usually pretty fast. Sometimes they get it within a couple of days. So just keep an eye out. Yeah. So if you can make it um in person, you are definitely welcome to check out the recordings on YouTube. And I believe that's it. Um, do you want to mention also if you if you are at CubeCon uh EU, um, we are going to be hanging out at the hotel observatory off and on. So if you're there, come say hi. Um, the observatory is always lots of fun. Um, because there's usually tons of hotel contributors, maintainers, etc. hanging out. So it's great to have like ad hoc conversations with folks. There's also going to be um if you're a CubeCon hotel project updates. Um so I strongly encourage folks to join that as well if they want to see like what's new and exciting in hotel. That's always a a fun session to attend. I think it's it's usually like half an hour um that it's slated for. So, and and also if you um if you made this recording but you have a friend who couldn't make it um you can replay on our LinkedIn um just share this the same link for the link the live um recording um with your friends or um this will also be available on our hotel YouTube channel um so tell your friends- official tell your friends. Tell your friends. All right. Thank you all so much and we will see you next time. Bye. [Music] +Hello. Welcome everyone to a new episode of Hotel Me, formerly known as the end user Q&A. I'm very excited to be here. Um, if you have been here before, I am Ree and I have Adriana here with me. I am coming to you live from Vancouver, Washington, not BC. Um, Adriana, where are you coming from? I am coming to you live from Toronto, Canada. So, we're at opposite ends of the coast, which is pretty We are. Yeah. And so, for those of you who are familiar with the flow, um, hang tight. We are going to bring our guests on pretty soon. Um, for those of you who are new, welcome. So, how this is going to go is we are going to bring on our two special guests from a an end user organization on shortly. We're going to have a quick introduction. Um, they'll introduce us to their tech stack and how long they've been using open telemetry. This one's gonna be a really interesting one because they were actually one of the first guests we talked to when we started this segment um back in 2022. So, I'm pretty excited to follow up with them and see, you know, how far they've come in their journey, what, you know, what kind of struggles they've had along the way and um what they've figured out. Um and then we'll get into some time for them to give us some feedback about the project. And we should also have some time at the end for any audience members to ask questions. Um, if you do have questions that come up as we go, feel free to pop them in the chat, whether you're on LinkedIn or YouTube, and we will get to them as appropriate um during the conversation. If not, we will wait until the end. And with that, I believe we can bring our guests on. Oh, and Adrian, do you want to talk about our resources just a little bit? Oh, yeah. So, uh, our resources, um, I believe this will I'm going to scan the QR code myself because I don't remember what this does. Um, this will take us to this takes us to open telemetry.io. Um, and we have a page in the hotel uh, site all about our um, all about our end user SIG. If you want to learn more about the end user SIG, we welcome you to join. I believe there's a a link on there also for joining our uh Slack channel on CNCF Slack. So, if you're already on CNCF Slack, um please come find us. We welcome everyone. Um people come along to ask questions. We might not have all the answers, but we can definitely direct you to where you need to be. Oh, and also we would love to know where you are listening from. So feel free to put that in the chat as well so we can we just like to know where people are listening from. And so the guests we have for you today are Jerome Johnson and columnist from relativity. Hello. Hi everyone. Howdy howdy. Hello. Would you we would love to um learn about you and can you tell us a little bit about your role in the company and kind of what relativity is focused on? You can go ahead, Cal. Alphabetical order and whatnot. Oh, okay. All right. Excellent. Very democratic. Yeah. So, a little bit about relativity. So, Relativity is a company that has a predominantly a SAS product that uh provides services to the legal community. So, we automate essentially all the uh processes around legal eiscocovery. So, exchange of documents and all of that. Um it is a SAS platform. We have a lot of companies around the world that actually use it and uh we as observability are really tuned toward how how the cloud our our SAS project is actually working at any particular point in time. Um as far as me I um I started with relativity about four almost five years ago now. I was brought in basically to upgrade uh the way we do telemetry and as Ree mentioned we were one of the probably the first first ones to have a chat with with you folks but also one of the first adopters of open telemetry in production. So we were very very early adopters of of open telemetry. Um I'm an architect here. Um and then I'll let Jerome introduce himself. Hello. As Cal said, we're on the observability team. My name is Jerome. Um, I've been at the company for about 10 years or so now. Uh, but, uh, I've only been on the observability team, specifically the one that's dealing with open telemetry for a little over a year now. So hopefully I can bring some perspective on like how easy it is to use from an on the internal side and how nice it is to so looking forward to chatting it up with you guys. Cool. That's so exciting. Well, um I think this is a great opportunity for us to uh start digging in and can um can either of you or both of you give us some insights into your tech stack and architecture? Yeah, so I could talk about that a bit and uh you know, so like I said, we were an early adopter of open telemetry. Uh what we had before that was an entirely bespoke I would say uh system for collecting telemetry from a lot of different sources. Um, in addition to that, um, we we did use New Relic at the time, but we also had a lot of other vendors in there, too. So, it was kind of a nightmare from an operational standpoint to actually do any uh troubleshooting and debugging because you had to go to lots of different sources to to sort of put all that telemetry together in your head. Um, at the time when I was hired, we we uh the company made sort of a decision to try to unify everything. We looked around to see what would be the best platform for doing that. Um and the one that sort of checked all the boxes even though it was really immature was open telemetry. So that was the one which we sort of pointed to to do that. Now internally we have a very complicated platform. Um and maybe this is time to bring up the diagram on on what we actually do now with um um with open telemetry. We have 1500 plus different services running on our platform and that is a huge mix of things. We are predominantly a Windows andnet shop. So most of that is actually Windows and .NET. Um but we have a mix of basically every other language in the world. Uh so we have a very very complicated tech stack and we need to pull in the telemetry from all those different things. So most of the complexity in our observability platform and I'll probably mention at various points. So if I say RELI, that is our observability platform that's built over open telemetry. Um most of the complexity is just being able to pull those signals in from lots of different sources and you see most of them here. We really prefer that people use the open telemetry SDKs now and we push that as hard as we can. But we still have people using our old platform which is relatively APM which we translate into open telemetry now. We have things like SNMP. uh we pull logging from lots and lots of different places and that's probably the part which is most most difficult and most uh heterogeneous in terms of the platform itself. We have a whole fleet of uh open telemetry collectors that we run in 20 20 different regions around the world. Each one of those is load balanced. So we're running we're running anywhere between uh depending on the time of day and all that between 200 and probably 800 individual collectors around the world to to pull all this telemetry in. Um and then we we send that to various uh data stores for operations. This is really new relic at this point. Um so that's where most of our uh engineers go to actually see what the what the telemetry is doing. We have a subset of people who use launch darkly and use that for future releases. So we send some set of telemetry there. Um and then we actually archive all of our telemetry for compliance reasons for a year and for forensic analysis and stuff like that. Uh we also have a reporting platform that uses the Azure blob storage basically as as its back end to to make that reporting for managers and things like that. Um so this makes it look simpler than it is. It's a lot more uh gnarly than than this diagram uh uh lets on. But we do have a large scale. I mean 20 plus regions around the world, 1500 services all feeding into this is is a fairly complicated platform. Dang, that was really cool actually. Um questions. So many questions. I do actually have a lot of um followup questions, but I also want to um get into why open telemetry like how like when exactly I know you mentioned, you know, kind of back in 2022 you you know your team was early adopters. Um how did you come upon open telemetry and like why did you decide like oh this makes sense for us? Yeah. So you know I think the thing which which sort of ticked all the boxes for us we we came from a place where we were having to maintain our own observability libraries um and given the fact that we werenet and then trying to diversify into lots of different languages uh meant that we were having to translate that thing again and again and again. Um and on top of that we also had uh well we were using New Relic at the time and we had a lot of the New Relic agents in the mix as well. So we had this this weird mix of our own libraries for some stuff. We had uh a mix of of things from open from New Relic uh with the uh with the agents and all of that kind of stuff. Um and we had a lot of people who were writing to New Relic directly from from their code as well uh as well as to other sources. So one of one of the things which was really complicated on our side was every time we wanted to move something or we wanted to change something it meant code changes. Um and that's something we wanted to avoid going forward. So we wanted to standardize on something which was ideally open source that we could include in our uh in our services one time and maintain that and then change what where we put things but didn't didn't have to change the code every time. So we're really looking for um you know something that would give us stability in terms of the codebase but allow us the flexibility to send things to other places as we needed to do that and open telemetry was the only one which sort of ticked those boxes for us at the time and the only worry we really had at the point was that it was it was a new project. Um there was a lot of risk in going that direction. uh we decided in the end it it wasn't an easy decision. We we actually discussed this a long time internally but we decided that it was worth the risk to to go ahead and adopt that because it looked like it was going to take tick the boxes for what we wanted to do there. Um you know so basically it's it's to isolate basically our code base from from the from what we do with the telemetry afterwards which was sort of the selling point for what we wanted to do there. That is very awesome. Um I don't know I don't know if Jerome has anything. Uh oh yeah I yeah just to add on to that um I can give somewhat of a perspective to what it was like before we had open telemetry like um I was serving on the performance team for a good number of years and um it was very challenging to do uh analysis of specific applications just because like Cal had mentioned things were it was kind of the wild west on how we handled metrics like performance metrics and monitoring are are somewhat look similar, a little different, but enough to where like every time we had to analyze a specific application, we would have to do a deep dive on a set application like what telemetry are you guys putting out? Oh, you're not putting out any at all. And then we'd have to like dig into the code and that would really muck up like how much time we could actually like drill down into what the core problem of said application performance uh perspective was. Usually it would take half the time we would do an analysis it would be like trying to understand what the product was and then actually coming up with a solution would be the the la latter 50 or 40% they have. So um since joining uh observability like and even like even though Cal showed a simplified version of um our reli stack and uh it's a like you said it's a bit more complicated once you drill down into it. I was amazed just how like how much cleaner it is to just understand like what our what relativity is doing as a whole. So, uh take that with what you want. That's very cool. Um now as a as a follow-up question, um one of the things that we are always curious about in the SIG is collector setup. Um what kind of you you mentioned I I believe um Cal mentioned that you guys manage a fleet of collectors. Um how do you manage that fleet of collectors? How many can you give us an idea of how many collectors? Is there a particular setup that you use? Like we we would love to know. Yeah. So, you know, I think um in terms of how things are usually deployed, I think you call it a gateway setup. So, all of our collectors are basically in a gateway setup. Um so, people send their telemetry to us, we process it and send it on. Um over the years, we've had more or less complicated pipelines, but in general, we have essentially a single set of collectors with a common configuration that handles all of that uh all of that uh telemetry. We um so in terms of the numbers of things so we are running over 20 plus regions each uh each region has somewhere between depending on the load between 10 and probably 50 to 100 uh instances themselves. So we're running at any point in time somewhere between 200 and 500 individual collectors. These are all running in Kubernetes. Um we are uh very focused on compute and Kubernetes. So that's where almost all these things are running. In terms of the deployment itself, we actually handle all of our deployments um uh via Helm charts. So we use Helm charts basically to deploy these all out to the um to the um uh Kubernetes clusters. Uh we do take pains to make sure that essentially all of our configurations are isolated. So our collectors except in a very few cases do not reach out to external services. We manage the configurations all locally to make sure that they are as available and uh as as available and robust as possible, right? Um so that's kind of how we do all of that. In terms of the pipelines themselves, um one of the other reasons why we went with open telemetry is we have some internal uh I would say attribute enrichments that we do specifically to our product and the way we use things. We actually implement those inside of the uh collector itself. Um so we have some custom processors which en enhance the data in various ways. We actually validate it. We actually drop data which is not uh which contains things which shouldn't be there for instance. Um so we do a lot of validation on the fly as things go through the the collectors themselves before we write it out to our various data stores. Um I think I I think that hit most of the points you're asking for. Did I miss any of the uh the things you're asking? I think I think you got most of them. And and yeah, actually one one follow-up question I had was uh also since you you mentioned you deploy your your collectors in Kubernetes via Helmcharts, have you used um the hotel operator to manage any of your collectors? Is that something you played with? We have not. The the hotel operator sort of came out after we were already doing stuff. So in fact, we uh we don't uh but not for any particularly good reason. No. Okay. Okay. Fair enough. So, it's it's that we had uh you consider it sort of tech debt. You know, we had the Helm chart and all of that beforehand. Um so, we've just continued doing that and we're kind of used to managing things that way. Um we actually will have an internal shift in in our deployment uh tooling uh coming up for the organization. So, that might be an that might be an opportunity to switch over how we're actually managing things. But at the moment, yeah, we're not we don't have any experience with the uh open telemetry operator at the moment. Got it. Got it. And I I uh another similar question. Um since you do meet manage a fleet of collectors, um have you played around with the opamp protocol for for doing that? We've not again again we've tried very hard to make our our fleet essentially uh standalone. So even for configurations and things like that, we don't actually reach out to anything else. We really deployed uh sort of static config maps for that. Um that that was basically a choice that we made um just to because we have such a large deployment and because it's distributed so widely, we wanted to make sure we were as um resilient against networking problems as possible. So even for things like configuration changes, we we push out new new configs for those things rather than than doing something that Got it. Got it. Do you have a It's not to not to say that's a bad idea. It's just it's a choice. It's a choice that we made. Of course. Of course. That makes sense. That makes sense. What about um maintaining the collector pipeline? Who is there a dedicated team? Is that just whoever is on the observability team or how do you maintain the local configurations and the pipelines? Yeah, so we do have an observability team. Um, so we are uh split between Poland and here um our other big engineering office is actually in Krakow, Poland. Um, so we have two uh two observability teams split between the two regions. Uh, each one has I think around five people at the moment. Um but collectively we are responsible for maintaining those pipelines and that configuration. Um almost I would say almost entirely we are responsible for that. There are a few areas where we share responsibility with certain places where we pull in configurations. One of them is our Kubernetes team. um they actually define part of our configuration for which metrics we actually pull into the global system rather than just keeping it local to their Prometheus instances. But by and large we are responsible for that entire configuration. Um and we try to avoid shared responsibility there just because we want to make sure that uh you know shared responsibility is always means no one's responsible. So we try to really avoid that. So, as you know, the project has matured um and it sounds like you've got a pretty good setup that work is working well for you guys right now. Um how has the implementation evolved over the years, you know, as um things have become GA? Have you added additional instrumentation signals? Um how has that looked as you've kind of grown alongside the project? Yeah. So, um yeah, it's actually an interesting question um in the sense that the thing that surprised me overall is that the deployment that we have hasn't really changed in terms of its architecture since the beginning. It's been the same architecture the entire time. Now, we've made tweaks here and there and we've simplified things, you know, as new features have come out and that kind of stuff, but by and large, it's stayed the same. and and that I think is amazing given the fact that it was really sort of pre-alpha when we started. Um so we've gone through you know we sort of gone through the uh the alpha beta you know and things are starting to become stable now and we've not really noticed uh any really heavy changes that we've needed needed to make in the platform. Now that being said I mean we have added new signals. So at the beginning of course it was metrics and traces that were were there at the beginning. Logs came in afterwards. Um we had logs coming into our system even earlier than that. So we we had some of our own receivers to do some of that before it was an official signal. So we've we've sort of folded those changes into the platform as they as they come along and it hasn't really been terribly painful in doing that. Um, so you know, as far as the community goes, the, you know, the the alpha beta stable kind of thing, you know, we've not seen really any negative consequences from that. We've had to update on occasion. Um, but by and large, those updates are pretty easy. and and Jerome I mean his he's been doing a lot of the following for the collect we have our own uh build of the collector uh which of course we try to keep in sync with the external one and Jerome has been doing quite a lot of that so you can probably give some feedback on how difficult that process is and and where we've run into issues. Oh yeah, I'm sure. I mean, for the most part, um, uh, it's, and I don't know if this speaks to open telemetry or the automation we put around it, but as far as just updating the collectors, it's been a relatively painless process. I mean, sure, um, there have been a few changes that we've had to uh, kind of team uh, just file down on as a team and like, hey, what path do we want to go down? But um as far as just like pushing out those changes, they've been pretty simple, especially with how elaborate and um detailed the white papers or the change logs have been out of open telemetry just explicitly telling hey these libraries or modules you've been using uh they here's an uh we're introducing a breaking change or an API is uh going to be deprecated. Uh but overall it's been uh quite e uh use of ease experience and um uh getting more people in our team to actually utilize that process has been uh fairly easy as well. Um, I I have a followup actually on on um on on building your your collectors because I think uh of I think out of all the people we've talked to, I think you guys might be the ones that have done uh have built your own collector, which makes me very excited. Um did you um in in terms of building like your your own dro um did you have like what was your experience around that like with as far as like the documentation provided on the hotel site because that that's another thing that we we want to hear from from um practitioners of OTEL is how how usable is is the documentation and and have you noticed an improvement in the documentation? So even um building like building your your own collector, how useful was that documentation? Did you have to do a little extra like Sherlock Holmesing to figure stuff out? And and also um were you able to uh build like a nice streamlined process for building the collector that uh uh like did you I guess have to go outside of the confines of of what was already documented to do that? Yeah. So um I'm not sure I'm going to be able to provide necessarily documenta you know feedback on the documentation but the process as a whole I mean we one of the things we started with was we had to start with our own uh our own build of the open open telemetry collector mainly because we do have custom custom receivers we have custom processors we have custom exporters as well and we kind of needed that at the beginning to cover some of our some of our use case so we were forced basically at the beginning to to build our own collector, right? Um, at the beginning it was very painful. It was basically us cloning the entire repository and then making the changes we needed to build it. Um, once once the collector builder came along um that made that whole process so much easier and it was it was a great addition to that uh that process. Um right now basically we have moved to a situation where we have our own repository. Um and basically we have the single config file for the uh for the collector builder that just tells us what we pull in from from contrib and from the uh the main open telemetry collector and the only thing we have in our repo at this point is our own custom stuff. Um so once the once the builder was there everything since then has been really really easy and smooth to to build our own stuff. Um we did that right at the beginning. So before there was documentation so um and it's worked since then. So I I can't say you know whether the current documentation is good there or not but we use the builder and it it works seamlessly for us. And uh just as a followup because you mentioned you you built your own components. Um, how how was that experience of of building your own components? Was that um was that tricky? Like what was the um uh because I that that's definitely something that um I I would say personally feels a little bit lacking in the hotel documentation for from building like your own receivers, processors, exporters kind of thing. How how is that for you? Yeah. So I think in terms of the team itself um I think the biggest hurdle there was was learning go we we weren't uh we weren't um so I think from the team standpoint that was sort of the biggest uh initial obstacle for that um all of us I think have some good experience with go uh now so I don't think that's an obstacle anymore um in terms of how to do it um we again were doing this very early. So it came came at a time when it was basically well let's take one which we know works copy it over and then make the changes make the changes you know that we need to do to make our own stuff right. Um since then since then the documentation has gotten much better and it's much easier to sort of start with with those things and get them done. Um, but you know the I must say the the process of just copying something that works and moving it over and making the changes you need that also worked pretty easily as well. So you know I don't think there's a huge for an experienced programmer I don't think it's a huge barrier to do those things. Um now there are some things which I would love to have better documentation on like in terms of internal telemetry how to you know how to hook in better to the internal telemetry and and that kind of stuff. Um we sort of worked it out uh you know by uh reverse engineering what was there but some some of the things like that would be much more helpful if uh they were better documented for instance. Thanks. Oh, I was just going to say um yes, I remember um that when we were catching up um a bit yesterday and we definitely want to get into more of like the feedback that you have for the project. Um there was a an audience question that I think is interesting from Sumit. uh how much data size do you consume and have you found the need to reduce it for example via tail sampling or other methods and from application logs and security logs perspective. Yeah. So uh in terms of the data that we push through the system we are somewhere around uh we write typically somewhere between 10 and 15 terabytes a day. So that's sort of the the uh the volume of the data that we we consume. Um the in terms of the split between logs and and and traces and metrics, it's not quite an even slip between them, but you know, it's we we we uh take all of those things in, right? Um so it's it's not so different between the between the three of them. Um in terms of the platform itself, for the stuff that flows through the collectors, we've never had an issue with scale. uh we've been able to take whatever people have thrown at us uh because we have autoscaling on all of our collectors and everything else. Um we've not run into any issues at that scale and there are times when that scale goes up by a factor or two because someone's doing something stupid. So you know uh so in terms of of tech and all of that things have just scaled correctly. The place where we've had to reduce data is mainly because we do push this data into New Relic and we pay for what we push into New Relic and we do have budget constraints. So we do manage essentially the volume of data we push into New Relic mainly for budget reasons rather than technical reasons. Um and in those cases yes we do sample down the traces. That's what gets sampled down most heavily in in the things that we we pull in. We obviously manage our logs by log levels. So we really do have dynamic ways of changing the log levels we use for that. Um typically coming from the Kubernetes platform as well. Um and metrics we tend to leave untouched. Um but there are cases where people abuse that and and we and we go after people. It tends to be rather than a proactive thing. It tends to be reactive. uh we see that uh we do look at who's who's pushing out data um and then go after them if we think that they're doing something unreasonable. Um but I will say in terms of tech, we've not run into any issues with scaling which I think is really amazing uh for for this for open telemetry in general. Um and most of the most of the things have been really uh based on budget rather than rather than the actual technical stack. Um, I wanted to uh ask a follow-up question to one of the comments that you made because actually overall um because the the vibe I get from from speaking with both of you guys is that your organization is very much into like very much supportive of of like the observability journey um and and open telemetry adoption. Can you talk a little bit more about like this culture of observability? Uh yeah, so you know the I was brought in basically when I when I started to to change the way we do observability at the at the organization you know and and a major part of that has been open telemetry and it's been really nice to see that the technical aspects of that have been have been actually the easiest parts uh you know the tech has just worked. uh you know not to say that there aren't hiccups and all of that but um you know the the tech stack is actually working really well and the data flows that we have in place I think have been uh have been really good. The harder part of this has been the cultural changes which you mentioned a bit. um and changing the way the organization deals with observability. Um so there's the you know resistant to migration. So you know there there are migrations there and people are resistant to changing code and we have a large code base. So that's one of the reasons why we still have some of our old libraries still producing some telemetry. Um so there there are res there has been resistance there. Um, and the the nice thing I've seen over the last few years is that as people get more and more experienced with the open telemetry and see what it can do for them and how it, you know, sort of simplifies the way they generate the telemetry, there's been less and less resistance going forward, which is which is a nice change to see, right? Um, so people are starting to see the benefits of this and they're starting to really, you know, uh, lean into open telemetry as a platform for that. Um the other thing I've seen on the other side of things is because we can write to various different places very easily. Um being able to do that has uh has sort of enabled people to to do nice new interesting things. And you know one of the things which is you know on the diagram I showed at the beginning was one of the places we write telemetry is launch darkly. Um, and they have a really cool feature where you can say, um, uh, it's called release guardian, and it's a way of actually, uh, changing a flag and having launch darkly ch make those changes automatically and progressively roll it out based on the telemetry it's seeing, right? So, we send spans there and if the spans look like we're getting a lot more errors or that the latency is getting larger or things like that, it will automatically roll back. And the fact that we could actually roll that out by just a configuration change on the open on the observability side is really cool because we've you know incorporated an entire another platform with our telemetry without having to change anything fundamental in our in our code and that uh for me is really really powerful and I think as people see things like that I think they're they're more convinced that this is a good choice. That's great. Um oh sorry go ahead. I don't know just going to give another before story as well. Um like like I mentioned uh I was on the performance team initially and we did have uh like Cal said like uh people are very uh hesitant for change. Uh like to give you some perspective like we did uh we had an initiative uh in the performance sphere to try to get more people to start using more performance telemetry and that dragged out for like almost a year and a half with teams like moaning and complaining all the way through uh just because of how non-industry standard it was and even at the end of it people weren't 100% sure like how much benefit it was bringing but Um right but to bring that to back to today we have another initiative uh along with open telemetry and we've gotten feedback basically just to get more telemetry uh maturity and coverage throughout all of our systems and services and we've gotten feedback from teams like almost immediately like how much they're benefiting from it as they mature their own uh application telemetry. uh and uh we we've seen like just like uh like response times to like issues that have come up uh that uh have brought about this because of open telemetry. So it's at least from someone who's been on both halves of this, it's like night and day. That's really exciting and you actually answered the question I was about to ask you. So that's great. Um Reese, do you have anything else that you wanted to touch upon? Um, I'll say quite a bit, but um, in the interest of time, um, you know, I did want to give some space for you to share additional feedback that we could share with the project maintainers. Um, I know you talked a little bit about, um, the issue with, um, conventions for internal data schemas. Um, so if you want to go into a little bit more about that and anything else that you would like to see improvements in or feature requests, things like that. Yeah. So, um, yeah. So, first let me preface this by, you know, I I think probably from what I've said already, you know, I am 100% in with open telemetry and I think it's a great platform and what the maintainers are doing is fantastic both on the documentation level and the code level, right? So, Um, that's all great. Um, and and this isn't feedback necessarily uh in terms of the actual uh collector implementations or open telemetry project as a whole, but I think it's sort of feedback on I think where we're going in our own journey and probably that has some bearing on other people as well as as we're generating more and more telemetry and it's becoming more and more useful for people. We have a lot of downstream consumers now. So we have an entire AI division who's now looking at our telemetry. We have people managers who are trying to do reporting off of this and getting standard data is becoming more and more important. Um one of the one of the things we built initially and that one of the reasons why we had to have a custom processor is we do have standards for the attributes which come into the system. So we enforce a sort of a global schema on everyone even though that's extensible. Um now how that fits in with the semantic conventions, how that fits in with you know the elastic common uh uh elastic common schema um in general how this fits in with sort of data contracts and how you have contracts between the producers of the data and the consumers of it. Um all those things are things we're thinking very hard about right now and we don't have good ideas on how to you know apply that across the board um and good ways and the level at which we need to do that. So you know I think discussions about uh the semantic conventions and the ECS and you know how to do things like data contracts with with open telemetry data I think is going to become more and more important. uh I see that in our own organization and it'd be good to come up with sort of common common themes there and common tooling and common ideas. Um so a little more convergence I think within the conver within the community there I think would go a long way. Um in terms of other feedback the tech stack I think uh in general is really really solid and you know the way things are managed I think is great. So I don't have a whole lot of feedback there. Um yeah so I think it's mainly looking forward and seeing how we can sort of use the technological base that we have and and come up with better ways of standardizing the data which is produced uh making it more useful for downstream consumers I think is the the main thing I would like to see discussed more right on thank you and I think we have a bit of time to catch up on some audience questions um so let's see one of them is from Doug Doug on YouTube and Doug wants to know with regards to collectors, their dream is that cloud providers will offer as a boring text similar to an SMTP or SFTP. Is Doug crazy to want that? Um we um the first question we ask any vendor now is do they support OTLP? Um, so you know, we treat that as our standard protocol and uh it's a negative uh a negative check mark against someone if they're not supporting it already. So we we're pushing all of our vendors and and various tech products and whatever to support that as much as possible. Um so I don't think that that's um I don't think that's a pipe dream. I think uh pushing people in that direction is is a good thing. Um as I said we are uh very heavily a Microsoft shop so we use Azure uh very much and the feedback we've consistently given them is uh we want you know Azure monitor and those types of things to produce open telemetry signals because we don't want to have to do that translation ourselves. Thank you. And Manas I see you have a question about OPM tutorials. we will respond directly with some resources for you um in the LinkedIn chat. Um in the meantime, there was another question. So, did you ever feel that there were classes of telemetry data that hotel didn't collect well and you needed to add another collection method such as eBPF? we've had to uh we've had to come up with some um some custom receivers and things like that that uh that weren't covered beforehand. One of the early things we did was there was no way to sort of uh provide a web hook into open telemetry at the beginning. There is now. Um so that's one of our one of our legacy receivers that we've had. Um and that was something that was not difficult to write but something that was sort of lacking. Um in terms of other classes of telemetry uh I would say that the one of the biggest gaps we've had internally as an organization is front-end telemetry. Um and that's something that we're actually closing now. We actually have a pro project in uh in currently going on basically to collect more of our front end telemetry uh with the open telemetry SDK the JavaScript one. Um so that was something that we internally was difficult to do for for many reasons and we're now sort of closing that gap. In terms of other types of cle telemetry that's been difficult. I don't think we've had any real issues. most of the things that we've had to collect if it's not OTLP or things like Splunk Hack or Fluent Bit and things like that where we've been able to get things into into our collectors fairly easily. Perfect. Thank you. And I think our time is just about up. Um were there any last well not last parting words? No just uh thanks thanks for inviting us I think is the is the the last thing from our side. Um and you know if anyone wants to reach out or whatever I am in the uh in the slack so feel free to reach out to me directly if you have questions or whatever happy to discuss what uh what we've done in detail if it's if it's helpful. Thank you so much for offering that and thank you so much Kyle and Jerome for being on and sharing your journey with us. Um we will link to our um hotel sig and user channel as well. Um so you can find all of us in there. Um you can also reach out to Cal directly via the CNCF Slack as well. Um, and so we do have a link manus for you um that we'll put up here in a second um that links to OPE documentation and oh well we might have time for one more quick question. This is kind of what do you think about auto instrumentation? Uh me personally I think it's a great thing uh you know and we use it where we can uh and in fact for the front end stuff we are planning to use some of that auto instrumentation there. Um the downside we've seen uh we have tried to use it and we do use it on the net side in the net SDK for uh HTTP requests and things like that. The the problem we've run into there is volume. So it it it tends to be difficult to sort of scale down to exactly which ones you want. So the auto instrumentation tends to be overly broad sometimes, but that's the only sort of downside we've seen with uh with it. Uh so in general, we you know, I think it's a great thing and if it works for you, then yeah, definitely use it. Awesome. Thank you. Well, uh, Adriana and I would like to thank you again so much for being on here. This has been really great and I'm sure I'll have um myself and other people will have more questions to learn more about open s implementation since you guys were early adopters. Um, we do have a couple things we want to share real quick before we head out. Um, hotel community day is coming up on June 25th, I believe, and we are currently accepting uh talk proposals. So, if you would like to submit a talk, we definitely encourage you to submit a topic for open telemetry day. It's going to be in Colorado as part of open source in Denver. It it's a collocca I think it's colllocated event of opensource summit North America in Denver, Colorado. Yes. So if you want to come out get um well I guess that's too late to ski but the beautiful mountains I guess. Yes. Um, also if you are going to or if you are already in Europe and you're planning to um head out to CubeCon EU in London in two weeks, less than two weeks. Yeah. Um, we would love to see you there. Adrienne and I will be speaking um at the event and we have a blog post. Yes. and we have a blog post um about the event that outlines all the open specific talks. Um the recordings if you're not able to make it will be available on the CNCF YouTube channel I think about one to two weeks after I think um depending. So yeah, they're they're usually pretty fast. Sometimes they get it within a couple of days. So just keep an eye out. Yeah. So if you can make it um in person, you are definitely welcome to check out the recordings on YouTube. And I believe that's it. Um, do you want to mention also if you if you are at CubeCon uh EU, um, we are going to be hanging out at the hotel observatory off and on. So if you're there, come say hi. Um, the observatory is always lots of fun. Um, because there's usually tons of hotel contributors, maintainers, etc. hanging out. So it's great to have like ad hoc conversations with folks. There's also going to be um if you're a CubeCon hotel project updates. Um so I strongly encourage folks to join that as well if they want to see like what's new and exciting in hotel. That's always a a fun session to attend. I think it's it's usually like half an hour um that it's slated for. So, and and also if you um if you made this recording but you have a friend who couldn't make it um you can replay on our LinkedIn um just share this the same link for the link the live um recording um with your friends or um this will also be available on our hotel YouTube channel um so tell your friends- official tell your friends. Tell your friends. All right. Thank you all so much and we will see you next time. Bye. diff --git a/video-transcripts/transcripts/2025-04-04T01:10:32Z-humans-of-otel-streaming-live-from-kubecon-with-austin-parker-marylia-gutierrez.md b/video-transcripts/transcripts/2025-04-04T01:10:32Z-humans-of-otel-streaming-live-from-kubecon-with-austin-parker-marylia-gutierrez.md index 9bfa3bb..e5a5159 100644 --- a/video-transcripts/transcripts/2025-04-04T01:10:32Z-humans-of-otel-streaming-live-from-kubecon-with-austin-parker-marylia-gutierrez.md +++ b/video-transcripts/transcripts/2025-04-04T01:10:32Z-humans-of-otel-streaming-live-from-kubecon-with-austin-parker-marylia-gutierrez.md @@ -10,119 +10,80 @@ URL: https://www.youtube.com/watch?v=EL0UkhvFAmY ## Summary -The live stream "Humans of OA" features Reese, a senior developer relations engineer at New Relic, along with colleagues Adriana and Marilia, who is a staff engineer at Grafana Labs. They discuss open telemetry, its growth, and contributions to observability in tech. Marilia shares her journey from an end user to a maintainer, emphasizing her roles in semantic conventions, localization, and the importance of community contributions. The conversation also touches on various roles within the open source community, the significance of documentation, and upcoming events like the OpenTelemetry Community Day. Additionally, they discuss the impact of AI on observability and the evolution of logging APIs within the OpenTelemetry project. The stream wraps up with reflections on the vibrant community at KubeCon EU and the importance of fostering new contributors in the tech space. +In this live stream from CubeCon EU, host Reese, a senior developer relations engineer at New Relic, is joined by colleagues Adriana Vila and Marilia, a staff engineer at Grafana Labs. The discussion centers around the OpenTelemetry project, its community, and the importance of localization in technology. Marilia shares her journey from being an end user to contributing to OpenTelemetry, emphasizing the significance of semantic conventions and the roles within the community, such as contributor and maintainer. The conversation also highlights the recent advancements in OpenTelemetry, including a new database monitoring release, ongoing efforts in localization for non-English speakers, and the impact of AI on observability tools. Austin Parker, the community manager for OpenTelemetry, joins later to discuss the project's growth, upcoming community events, and the evolution of logging infrastructure. Overall, the stream showcases the vibrant OpenTelemetry community and the collaborative efforts in improving observability practices. -# Humans of OpenTelemetry Live Stream Transcript +## Chapters -Hello, and welcome to a live stream of Humans of OpenTelemetry! We are excited to be here, despite some minor subtitle issues. My name is Reese, and I am a Senior Developer Relations Engineer at New Relic. I’m joined by my lovely industry colleagues, Adriana and Marilia, who is our first guest today. +00:00:00 Introductions to the livestream and guests +00:02:30 Marilia discusses her background and role at Grafana Labs +00:05:15 Overview of OpenTelemetry's focus and recent updates +00:08:00 Discussion on the Database Semantic Convention Special Interest Group (SIG) +00:12:30 Explanation of contributor roles in the OpenTelemetry community +00:16:00 Marilia talks about the importance of localization in documentation +00:20:00 Overview of the Contributor Experience SIG and its goals +00:24:00 Austin Parker joins the discussion as the community manager +00:30:00 Discussion on AI's potential impact on OpenTelemetry +00:35:00 Overview of upcoming OpenTelemetry events and community days -**Adriana:** -Hey everyone! Thanks for tuning in. My name is Adriana Vila, and I work alongside Reese. +# Humans of OA Live Stream Transcript -**Marilia:** -Hi everyone! My name is Marilia, and I am a Staff Engineer at Grafana Labs, also involved in OpenTelemetry. I work across a few different groups, serving as a maintainer for the contributor experience and an approver. +**Hello, welcome to a live stream of Humans of OA!** We are excited to be here despite some subtitle issues. My name is Reese, and I am a Senior Developer Relations Engineer at New Relic. I’m joined by my lovely industry colleagues, Adriana and Marilia, who is our first guest. -Reese: -That's fantastic! We’re really curious about your journey. You started as an end-user and then began contributing. Can you tell us about that transition? +**Adriana:** Hey everyone, thanks for tuning in! My name is Adriana Vila, and I work alongside Reese. -Marilia: -Sure! At my previous job, I was responsible for observability at Cockroach Labs, where I managed the observability team. This experience got me learning about OpenTelemetry, as we wanted to use it for monitoring outside the database. We created some basic endpoints, but my current focus is solely on OpenTelemetry, which allows me to have fun with it every day. +**Marilia:** Hi everyone! My name is Marilia, and I am a Staff Engineer at Grafana Labs, working with OpenTelemetry. I’m involved in a few different groups; I am a maintainer for the contributor experience and an approver. -Reese: -That sounds awesome! What kind of technologies are you working with now? +**Reese:** That’s amazing! You started as an end user and eventually began contributing. Can you tell us more about your journey? -Marilia: -I'm involved with Java and .NET, ensuring that all SDKs are on the same level. We also have ownership over the PostgreSQL plugin, which I’ve been touching lately. +**Marilia:** Sure! In my prior job at Cockroach Labs, I was responsible for observability. I began learning about OpenTelemetry because we wanted to use it outside of the database. We created some basic endpoints, but my team focused solely on OpenTelemetry. Now, I get to have fun with it every day! -Reese: -Actually, you gave a talk yesterday about database monitoring, right? +**Reese:** That’s awesome! What are some of the programming languages and SDKs you work with? -Marilia: -Yes! We just released the release candidate this week, which means by the end of the month, we will mark it as stable. People can start using it without concerns. +**Marilia:** Currently, I’m focusing on Java and .NET, ensuring that all SDKs are on the same level. I also manage the Postgres plugin, which I have been working on. -Reese: -You mentioned you're part of the semantic conventions SIG for databases. Can you explain the distinction between that and the general semantic conventions SIG? +**Reese:** You recently gave a talk about database monitoring, right? -Marilia: -Sure! The general semantic conventions SIG includes people from all different areas, which can be a bit overwhelming. So, we created a separate SIG for databases to focus on specific updates and discussions relevant to that area. This allows us to bring in experts and prioritize our work better. +**Marilia:** Yes! We just released the release candidate this week, and by the end of the month, it will be marked as stable. This means people can start using it without concerns. -Reese: -That makes sense! And can you give us an overview of the different roles within the community? +**Reese:** You mentioned you’re part of the semantic convention SIG for databases. Can you explain how that differs from the general semantic conventions SIG? -Marilia: -Absolutely! You start as a contributor. You don’t need to be a member of the CNCF to contribute; just open a PR to become an official contributor. As you start helping out, you can become a triager, which involves managing issues. The next step is becoming an approver, which means your approval counts for merging PRs. Finally, maintainers can merge changes and create releases while considering input from contributors. +**Marilia:** Typically, the general semantic conventions include representatives from various areas, which can lead to discussions that may not be relevant to everyone. By creating separate SIGs for specific areas like databases, HTTP, and RPC, we can focus on topics relevant to those who have experience in that field. This helps keep the work focused and easier to prioritize. -Reese: -That’s a great overview! You also mentioned working on Portuguese localization. Why is localization important? +**Reese:** That makes sense! You have a lot of responsibilities in the community. Could you explain the distinctions between your roles as a contributor, approver, and maintainer? -Marilia: -Localization breaks down language barriers that can hinder learning tech concepts. Many people don't speak English as their first language, which makes documentation less accessible. I've been involved in translating documentation and helping others in Brazil understand OpenTelemetry better. +**Marilia:** Absolutely! You can become a contributor without being a member of the CNCF. After making a few contributions, you can ask to become an official member. Once you start helping out by reviewing PRs and commenting, you can achieve the status of triager, where you help manage incoming issues. From there, you can become an approver, meaning your approval counts towards merging PRs. Finally, maintainers are responsible for merging changes and creating releases, while also gathering input from contributors. -Reese: -Has this work improved your own Portuguese as well? +**Reese:** That’s a great overview! Speaking of contributions, I know you’re also involved in Portuguese localization. Can you tell us why it's important and how you got involved? -Marilia: -Definitely! It’s funny because sometimes I have to give talks in Portuguese, and it’s a challenge. We also collaborate with teams working on Spanish, French, and Japanese translations. +**Marilia:** Localization is crucial because language can be a barrier. Many people learning tech concepts may not speak English as their first language, making documentation less accessible. I started a blog to help with translations and worked on reviewing materials. It’s important to decide what to translate and what to leave in English, as some concepts don’t have a direct translation. This work helps break down barriers and brings more people into the community. -Reese: -That’s exciting! You mentioned that localized pages now have language selection options. +**Reese:** It sounds like a lot of work! Has this experience helped improve your own Portuguese? -Marilia: -Yes! We’ve worked hard on that, and it's great to see more content becoming accessible to non-English speakers. +**Marilia:** Yes, it has! Living abroad for so long has made it a challenge to keep my language skills sharp, especially when giving talks in Portuguese. We also have contributors working on Spanish, French, and Japanese localization, so we’re constantly learning from each other. -Reese: -What advice would you give to newcomers who want to contribute? +**Reese:** That's fantastic! As we wrap up, what advice would you give to someone looking to start contributing? -Marilia: -There’s something for everyone! Start by joining calls and listening to discussions. Remember that open source is different from day-to-day jobs, so approach it with an open mind. +**Marilia:** There’s always something for everyone, so just try to get involved! Open source is different from a daily job, so it's important to keep that in mind. Be proactive, and don’t hesitate to reach out to the community for guidance. -Reese: -Great tips! I also want to plug the end-user SIG, which is welcoming new contributors. If you’re interested, check the show notes for more information. +**Reese:** That’s great advice. Thank you so much for sharing your insights today, Marilia! We’ll continue our conversation shortly as we bring on our next guest, Austin Parker, the community manager for OpenTelemetry. -**[Transition to next guest]** +**Austin:** Hi everyone! I'm excited to be here. I’ve been managing the community and working on various projects, including the OpenTelemetry website. -Reese: -We're excited to welcome our next guest, Austin Parker, the community manager for OpenTelemetry. +**Reese:** It's great to have you! What changes have you seen in the community since it started? -Austin: -Hi everyone! Yes, I’m still the community manager and a member of the OpenTelemetry governance committee. I’ve been involved in various projects, including the OpenTelemetry website and demo. +**Austin:** There’s been significant growth! When we first came to KubeCon, many people were using OpenTelemetry, and the project updates room has grown substantially over the years. It's exciting to see new vendors and commercial solutions built on top of OpenTelemetry. -Reese: -What have you seen as the biggest changes in OpenTelemetry since its inception? +**Reese:** It sounds like the excitement is palpable! So many people are getting involved. What do you see as the future of OpenTelemetry? -Austin: -The growth has been incredible! When we first came to KubeCon, so many hands went up when asking who uses OpenTelemetry. The project has expanded significantly, and we now have a larger room for discussions, which is great to see. +**Austin:** AI is going to play a big role. We’re looking into how to improve documentation for LLMs, as well as using AI to reduce toil in observability tasks, like migrations. This will help make the process smoother for developers. -Reese: -What excites you most about the future of OpenTelemetry? +**Reese:** That’s a fascinating perspective! As we look ahead, what upcoming events should we be aware of? -Austin: -I think the opportunities for integration with other projects are really exciting. We’re seeing more CNCF projects natively integrate OpenTelemetry, which pushes the community further. +**Austin:** We have the OpenTelemetry Community Day in Denver scheduled for June, alongside the Open Source Summit. We're also planning to have a community day in Europe, so stay tuned for more information! -Reese: -What’s happening currently in the project? - -Austin: -We’re working on a system-level profiler using eBPF, and we’re evolving our logging infrastructure to support structured logging events. There’s a lot in progress, including the release of the JavaScript SDK 2.0. - -Reese: -Can you give us updates on upcoming events? - -Austin: -Yes! We have the OpenTelemetry Community Day in Denver, Colorado, coming up in June, and we’re looking into a potential community day in Europe as well. - -Reese: -That sounds amazing! It’s great to see so many new contributors joining the community. - -Austin: -Absolutely! It feels like a family reunion every time we gather, and it’s exciting to see friendships and careers grow from this community. - -Reese: -Thank you all for joining us today. A special thanks to our guests, Marilia and Austin, and a shout-out to our behind-the-scenes producer, Henrik. We’ll see you next time! - -**[Music fades out]** +**Reese:** Fantastic! Thank you both for joining us today, and thank you to our behind-the-scenes crew for making this possible. We appreciate everyone tuning in, and we hope to see you next time! ## Raw YouTube Transcript -Hello, welcome to a live stream of humans of OA. We are coming to you live from what had a little bit of subtitle issues, but we are here and we're so excited. My name is Reese. I am a senior developer relations engineer at New Relic and I am joined here by my lovely industry colleague Adriana and Marilia who is our first guest. Adriana. Hey everyone. Um, thanks for tuning in. My name is Adriana Vila. I work alongside Reese and the Hi everyone. My name is Marilis. I am a staff stops engineer at Graphana Labs and also in open telemetry. I work in a few different groups. I am a maintainer for the contributor experience and I'm an approver for if you will earlier about all the different parts that you're involved in and we're really curious because you came you started as an end user um and then eventually like started contributing. Yeah. And so we would love to on my prior job I was responsible for the observability. I used to work on cockroach labs and I was the manager for the observability there. So I started learning like I learning about hotel specifically because there we did would be able to use that like maybe someone want to see some of those things outside of the database. So we created like some endpoints here and there but it was like very basic stuff like this team is to work only on open telemetry. So your focus is just open telemetry. So this is my dayto-day I can have fun and just be on it. That's awesome. I didn't realize that. That's so cool. And and what would that new like Java and net and I want to make sure that all SDKs are on the same level and the other two were a little here. What the are the things that we can add? So we have owner for the Postgress plugin that I was already like touching and there. That's awesome. And actually you gave a talk um yesterday about database monitoring. Exactly. So one of the other big ones. So fresh news. We just this week we released the was the release candidate too. So that means by the end of the month we're going to mark as stable. So people can really start using without any concerns. So um you know you you mentioned that you're you're in the semantic convention sig for for databases and there's like a general semantic convention sig correct. What is it about the database semantic convention sig that's sig spun off? Yeah. So usually you have the general semantic conversion and on that one you have sometimes people from all of the other semantics that can give like an updates or discuss this particular name of the metric and spend an hour when you have a lot of people on the code they're not related to databases so they don't have any input to give it might be waste of time so it's kind of like you spin up like different semantic conventions for each of like oh we want to focus on database we want to focus on the HTTP we want to focus RPC See, so you now you can bring people that have experience on that area and talk about it. The same way you can think about if you would have a SI for SDK, you have so many languages. So we have now one S for each languages. So it's kind of like the same idea. Yeah. Help like keep it like the work focused and easy to prioritize. Yeah. And then we can always have the updates that we can bring to the main semantic one because the when they generate like a version is for all semantic. is not as specific for the database. So it's you just merge that repo and whenever new version come in might be bringing updates for the database might be from the HTTP and so on. So we just like can bring the update like to the main ones like hey this one we're marking as stable. We have these updates and things like that. And then you mentioned and I'm glad you uh were able to rattle off the long list of titles that you have in the community because I was like I am not going to remember all these but they're so cool. Um so there's like different titles. Um there's a prover then a prover then a tainer. Okay perfect. And then could you explain to us briefly like what each what's the distinction between each role? Yeah. So you started first the like as a contributor. So become so you can become a contributor not being a member of CNCF at all. So when you do a couple of uh contributor contributions you ask to become a member which is something I think everybody should do if you are doing because it's a very easy to join. You just have to open a VR say like hey I want to become a official member of CNCF and then oh hotel and then you are far you're officially a contributor then you pick something that is more interested to you so for example I really like this language I or I really like this specific component start joining the sex just to learn more about it see how you can help and when you start like really helping out start reviewing PRs putting comments like creating issues for things that you mind and with time you get the status of triager which is somebody that is helping when a lot of issues coming up you help exactly triage those issues and see like things that make sense things that actually they were not using correctly or there are actual bugs then from that you can become an approver that when you approve a PR your approval actually counts because people can approve but to have be able to merge you need to have people with permission giving the approval and being able able to merge. So the approver now have this whatever we approve it counts and somebody who has the permission can just merge in. And then you have the final one which is maintainer. So is the people that can actually merge usually the ones creating the releases uh creating like road maps but they do get input from everybody that that is contributing. That's so exciting. That is really cool. Thanks for the overview. Yeah. Um the other thing that we wanted to ask um you know you mentioned that you're working in the uh Portuguese localization. Um talk a little bit about that like talk about why it's important to have localization how you got involved with that specifically how that's been. Yeah. So I think it's really important because it's a barrier like the language and it's already hard for you to like completely learn something completely new like on tech but imagine if you don't even know the language and a lot of other countries like they're the first language not English it's not always you're going to find this documentation and not easily accessible for everybody. So actually I started like a while ago. I created like my own blog post and I was like everything that I'm created there I'm going to create and where people that actually met when I did like conference dur days in Brazil and there was the bad group. So I started like oh I'm going to help them out. So I like to like review a lot of things and the good things a lot of things in English that you learn like just experience living. So I can help out with like the translation like the localization and also decisions like what are the things that you don't translate at all because we're going to say tracer we don't that makes sense because at the same time you don't want to translate everything because then when they're going to search those word don't exist anywhere but we have a lot of yeah but yeah I think it's like really good to just break this barrier and bring a lot of people they would have no idea. Yeah, that's so great. That's that's a lot of work. Um I guess in in some ways has it like improved your Portuguese as a as a result. So yeah, it's always funny because like living outside for so long and sometimes you I'm going to give a talk like in Portuguese and I was like here so I know the right thing to use but it's always always a challenge but we also try because there's not only Portuguese we also have like people work on the Spanish, French and Japanese. So do completely different than the French know. We also have this thing like which is like similar to the same. So we have the documentation one but we have different groups. So every time a docs page comes out um you like the localization s translate. Yeah. So that is the idea. So right now that people are accessing more things like that. So we start with those ones and like the same thing for SDKs. I see which ones people are like downloading more. So we and some come in and do an updates on the original one. So we we do have something to because part of the when you do a localization there is a new kind of like tag that you had to put in what was the commit that you translated. Okay? Because it's really hard for people to know that somebody did an update somewhere. uh you had to really be paying attention. So the idea is to have something that would tell us, hey those are behind just catch up and things like that. So we were always up to date and Oh, sorry. Go ahead. I was curious cuz um when I was asking you um you know all your different that was like the first time I'd heard of that specifically cuz I know we have like translated some um some of our documentation but I didn't realize there were like specific teams working on these languages. Um so the localization sigs they are only translating um documentation or what all involves so like the official page like the open telemetry.io All the pages there are are the ones. If you look at the now at the top that was didn't have anything. Now if you're looking at the page the top you have a select there that you can change the language that you want. So whenever the page is translated now you have so cool learned something new today. I then joke that I have a PR with the translation to the Japanese even though I don't speak any Japanese and sometimes it goes wrong there too. Uhhuh. So it's a way to catch up. So like and then when they was like, "Okay, so the I was like, "Oh, let me see if any other languages translated the same thing wrong." Async or something. Okay. Yeah, that's pretty big. Yeah. And then it was good that it was just like a word and I translated and I put a PR and I tagged the reviewers for Japanese. I was like, I don't speak any Japanese. I put it this thing that I think is the right tradition. Like there was a part for the translation, but we even like looking for we want to add also more examples because we don't have a lot of examples. If there is something like for example you never did so you try it yourself and you might for the future one that are coming that they know like more examples more things to try it on. Yeah, man. This is so exciting. I'm learning all kinds of new things. I know. Yeah. So yeah, so many things that we're like when people ask like how do I start like so many of you're going to be able to find somewhere something that is interesting. There's always something for you, right? Yeah, it's really hard and just try joining just like listening just but at the same time like keep in mind that open source is different than a dayto-day job cuz sometimes what I see people find things was like how I I so I found the things that like oh they need help with this they need help with that and then the things that I already have priorities like from even coming like from hey can you take a look at this up for grabs a few reples have them uh so we have things like for example contribute fast that we did here and a lot of them got tagged with country fest but not all of them got actually work on so those are the ones that you know and you're going to find something and people in the community are very helpful like tag them on slack on the issue itself they are happy to give you guidance as well yeah and I'll do a shameless plug for the end user sig um as well we are always happy to have contributors um you can find more information in our show notes part of like the end user saying the other thing I want as the is the contributor experience. So if you want to like I want to contribute like what is missing. So for example one of the project that I did that was so like how do I start? I don't know how to set up locally. There was no like what is the dependency that you need. So there was like nothing there. So I work like as a mentor for the AI program template that all repos could use and started creating like the PR so people have at least the basic things that they need to just at least start having things. Oh, that's amazing. And and really quickly, actually, I want to ask about the contributor experience SIG. Um because that one's a relatively new SIG, right? Yes. Um can you tell us a little bit about that SIG, how it got started? Um and how you became a M. Was it because we saw this need of people complaining that they don't know how to start? They don't always understand like the path for like trier or like how like there is nobody there to guide. Okay, if nobody is going to be there for you, are you able to find all the documentation that you need to be able to follow the path that you want or like we didn't even know the challenges people were having a few of the things and we do like also surveys to find out people like how are you feeling like being a contributor and we started tackle the things like were the most concerned like documentation was the top one so this is why I started that project and container almost right away. What are some of the upcoming things for the contributor experiencing? So for this one, I just finished this part like this month for the template. So my next meeting is going to be actually picking up what is going to be the next one. Uh but we do have like issues open that is just helping out like clarify things to people. Uh and if anyone like also if you're listening if you have something that is really like bothering you and you need feel free to like message us or like open issue directly on the report there we can't prioritize because we have a list there but we are just prioritizing ourself because we don't have necessarily feedback things but react like thumbs up or really want this kind of thing and we will be able to put it in first. Awesome. And I think we're almost ready for our next guest. Yeah, I think we're so I guess we will continue our conversation. I I guess as as we get ready to bring on our next guest experience, they will be able to share like people just joining and people that have a lot of experience will be able to guide you as well just as I join the calls and you're going to be able to and they are Austin Parker. I'm just going to get the surprise out of the way. uh community manager for open telemetry and we are so excited to have him on. How am I still community manager? Wait, are you no longer the community manager? What? You're still community manager? Yeah, I think I am. If you go and check the uh check the repo, I probably am. I mean, I do both jobs still. So, io and a member of the open tree governance uh committee. I also have been a maintainer on the open country website uh open country demo. I've working group I guess you could say like where we combine of open census maintainers and open uh tracing maintainers where we you know came in and and worked with some people. Um yeah, what have you seen as like the biggest changes since things started? Like when when we first came to CubeCon, um zero day event, you know, ask people who's how many people are using hotel and it's like every hand in the room is going up, right? Like the growth of Yeah. which 2022 um the project updates room, you know, the project was given like a really small room. It was like pretty packed. Yeah. And over the years I've seen that room like get bigger and bigger. Yeah. I know they probably like a 200 seat room and there are still people standing in the back, right? Um there was also a lot of you know there was a keynote yesterday that was actually the thing that I've been most excited about the growth of hotel isn't open telemetry necessarily but the opportunities that we see other people kind of taking and running with right um projects like Percy's new vendors new commercial solutions built on top of open telemetry have been really exciting to see Um, and I think, you know, that that's the sort of stuff that you Yeah. Yeah. Definitely. Yeah. It's been exciting to see it grow. I I remember even uh like my first CubeCon was uh Detroit and there was uh there was an open observability day and then there was an hotel unplugged with CubeCon North America and EU, which I'm super stoked about. Um what um super busy too. We have hundreds and hundreds of people showing up for those. Yeah. It was great. Yeah. Thank you. I mean, I feel like it's also one of those things where it's kind like it's you can't really do three tracks, right? Like that's a bit like two is good. Seems like a good number, but there's observability talks, you know, throughout the week at CubeCon. I don't know. This this used to be the Kubernetes club, right? This is the operations uh the SRE kids. And now, you know, security has a big presence in the CNCF. I think what we're seeing what I what I think is important is you in a lot of ways if you think about how a lot of technology that we still rely on today you know was built and maintained and came right maybe someone maybe someone and if that's you more power to you um to America we might need your help um things became the facto standards through whatever you know combination of factors happened right like I think what's cool about C4 vendor you happen to use or so on and so forth now the downside is you know in 25 years my kid's going to copy me and be like what the what's this open telemetry crap dad um don't metaverse open telemetry now oh man also Speaking of the future, yeah, we were curious about how do you think AI might impact open telemetry? How it might how it might impact or have an impact or Yeah. Uh there's a lot of ways. I think one thing I was So, it's funny cuz I was just talking to someone about this. Um, a project I've been working on kind of on the side is how to make better docs for LLMs because the documentation that we write for human beings is really good for human beings and LLMs are not human beings. They are something else. So you you need to kind of give them documentation in a way that is similar to how you would give it to a human but but distinct enough that it's not you know it's trickier than just saying like oh go read this web page, right? cuz the web page has a lot of stuff that's for humans. We care about things like font sizes and colors and we care a lot about sort of the organization of information and we want to have pages that are single concept, right? so that you can focus on like what is the goal of this page and but then LM it's there's a distinction you need um you can you like one interesting thing if you think about writing a document you know writing documentation is you're not supposed to use um even if there's a more specific word like use more words not less right you have to kind of hit it at whatever reading level you expect people to be at even for that kind of stuff but with LMS actually don't have that disadvantage right like you can give it a very large flowery word and if it's the most specific word that's actually helpful because of the way the semantic search works. Um precision in language is actually really important. So you wind up needing documentation that is the same documentation you give to humans because the concepts need all need to all match but is organized and structured in a different way is much longer has kind of everything in one big chunk is optimized for the amount of tokens and the um semantic values of those tokens. But the advantage of doing this right of thinking about how do I give the LM documentation is that LLMs are remarkable you know especially if you're using them as part of like AI assisted coding are remarkable at reducing toil. One of the things that I think you know all of us here have been working in observability for years and what is like the one thing nobody wants to do? Nobody wants to do a migration, right? No, you go tell someone like, "Oh, here's the new thing." They have to rewrite all this instrumentation, all these logs, all this stuff. They just be like, "Thanks, I'll pass." It's cuz it's a toil. It doesn't make sense, right? The existing stuff works well enough. It's maybe it could be better, but you know, we're not going to go dedicate however many engineers lives for three months to rewrite all of our logging statements. But what if you just have an AI do it, right? The AI doesn't care. Doesn't complain. you if you give it good instructions and good rules, it's able to, you know, make good deci, you know, make pretty good decisions. Um, and it's not like you're replacing human effort, right? You're not replacing the programmer. You're not replacing the people that are responsible to maintain the system. You're easing the burden of modernizing what they're trying to do. And so you're actually benefiting those people a lot cuz even with you know improvements in AI powered anomaly detection whatever right like we've been doing AI and observability for a while um LLM's just advance it a little bit maybe a lot we'll see but even nonAI stuff right if you think about traditional sort of anomaly detection and heristic based detection it's still machine learning LM machine learning. It's all machine learning. We've been doing machine learning for a while. And when you get to a certain size and complexity of your systems, you have to have it because there's just too much data for humans to process and and go through. So let's figure out how we can you you know that to me is like the impact of AI on observability on open telemetry is we can make it easier for the a make it easier for you those your observability tooling to understand hotel and interpret what's happening your system better and at the end of the day give you more time to do we had talked about um hotel project updates um which took place yesterday right that sounds right I'm losing track I'm losing Yeah, it is. It was yesterday milk. That doesn't sound like a good bagel. It's I wish it said bagel clock. This is probably a great This is probably a great visual bit for the And it's been It attracted my attention ever since I came up here and now it's like I've been obsessed. I've been waiting for the appropriate moment to drop the bagel clock into the conversation. You did it. There you go. This is what we call commitment to the bed. Yes. I update. Um, yeah, project update. So, um, can you give folks a, uh, open telemetry right now is the second or first biggest project in the CNCF, depending on how you count it, but contributing um, across, you know, dozens and dozens of repositories. We're maintaining APIs, SDKs, tools in, you know, dozen plus languages. A lot's going on, right? And at this point, the project is really too big almost to have kind of a single narrative of whatever we're doing, but there's a few areas that we wanted to focus on. We're starting to see a lot of great adoption of open telemetry by the sort of broader community outside of uh just so we're starting to see more CNCF projects natively integrate open telemetry. um we're starting to see are integrating open telemetry into their frameworks and into the in Dino's case into the runtime itself, right? So if you're writing a JavaScript app and you're using Dino, you pass in a config, you don't have to do anything, which is great. That's the vision for the project. So um beyond that, you know, beyond the kind of growth we're seeing in adoption, we're seeing a few, you know, longerterm projects that we are proceeding uh along. So one thing that we've been working on over the past 6 to 8 months, we've had a lot of progress there. We have a system level profiler that is being worked on. It uses EVPF and other various other technologies to let you use profiling across your services on a [Applause] node. That is still in alpha like it's not done. It's not ready but pretty soon um that should be ready for people to start banging on. Another important thing we're doing is that we are evolving our uh logging infrastructure or logging APIs. So tradition originally we would just bridge to your existing logging API because there's a lot of those there's logj there's various facads in go and net and wherever but one of the pieces of feedback we were going out and finding these sort of consistent metadata across services and libraries and domains is people need people needed a structured a way to emit structured events, right? Things like that you and I would probably we would call the LOG. Some people would call it an event. And one thing I have learned is that uh the third rail of observability is talking about logging at all because people are very very precious about what lo what the word logs means to them. I've noticed. Oh, true. Yeah. It's you don't mess with people's logs. Yes. So what we've kind of come to realize through this whole process is that we need some sort of API level answer to that and we're pretty close to have you know we have some OTAs and some specs in flight on this but the idea is that there will be a open telemetry logging API um that will exist to let you emit structured events and a structured event is really just a fancy way of saying a structured log that has a known schema right in the same way that semantic conventions in hotel let you apply schemas to your telemetry to your logs or sorry to your metrics and traces. You'll be able to say hey here's a client side ROM event or here's a you know out of memory exception or or any of the various things that can happen. Um, you'll be able to say, "Hey, here's a generative AI prompt, for example." And what we'll do is you'll be able to either take that and use it like you would use a span event today and bundle it in with the span or you'll be able to emit it separately through the log record sign through the logging signal and then have your backend either stitch them together or process them independently or do do whatever, right? like once once it's out of our hands, we we don't really care what you do with it. Um but that that's probably the two big inflight things I would say. Beyond that, um a lot of work is happening on other things. Stabilizing the collector, stabilizing various other SDKs and APIs. Um shout out to our JavaScript uh SIG which just released uh JS SDK 2.0 know which uh my understanding is this fixes a lot of problems that people have had with especially with bundling it um and things around ESM modules. I don't quite know what all that is. It sounds very scary. The JS devs ensure me it's very important but but seriously I I think it's actually a really good sign of the health of that project, right? That they have been able to get enough feedback about like hey these are the decisions that worked and didn't work to create a 2.0 know and then for an end user I was actually talking to someone Monday um Sunday at cloud native rejects about this who maintains a integration into open telemetry into his company's um product and he was like oh yeah the migration was like 5 minutes right wow because the API and the SDK are independent in so an SDK change is really very you know it's not a hu it's not a lot that you have to do to take those updates so that's something that for obviously other maintainers can't may or may not decide to do it but we're you know it certainly seems that a lot of maintainers are thinking about well maybe it's a good idea to go back it's been 5 6 years right like you can learn a lot you get a lot of great feedback over that time and there's things that we would probably do differently in every language if we had a doover so thanks to the hotel architecture you can kind of get that doover which is cool that's great um how are We on time. I'm not sure cuz we have uh Are we on time, producer? Okay. Well, I guess this could be a good opportunity for us to plug some upcoming stuff like Hotel Community Day. Yes, hotel community day in Denver, Colorado coming up this summer. Um June [Music] 26 20 25th or 26th it's the same week as open source summit open source summit go look on the web um the CFP is closed that unfortunately but we will be uh we should be announcing the schedule on that here pretty soon um even if you aren't planning on you know plan on speaking highly recommend everyone uh that's in the US North America to come out to that it's going to be a great And we are also, no promises, but we're trying to do a community day in Europe this year. So stay tuned. If not this year, definitely next year. Is it going to be part of Open Source Summit EU or that that would be the plan, but nothing's in set in stone. Um but we've definitely one of the fun facts at the um project update is that about 50% of our contributors are actually not in the US in hotel um and we've seen now that's US and then everywhere else right so but if you look at if you break it down by like region so you go like North America EMIA APAC other then we like the line for EMIA is just like doing this and the US one is kind of doing this a little bit. So, they're starting to they're starting to get closer and closer together. But, we've definitely, you know, one of the things I always love coming to uh CubeCon EU is we have so many, you know, our user community is so, you know, so vibrant here. Um, we have so many maintainers whose work is just fantastic and we really want to support our European um, end user and contributor community. So, we're very strongly going to be figuring out how to do a European community day. Awesome. Oh, that's awesome. Yeah. I always love the vibe at the CubeCon EU is just very vibrant. And I think um at the keynote um they mentioned this was the biggest CubeCon so far. 12,000. Yeah. Over 13,000 people. Over 13,000. That's the number I heard. Damn, that is wild. Yeah. No, they I mean I And it's like how's your CubeCon? and they say it's like, "Oh, it's my first." I'm like, "You're going to have that. Bring lots of water. Bring lots of water." But I love I love that we're still that new people are still coming into this community, right? That we're really cool to see the growth of, you know, this community, right? Like to see it expand, to see it bring in new new people, right? Like Yeah. Oh my god. There's a ton of people that we know that all the three of us here know that careers, friendships, right? friendships like that's given people the opportunity to really, you know. Yeah. Um that's neat. Yeah. Also, it feels like a family reunion every time we're together, right? Yeah. No, it's great. It's it's like a family reunion with like a 10,000. Yeah. So, a bunch of batteries and then we get on the other side. They have the retail technology exhibition. Yep. They actually get a red carpet. Very loud bugers walking around which was um exciting. It was definitely unexpected. Yeah, I know. We were walking yesterday and we're like well the tube it was like there's like two stops. Yeah, there's a station either end. Yeah. Yeah. Yeah. And I I heard that um it's actually faster if you time it right to take the tube. If you hit if you hit the um it also depends on where you start from, but if you're from door to door, it's definitely faster. Yeah, it's nice. You just tap in and out. So Oh, I know. Yeah, it's really card. Discovered that. Oh, is that better than the I just use Google Pay. Yeah, you can use your phone. Yeah, you can use your phone. Yeah. Yeah. Yeah. Works well. Yeah, we have that in Toronto. I mean, we have we have uh tap in New York. They finally um it's Omni now. Omny instead of this for another episode um live stream for you from CubeCon EU. Sorry guys, it's been a long week. Um thank you so much for joining us. We will have um our lovely guests. Um again, I'm Ree. This is Adriana. Thank you so much Austin for being here and Maria for um being here earlier. And also shout out to our behind the scenes. Um but he is the one producing all this for you. All of our streams. Yes. Henrik said from Dana Trace. Thank you so much and we will see you next time. [Music] +Hello, welcome to a live stream of humans of OA. We are coming to you live from what had a little bit of subtitle issues, but we are here and we're so excited. My name is Reese. I am a senior developer relations engineer at New Relic and I am joined here by my lovely industry colleague Adriana and Marilia who is our first guest. Adriana. Hey everyone. Um, thanks for tuning in. My name is Adriana Vila. I work alongside Reese and the Hi everyone. My name is Marilis. I am a staff stops engineer at Graphana Labs and also in open telemetry. I work in a few different groups. I am a maintainer for the contributor experience and I'm an approver for if you will earlier about all the different parts that you're involved in and we're really curious because you came you started as an end user um and then eventually like started contributing. Yeah. And so we would love to on my prior job I was responsible for the observability. I used to work on cockroach labs and I was the manager for the observability there. So I started learning like I learning about hotel specifically because there we did would be able to use that like maybe someone want to see some of those things outside of the database. So we created like some endpoints here and there but it was like very basic stuff like this team is to work only on open telemetry. So your focus is just open telemetry. So this is my dayto-day I can have fun and just be on it. That's awesome. I didn't realize that. That's so cool. And and what would that new like Java and net and I want to make sure that all SDKs are on the same level and the other two were a little here. What the are the things that we can add? So we have owner for the Postgress plugin that I was already like touching and there. That's awesome. And actually you gave a talk um yesterday about database monitoring. Exactly. So one of the other big ones. So fresh news. We just this week we released the was the release candidate too. So that means by the end of the month we're going to mark as stable. So people can really start using without any concerns. So um you know you you mentioned that you're you're in the semantic convention sig for for databases and there's like a general semantic convention sig correct. What is it about the database semantic convention sig that's sig spun off? Yeah. So usually you have the general semantic conversion and on that one you have sometimes people from all of the other semantics that can give like an updates or discuss this particular name of the metric and spend an hour when you have a lot of people on the code they're not related to databases so they don't have any input to give it might be waste of time so it's kind of like you spin up like different semantic conventions for each of like oh we want to focus on database we want to focus on the HTTP we want to focus RPC See, so you now you can bring people that have experience on that area and talk about it. The same way you can think about if you would have a SI for SDK, you have so many languages. So we have now one S for each languages. So it's kind of like the same idea. Yeah. Help like keep it like the work focused and easy to prioritize. Yeah. And then we can always have the updates that we can bring to the main semantic one because the when they generate like a version is for all semantic. is not as specific for the database. So it's you just merge that repo and whenever new version come in might be bringing updates for the database might be from the HTTP and so on. So we just like can bring the update like to the main ones like hey this one we're marking as stable. We have these updates and things like that. And then you mentioned and I'm glad you uh were able to rattle off the long list of titles that you have in the community because I was like I am not going to remember all these but they're so cool. Um so there's like different titles. Um there's a prover then a prover then a tainer. Okay perfect. And then could you explain to us briefly like what each what's the distinction between each role? Yeah. So you started first the like as a contributor. So become so you can become a contributor not being a member of CNCF at all. So when you do a couple of uh contributor contributions you ask to become a member which is something I think everybody should do if you are doing because it's a very easy to join. You just have to open a VR say like hey I want to become a official member of CNCF and then oh hotel and then you are far you're officially a contributor then you pick something that is more interested to you so for example I really like this language I or I really like this specific component start joining the sex just to learn more about it see how you can help and when you start like really helping out start reviewing PRs putting comments like creating issues for things that you mind and with time you get the status of triager which is somebody that is helping when a lot of issues coming up you help exactly triage those issues and see like things that make sense things that actually they were not using correctly or there are actual bugs then from that you can become an approver that when you approve a PR your approval actually counts because people can approve but to have be able to merge you need to have people with permission giving the approval and being able able to merge. So the approver now have this whatever we approve it counts and somebody who has the permission can just merge in. And then you have the final one which is maintainer. So is the people that can actually merge usually the ones creating the releases uh creating like road maps but they do get input from everybody that that is contributing. That's so exciting. That is really cool. Thanks for the overview. Yeah. Um the other thing that we wanted to ask um you know you mentioned that you're working in the uh Portuguese localization. Um talk a little bit about that like talk about why it's important to have localization how you got involved with that specifically how that's been. Yeah. So I think it's really important because it's a barrier like the language and it's already hard for you to like completely learn something completely new like on tech but imagine if you don't even know the language and a lot of other countries like they're the first language not English it's not always you're going to find this documentation and not easily accessible for everybody. So actually I started like a while ago. I created like my own blog post and I was like everything that I'm created there I'm going to create and where people that actually met when I did like conference dur days in Brazil and there was the bad group. So I started like oh I'm going to help them out. So I like to like review a lot of things and the good things a lot of things in English that you learn like just experience living. So I can help out with like the translation like the localization and also decisions like what are the things that you don't translate at all because we're going to say tracer we don't that makes sense because at the same time you don't want to translate everything because then when they're going to search those word don't exist anywhere but we have a lot of yeah but yeah I think it's like really good to just break this barrier and bring a lot of people they would have no idea. Yeah, that's so great. That's that's a lot of work. Um I guess in in some ways has it like improved your Portuguese as a as a result. So yeah, it's always funny because like living outside for so long and sometimes you I'm going to give a talk like in Portuguese and I was like here so I know the right thing to use but it's always always a challenge but we also try because there's not only Portuguese we also have like people work on the Spanish, French and Japanese. So do completely different than the French know. We also have this thing like which is like similar to the same. So we have the documentation one but we have different groups. So every time a docs page comes out um you like the localization s translate. Yeah. So that is the idea. So right now that people are accessing more things like that. So we start with those ones and like the same thing for SDKs. I see which ones people are like downloading more. So we and some come in and do an updates on the original one. So we we do have something to because part of the when you do a localization there is a new kind of like tag that you had to put in what was the commit that you translated. Okay? Because it's really hard for people to know that somebody did an update somewhere. uh you had to really be paying attention. So the idea is to have something that would tell us, hey those are behind just catch up and things like that. So we were always up to date and Oh, sorry. Go ahead. I was curious cuz um when I was asking you um you know all your different that was like the first time I'd heard of that specifically cuz I know we have like translated some um some of our documentation but I didn't realize there were like specific teams working on these languages. Um so the localization sigs they are only translating um documentation or what all involves so like the official page like the open telemetry.io All the pages there are are the ones. If you look at the now at the top that was didn't have anything. Now if you're looking at the page the top you have a select there that you can change the language that you want. So whenever the page is translated now you have so cool learned something new today. I then joke that I have a PR with the translation to the Japanese even though I don't speak any Japanese and sometimes it goes wrong there too. Uhhuh. So it's a way to catch up. So like and then when they was like, "Okay, so the I was like, "Oh, let me see if any other languages translated the same thing wrong." Async or something. Okay. Yeah, that's pretty big. Yeah. And then it was good that it was just like a word and I translated and I put a PR and I tagged the reviewers for Japanese. I was like, I don't speak any Japanese. I put it this thing that I think is the right tradition. Like there was a part for the translation, but we even like looking for we want to add also more examples because we don't have a lot of examples. If there is something like for example you never did so you try it yourself and you might for the future one that are coming that they know like more examples more things to try it on. Yeah, man. This is so exciting. I'm learning all kinds of new things. I know. Yeah. So yeah, so many things that we're like when people ask like how do I start like so many of you're going to be able to find somewhere something that is interesting. There's always something for you, right? Yeah, it's really hard and just try joining just like listening just but at the same time like keep in mind that open source is different than a dayto-day job cuz sometimes what I see people find things was like how I I so I found the things that like oh they need help with this they need help with that and then the things that I already have priorities like from even coming like from hey can you take a look at this up for grabs a few reples have them uh so we have things like for example contribute fast that we did here and a lot of them got tagged with country fest but not all of them got actually work on so those are the ones that you know and you're going to find something and people in the community are very helpful like tag them on slack on the issue itself they are happy to give you guidance as well yeah and I'll do a shameless plug for the end user sig um as well we are always happy to have contributors um you can find more information in our show notes part of like the end user saying the other thing I want as the is the contributor experience. So if you want to like I want to contribute like what is missing. So for example one of the project that I did that was so like how do I start? I don't know how to set up locally. There was no like what is the dependency that you need. So there was like nothing there. So I work like as a mentor for the AI program template that all repos could use and started creating like the PR so people have at least the basic things that they need to just at least start having things. Oh, that's amazing. And and really quickly, actually, I want to ask about the contributor experience SIG. Um because that one's a relatively new SIG, right? Yes. Um can you tell us a little bit about that SIG, how it got started? Um and how you became a M. Was it because we saw this need of people complaining that they don't know how to start? They don't always understand like the path for like trier or like how like there is nobody there to guide. Okay, if nobody is going to be there for you, are you able to find all the documentation that you need to be able to follow the path that you want or like we didn't even know the challenges people were having a few of the things and we do like also surveys to find out people like how are you feeling like being a contributor and we started tackle the things like were the most concerned like documentation was the top one so this is why I started that project and container almost right away. What are some of the upcoming things for the contributor experiencing? So for this one, I just finished this part like this month for the template. So my next meeting is going to be actually picking up what is going to be the next one. Uh but we do have like issues open that is just helping out like clarify things to people. Uh and if anyone like also if you're listening if you have something that is really like bothering you and you need feel free to like message us or like open issue directly on the report there we can't prioritize because we have a list there but we are just prioritizing ourself because we don't have necessarily feedback things but react like thumbs up or really want this kind of thing and we will be able to put it in first. Awesome. And I think we're almost ready for our next guest. Yeah, I think we're so I guess we will continue our conversation. I I guess as as we get ready to bring on our next guest experience, they will be able to share like people just joining and people that have a lot of experience will be able to guide you as well just as I join the calls and you're going to be able to and they are Austin Parker. I'm just going to get the surprise out of the way. uh community manager for open telemetry and we are so excited to have him on. How am I still community manager? Wait, are you no longer the community manager? What? You're still community manager? Yeah, I think I am. If you go and check the uh check the repo, I probably am. I mean, I do both jobs still. So, io and a member of the open tree governance uh committee. I also have been a maintainer on the open country website uh open country demo. I've working group I guess you could say like where we combine of open census maintainers and open uh tracing maintainers where we you know came in and and worked with some people. Um yeah, what have you seen as like the biggest changes since things started? Like when when we first came to CubeCon, um zero day event, you know, ask people who's how many people are using hotel and it's like every hand in the room is going up, right? Like the growth of Yeah. which 2022 um the project updates room, you know, the project was given like a really small room. It was like pretty packed. Yeah. And over the years I've seen that room like get bigger and bigger. Yeah. I know they probably like a 200 seat room and there are still people standing in the back, right? Um there was also a lot of you know there was a keynote yesterday that was actually the thing that I've been most excited about the growth of hotel isn't open telemetry necessarily but the opportunities that we see other people kind of taking and running with right um projects like Percy's new vendors new commercial solutions built on top of open telemetry have been really exciting to see Um, and I think, you know, that that's the sort of stuff that you Yeah. Yeah. Definitely. Yeah. It's been exciting to see it grow. I I remember even uh like my first CubeCon was uh Detroit and there was uh there was an open observability day and then there was an hotel unplugged with CubeCon North America and EU, which I'm super stoked about. Um what um super busy too. We have hundreds and hundreds of people showing up for those. Yeah. It was great. Yeah. Thank you. I mean, I feel like it's also one of those things where it's kind like it's you can't really do three tracks, right? Like that's a bit like two is good. Seems like a good number, but there's observability talks, you know, throughout the week at CubeCon. I don't know. This this used to be the Kubernetes club, right? This is the operations uh the SRE kids. And now, you know, security has a big presence in the CNCF. I think what we're seeing what I what I think is important is you in a lot of ways if you think about how a lot of technology that we still rely on today you know was built and maintained and came right maybe someone maybe someone and if that's you more power to you um to America we might need your help um things became the facto standards through whatever you know combination of factors happened right like I think what's cool about C4 vendor you happen to use or so on and so forth now the downside is you know in 25 years my kid's going to copy me and be like what the what's this open telemetry crap dad um don't metaverse open telemetry now oh man also Speaking of the future, yeah, we were curious about how do you think AI might impact open telemetry? How it might how it might impact or have an impact or Yeah. Uh there's a lot of ways. I think one thing I was So, it's funny cuz I was just talking to someone about this. Um, a project I've been working on kind of on the side is how to make better docs for LLMs because the documentation that we write for human beings is really good for human beings and LLMs are not human beings. They are something else. So you you need to kind of give them documentation in a way that is similar to how you would give it to a human but but distinct enough that it's not you know it's trickier than just saying like oh go read this web page, right? cuz the web page has a lot of stuff that's for humans. We care about things like font sizes and colors and we care a lot about sort of the organization of information and we want to have pages that are single concept, right? so that you can focus on like what is the goal of this page and but then LM it's there's a distinction you need um you can you like one interesting thing if you think about writing a document you know writing documentation is you're not supposed to use um even if there's a more specific word like use more words not less right you have to kind of hit it at whatever reading level you expect people to be at even for that kind of stuff but with LMS actually don't have that disadvantage right like you can give it a very large flowery word and if it's the most specific word that's actually helpful because of the way the semantic search works. Um precision in language is actually really important. So you wind up needing documentation that is the same documentation you give to humans because the concepts need all need to all match but is organized and structured in a different way is much longer has kind of everything in one big chunk is optimized for the amount of tokens and the um semantic values of those tokens. But the advantage of doing this right of thinking about how do I give the LM documentation is that LLMs are remarkable you know especially if you're using them as part of like AI assisted coding are remarkable at reducing toil. One of the things that I think you know all of us here have been working in observability for years and what is like the one thing nobody wants to do? Nobody wants to do a migration, right? No, you go tell someone like, "Oh, here's the new thing." They have to rewrite all this instrumentation, all these logs, all this stuff. They just be like, "Thanks, I'll pass." It's cuz it's a toil. It doesn't make sense, right? The existing stuff works well enough. It's maybe it could be better, but you know, we're not going to go dedicate however many engineers lives for three months to rewrite all of our logging statements. But what if you just have an AI do it, right? The AI doesn't care. Doesn't complain. you if you give it good instructions and good rules, it's able to, you know, make good deci, you know, make pretty good decisions. Um, and it's not like you're replacing human effort, right? You're not replacing the programmer. You're not replacing the people that are responsible to maintain the system. You're easing the burden of modernizing what they're trying to do. And so you're actually benefiting those people a lot cuz even with you know improvements in AI powered anomaly detection whatever right like we've been doing AI and observability for a while um LLM's just advance it a little bit maybe a lot we'll see but even nonAI stuff right if you think about traditional sort of anomaly detection and heristic based detection it's still machine learning LM machine learning. It's all machine learning. We've been doing machine learning for a while. And when you get to a certain size and complexity of your systems, you have to have it because there's just too much data for humans to process and and go through. So let's figure out how we can you you know that to me is like the impact of AI on observability on open telemetry is we can make it easier for the a make it easier for you those your observability tooling to understand hotel and interpret what's happening your system better and at the end of the day give you more time to do we had talked about um hotel project updates um which took place yesterday right that sounds right I'm losing track I'm losing Yeah, it is. It was yesterday milk. That doesn't sound like a good bagel. It's I wish it said bagel clock. This is probably a great This is probably a great visual bit for the And it's been It attracted my attention ever since I came up here and now it's like I've been obsessed. I've been waiting for the appropriate moment to drop the bagel clock into the conversation. You did it. There you go. This is what we call commitment to the bed. Yes. I update. Um, yeah, project update. So, um, can you give folks a, uh, open telemetry right now is the second or first biggest project in the CNCF, depending on how you count it, but contributing um, across, you know, dozens and dozens of repositories. We're maintaining APIs, SDKs, tools in, you know, dozen plus languages. A lot's going on, right? And at this point, the project is really too big almost to have kind of a single narrative of whatever we're doing, but there's a few areas that we wanted to focus on. We're starting to see a lot of great adoption of open telemetry by the sort of broader community outside of uh just so we're starting to see more CNCF projects natively integrate open telemetry. um we're starting to see are integrating open telemetry into their frameworks and into the in Dino's case into the runtime itself, right? So if you're writing a JavaScript app and you're using Dino, you pass in a config, you don't have to do anything, which is great. That's the vision for the project. So um beyond that, you know, beyond the kind of growth we're seeing in adoption, we're seeing a few, you know, longerterm projects that we are proceeding uh along. So one thing that we've been working on over the past 6 to 8 months, we've had a lot of progress there. We have a system level profiler that is being worked on. It uses EVPF and other various other technologies to let you use profiling across your services on a node. That is still in alpha like it's not done. It's not ready but pretty soon um that should be ready for people to start banging on. Another important thing we're doing is that we are evolving our uh logging infrastructure or logging APIs. So tradition originally we would just bridge to your existing logging API because there's a lot of those there's logj there's various facads in go and net and wherever but one of the pieces of feedback we were going out and finding these sort of consistent metadata across services and libraries and domains is people need people needed a structured a way to emit structured events, right? Things like that you and I would probably we would call the LOG. Some people would call it an event. And one thing I have learned is that uh the third rail of observability is talking about logging at all because people are very very precious about what lo what the word logs means to them. I've noticed. Oh, true. Yeah. It's you don't mess with people's logs. Yes. So what we've kind of come to realize through this whole process is that we need some sort of API level answer to that and we're pretty close to have you know we have some OTAs and some specs in flight on this but the idea is that there will be a open telemetry logging API um that will exist to let you emit structured events and a structured event is really just a fancy way of saying a structured log that has a known schema right in the same way that semantic conventions in hotel let you apply schemas to your telemetry to your logs or sorry to your metrics and traces. You'll be able to say hey here's a client side ROM event or here's a you know out of memory exception or or any of the various things that can happen. Um, you'll be able to say, "Hey, here's a generative AI prompt, for example." And what we'll do is you'll be able to either take that and use it like you would use a span event today and bundle it in with the span or you'll be able to emit it separately through the log record sign through the logging signal and then have your backend either stitch them together or process them independently or do do whatever, right? like once once it's out of our hands, we we don't really care what you do with it. Um but that that's probably the two big inflight things I would say. Beyond that, um a lot of work is happening on other things. Stabilizing the collector, stabilizing various other SDKs and APIs. Um shout out to our JavaScript uh SIG which just released uh JS SDK 2.0 know which uh my understanding is this fixes a lot of problems that people have had with especially with bundling it um and things around ESM modules. I don't quite know what all that is. It sounds very scary. The JS devs ensure me it's very important but but seriously I I think it's actually a really good sign of the health of that project, right? That they have been able to get enough feedback about like hey these are the decisions that worked and didn't work to create a 2.0 know and then for an end user I was actually talking to someone Monday um Sunday at cloud native rejects about this who maintains a integration into open telemetry into his company's um product and he was like oh yeah the migration was like 5 minutes right wow because the API and the SDK are independent in so an SDK change is really very you know it's not a hu it's not a lot that you have to do to take those updates so that's something that for obviously other maintainers can't may or may not decide to do it but we're you know it certainly seems that a lot of maintainers are thinking about well maybe it's a good idea to go back it's been 5 6 years right like you can learn a lot you get a lot of great feedback over that time and there's things that we would probably do differently in every language if we had a doover so thanks to the hotel architecture you can kind of get that doover which is cool that's great um how are We on time. I'm not sure cuz we have uh Are we on time, producer? Okay. Well, I guess this could be a good opportunity for us to plug some upcoming stuff like Hotel Community Day. Yes, hotel community day in Denver, Colorado coming up this summer. Um June 26 20 25th or 26th it's the same week as open source summit open source summit go look on the web um the CFP is closed that unfortunately but we will be uh we should be announcing the schedule on that here pretty soon um even if you aren't planning on you know plan on speaking highly recommend everyone uh that's in the US North America to come out to that it's going to be a great And we are also, no promises, but we're trying to do a community day in Europe this year. So stay tuned. If not this year, definitely next year. Is it going to be part of Open Source Summit EU or that that would be the plan, but nothing's in set in stone. Um but we've definitely one of the fun facts at the um project update is that about 50% of our contributors are actually not in the US in hotel um and we've seen now that's US and then everywhere else right so but if you look at if you break it down by like region so you go like North America EMIA APAC other then we like the line for EMIA is just like doing this and the US one is kind of doing this a little bit. So, they're starting to they're starting to get closer and closer together. But, we've definitely, you know, one of the things I always love coming to uh CubeCon EU is we have so many, you know, our user community is so, you know, so vibrant here. Um, we have so many maintainers whose work is just fantastic and we really want to support our European um, end user and contributor community. So, we're very strongly going to be figuring out how to do a European community day. Awesome. Oh, that's awesome. Yeah. I always love the vibe at the CubeCon EU is just very vibrant. And I think um at the keynote um they mentioned this was the biggest CubeCon so far. 12,000. Yeah. Over 13,000 people. Over 13,000. That's the number I heard. Damn, that is wild. Yeah. No, they I mean I And it's like how's your CubeCon? and they say it's like, "Oh, it's my first." I'm like, "You're going to have that. Bring lots of water. Bring lots of water." But I love I love that we're still that new people are still coming into this community, right? That we're really cool to see the growth of, you know, this community, right? Like to see it expand, to see it bring in new new people, right? Like Yeah. Oh my god. There's a ton of people that we know that all the three of us here know that careers, friendships, right? friendships like that's given people the opportunity to really, you know. Yeah. Um that's neat. Yeah. Also, it feels like a family reunion every time we're together, right? Yeah. No, it's great. It's it's like a family reunion with like a 10,000. Yeah. So, a bunch of batteries and then we get on the other side. They have the retail technology exhibition. Yep. They actually get a red carpet. Very loud bugers walking around which was um exciting. It was definitely unexpected. Yeah, I know. We were walking yesterday and we're like well the tube it was like there's like two stops. Yeah, there's a station either end. Yeah. Yeah. Yeah. And I I heard that um it's actually faster if you time it right to take the tube. If you hit if you hit the um it also depends on where you start from, but if you're from door to door, it's definitely faster. Yeah, it's nice. You just tap in and out. So Oh, I know. Yeah, it's really card. Discovered that. Oh, is that better than the I just use Google Pay. Yeah, you can use your phone. Yeah, you can use your phone. Yeah. Yeah. Yeah. Works well. Yeah, we have that in Toronto. I mean, we have we have uh tap in New York. They finally um it's Omni now. Omny instead of this for another episode um live stream for you from CubeCon EU. Sorry guys, it's been a long week. Um thank you so much for joining us. We will have um our lovely guests. Um again, I'm Ree. This is Adriana. Thank you so much Austin for being here and Maria for um being here earlier. And also shout out to our behind the scenes. Um but he is the one producing all this for you. All of our streams. Yes. Henrik said from Dana Trace. Thank you so much and we will see you next time. diff --git a/video-transcripts/transcripts/2025-05-01T06:00:52Z-otel-me-with-eromosele-akhigbe.md b/video-transcripts/transcripts/2025-05-01T06:00:52Z-otel-me-with-eromosele-akhigbe.md index b9dba7d..d8096ac 100644 --- a/video-transcripts/transcripts/2025-05-01T06:00:52Z-otel-me-with-eromosele-akhigbe.md +++ b/video-transcripts/transcripts/2025-05-01T06:00:52Z-otel-me-with-eromosele-akhigbe.md @@ -10,99 +10,145 @@ URL: https://www.youtube.com/watch?v=KzqY4roXhHs ## Summary -In this episode of "O Tell Me," host Adriana Vila, along with co-host Victoria, interviews Mole Aigbe, a developer advocate at Step Security and an OpenTelemetry community member. Mole shares his journey into the world of open source through the Outreachy program, which supports individuals from underrepresented communities in contributing to open source projects. He discusses the challenges he faced while learning Golang and OpenTelemetry, including the application process for Outreachy and his experiences creating his first pull request (PR) and an exporter for the OpenTelemetry Collector. Mole emphasizes the importance of mentorship and community support, highlighting how he overcame imposter syndrome and gained confidence through collaboration. He also provides valuable feedback on improving documentation for OpenTelemetry, advocating for more visual resources and clearer guidance for newcomers. The conversation underscores the significance of community engagement and advocacy in fostering inclusivity in tech. +In this episode of "O Tell Me," host Adriana Vila, along with co-host Victoria, engages in a discussion with guest Mole Aigbe, a developer advocate at Step Security and a member of the OpenTelemetry community. They explore Mole's journey into open source through the Outreachy program, which supports underrepresented communities in contributing to open source projects. Mole shares insights on the challenges of applying for Outreachy, the application process, and his experiences learning Go and contributing to OpenTelemetry, including building an exporter for the OpenTelemetry Collector. He emphasizes the importance of supportive mentors and the community's welcoming nature, as well as the value of studying existing code to accelerate learning. The conversation also touches on the need for improved documentation and resources for newcomers to OpenTelemetry. The episode concludes with announcements about upcoming events and opportunities for community engagement. -# O Tell Me: Episode with Mole Aigbe +## Chapters -[Music] +Here are 10 key moments from the livestream along with their timestamps: -**Adriana Vila**: Hey everyone, welcome to the latest edition of O Tell Me. My name is Adriana Vila, and I am one of the maintainers of the OpenTelemetry End User SIG. I'm super excited to have folks joining us here! Please feel free to share where you're joining from in the chat. I'm joining from Toronto, Canada. +00:00:00 Introductions and welcome message +00:02:30 Introduction of co-host Victoria +00:04:00 Introduction of guest Mole Aigbe +00:06:00 Overview of the Outreachy program +00:10:30 Mole's initial thoughts on open source before Outreachy +00:15:00 Discussion on the application process for Outreachy +00:20:00 Mole shares his first contribution experience +00:30:00 Mole talks about building an exporter for OpenTelemetry +00:40:00 Insights on learning Go programming language +00:50:00 Discussion about the OpenTelemetry community and its support -I have some very special guests with me today. First of all, I have Victoria, who is co-hosting with me. Victoria, why don't you introduce yourself? +Feel free to ask if you need any more details! -**Victoria**: Hi everyone! I'm Victoria, a user experience designer currently interning with the Prometheus project as an LFM. My project has got me interfacing with a lot of OpenTelemetry members, which is how I learned about this program. I'm looking forward to interviewing and learning all about Mole's experience. +# O Tellme Episode Transcript -**Adriana**: Awesome! And now it's time to introduce our guest, Mole Aigbe. +**Adriana:** +Hey everyone, welcome to the latest edition of O Tellme. My name is Adriana Vila, and I am one of the maintainers of the OpenTelemetry End User SIG. I'm super excited to have folks joining us here, and please feel free in the chat to just say where you're joining from. I'm joining from Toronto, Canada. -**Mole Aigbe**: Hello everyone! It's so nice to be here. Thank you so much, Adriana. My name is Mole Aigbe. I am a developer advocate at Sematext and also an OpenTelemetry member. I was a former Outreachy intern, and that’s where I got introduced to OpenTelemetry. I'm so excited to share my experience with OpenTelemetry here today. Thank you for having me! +I have some very special folks here joining today. First of all, I have Victoria, who is co-hosting with me. So, Victoria, why don't you introduce yourself? -**Adriana**: Amazing! So, Mole, you mentioned Outreachy in your introduction. Why don't you tell folks a little bit about the Outreachy program? +**Victoria:** +Hi everyone! I'm Victoria. I'm a user experience designer, currently interning with the Prometheus project as an LFM. My project has got me interfacing with a lot of OpenTelemetry members, which is how I learned about this program. I'm looking forward to interviewing and learning all about this experience. -**Mole**: Sure! The Outreachy program is an initiative aimed at helping individuals from underrepresented and marginalized communities get into open source. They encourage these individuals to take their first steps into open source and provide incentives to assist them in doing so. Outreachy isn't just for open source; it's also for open science, covering diverse projects beyond just tech. Personally, my passion lies in tech, so that’s how I got to know about OpenTelemetry and the amazing community it has. +**Adriana:** +Awesome. And now it's time to introduce our guest, Mole. -**Adriana**: Did you have any awareness of open source before you applied for Outreachy, or was Outreachy your introduction to it? +**Mole:** +Hello everyone, it's so nice to be here. Thank you so much, Adriana. My name is Mole Aigbe. I am a developer advocate at Sematext and also an OpenTelemetry member. I was a former Outreachy intern, and that's where I got introduced to OpenTelemetry. I'm so excited to share my experience with OpenTelemetry here today. Thank you for having me. -**Mole**: I always knew that open source existed, but I never thought someone like me could contribute. I believed that only people from big companies like Microsoft or Google could do it. I would watch YouTube videos on how to contribute and get scared, so I just ran away. However, Outreachy provided a platform with mentors who guided us through the step-by-step process of making our first contributions. Once I made that first contribution, I realized it wasn’t as hard as I thought; it was all in my head! +**Adriana:** +Amazing! Yeah, I'm super excited. You mentioned Outreachy in your intro. Why don't you tell folks a little bit about the Outreachy program? -**Adriana**: That's amazing! Could you tell us more about the application process? What was it like? +**Mole:** +Sure! The Outreachy program is an initiative aimed at helping people from underrepresented communities—marginalized and overlooked communities—get into open source. They encourage them to take their first steps into open source and provide incentives to assist them in doing so. It's not just for open source; it's also for open science. They have projects covering biological topics, geomaps, and various diverse areas, not just open source projects. However, my love is in tech, and I love open source. That's where I got to know about OpenTelemetry and the amazing community. -**Mole**: I almost didn't apply because I thought, "This is open source; it's for the cool kids." But a good friend encouraged me to just apply, saying, "What's the worst that could happen?" So I decided to take that step. The application process involves writing five essays explaining why you believe you should be awarded the internship. Outreachy is not looking for the most skilled individuals; they focus on supporting people from marginalized backgrounds. The application is read by real people who are looking for those they can support. +**Adriana:** +Were you aware of open source before you applied for Outreachy, or was Outreachy like your introduction to open source? -Once you write the essays, there are two phases: the essay phase and the work phase. I struggled a lot during the work phase because I was learning Golang at the same time. I had just started learning about DevOps in December and had to learn Golang and contribute simultaneously. Thankfully, with the help of my mentors, I was able to make significant contributions and eventually got my internship! +**Mole:** +I always knew open source existed. I knew there were smart people doing awesome work in that space. But I never believed someone like me could contribute to open source. I thought, "What do I know?" I didn't work at Microsoft or Google; I didn't have that knowledge or experience. I would often use YouTube to figure out how to contribute to open source, but I'd get scared and run away. Outreachy provided a platform with mentors who directed us step-by-step on how to make our first contributions. Once I made my first contribution, I realized it wasn't that hard; it was all in my head. -**Adriana**: That's fantastic! I can't underscore enough the importance of programs like Outreachy because they provide opportunities for individuals who might not otherwise have them. How long was your internship? +**Adriana:** +That’s amazing! Could you tell us more about the application process? What was it like? -**Mole**: The internship was for three months, but the entire application process took about six months. +**Mole:** +Sure! I almost didn't apply because of the same mentality I had—that open source was for the "cool kids." But my good friend encouraged me to apply, saying, "What's the worst that can happen? They tell you no, and you move on." So, I decided to take that step. The application process involves writing five essays about why you believe you should be awarded the internship. They're not looking for highly skilled people; they want to support those from minority backgrounds. -**Adriana**: How did you get paired up with a mentor during the internship? +Once you write the essays, there's a two-phase process. The first phase is the essay phase, followed by a phase where you have to do the work. I remember wanting to give up many times because I didn't know Go, which is the language most cloud-native applications are written in. I started learning Go just two months before the internship while trying to learn about OpenTelemetry. It was hectic, but with the help of mentors and resources, I managed to get my contributions in. -**Mole**: When you pick a project, each project has mentors who volunteer to guide participants. For OpenTelemetry, my mentors were Jurassi and Yuri, who were incredibly supportive throughout my journey. +**Adriana:** +That's really interesting. I can't underscore enough the importance of programs like Outreachy. They provide opportunities to people who might not otherwise have them. How long was your internship? -**Adriana**: OpenTelemetry is a massive project. What specific areas did you contribute to? +**Mole:** +The internship lasted for three months. However, if you include the entire application process, it was about six months in total. -**Mole**: Initially, I found it overwhelming to choose a repo to contribute to. I explored various repositories and eventually found solace in the collector contrib. During Outreachy, I worked on building a logging bridge in Golang and successfully built the Zero Log bridge for Golang by the end of my internship. +**Adriana:** +How do you get paired up with a mentor? -**Adriana**: That's incredible! Can you share how your internship helped you secure your first job? +**Mole:** +When you pick a project, each project has mentors who volunteer to guide participants. In my case, Jurassi and Yuri volunteered to be mentors for the OpenTelemetry project, so they became my mentors. -**Mole**: After doing a lot of work, I started writing articles for a collaborative observability platform called Signals, which supports OpenTelemetry natively. My former boss, the CEO of Sematext, saw some of my articles and reached out to me. We had a conversation about my experience with OpenTelemetry, and I ended up working on building an exporter for their platform to receive data from OpenTelemetry directly. That's how I got my first job! +**Adriana:** +What specific areas did you contribute to in OpenTelemetry? -**Adriana**: That's a great story and really highlights how putting yourself out there can lead to amazing opportunities. What resources did you use to learn OpenTelemetry? +**Mole:** +Before the internship, I explored various repos on GitHub, but I found solace in the Collector contrib. The project I worked on during Outreachy was building a logging bridge in Go. By the end of the internship, I successfully built the Zero log bridge for Go. -**Mole**: I started by Googling OpenTelemetry and looking at the documentation, but I found it overwhelming. Since I’m a visual learner, I turned to YouTube and found Henrik's channel. His videos provided a lot of context. I also read Adriana’s blogs, which helped me navigate the documentation better. +**Adriana:** +Can you share how the Outreachy internship helped you secure your first job? -**Adriana**: That's great to hear! What was your experience like creating your first PR? +**Mole:** +I had done a lot of work and was writing articles for Sematext. The CEO, Otus, saw some of my articles and reached out to me about integrating their platform with OpenTelemetry. I showed him what I did, and he offered me a position to build an exporter for their platform using the OpenTelemetry Collector. That’s how I got my first job. -**Mole**: Creating my first PR was a journey! I spent about two weeks learning and studying other people's PRs. I read through closed issues and the corresponding PRs to understand how to contribute. Eventually, I found an issue I could tackle and reached out to the person to ask for assignment. I made mistakes, but the community was supportive. After two days, my PR got merged, and it felt like a huge milestone! +**Adriana:** +That’s amazing! It really underscores how Outreachy gave you the tools to contribute to OpenTelemetry, but you also put yourself out there by writing blog posts. What were your resources for learning OpenTelemetry? -**Adriana**: That's such a glorious feeling! You mentioned working on the OpenTelemetry exporter. What was that experience like? +**Mole:** +I initially stumbled upon the OpenTelemetry documentation, but it felt overwhelming. As a visual learner, I turned to YouTube and found Henrik's channel, which provided a lot of context. I also read Adriana's blogs, which were immensely helpful. After getting a foundational understanding, I could go back to the documentation, knowing what I was looking for. -**Mole**: Building the exporter was challenging because there was little documentation on how to create one. I had to study existing exporters and adapt them to my needs. I utilized the OpenTelemetry Collector Builder and found resources on YouTube, which were incredibly helpful. After much trial and error, I managed to get my first metrics and logs into Sematext! +**Adriana:** +What specific resources were most useful for you? -**Adriana**: It sounds like a significant learning experience! How did you learn Golang for this project? +**Mole:** +For videos, I would definitely go with Henrik’s channel. For text, I would recommend Adriana's blog on Medium. -**Mole**: I learned Golang in a very hands-on way, as I had to apply it to real-life projects. My teammates at Sematext were instrumental in guiding me, helping me understand what works in production. I also took a course that provided me with a solid context for using Golang effectively. +**Adriana:** +What was your experience like creating your first PR? -**Adriana**: Have you thought about documenting your journey or the process of creating the exporter? +**Mole:** +Creating my first PR was a whole experience. I spent about two weeks learning and then realized I needed to make my first PR. I studied other people's PRs and found an issue that I could tackle. With the guidance of others, I was able to submit my first PR, and when it got merged, I was ecstatic! -**Mole**: I considered writing an article, but I realized that showing how I built it might not be useful to others because each context is different. Instead, I would encourage people not to give up and to study others' code as it has significantly helped me in my journey. +**Adriana:** +Can you tell us about your experience creating the OpenTelemetry exporter? -**Adriana**: That's great advice! What feedback do you have for improving the OpenTelemetry documentation? +**Mole:** +When I started working on the exporter, I found that there was no documentation on how to create one. I had to study existing exporters and use the OpenTelemetry Collector Builder. It was a lot of trial and error, but eventually, after much effort, I succeeded in building the exporter. -**Mole**: I believe there should be more video documentation and clearer guidelines for building exporters. While the documentation is comprehensive, it can be overwhelming for beginners, so breaking it down or providing more visual aids would be beneficial. +**Adriana:** +What was your experience learning Go for this project? -**Adriana**: Absolutely! And what has your experience been like within the OpenTelemetry community? +**Mole:** +Learning Go was challenging because I had to apply it directly to real-life projects. I made many mistakes, but my team at Sematext was supportive. I took a course that provided context, which helped me understand the language better. It was a tough but rewarding experience. -**Mole**: The community is incredibly supportive and welcoming. My first interaction was a bit intimidating, but everyone was friendly and encouraging. I connected with my mentors and other community members, which made the experience much more enjoyable. I even attended KubeCon and met many people in person, which further solidified my appreciation for the community. +**Adriana:** +What plans do you have for documenting the exporter for future users? -**Adriana**: That's wonderful to hear! Have you made any recent contributions to the community? +**Mole:** +While there is a template for exporter documentation, I think it would be more beneficial to write about the thought process rather than a step-by-step guide, as each context is different. Additionally, I believe sharing experiences and practical advice would be more helpful for new learners. -**Mole**: Yes! I've been advocating for OpenTelemetry in Nigeria, attending conferences, and helping others get started with OpenTelemetry and open source. I share my experiences and practical steps I took to help others in their journeys. +**Adriana:** +What feedback do you have for the OpenTelemetry documentation? -**Adriana**: That's amazing! It highlights that contributing to OpenTelemetry isn't just about code; it's also about advocacy and awareness. As we wrap up, does anyone from the audience have questions? +**Mole:** +I think creating video documentation would be beneficial for visual learners. The documentation can be overwhelming, so having video resources would help bridge the gap for different kinds of learners. -[Music] +**Adriana:** +What was your first impression of the OpenTelemetry community? -**Adriana**: It seems we don’t have any questions. Before we go, I want to remind everyone about the Open Source Summit in Denver, Colorado, and the Open Observability Con. Also, if you're interested in joining the OpenTelemetry End User SIG, feel free to check out our GitHub repo and get involved. +**Mole:** +I found the community to be very welcoming and supportive. I felt encouraged from the start, especially from my mentors. The OpenTelemetry community is caring, and I love being a part of it. Meeting everyone in person at KubeCon was a fantastic experience. -Thank you so much, Mole, for joining us today, and thank you, Victoria, for co-hosting with me. This has been a lot of fun! +**Adriana:** +Have you made any recent contributions to the OpenTelemetry community? -**Mole**: Thank you for having me! +**Mole:** +Yes! I’ve been advocating for OpenTelemetry in Nigeria. I realized it wasn't a widely discussed topic here, so I started attending conferences to share knowledge about OpenTelemetry and how to get started with open source. I'm also still working on pushing the exporter into the OpenTelemetry Collector contrib. -**Victoria**: Thanks, everyone! +--- -[Music] +**Adriana:** +Thank you so much, Mole, for sharing your experiences. And thank you, Victoria, for co-hosting with me today. We will see everyone at the next event! ## Raw YouTube Transcript -[Music] Hey everyone, welcome to the latest edition of O tellme. My name is Adriana Vila. I am one of the maintainers of the open telemetry enduser SIG. Um, super excited to have folks joining us here and please feel free in the chat to just say where you're joining from. I'm joining from Toronto, Canada. Um, I have some very special folks here joining today. First of all, um, I have Victoria who is co-hosting with me. So, Victoria, why don't you introduce yourself? Hi everyone. U, I'm Victoria. I'm a user experience designer. I'm currently interning with the Prometheus project as an LFM. Uh so from my project has got me interfacing with a lot of hotel members which is how I learn about this program. Uh and I'm looking forward to interviewing and learning all about his experience. Awesome. And now it's time to introduce our guest for mele. Hello everyone. It's so nice to be here. Thank you so much Adriana. Um my name is Mole Aigbe. I am a developer advocate at step security. I'm also an open telemetry member. I was a former algi intern and that's where I got introduced to open telemetry and I'm so excited to share my experience with hotel here today. Thank you for having me. Amazing. Yeah, super excited. So um well you know you mentioned um outreachy in your intro why don't you tell folks a little bit about the outreachy program. Okay. So, um the outreach program is is an initiative um that is that their aim is to help people that underrepresented communities, you know, marginalized, you know, look down on communities, you know, get into open source. They encourage them to get into open source to put their first leg into open source and then they also um give some kind of, you know, incentive, you know, to you know, assist them, right? to get their first leg in into open source you know the open source community at large. So that is what is that is what alry is and it's not just for open source you know it's for both open source and open science. So they have projects about um that covers biological things um projects that covers um geom maps you know different diverse projects not just open source projects but my my love since I'm a tech guy uh I love open source and that's where I got to know about open source and the amazing open telemetry community. Yeah. So did you were you aware of like open source before you you applied for outreachy or or like or or was outreachy like your awareness into open source then? Okay. So, yeah. So, I I always knew that open source existed, right? I always knew that, okay, there's something out there in the world called open source that some cool dudes that very smart, very intelligent are, you know, doing awesome work in, right? So, but I just never believed that someone like me could contribute to open source because I was like, bro, what do you know? You you do not work in Microsoft. You don't work in Google. You don't have that knowledge, man. You don't have experience. So just chill when you're you know when you are skilled enough maybe you can start you know playing in the big leagues right so open source was something I I remember you know several times I would use YouTube to how to contribute to open source and then I'll try to watch some videos I'll get scared and then I'll run away right so but ali you know provided that platform you know so we had mentors that actually directed and showed us you know the step-by-step process on how to make your first contribution and then once you make your first contribution realize that wait a second man it's not that it's not that hard it's it's not that hard it's all in your head right and then once you make that first contribution you know you keep I just kept making and making and then it just became so cool and cool so yeah so that's how I got to learn about I knew about open source but the altitude gave me the platform to take that first step into open source amazing Yeah, really interesting. Um, could you tell us more about the application process? What was it like? Okay. Uh, so first of all, I almost did not apply for uh I almost I almost did not apply for because I also had that whole mentality that okay man this is open source. This is for the cool kids. I'm not a cool kid, right? So I was like you know but my friend u my very good friend she was like no you can do it. you can just just apply. What's the what's what's the worst that can happen? They tell you no, you go back, you cry, and you're fine. So, I decided to take that step. I applied. So, what the application process is like is that you have to write like um five you have to write like five essays on um why you believe that you should be awarded um the the internship because like I said, the internship is not about they're not looking for very skilled people. They're looking to actually support, you know, people that are that come from, you know, the minorities of the society, like people that you would hear about on a normal on a regular day, right? Like people that um that are looked down on, you know, that people don't even like the big the big leagues don't even look at in the society. So that's their goal, right? So those are the people they are trying to support. So you can be quote unquote very smart or high and mighty and alone accept you. But you know if you feel like man who will hear my voice no one cares about me I'll say go for alry like they care like the people actually sit down and read people's essays to see and to look for those kind of people that they know that they can support. So I know people have heard a lot of things about job applications that they use one AI tracker to just wipe people applications but I can tell you from experience that out is not like that. real people actually read your essays and you can tell your story and through your story you know you might get a chance to you know walk into and begin your own open source journey. So you have to write five essays. Once you write the five essays you get into the first phase and then there's a actually two stages. The first phase is the essay phase where you have to you know prove that you need you know the internship and the second phase is where you actually have to now do the work and um in doing that work I'll say there a million times I wanted to give up because I did not know Golang so that's the language that you know most cloud native applications are written Golang I December like December 2023 that's two months before I was learning about um DevOps. So I just in that February I started learning Golang. So I was I was trying to learn open telemetry. I was trying to learn Golang and I was trying to contribute all in one month. So it was a whole lot. It was a whole lot. But you know I always I always say shout out to um Henrik and Adriana. their content really, you know, ramped me up, you know, to speed on how to on first understand the project and then make some reasonable contributions. So then I got the I was able to get like 70 hours merged and then I also spoke at and then my mentors were like, "You're so cool. Come on board." My mentors, by the way, Jurassic and Yuri are the coolest guys out there. Shout out to them. I love them so much with all my heart and I'm always grateful to them for the forever they will be part of my tech journey. I always mention their names everywhere I go, you know, as the ones that, you know, helped me like a baby and, you know, trained me up to be this man that I am now. So, yeah. So, that's like a summary of the application process. Yeah, that's so great. And you know, I can't underscore enough the importance of programs like Outreachy because they give opportunities to people who otherwise wouldn't necessarily have gotten the opportunity and and especially like nowadays I'm going to get on my little soap box and and talk about the fact that like so many DEI programs are getting cut and there's so much I don't know there's like global hate towards DEI programs and like so it's so nice to have a reminder that um programs like Outreach exist. They benefit folks. Um they elevate folks that who are awesome who we wouldn't necessarily have known about. And and so like I I love that. I love that you got so much out of the program. Um I love that the program exists. Um I wanted to ask you um so how long how long was your how long was your internship? Okay, so the internship was for three months. Um but if I want to combine the whole application process everything together was like six months right but officially the internship is just for three months you know so that's how long the internship was awesome and uh you talked a little bit about um about your mentors how do you get how do you get paired up with a mentor okay so um when you pick a project every project has you know mentors for the project right so when I picked the project open telemetry it was Jurassi and um Yuri that were they actually volunteer so it's not so they actually volunteered to be mentors for a particular project right and so they volunteered to be mentors of the open telemetry project at that time and then that's how they became my mentors cool Okay. So I would like to ask I know I know that hotel is a very big project. So what specific areas of hotel what specific areas did you contribute to doing? Okay. Um so the I'll go before the actual internship I I remember scrolling through the different repos on GitHub and I was like oh my god where do I even start from? And I I'll say I found I tried some things in the golang sig the go sig um the go the golang instrumentation but my go was not very very strong so I think no no PR got merged there but then I now found a lot of solace in the collector contrip um sig um the collector contrib right and that's why I made um some contributions to to get into the outri and in outrichy the project that I worked on was building a login bridge in Golang and by the end of the internship I was able to build the zero log bridge um for Golang. Yeah. So that's those are the projects that I'm working on and then currently I still contribute to the collector contributions. Wow. But like it's funny to see how you went from not knowing about school to completing your internship to to building something that the community is making. That's that's huge. So So I'm um uh Yeah. Could you could you share with us how the African internship helped you secure your first job? Okay. Uh so how Okay. What happened was I had done a lot of work and then I was writing articles for signals. Um signals very cool collab observability platform also um support open telemetry natively. So I was writing articles for signals and I think my former boss um Otus the CEO of sematex saw some of the articles that I wrote and you know had a conversation with me that oh hey man you worked on open telemetry right? I'm like yeah I did. What did you do? I showed him what I did. We got on a call um I did some demos on how auto instrumentation works. He was like wow this is so cool. I was like yeah it's so cool. Oh, open telemetry is so cool. And he was like, okay, that they want to um, you know, integrate their platform with open telemetry and and am I up for it? I'm like, okay, sure, I'm up for it. Let's go. You know, and uh, so in seatex what I was working on was building an exporter for their platform to so that they can receive, you know, data from open hotel data into their platform directly using the collector open telemetry collector. So that so that was the project I worked on while at seat text and it was it was very stressful but it was very cool. It was very cool in the end. Yeah. So yeah. So that's that's how I got my first job. The contributions I did and the some demos I did about hotel. That's what I got my first job. Yeah. That's amazing. And I think you know it really underscores like two things. First of all, like outreachy kind of gave you the tools, right, to um to contribute to open source, contribute to open telemetry, but then you did another awesome thing, which is putting yourself out there, writing these blog posts, um which is so so important. And it's so hard sometimes to like put ourselves out there. Um and you know and we I think we all in tech suffer um from imposttor syndrome at some point or another maybe constantly feel it comes in waves for me. Um and you just have to push past that imposttor syndrome and do it anyway. Um so mega kudos to what you're doing. Now you've made a few references about um you know learning learning open telemetry learning go what did you use for um can you dig a little bit deeper on on on how you learned um open tele okay yeah so I'll start from my first introduction to open telemetry I just Google open telemetry enter and I saw the documentation I was like oh this looks so cool and then I tried to read the docs and my brain wanted to blow So I went to so I'm more of a visual learner, right? So I went to YouTube and I typed what is open telemetry and um Henrik um he is the the owner or you're the owner of the channel and is it observable right and I was just watching a lot of his videos at that time and I would say those videos gave me a lot of context right because how I learned is I first I I like specific information before general information so I use his videos a lot to ramp up on my knowledge on hotel And then our wonderful uh co-host here, Hadriana. Her blogs, her blogs are so awesome. I I read a lot of our blogs, you know, at that time just to ramp up because the documentation was very overwhelming for me. So after reading those two things, I then began to go to the docs and actually, you know, narrow down on what I was looking for. So I really knew what I was looking for because documentation sometimes can be very overwhelming because there's so much information packed in the documentation and it's trying to you know talk to different classes of people both beginners, intermediate and advanced people. So beginners sometimes always feel overwhelmed right from those documentation. But I'll say um when it came to running the open telemetry demo the docs did a good job. Yeah did a good pretty good job on how to run the open telemetry demo which was pretty cool. But yeah, so those were the resources I I used. I bought more YouTube videos, but these are the resources that really, you know, accelerated my learning journey to make that first contribution to telemetry. Yeah. Wow, that's great. Personally, I'm more I'm more of a reader like I prefer to consume text based resources than watching. Cool. I'm with you on that. So that's interesting interesting to know. Good for you. So of all these resources that you mentioned, the YouTube videos, which of them would you say was the most um useful for you to learn about? So you you want me to pick one? [Laughter] You want me one? one or two if it's too hard to choose. If you tell me to pick one, I'm I'm very biased. I always go for videos. So, you tell me to pick one, I'll go for Henrik's channel. Hands up. I mean, Henrik produces amazing content, so I I can definitely vouch for that. Yep. Yeah. So, for Okay, so I won't pick one. I'll divide them into sections. So, for video, I'll say Henrik's channel. Is it observable? If you haven't, go and check it out. It's awesome. And then for text, Adriana's blog on Medium. I don't Adriana Adriana's blog. I don't know if she's hearing me right now. Adriana's blog. Yeah. So, yeah, you should also check that out, right? Yeah. So, those are the two most useful resources. Uh, yeah, that's cool. Yeah. Okay. Um, now you said you you have a preference for the videos, right? Is there a reason why? Okay. So, is there something particular about those? Yeah. Okay. So, uh, so how I how I reason is I'm a very imaginative thinker. Like I I can sit down and just go on and just be imagining a lot of things. So, I like things I like pictures, visual things because if I'm just reading a bunch of text of of a concept I haven't understood yet, I get bored like I will just easily get bored and do off. So, if I'm watching a video, you know, the sound of the person, the tone, the the infographics, the pictures, you know, just keeps me excited to keep going and keep going and keep going and keep going and before I know it, I've tricked my brain into learning a new concept, right? So, yeah. So that's just what that's just what works for me. I've observed it over time. Yeah, that's it. That's awesome. Yeah, makes sense. Um the the next question that we were wondering about, you know, you mentioned that you use the uh the hotel ducks as a as a reference and you know, one of the one of the goals of these sessions um that we have like with hotel me is also to provide feedback um to the hotel community on on areas for improvement. Um, is there anything like why is it that the you found um third-party um documentation more useful than the official hotel docs? Um, and what made you go back to them afterwards? And and what would you So this like multi-art question, sorry. And what what would you what would you do to improve the docs experience if it if it were up to you as as like a new learner? Okay. Uh so sorry this is an issue with multiple questions. The first question is I hope I remember them. The first question is um what made me uh feedback for the docs or maybe leave the docs or Yeah. Why why did you choose why did you choose uh third party documentation uh over over the hotel docs for starters? Okay. So, uh well because number one, there's no video documentation yet. That's why fair. Yeah. Fair. Yeah. But um also when I'm reading someone's content, right? Like I'm learning your content or someone else out there. I am not just reading like I can feel I'm feeling I feel human connection like I can like you can say something like ah I got stuck at this. this is what I had to do and then this got you know solved but in a documentation it's just man it's just it's like reading a textbook you know there you know do this do this do this do that move on you know but you know when you're writing your content you can say oh do this if this does not work try this you try this out if this does not work try this out so I'll say that's why I preferred you know to go for like I said the when I enter the document doumentation I I got overwhelmed cuz I was scrolling through a lot of things I was scrolling through then I didn't understand what instrumentation was and then documentation has so many links that link you to so many things and before you know you're lost in the rabbit hole of the documentation so I'll be reading something I'm like instrumentation I'll see a link I'll click the link I'm going to you know a different entire part and then before you know I was like where am I what am I learning so I got overwhelmed right so so how would I if I had that opportunity How would I okay what what made me go back to the docs? What made me go back to the docs is I already had like I already built a foundation. So I was not overwhelmed when I saw what is instrumentation or when I saw what is a collector. So imagine someone that is new that does not know anything about a collector trying to learn open telemetry is and then he's seen open telemetry what is open telemetry you can't really define open telemetry without the four components instrumentation collector and he's like what what is open telemetry operator? Oh my gosh, he's seen Kubernetes like where am I, you know? So, um I already got that foundational knowledge. So, I could now go and head for the docs, right? And because now I already knew what I was looking for in the documentation. So, let's say I want to do I want to do a particular task. I can just search for that particular part in the docs and focus on just that side. And um so what would I do to improve the doc experience? I'll say I'll make a video documentation for you know we for for those who uh do follow the uh hotel content we do have our first video like instructional video on the uh hotel YouTube channel. So um and and there's there will be more to come. So that's great feedback. Yes. So you can also um you know because there are two different kinds of learners. they are visual learners and they are, you know, text based learners. So, they're satisfying both at the same time. So, that's what I was saying. Yep. Yep. That is a super fair point and I I I like that. Um it it's interesting too like what you were saying um that basically use the um the hotel docs for like more of a a directed directed search, right? like first understand stuff um from people's experiences and then okay I want to learn about this I will go into the docs to learn about that that's kind of that that's really cool that's the approach that's worked for me um to learn everything and that's what I'll keep using by the way Adrian I started learning Kubernetes just a side side note oh right on good luck good luck I'll journey It's a journey. I think you'll find it fun though. You're a problem solver. Thank you. Yeah. Yeah. Um, could you tell us what your experience was like creating your first PR? Ah, okay. We're here. So, creating my first PR, uh, I'll say it was a whole experience, right? I so after watching some videos, reading some documentation I think I did this for for two weeks I was just learning a whole lot and then I was like you know what you need to make your first PR like you can't waste any more time and then so I went to the the um the repos and I was like okay so which repo do I pick I think I think that that that was the first issue I had you know so open telemetry has a bunch of repos so I was like so which one do I go for so I think some for a while I was just lost in do I contribute to NodeJS do I know node I don't know node let me go to collector do I know I'm like okay then so what I started doing um was I started reading other people's PRs so I would literally go to an issue that is closed go under the PR you know because when an issue is closed the link to the PR that you know close the issue will always be there I'll literally go and be studying their code so like that's That's why I did a lot I don't think I've said share this anywhere before but that's why I did a lot I would go and literally be studying people's PR studying their code studying how they you know framed things okay this is how you write this in go because remember I was still learning go at this point so it's so so it was like a it was like a I was like um learning so much in one month because I was studying people's code I was doing a course at the same time right so and then after studying people's peer I was over a while I now gained some confidence I now I found an issue that someone had done something similar you know that's the beautiful thing about open source sometimes you want to update you know a bunch of components at the same time and I saw a similar issue to one of the peers I had studied and I was like you know I can tackle this and I asked you know I asked hello my name is this can I please be assigned to this issue I didn't think the person will answer me and he was like oh yeah sure why not go ahead I think I think his username is MX-Psi I don't know his real MX-PS psi sorry and he was like why not go ahead and you know I tried it out I made a m I made mistakes obviously because I was new to go I didn't understand the whole lint check thing but he then he you know like I said open telemetry community is so wonderful um they do not look down on you they don't look down on where you are as long as you're honest right don't come and prove to be a senior developer when you actually not like come clean I told the guy man I haven't done this before please help my life. I went to his DMs and he was like, "Okay, do this, do this, do this." And you know, at the end of, you know, I think after two days, he got merged. I was like, "Yay, my first PR." And then from there it just it was just like a full journey, right? It was a full journey, you know, from there. Really really nice journey. So that was my my summarized experience. I don't want to go into the details of me crying and, you know, want to tear my Yeah, that was that was just a summarized experience. Yeah, I I I have to agree with you that like that first PR getting merged is like the most glorious experience ever. But I will tell you like every time I have a PR merged, I do like a little happy dance at home. It's like, oh my god, I'm up for something. By the way, I I I gotta say like hats off to you um in your approach to to working on issues too, like studying the the PRs and the code behind the PRs. you you have the heart and soul of a software engineer. It's like you were made for this. That's amazing. And I I think this is a good lead in um also to our our uh next um area that we want to talk about which is um I believe you mentioned earlier that you uh did some work on creating an hotel exporter uh for the collector. So um do talk about that. What was that experience like? Okay, I have a lot of funny experiences. Sorry. Um, so when I got the project um with my former company, Simex, uh I was like, okay, let's do this, right? How hard can it be? So I went to the documentation and there was no documentation on how to create an exporter, right? uh the only documentation was was how to create a receiver and I was trying to understand it and it wasn't working because one one thing I now realized later that I didn't know then was that an exporter there's no really after there's no standard way of creating an exporter well there there's a template you have to follow but there's no standardized way because you know exporters depend on your vendor back end so in semantex um the we're receiving logs over influx live protocol call and then we were sorry receiving metrics over influx 5 protocol and then we were in we're receiving um logs over um what's that thing called what's that thing called elastic yeah elastic bulk index format right so I had to build an exporter that would convert hotel data to these two sources so what I what I did like I did for my other PRs let me just summarize you know like I said I would avoid all the days of shouting and panicking and running around and I'll go straight to the point. I started studying other other people's exporter. So I studied influx DB exporter because that way our metrics you know that's what our metrics exp the metric side of exporter would look like. Then I studied elastic elastic search exporter like I did a lot of study. I tried to break down a lot of things with my the help of my assistant cha [Laughter] great assistant. Yeah. So I I I broke I try to break it down. I try to um fit things in our own context. So how do I take this part from here, put it here, remove this part, you know? So it was a whole trial and error. I did a lot of trial and error. Um there was really no content that I could that I could look to that okay this is what assisted me in this journey. Oh sorry pardon me on that. I remember the open telemetry collector builder. So while learning while learning how to build um the exporter I had to use the OCB open telemetry collector builder and that's when I came across bind planes content on YouTube also really we had they have a full series on open telemetry collector very wonderful series I enjoyed every build so that's why I learned how to use the open telemetry collector builder so yeah so that really helped and the OCB is very fun it's so cool man it's like you can build your own custom collector you know with whatever component you want like you It's so it's cool. It's like play-doh, right? You can build as you want, play with it as you want, connect things, join things, break things, and you know, it's fine. So I So I learned how to use the OCB. And then in time, after lots of trial and error, I got my first metric in to SEAR text and it siting exciting feeling, right? And then after a while again got the logs in and then awesome. Then I I built the exporter. So that was that was the that was the summarized version of the experience of building the exporter. Yeah, it's really interesting. Uh believe me, I'm sharing in your excitement right now. I can I can only imagine. Um so for our next question could you tell us a bit about learning go to be able to work on this hotel exporter that you just talked about. So learning go has been a very interesting journey because it has I I did not learn go the normal way right the normal way just watching a course and just building some you know interesting projects you know maybe to-do app stuff. I learned go the hard way because I was actually actually had to like I was learning it and I had to apply it directly into real life projects that go into production, right? So I I remember I I had a help I had a lot of help from the people I worked with at Semantex. Bora Bora Akshhat really cool guys part of my team. So they were so so I'll make mistakes. They were like no they'll be like no error this is not the right way to do this. I know this is the YouTube way, this is the um course way, but in production this would break, this would fail. So I say I learned go handon you know contributing to things right. So it was a very interesting journey. I actually had I got a course from code cloud gave me a lot of like I said I like contextual things. So I had a full context of what go looks like. So I could understand code if you give me any Golan code I can know okay this is what this is what this is struck this is an interface this is what this does this what should do but then I had a lot of you know building things in real life and I feel like that's really the best way to learn you know although it's very stressful uh you would feel you feel very dumb sometimes because some things will I remember sometimes at night I'll literally be up by 2:00 a.m. time and detest keep failing and I'm like please just run please just run I'll literally be praying I'm praying over and over and over and over again and then finally run and I'll be like yay let's go another milestone achieved so so that I'll say that that was my whole experience learning golang and you know becoming that golang engineer yeah cool um throughout Throughout our conversation, you've talked a lot about the resources that helped you to learn open telemetry, that helped you to learn Google and helped you to build the auto exporter. Now, could you tell me what uh what plan you had for documenting this this exporter that you that you when you said so that other people Okay. So, what do you mean document it? Do you mean like you want me to document my journey or how I built a portment? Can you clarify documentation for Sorry, I I I didn't get you for I lost you for like two seconds. Sorry. Yeah, I said documentation for the exporter itself for those who who would be using it. Yeah. Okay. So, I think um when when you're contributing an exporter, there's actually a template of documentation that you actually have to, you know, just do. Well, I'll I'll answer your question from the standpoint because I think Adriana and I spoke about this um I think last time and uh so even if I document and she actually put an idea that I should write an article on how it is not really possible to show you how to build an exporter because I tried actually sat down after building exporter I was like okay how can I help the community with this and I'm like even if I show you how I did it it's not going to be useful to you because your context might be different, right? It might be different. So I what I would say is maybe I the the article I can write on or maybe the documentation will be don't give up. Try try try again. Even when it fails, keep trying. It's going to work. And then another thing I'll say is study study other code. Stud study other people's code. Like I'll say that's one thing has helped me in my whole coding journey and has really accelerated my speed is that I I um always studied people's code and understood the patterns in the code. So from understanding those patterns I can now create my own you know find how to remove parts and then join to what I'm looking for. So yeah. So it's more of a a process rather because I I guess from the sounds of it like um I think the other two components um uh receivers and and processors I think there is like some documentation around around creating those because it's more formulaic right whereas exporters are not but then I guess the thing that's in common is like what's the thought process that you need for creating it but not this is the exporter template Yeah, this is step step by step. Now, yeah, I I think that would be such a great blog post to write. Um because I think, you know, again, it all goes back to what you were saying too about like um some of these third party sources are about personal experiences and having that human touch, human connection. And I think that would be so beneficial. Um, it would also be interesting, I'm just giving an idea, um, to put in the docs, um, something along the lines of like why is it that we don't have a template for creating hotel exporters? Because that was like I remember I at some point I uh, I was I was looking into that as well and I'm like, I don't get it. Why Why is there no template for this? Yeah, absolutely. Yeah, there's no official template. I think it's it's when you start and then you know you speak to the the um the maintainers they not tell you oh no no no this should be here this shouldn't be here I'll be like okay let's sh change it things up right so I think yeah I think there should be an official like okay this is the maybe we can't help you to build the full thing but this these are the things you need you need a config file you need this file you need a factory go file you know things around that so yeah yeah I think Uh that's great. Um and then the uh the other thing that you mentioned that um I want to dig into is the hotel collector builder. Um and specifically like what was your um what was your experience around that? Cuz you know I I've played around with the builder. You've played around with the builder. Was it was it as easy as you thought it would be? Um and if not, what would you say is some feedback that you would like to give on what can be done to make it easier? Okay. So, um you know, in my journey, and I'm going to be very very honest here, right? In my journey with um learning to try to get the um understand the exporter. I didn't really get um any good information from the documentation. So, I'll say for for a while I just did not look at the docs anymore. I just went to YouTube and was doing the research. So that's why I found um bind plane right and then they did a whole video on how to build your own custom collector like 45 46 minutes video and that basically showed me okay yeah so this this works this can do this and then I think also the readme files so I think there's even some like some of the like the open entry collector builder it's more documented in the readme file than well at that time than the documentation itself. So I I think I got more value from the um from the readme file from the readme files at that time than the actual documentation. I don't know if it has been updated yet. I'll probably check but yeah. So I I did just uh get a PR merged in the uh in the hotel docs um about like cuz when I was doing it um I'm like okay I built this DRO how the hell do I containerize it? and there was no documentation around that and I had to ask and then I wrote a blog post about it but I'm like it should be in the docs too. So I just had that PR merged I think today or yesterday. So that's cool. That's so cool. Yeah. So that was my experience with the OCD telemetry. Awesome. All right. Uh I think I have just one more question. Could you could you tell us a bit about the hotel community and I what's your Yeah, I I would like to know your first impression of the community. How you able to collaborate with the community from the time you contributed to Yeah. as an agent and beyond that beyond the Yeah. Yeah. Okay. Um so the open telemetry community I love you all so much but uh so my my first my first experience with them was uh I introduced myself hello guys I am an I am so I used to be introverted some sort of way but the people don't really believe because you know I'm I'm very outspoken but I used to be introverted right so I was kind of shy uh because I was like okay I'm a newbie I don't really know much about stuff I don't know anything about open source who listened to me right and I was like hello my name is I'm from my Nigeria nice to meet you all and people said hi back right people people waved back oh nice to meet you welcome to open tele community and I was like oh that's so cool and then um I'll say when I stumbled into a lot of like back to my mentors right Yuri so Jorassi was um Yuri was my major mentor for the for the outreach internship so we had a lot of calls, right? And in those calls, he just keep encouraging me that I know this is tough, but don't worry, you're going to get it. Just keep just keep pushing, you know, keep trying. I said, I believe in you. You know, he's come, I believe in you. I know you can do this. So, you know, he was always just encouraging me. And then even not just my mentors, you know, it was like a collective effort. So, like I said, I've said this before, I'll go to some people's DMs and I'll be like I'll be crying that please help me. I don't understand this p this issue. How do I solve it? You've already assigned it to me. I tried this. It did not work. And they'll be like, "Okay, you know what? Try this. Make this change." And then I'll keep disturbing them and they'll keep replying. And I remember when I wrote my first blog my first blog post on on open telemetry and I had already like read some bunch of Adriana's blog posts and I was like I'm going to send I was like I'm going to send a DM to Adriana because I like to hear her thoughts on my blog post and I was like hello Adriana hi I'm Shington please can you review my blog post for me and she and she replied and she went I reviewed my blog post and I was like wow she doesn't know me from anywhere you know uh we are not in the same continent or in different from different tribes and she's actually willing to you know assist right so I was like oh that's so cool so I would say open community they are very loving I can't really vouch for a lot of open source communities right but I can vouch for the open telemetry community that they really cool and it's a very nice community to join by the way I'm now a member of the open community so you can also text me I'm very cool if you have questions about and she can text me. I'm cool. Like they are also cool, right? So the community is loving, nice, really caring and I love it. But then when I got to CubeCon and I met everyone in person, it was like way cooler. I met Adriana in person. I met Heric. I met Tras and some other really awesome people. It was really nice. So yeah, they're really nice community to join and be part of. Yeah, I'm I'm going to completely agree with everything just said because I just recently joined the community too and everyone has been about Adriana plus one to that. She's very cool. Yep. And and it's been nice getting to interact with her. Uh one more question before we go. Um have have you made any recent I believe as an hotel community member now you have to contribute to hotel community. So have you had any recent um contributions to the community that you like to talk about? Yeah. Okay. So um what I have been doing majorly right now is um so in in the Nigerian space I think when I came back from CubeCon I think when I started learning the whole open telemetry thing I realized that open telemetry was not a topic that was spoken about a lot in my you know my country locally you know Africa because Kubernetes is very popular here everyone knows Kubernetes right so um open telemetry is not really heard a lot so I I started advocating a lot about I think that's how I even slided into my developer advocate role. So currently I'm a developer advocate. I'm not a full software engineer anymore. Um so I started you know attending conferences um talking about open telemetry showing people how to get started with open telemetry how to um get into open source you know that those because I experienced you know I'll say I experienced the the dramas the struggles you know so I I understand when people come to me and tell me I'm so confused about open source how do I start so I understand yeah and I can also give them practical steps I took to you know get started so I would say I've been doing more of that and then I'm actually still the exporter has not it has been built fully but I'm still pushing it into you know because open source takes time you can't just throw things inside there's a whole process so the exporter is still being pushed into um the the open telemetry collector contrib so I'll say that is the in aside um related to code that those are the contributions that I've been making but then I've been doing more of advocacy you know about open telemetry you know in my Yeah, that's that's amazing. And I think that's that makes a a really good point as well. Um because contributing to open telemetry isn't just about the PRs that you're making. It's also about the advocacy um the bringing that awareness. So I think that's that's great. And um I think this wraps up the the the main Q&A and we do have a few uh minutes left. So if anyone from the audience has any questions that they would like to ask at this point um we are happy to take your questions. I feel like we need some music. This is where you take out the guitar for this for this. Dang it. Who the with the music? Is that you? Was that I think that was Henrik, our amazing producer. Well, it doesn't look like we have any questions. So, um, what I will say is we'll I guess we'll wrap up, but before we go, um, a few quick announcements, promotions. Um, first of all, for anyone who is planning on attending OpenSource Summit in Denver, Colorado, there's also going to be as as a coll-located event, there's an open observability con and hotel community day that is going on. Um, the CFPs for hotel community day are closed, but open observability con CFPs are still open. Um, and that's going to be at the end of June. So, um, feel free to submit CFPs to attend if you're in the area if you're planning on attending. Open source summit also is like lots of fun. Um, it's like CubeCon light. It's all the awesome people who attend without like without the flash. So, uh, it's tons of fun. Um the other thing um for anyone who is interested um as I mentioned I'm one of the maintainers of the hotel and user sig. Victoria is a newly joined member of the hotel and user sig. So um if you're interested in joining us seeing what we're up to um Henrik just flashed a link for our GitHub repo. Um we do a bunch of things like these types of events. We have hotel me. We have hotel and practice where if anyone who's watching is interested um if you have like a topic uh an observability hotel topic that you'd like to present on maybe you're testing out a a topic for a conference for a talk um want to flesh it out use us um just DM us uh on the hotel end user SIG Slack um and just you know just let us know if you're interested in in in uh doing a presentation for hotel in practice and we also run uh we also uh join forces with the various hotel sigs to run surveys. So um stay tuned there will be more surveys heading your way. The surveys are a great way to for us to get um feedback from our user community on ways to improve the SIGs. So that information is super important, super valuable um to the community. So thank you to the folks who have responded to the surveys. Thank you to the folks who have engaged in the hotel end user SIG. Um also um for those who aren't able to attend and for those who were able to attend, tell your friends that this video will be posted um on our hotel YouTube channel. It's hotel-official on YouTube. Um you can catch all of our previous hotel and practice and hotel me um sessions. And also we do have our first how-to hotel video on on the hotel YouTube channel. So for for those who are visual learners, this is your opportunity to check that out. Um, thank you so much Edoselle for joining and for Victoria for co-hosting uh with me today. This has been lots of fun. Always enjoy having these conversations and um we will see everyone at the next event. Sure. Thanks, Adriana. Thank you. Thanks, Father. Thanks, Victoria. It was nice meeting you. [Music] +Hey everyone, welcome to the latest edition of O tellme. My name is Adriana Vila. I am one of the maintainers of the open telemetry enduser SIG. Um, super excited to have folks joining us here and please feel free in the chat to just say where you're joining from. I'm joining from Toronto, Canada. Um, I have some very special folks here joining today. First of all, um, I have Victoria who is co-hosting with me. So, Victoria, why don't you introduce yourself? Hi everyone. U, I'm Victoria. I'm a user experience designer. I'm currently interning with the Prometheus project as an LFM. Uh so from my project has got me interfacing with a lot of hotel members which is how I learn about this program. Uh and I'm looking forward to interviewing and learning all about his experience. Awesome. And now it's time to introduce our guest for mele. Hello everyone. It's so nice to be here. Thank you so much Adriana. Um my name is Mole Aigbe. I am a developer advocate at step security. I'm also an open telemetry member. I was a former algi intern and that's where I got introduced to open telemetry and I'm so excited to share my experience with hotel here today. Thank you for having me. Amazing. Yeah, super excited. So um well you know you mentioned um outreachy in your intro why don't you tell folks a little bit about the outreachy program. Okay. So, um the outreach program is is an initiative um that is that their aim is to help people that underrepresented communities, you know, marginalized, you know, look down on communities, you know, get into open source. They encourage them to get into open source to put their first leg into open source and then they also um give some kind of, you know, incentive, you know, to you know, assist them, right? to get their first leg in into open source you know the open source community at large. So that is what is that is what alry is and it's not just for open source you know it's for both open source and open science. So they have projects about um that covers biological things um projects that covers um geom maps you know different diverse projects not just open source projects but my my love since I'm a tech guy uh I love open source and that's where I got to know about open source and the amazing open telemetry community. Yeah. So did you were you aware of like open source before you you applied for outreachy or or like or or was outreachy like your awareness into open source then? Okay. So, yeah. So, I I always knew that open source existed, right? I always knew that, okay, there's something out there in the world called open source that some cool dudes that very smart, very intelligent are, you know, doing awesome work in, right? So, but I just never believed that someone like me could contribute to open source because I was like, bro, what do you know? You you do not work in Microsoft. You don't work in Google. You don't have that knowledge, man. You don't have experience. So just chill when you're you know when you are skilled enough maybe you can start you know playing in the big leagues right so open source was something I I remember you know several times I would use YouTube to how to contribute to open source and then I'll try to watch some videos I'll get scared and then I'll run away right so but ali you know provided that platform you know so we had mentors that actually directed and showed us you know the step-by-step process on how to make your first contribution and then once you make your first contribution realize that wait a second man it's not that it's not that hard it's it's not that hard it's all in your head right and then once you make that first contribution you know you keep I just kept making and making and then it just became so cool and cool so yeah so that's how I got to learn about I knew about open source but the altitude gave me the platform to take that first step into open source amazing Yeah, really interesting. Um, could you tell us more about the application process? What was it like? Okay. Uh, so first of all, I almost did not apply for uh I almost I almost did not apply for because I also had that whole mentality that okay man this is open source. This is for the cool kids. I'm not a cool kid, right? So I was like you know but my friend u my very good friend she was like no you can do it. you can just just apply. What's the what's what's the worst that can happen? They tell you no, you go back, you cry, and you're fine. So, I decided to take that step. I applied. So, what the application process is like is that you have to write like um five you have to write like five essays on um why you believe that you should be awarded um the the internship because like I said, the internship is not about they're not looking for very skilled people. They're looking to actually support, you know, people that are that come from, you know, the minorities of the society, like people that you would hear about on a normal on a regular day, right? Like people that um that are looked down on, you know, that people don't even like the big the big leagues don't even look at in the society. So that's their goal, right? So those are the people they are trying to support. So you can be quote unquote very smart or high and mighty and alone accept you. But you know if you feel like man who will hear my voice no one cares about me I'll say go for alry like they care like the people actually sit down and read people's essays to see and to look for those kind of people that they know that they can support. So I know people have heard a lot of things about job applications that they use one AI tracker to just wipe people applications but I can tell you from experience that out is not like that. real people actually read your essays and you can tell your story and through your story you know you might get a chance to you know walk into and begin your own open source journey. So you have to write five essays. Once you write the five essays you get into the first phase and then there's a actually two stages. The first phase is the essay phase where you have to you know prove that you need you know the internship and the second phase is where you actually have to now do the work and um in doing that work I'll say there a million times I wanted to give up because I did not know Golang so that's the language that you know most cloud native applications are written Golang I December like December 2023 that's two months before I was learning about um DevOps. So I just in that February I started learning Golang. So I was I was trying to learn open telemetry. I was trying to learn Golang and I was trying to contribute all in one month. So it was a whole lot. It was a whole lot. But you know I always I always say shout out to um Henrik and Adriana. their content really, you know, ramped me up, you know, to speed on how to on first understand the project and then make some reasonable contributions. So then I got the I was able to get like 70 hours merged and then I also spoke at and then my mentors were like, "You're so cool. Come on board." My mentors, by the way, Jurassic and Yuri are the coolest guys out there. Shout out to them. I love them so much with all my heart and I'm always grateful to them for the forever they will be part of my tech journey. I always mention their names everywhere I go, you know, as the ones that, you know, helped me like a baby and, you know, trained me up to be this man that I am now. So, yeah. So, that's like a summary of the application process. Yeah, that's so great. And you know, I can't underscore enough the importance of programs like Outreachy because they give opportunities to people who otherwise wouldn't necessarily have gotten the opportunity and and especially like nowadays I'm going to get on my little soap box and and talk about the fact that like so many DEI programs are getting cut and there's so much I don't know there's like global hate towards DEI programs and like so it's so nice to have a reminder that um programs like Outreach exist. They benefit folks. Um they elevate folks that who are awesome who we wouldn't necessarily have known about. And and so like I I love that. I love that you got so much out of the program. Um I love that the program exists. Um I wanted to ask you um so how long how long was your how long was your internship? Okay, so the internship was for three months. Um but if I want to combine the whole application process everything together was like six months right but officially the internship is just for three months you know so that's how long the internship was awesome and uh you talked a little bit about um about your mentors how do you get how do you get paired up with a mentor okay so um when you pick a project every project has you know mentors for the project right so when I picked the project open telemetry it was Jurassi and um Yuri that were they actually volunteer so it's not so they actually volunteered to be mentors for a particular project right and so they volunteered to be mentors of the open telemetry project at that time and then that's how they became my mentors cool Okay. So I would like to ask I know I know that hotel is a very big project. So what specific areas of hotel what specific areas did you contribute to doing? Okay. Um so the I'll go before the actual internship I I remember scrolling through the different repos on GitHub and I was like oh my god where do I even start from? And I I'll say I found I tried some things in the golang sig the go sig um the go the golang instrumentation but my go was not very very strong so I think no no PR got merged there but then I now found a lot of solace in the collector contrip um sig um the collector contrib right and that's why I made um some contributions to to get into the outri and in outrichy the project that I worked on was building a login bridge in Golang and by the end of the internship I was able to build the zero log bridge um for Golang. Yeah. So that's those are the projects that I'm working on and then currently I still contribute to the collector contributions. Wow. But like it's funny to see how you went from not knowing about school to completing your internship to to building something that the community is making. That's that's huge. So So I'm um uh Yeah. Could you could you share with us how the African internship helped you secure your first job? Okay. Uh so how Okay. What happened was I had done a lot of work and then I was writing articles for signals. Um signals very cool collab observability platform also um support open telemetry natively. So I was writing articles for signals and I think my former boss um Otus the CEO of sematex saw some of the articles that I wrote and you know had a conversation with me that oh hey man you worked on open telemetry right? I'm like yeah I did. What did you do? I showed him what I did. We got on a call um I did some demos on how auto instrumentation works. He was like wow this is so cool. I was like yeah it's so cool. Oh, open telemetry is so cool. And he was like, okay, that they want to um, you know, integrate their platform with open telemetry and and am I up for it? I'm like, okay, sure, I'm up for it. Let's go. You know, and uh, so in seatex what I was working on was building an exporter for their platform to so that they can receive, you know, data from open hotel data into their platform directly using the collector open telemetry collector. So that so that was the project I worked on while at seat text and it was it was very stressful but it was very cool. It was very cool in the end. Yeah. So yeah. So that's that's how I got my first job. The contributions I did and the some demos I did about hotel. That's what I got my first job. Yeah. That's amazing. And I think you know it really underscores like two things. First of all, like outreachy kind of gave you the tools, right, to um to contribute to open source, contribute to open telemetry, but then you did another awesome thing, which is putting yourself out there, writing these blog posts, um which is so so important. And it's so hard sometimes to like put ourselves out there. Um and you know and we I think we all in tech suffer um from imposttor syndrome at some point or another maybe constantly feel it comes in waves for me. Um and you just have to push past that imposttor syndrome and do it anyway. Um so mega kudos to what you're doing. Now you've made a few references about um you know learning learning open telemetry learning go what did you use for um can you dig a little bit deeper on on on how you learned um open tele okay yeah so I'll start from my first introduction to open telemetry I just Google open telemetry enter and I saw the documentation I was like oh this looks so cool and then I tried to read the docs and my brain wanted to blow So I went to so I'm more of a visual learner, right? So I went to YouTube and I typed what is open telemetry and um Henrik um he is the the owner or you're the owner of the channel and is it observable right and I was just watching a lot of his videos at that time and I would say those videos gave me a lot of context right because how I learned is I first I I like specific information before general information so I use his videos a lot to ramp up on my knowledge on hotel And then our wonderful uh co-host here, Hadriana. Her blogs, her blogs are so awesome. I I read a lot of our blogs, you know, at that time just to ramp up because the documentation was very overwhelming for me. So after reading those two things, I then began to go to the docs and actually, you know, narrow down on what I was looking for. So I really knew what I was looking for because documentation sometimes can be very overwhelming because there's so much information packed in the documentation and it's trying to you know talk to different classes of people both beginners, intermediate and advanced people. So beginners sometimes always feel overwhelmed right from those documentation. But I'll say um when it came to running the open telemetry demo the docs did a good job. Yeah did a good pretty good job on how to run the open telemetry demo which was pretty cool. But yeah, so those were the resources I I used. I bought more YouTube videos, but these are the resources that really, you know, accelerated my learning journey to make that first contribution to telemetry. Yeah. Wow, that's great. Personally, I'm more I'm more of a reader like I prefer to consume text based resources than watching. Cool. I'm with you on that. So that's interesting interesting to know. Good for you. So of all these resources that you mentioned, the YouTube videos, which of them would you say was the most um useful for you to learn about? So you you want me to pick one? You want me one? one or two if it's too hard to choose. If you tell me to pick one, I'm I'm very biased. I always go for videos. So, you tell me to pick one, I'll go for Henrik's channel. Hands up. I mean, Henrik produces amazing content, so I I can definitely vouch for that. Yep. Yeah. So, for Okay, so I won't pick one. I'll divide them into sections. So, for video, I'll say Henrik's channel. Is it observable? If you haven't, go and check it out. It's awesome. And then for text, Adriana's blog on Medium. I don't Adriana Adriana's blog. I don't know if she's hearing me right now. Adriana's blog. Yeah. So, yeah, you should also check that out, right? Yeah. So, those are the two most useful resources. Uh, yeah, that's cool. Yeah. Okay. Um, now you said you you have a preference for the videos, right? Is there a reason why? Okay. So, is there something particular about those? Yeah. Okay. So, uh, so how I how I reason is I'm a very imaginative thinker. Like I I can sit down and just go on and just be imagining a lot of things. So, I like things I like pictures, visual things because if I'm just reading a bunch of text of of a concept I haven't understood yet, I get bored like I will just easily get bored and do off. So, if I'm watching a video, you know, the sound of the person, the tone, the the infographics, the pictures, you know, just keeps me excited to keep going and keep going and keep going and keep going and before I know it, I've tricked my brain into learning a new concept, right? So, yeah. So that's just what that's just what works for me. I've observed it over time. Yeah, that's it. That's awesome. Yeah, makes sense. Um the the next question that we were wondering about, you know, you mentioned that you use the uh the hotel ducks as a as a reference and you know, one of the one of the goals of these sessions um that we have like with hotel me is also to provide feedback um to the hotel community on on areas for improvement. Um, is there anything like why is it that the you found um third-party um documentation more useful than the official hotel docs? Um, and what made you go back to them afterwards? And and what would you So this like multi-art question, sorry. And what what would you what would you do to improve the docs experience if it if it were up to you as as like a new learner? Okay. Uh so sorry this is an issue with multiple questions. The first question is I hope I remember them. The first question is um what made me uh feedback for the docs or maybe leave the docs or Yeah. Why why did you choose why did you choose uh third party documentation uh over over the hotel docs for starters? Okay. So, uh well because number one, there's no video documentation yet. That's why fair. Yeah. Fair. Yeah. But um also when I'm reading someone's content, right? Like I'm learning your content or someone else out there. I am not just reading like I can feel I'm feeling I feel human connection like I can like you can say something like ah I got stuck at this. this is what I had to do and then this got you know solved but in a documentation it's just man it's just it's like reading a textbook you know there you know do this do this do this do that move on you know but you know when you're writing your content you can say oh do this if this does not work try this you try this out if this does not work try this out so I'll say that's why I preferred you know to go for like I said the when I enter the document doumentation I I got overwhelmed cuz I was scrolling through a lot of things I was scrolling through then I didn't understand what instrumentation was and then documentation has so many links that link you to so many things and before you know you're lost in the rabbit hole of the documentation so I'll be reading something I'm like instrumentation I'll see a link I'll click the link I'm going to you know a different entire part and then before you know I was like where am I what am I learning so I got overwhelmed right so so how would I if I had that opportunity How would I okay what what made me go back to the docs? What made me go back to the docs is I already had like I already built a foundation. So I was not overwhelmed when I saw what is instrumentation or when I saw what is a collector. So imagine someone that is new that does not know anything about a collector trying to learn open telemetry is and then he's seen open telemetry what is open telemetry you can't really define open telemetry without the four components instrumentation collector and he's like what what is open telemetry operator? Oh my gosh, he's seen Kubernetes like where am I, you know? So, um I already got that foundational knowledge. So, I could now go and head for the docs, right? And because now I already knew what I was looking for in the documentation. So, let's say I want to do I want to do a particular task. I can just search for that particular part in the docs and focus on just that side. And um so what would I do to improve the doc experience? I'll say I'll make a video documentation for you know we for for those who uh do follow the uh hotel content we do have our first video like instructional video on the uh hotel YouTube channel. So um and and there's there will be more to come. So that's great feedback. Yes. So you can also um you know because there are two different kinds of learners. they are visual learners and they are, you know, text based learners. So, they're satisfying both at the same time. So, that's what I was saying. Yep. Yep. That is a super fair point and I I I like that. Um it it's interesting too like what you were saying um that basically use the um the hotel docs for like more of a a directed directed search, right? like first understand stuff um from people's experiences and then okay I want to learn about this I will go into the docs to learn about that that's kind of that that's really cool that's the approach that's worked for me um to learn everything and that's what I'll keep using by the way Adrian I started learning Kubernetes just a side side note oh right on good luck good luck I'll journey It's a journey. I think you'll find it fun though. You're a problem solver. Thank you. Yeah. Yeah. Um, could you tell us what your experience was like creating your first PR? Ah, okay. We're here. So, creating my first PR, uh, I'll say it was a whole experience, right? I so after watching some videos, reading some documentation I think I did this for for two weeks I was just learning a whole lot and then I was like you know what you need to make your first PR like you can't waste any more time and then so I went to the the um the repos and I was like okay so which repo do I pick I think I think that that that was the first issue I had you know so open telemetry has a bunch of repos so I was like so which one do I go for so I think some for a while I was just lost in do I contribute to NodeJS do I know node I don't know node let me go to collector do I know I'm like okay then so what I started doing um was I started reading other people's PRs so I would literally go to an issue that is closed go under the PR you know because when an issue is closed the link to the PR that you know close the issue will always be there I'll literally go and be studying their code so like that's That's why I did a lot I don't think I've said share this anywhere before but that's why I did a lot I would go and literally be studying people's PR studying their code studying how they you know framed things okay this is how you write this in go because remember I was still learning go at this point so it's so so it was like a it was like a I was like um learning so much in one month because I was studying people's code I was doing a course at the same time right so and then after studying people's peer I was over a while I now gained some confidence I now I found an issue that someone had done something similar you know that's the beautiful thing about open source sometimes you want to update you know a bunch of components at the same time and I saw a similar issue to one of the peers I had studied and I was like you know I can tackle this and I asked you know I asked hello my name is this can I please be assigned to this issue I didn't think the person will answer me and he was like oh yeah sure why not go ahead I think I think his username is MX-Psi I don't know his real MX-PS psi sorry and he was like why not go ahead and you know I tried it out I made a m I made mistakes obviously because I was new to go I didn't understand the whole lint check thing but he then he you know like I said open telemetry community is so wonderful um they do not look down on you they don't look down on where you are as long as you're honest right don't come and prove to be a senior developer when you actually not like come clean I told the guy man I haven't done this before please help my life. I went to his DMs and he was like, "Okay, do this, do this, do this." And you know, at the end of, you know, I think after two days, he got merged. I was like, "Yay, my first PR." And then from there it just it was just like a full journey, right? It was a full journey, you know, from there. Really really nice journey. So that was my my summarized experience. I don't want to go into the details of me crying and, you know, want to tear my Yeah, that was that was just a summarized experience. Yeah, I I I have to agree with you that like that first PR getting merged is like the most glorious experience ever. But I will tell you like every time I have a PR merged, I do like a little happy dance at home. It's like, oh my god, I'm up for something. By the way, I I I gotta say like hats off to you um in your approach to to working on issues too, like studying the the PRs and the code behind the PRs. you you have the heart and soul of a software engineer. It's like you were made for this. That's amazing. And I I think this is a good lead in um also to our our uh next um area that we want to talk about which is um I believe you mentioned earlier that you uh did some work on creating an hotel exporter uh for the collector. So um do talk about that. What was that experience like? Okay, I have a lot of funny experiences. Sorry. Um, so when I got the project um with my former company, Simex, uh I was like, okay, let's do this, right? How hard can it be? So I went to the documentation and there was no documentation on how to create an exporter, right? uh the only documentation was was how to create a receiver and I was trying to understand it and it wasn't working because one one thing I now realized later that I didn't know then was that an exporter there's no really after there's no standard way of creating an exporter well there there's a template you have to follow but there's no standardized way because you know exporters depend on your vendor back end so in semantex um the we're receiving logs over influx live protocol call and then we were sorry receiving metrics over influx 5 protocol and then we were in we're receiving um logs over um what's that thing called what's that thing called elastic yeah elastic bulk index format right so I had to build an exporter that would convert hotel data to these two sources so what I what I did like I did for my other PRs let me just summarize you know like I said I would avoid all the days of shouting and panicking and running around and I'll go straight to the point. I started studying other other people's exporter. So I studied influx DB exporter because that way our metrics you know that's what our metrics exp the metric side of exporter would look like. Then I studied elastic elastic search exporter like I did a lot of study. I tried to break down a lot of things with my the help of my assistant cha great assistant. Yeah. So I I I broke I try to break it down. I try to um fit things in our own context. So how do I take this part from here, put it here, remove this part, you know? So it was a whole trial and error. I did a lot of trial and error. Um there was really no content that I could that I could look to that okay this is what assisted me in this journey. Oh sorry pardon me on that. I remember the open telemetry collector builder. So while learning while learning how to build um the exporter I had to use the OCB open telemetry collector builder and that's when I came across bind planes content on YouTube also really we had they have a full series on open telemetry collector very wonderful series I enjoyed every build so that's why I learned how to use the open telemetry collector builder so yeah so that really helped and the OCB is very fun it's so cool man it's like you can build your own custom collector you know with whatever component you want like you It's so it's cool. It's like play-doh, right? You can build as you want, play with it as you want, connect things, join things, break things, and you know, it's fine. So I So I learned how to use the OCB. And then in time, after lots of trial and error, I got my first metric in to SEAR text and it siting exciting feeling, right? And then after a while again got the logs in and then awesome. Then I I built the exporter. So that was that was the that was the summarized version of the experience of building the exporter. Yeah, it's really interesting. Uh believe me, I'm sharing in your excitement right now. I can I can only imagine. Um so for our next question could you tell us a bit about learning go to be able to work on this hotel exporter that you just talked about. So learning go has been a very interesting journey because it has I I did not learn go the normal way right the normal way just watching a course and just building some you know interesting projects you know maybe to-do app stuff. I learned go the hard way because I was actually actually had to like I was learning it and I had to apply it directly into real life projects that go into production, right? So I I remember I I had a help I had a lot of help from the people I worked with at Semantex. Bora Bora Akshhat really cool guys part of my team. So they were so so I'll make mistakes. They were like no they'll be like no error this is not the right way to do this. I know this is the YouTube way, this is the um course way, but in production this would break, this would fail. So I say I learned go handon you know contributing to things right. So it was a very interesting journey. I actually had I got a course from code cloud gave me a lot of like I said I like contextual things. So I had a full context of what go looks like. So I could understand code if you give me any Golan code I can know okay this is what this is what this is struck this is an interface this is what this does this what should do but then I had a lot of you know building things in real life and I feel like that's really the best way to learn you know although it's very stressful uh you would feel you feel very dumb sometimes because some things will I remember sometimes at night I'll literally be up by 2:00 a.m. time and detest keep failing and I'm like please just run please just run I'll literally be praying I'm praying over and over and over and over again and then finally run and I'll be like yay let's go another milestone achieved so so that I'll say that that was my whole experience learning golang and you know becoming that golang engineer yeah cool um throughout Throughout our conversation, you've talked a lot about the resources that helped you to learn open telemetry, that helped you to learn Google and helped you to build the auto exporter. Now, could you tell me what uh what plan you had for documenting this this exporter that you that you when you said so that other people Okay. So, what do you mean document it? Do you mean like you want me to document my journey or how I built a portment? Can you clarify documentation for Sorry, I I I didn't get you for I lost you for like two seconds. Sorry. Yeah, I said documentation for the exporter itself for those who who would be using it. Yeah. Okay. So, I think um when when you're contributing an exporter, there's actually a template of documentation that you actually have to, you know, just do. Well, I'll I'll answer your question from the standpoint because I think Adriana and I spoke about this um I think last time and uh so even if I document and she actually put an idea that I should write an article on how it is not really possible to show you how to build an exporter because I tried actually sat down after building exporter I was like okay how can I help the community with this and I'm like even if I show you how I did it it's not going to be useful to you because your context might be different, right? It might be different. So I what I would say is maybe I the the article I can write on or maybe the documentation will be don't give up. Try try try again. Even when it fails, keep trying. It's going to work. And then another thing I'll say is study study other code. Stud study other people's code. Like I'll say that's one thing has helped me in my whole coding journey and has really accelerated my speed is that I I um always studied people's code and understood the patterns in the code. So from understanding those patterns I can now create my own you know find how to remove parts and then join to what I'm looking for. So yeah. So it's more of a a process rather because I I guess from the sounds of it like um I think the other two components um uh receivers and and processors I think there is like some documentation around around creating those because it's more formulaic right whereas exporters are not but then I guess the thing that's in common is like what's the thought process that you need for creating it but not this is the exporter template Yeah, this is step step by step. Now, yeah, I I think that would be such a great blog post to write. Um because I think, you know, again, it all goes back to what you were saying too about like um some of these third party sources are about personal experiences and having that human touch, human connection. And I think that would be so beneficial. Um, it would also be interesting, I'm just giving an idea, um, to put in the docs, um, something along the lines of like why is it that we don't have a template for creating hotel exporters? Because that was like I remember I at some point I uh, I was I was looking into that as well and I'm like, I don't get it. Why Why is there no template for this? Yeah, absolutely. Yeah, there's no official template. I think it's it's when you start and then you know you speak to the the um the maintainers they not tell you oh no no no this should be here this shouldn't be here I'll be like okay let's sh change it things up right so I think yeah I think there should be an official like okay this is the maybe we can't help you to build the full thing but this these are the things you need you need a config file you need this file you need a factory go file you know things around that so yeah yeah I think Uh that's great. Um and then the uh the other thing that you mentioned that um I want to dig into is the hotel collector builder. Um and specifically like what was your um what was your experience around that? Cuz you know I I've played around with the builder. You've played around with the builder. Was it was it as easy as you thought it would be? Um and if not, what would you say is some feedback that you would like to give on what can be done to make it easier? Okay. So, um you know, in my journey, and I'm going to be very very honest here, right? In my journey with um learning to try to get the um understand the exporter. I didn't really get um any good information from the documentation. So, I'll say for for a while I just did not look at the docs anymore. I just went to YouTube and was doing the research. So that's why I found um bind plane right and then they did a whole video on how to build your own custom collector like 45 46 minutes video and that basically showed me okay yeah so this this works this can do this and then I think also the readme files so I think there's even some like some of the like the open entry collector builder it's more documented in the readme file than well at that time than the documentation itself. So I I think I got more value from the um from the readme file from the readme files at that time than the actual documentation. I don't know if it has been updated yet. I'll probably check but yeah. So I I did just uh get a PR merged in the uh in the hotel docs um about like cuz when I was doing it um I'm like okay I built this DRO how the hell do I containerize it? and there was no documentation around that and I had to ask and then I wrote a blog post about it but I'm like it should be in the docs too. So I just had that PR merged I think today or yesterday. So that's cool. That's so cool. Yeah. So that was my experience with the OCD telemetry. Awesome. All right. Uh I think I have just one more question. Could you could you tell us a bit about the hotel community and I what's your Yeah, I I would like to know your first impression of the community. How you able to collaborate with the community from the time you contributed to Yeah. as an agent and beyond that beyond the Yeah. Yeah. Okay. Um so the open telemetry community I love you all so much but uh so my my first my first experience with them was uh I introduced myself hello guys I am an I am so I used to be introverted some sort of way but the people don't really believe because you know I'm I'm very outspoken but I used to be introverted right so I was kind of shy uh because I was like okay I'm a newbie I don't really know much about stuff I don't know anything about open source who listened to me right and I was like hello my name is I'm from my Nigeria nice to meet you all and people said hi back right people people waved back oh nice to meet you welcome to open tele community and I was like oh that's so cool and then um I'll say when I stumbled into a lot of like back to my mentors right Yuri so Jorassi was um Yuri was my major mentor for the for the outreach internship so we had a lot of calls, right? And in those calls, he just keep encouraging me that I know this is tough, but don't worry, you're going to get it. Just keep just keep pushing, you know, keep trying. I said, I believe in you. You know, he's come, I believe in you. I know you can do this. So, you know, he was always just encouraging me. And then even not just my mentors, you know, it was like a collective effort. So, like I said, I've said this before, I'll go to some people's DMs and I'll be like I'll be crying that please help me. I don't understand this p this issue. How do I solve it? You've already assigned it to me. I tried this. It did not work. And they'll be like, "Okay, you know what? Try this. Make this change." And then I'll keep disturbing them and they'll keep replying. And I remember when I wrote my first blog my first blog post on on open telemetry and I had already like read some bunch of Adriana's blog posts and I was like I'm going to send I was like I'm going to send a DM to Adriana because I like to hear her thoughts on my blog post and I was like hello Adriana hi I'm Shington please can you review my blog post for me and she and she replied and she went I reviewed my blog post and I was like wow she doesn't know me from anywhere you know uh we are not in the same continent or in different from different tribes and she's actually willing to you know assist right so I was like oh that's so cool so I would say open community they are very loving I can't really vouch for a lot of open source communities right but I can vouch for the open telemetry community that they really cool and it's a very nice community to join by the way I'm now a member of the open community so you can also text me I'm very cool if you have questions about and she can text me. I'm cool. Like they are also cool, right? So the community is loving, nice, really caring and I love it. But then when I got to CubeCon and I met everyone in person, it was like way cooler. I met Adriana in person. I met Heric. I met Tras and some other really awesome people. It was really nice. So yeah, they're really nice community to join and be part of. Yeah, I'm I'm going to completely agree with everything just said because I just recently joined the community too and everyone has been about Adriana plus one to that. She's very cool. Yep. And and it's been nice getting to interact with her. Uh one more question before we go. Um have have you made any recent I believe as an hotel community member now you have to contribute to hotel community. So have you had any recent um contributions to the community that you like to talk about? Yeah. Okay. So um what I have been doing majorly right now is um so in in the Nigerian space I think when I came back from CubeCon I think when I started learning the whole open telemetry thing I realized that open telemetry was not a topic that was spoken about a lot in my you know my country locally you know Africa because Kubernetes is very popular here everyone knows Kubernetes right so um open telemetry is not really heard a lot so I I started advocating a lot about I think that's how I even slided into my developer advocate role. So currently I'm a developer advocate. I'm not a full software engineer anymore. Um so I started you know attending conferences um talking about open telemetry showing people how to get started with open telemetry how to um get into open source you know that those because I experienced you know I'll say I experienced the the dramas the struggles you know so I I understand when people come to me and tell me I'm so confused about open source how do I start so I understand yeah and I can also give them practical steps I took to you know get started so I would say I've been doing more of that and then I'm actually still the exporter has not it has been built fully but I'm still pushing it into you know because open source takes time you can't just throw things inside there's a whole process so the exporter is still being pushed into um the the open telemetry collector contrib so I'll say that is the in aside um related to code that those are the contributions that I've been making but then I've been doing more of advocacy you know about open telemetry you know in my Yeah, that's that's amazing. And I think that's that makes a a really good point as well. Um because contributing to open telemetry isn't just about the PRs that you're making. It's also about the advocacy um the bringing that awareness. So I think that's that's great. And um I think this wraps up the the the main Q&A and we do have a few uh minutes left. So if anyone from the audience has any questions that they would like to ask at this point um we are happy to take your questions. I feel like we need some music. This is where you take out the guitar for this for this. Dang it. Who the with the music? Is that you? Was that I think that was Henrik, our amazing producer. Well, it doesn't look like we have any questions. So, um, what I will say is we'll I guess we'll wrap up, but before we go, um, a few quick announcements, promotions. Um, first of all, for anyone who is planning on attending OpenSource Summit in Denver, Colorado, there's also going to be as as a coll-located event, there's an open observability con and hotel community day that is going on. Um, the CFPs for hotel community day are closed, but open observability con CFPs are still open. Um, and that's going to be at the end of June. So, um, feel free to submit CFPs to attend if you're in the area if you're planning on attending. Open source summit also is like lots of fun. Um, it's like CubeCon light. It's all the awesome people who attend without like without the flash. So, uh, it's tons of fun. Um the other thing um for anyone who is interested um as I mentioned I'm one of the maintainers of the hotel and user sig. Victoria is a newly joined member of the hotel and user sig. So um if you're interested in joining us seeing what we're up to um Henrik just flashed a link for our GitHub repo. Um we do a bunch of things like these types of events. We have hotel me. We have hotel and practice where if anyone who's watching is interested um if you have like a topic uh an observability hotel topic that you'd like to present on maybe you're testing out a a topic for a conference for a talk um want to flesh it out use us um just DM us uh on the hotel end user SIG Slack um and just you know just let us know if you're interested in in in uh doing a presentation for hotel in practice and we also run uh we also uh join forces with the various hotel sigs to run surveys. So um stay tuned there will be more surveys heading your way. The surveys are a great way to for us to get um feedback from our user community on ways to improve the SIGs. So that information is super important, super valuable um to the community. So thank you to the folks who have responded to the surveys. Thank you to the folks who have engaged in the hotel end user SIG. Um also um for those who aren't able to attend and for those who were able to attend, tell your friends that this video will be posted um on our hotel YouTube channel. It's hotel-official on YouTube. Um you can catch all of our previous hotel and practice and hotel me um sessions. And also we do have our first how-to hotel video on on the hotel YouTube channel. So for for those who are visual learners, this is your opportunity to check that out. Um, thank you so much Edoselle for joining and for Victoria for co-hosting uh with me today. This has been lots of fun. Always enjoy having these conversations and um we will see everyone at the next event. Sure. Thanks, Adriana. Thank you. Thanks, Father. Thanks, Victoria. It was nice meeting you. diff --git a/video-transcripts/transcripts/2025-05-02T18:18:11Z-even-more-humans-of-opentelemetry-kubecon-eu-2025.md b/video-transcripts/transcripts/2025-05-02T18:18:11Z-even-more-humans-of-opentelemetry-kubecon-eu-2025.md index 41d148f..f9a2c59 100644 --- a/video-transcripts/transcripts/2025-05-02T18:18:11Z-even-more-humans-of-opentelemetry-kubecon-eu-2025.md +++ b/video-transcripts/transcripts/2025-05-02T18:18:11Z-even-more-humans-of-opentelemetry-kubecon-eu-2025.md @@ -10,89 +10,80 @@ URL: https://www.youtube.com/watch?v=eZ3OrhxUAmU ## Summary -In this YouTube video, various experts in the field of observability and OpenTelemetry share their experiences and insights. Presenters include Marylia Gutierrez from Grafana Labs, Adriel Perkins from Liatrio, Jamie Danielson from Honeycomb, and others, each discussing their roles and contributions to OpenTelemetry. The main topics explored include the importance of observability in understanding complex systems, the evolution and significance of OpenTelemetry as an open-source framework for telemetry data, and personal anecdotes about their journeys into the observability space. They highlight the collaborative nature of the OpenTelemetry community, the utility of different telemetry signals like traces and metrics, and the critical role of observability in improving software engineering practices. Overall, the discussion emphasizes the value of community-driven open standards in enhancing observability across diverse applications and infrastructures. +In this YouTube video, a diverse group of professionals, including Marylia Gutierrez from Grafana Labs, Adriel Perkins from Liatrio, and Alolita Sharma from Apple, discuss their experiences and insights regarding OpenTelemetry and observability in software engineering. They explore the importance of OpenTelemetry as a vendor-neutral standard that enhances collaboration among various companies and contributes to a more effective observability ecosystem. Key topics include the evolution of their involvement in OpenTelemetry, the significance of observability in understanding complex systems, and the role of different telemetry signals like metrics and traces in improving application performance. The participants emphasize the community aspect of OpenTelemetry, highlighting how collaboration and shared goals drive innovation and problem-solving in the field of observability. -# Introduction +## Chapters -My name is **Marylia Gutierrez**, and I'm a staff software engineer at Grafana Labs. I also work on a few different groups in **OpenTelemetry**. +Here are 10 key moments from the livestream with their corresponding timestamps: -My name is **Adriel Perkins**, and I'm a principal engineer at a consulting company in the United States called **Liatrio**. We're both end users, but I am also a contributor to the OpenTelemetry project. I'm co-lead of the **CI/CD SIG** with **Dotan Horvitz**, a **CNCF Ambassador**. We work on the Collector and the specification repository. +00:00:00 Introductions of speakers and their roles in OpenTelemetry +00:05:30 Personal journeys into OpenTelemetry +00:10:15 The importance of OpenTelemetry in centralizing data sources +00:14:20 Observability: definition and personal significance +00:20:40 How OpenTelemetry facilitates collaboration across companies +00:25:10 The role of OpenTelemetry in cloud-native applications +00:30:00 Discussion on the OpenTelemetry protocol and its impact +00:35:45 Favorite telemetry signals: metrics vs. traces +00:40:00 The community aspect of OpenTelemetry and its collaborative nature +00:45:30 Insights on the future of observability and OpenTelemetry's role in it -I’m **Hanson Ho**, and I focus on Android development, specifically observing the Android ecosystem at **Embrace**. +# OpenTelemetry Community Insights -I’m **Jamie Danielson**, an engineer at **Honeycomb**, where I work on instrumentation libraries and OpenTelemetry. +## Introductions +- **Marylia Gutierrez**: Staff Software Engineer at Grafana Labs, involved with various groups in OpenTelemetry. +- **Adriel Perkins**: Principal Engineer at Liatrio, contributor to the OpenTelemetry project, co-lead of the CI/CD SIG alongside Dotan Horvitz. +- **Hanson Ho**: Works on Android observability at Embrace. +- **Jamie Danielson**: Engineer at Honeycomb, focusing on instrumentation libraries and OpenTelemetry. +- **Mikko Viitanen**: Product Manager at Dynatrace, maintainer of the OTel Demo App. +- **Damien Matheiu**: Maintainer for OpenTelemetry Go, code owner for profiling in the Collector, and approver for the eBPF profiler. +- **Jacob Aronoff**: CTO at Omelet, a startup focused on observability and telemetry pipelines. +- **Alolita Sharma**: Leads AI/ML observability engineering at Apple. -I’m **Mikko Viitanen**, a product manager for **Dynatrace**, and I'm also a maintainer in the **OTel Demo App**. +## Early Involvement in OpenTelemetry +Alolita Sharma mentioned her first encounter with OpenTelemetry while exploring an enterprise observability solution. She found it to be an excellent way to centralize data from various sources, allowing for a holistic view of observability. -I am **Damien Matheiu**, and I do many things at OpenTelemetry. I am a maintainer for OpenTelemetry Go, a code owner for everything profiling on the Collector, and an approver for the **eBPF profiler**. +At Embrace, the team sought to better serve the community, leading them to OpenTelemetry as an open framework for contributing to a common standard. This transition allowed them to expand their data collection capabilities without drastically modifying their SDK. -My name is **Jacob Aronoff**, and I am the CTO at **Omelet**, a new startup focused on observability, telemetry pipelines, and OpenTelemetry. +Jamie Danielson began working with OpenTelemetry more extensively in 2021 at Honeycomb, focusing on the Collector and instrumentation libraries. -Hi everyone, I'm **Alolita Sharma**, and I lead **AIML** at **Apple** for observability engineering and infrastructure. +Damien Matheiu started his journey in OpenTelemetry around three years ago, contributing to the OTel Demo App which provided a hands-on learning experience with instrumentation and the Collector configuration. -# Introduction to OpenTelemetry +Jacob Aronoff began his work with OpenTelemetry while at Lightstep, contributing to the telemetry pipelines team and focusing on Kubernetes aspects of the OpenTelemetry Operator. -I started in OpenTelemetry after working in the observability world for a while. Eventually, I discovered OpenTelemetry and found it to be a great way to work without dependencies. My first involvement was when I was asked to look at an observability enterprise solution, which led me to the Collector. This was due to the need for centralizing data from various sources to obtain a holistic view. I thought, “This stuff is really, really cool.” +## Perspectives on Observability +### What Does Observability Mean? +Observability allows engineers to discover insights they didn't initially know they needed. It enables continuous learning and improvement by providing visibility into the system's health. -**Embrace** sought to better serve the community, and OpenTelemetry was an obvious choice as it is an open framework that allows contributions to a common standard. We had a proprietary SDK collecting signals sent to our own servers, but with OpenTelemetry, we could expand our data destinations to open-source collectors without extensive changes to the SDK. +**Key Insights:** +- Observability is about asking insightful questions and acting on the answers. +- It helps understand application behavior and operational health. +- Observability is crucial in pinpointing issues within complex systems, such as telecom networks, where it was essential to resolve call drop issues. -When I started looking into OpenTelemetry, I realized that not many people were focusing on mobile, and since then, interest has significantly grown. +### Personal Experiences +Many speakers shared their frustrations with traditional logging and the challenges of convincing teams to adopt more holistic observability practices. They emphasized the importance of continuous learning and the ability to interpret data effectively. -In 2021, when I joined **Honeycomb**, my team began working more with OpenTelemetry and instrumentation libraries. I started working on the Collector and Java before settling into OpenTelemetry JavaScript, where I have been for the last three years, becoming an approver and more recently a maintainer. +**Examples:** +- **Damien** shared his transition from SRE roles to focusing on observability to improve incident response. +- **Alolita** highlighted her commitment to learning and improving through observability. -I began my OpenTelemetry journey around three years ago with small contributions to the OTel Demo App, which I found to be an excellent place to learn about instrumentation and the Collector. +## The Role of OpenTelemetry +OpenTelemetry is viewed as a central component in observability stacks. It provides a vendor-neutral standard that allows different contributors to collaborate and build solutions that benefit the entire community. -I was previously on an observability team at a different company and was frustrated because I believed we should use tracing alongside logs. It was challenging to convince the engineering team, so as a New Year’s resolution in 2022, I decided to contribute to the OpenTelemetry Go repository, which eventually led to a full-time job in OpenTelemetry. +**Highlights:** +- OpenTelemetry fosters collaboration among competitors, creating a shared understanding and vocabulary for observability. +- It allows end users to instrument their applications seamlessly across different vendors. -I also started at **Lightstep**, working on the telemetry pipelines team and contributing to the OpenTelemetry Operator, focusing on upgrading the target allocator that handles horizontally sharding of scrape targets for Prometheus. +## Favorite Telemetry Signals +The community shared their preferences for telemetry signals, often highlighting the importance of tracing and metrics: -I got involved with OpenTelemetry over six years ago while at **AWS**, where I had always participated in open-source projects. As we built new Kubernetes-native services, we decided to engage with open-source observability projects, with OpenTelemetry being at the forefront after the merger of OpenTracing and OpenCensus. +- **Tracing** is favored for its ability to provide end-to-end visibility in applications, helping teams to pinpoint issues effectively. +- **Metrics** serve as a foundational signal, offering a straightforward way to introduce observability concepts to new users. -# Understanding Observability +**Conclusions:** +- Many believe that while all telemetry signals are important, **tracing** stands out due to its powerful insights and versatility. +- The combination of **metrics** and **traces** is crucial for understanding application behavior and infrastructure performance. -To me, observability means being able to discover things you didn't know you needed to know, aided by extensive information. However, having information alone is not enough; understanding how to interpret it is crucial. - -Observability allows for continuous learning and improvement. The more I discover, the more I realize there is to learn, both technically and socially. It enhances my engineering skills and facilitates discovery throughout the software development lifecycle. - -**Observability** can be boiled down to asking questions and obtaining answers, especially those you didn't initially think to ask. It's about acting on and learning from data to understand your system better. - -To me, observability means having insights into how your applications and systems work, providing visibility into unknown factors until issues arise. - -I came across observability years ago while working with telecom networks, where it was essential for troubleshooting dropped calls across multiple nodes. Without observability, pinpointing issues would be impossible. - -Before focusing on observability, I spent many years in SRE roles, dealing with incident management and root cause analysis, often during challenging situations. My work in observability aims to ensure that if I return to SRE roles, it will be easier for others. - -Observability is knowing the health of your servers, similar to how a car’s dashboard indicates its operational status. This analogy applies on a much larger scale in server management. - -For cloud-native infrastructure and applications, observability is essential. With the complexity of microservices, having observability integrated from day one is crucial to ensure that applications and infrastructure function correctly. - -OpenTelemetry enables users to build observability solutions seamlessly without worrying about specific protocols for different types of data. - -# The Role of OpenTelemetry - -To me, **OpenTelemetry** represents a community effort where competing companies collaborate on a vendor-neutral standard that benefits everyone. It allows for easier instrumentation of applications without needing to deal with different agents when switching vendors. - -OpenTelemetry is about community and collaboration, solving engineering problems while sharing a common understanding across various languages and frameworks. This collaborative spirit is rare in today's environment, where competitors often remain in their corners. - -OpenTelemetry is a vast ecosystem that provides a unified approach to collecting and managing data, making it easier for engineers to work across different companies. It’s about having a standard way to do things, which saves time and effort. - -The project has nearly 80 repositories and fosters partnerships between vendors and end users, solving technical challenges collaboratively. OpenTelemetry not only builds collection architectures for cloud-native applications but also promotes interoperability through open standards like the OpenTelemetry protocol. - -This protocol enables users to develop observability solutions without worrying about the specifics of metrics, logs, or traces, allowing for a more streamlined experience. - -I love working on the various components of OpenTelemetry, particularly the Collector, where I focus on improving integrations and performance features. - -# Favorite Telemetry Signals - -My favorite telemetry signal is **metrics**, as they serve as a simple entry point for understanding observability. They provide a number that can be enriched with attributes, making them accessible to those new to the field. - -However, I have grown to appreciate **tracing** even more. It allows for comprehensive visibility into service interactions, enhancing understanding of application flows. - -Many of my colleagues share similar sentiments, emphasizing the power of traces for pinpointing issues and gaining insights into system performance. Tracing provides a waterfall view that aids in identifying bottlenecks or slow responses. - -While I value both profiling and tracing, I lean towards tracing as it addresses issues that arise during operations. It encapsulates the essence of observability, allowing for deep insights into application behavior. - -In conclusion, **tracing** stands out as my favorite telemetry signal due to its ability to derive metrics and logs and facilitate visualizations that enhance both high-level observability goals and low-level debugging efforts. +In conclusion, the OpenTelemetry community is marked by collaboration, shared learning, and a commitment to improving observability across diverse systems. The participants view OpenTelemetry as an integral tool that not only enhances their work but also contributes to the broader observability landscape. ## Raw YouTube Transcript diff --git a/video-transcripts/transcripts/2025-05-07T18:47:21Z-otel-night-berlin-v2025-05-spring-starter-for-opentelemetry.md b/video-transcripts/transcripts/2025-05-07T18:47:21Z-otel-night-berlin-v2025-05-spring-starter-for-opentelemetry.md index 3a270ce..8bc9a43 100644 --- a/video-transcripts/transcripts/2025-05-07T18:47:21Z-otel-night-berlin-v2025-05-spring-starter-for-opentelemetry.md +++ b/video-transcripts/transcripts/2025-05-07T18:47:21Z-otel-night-berlin-v2025-05-spring-starter-for-opentelemetry.md @@ -10,125 +10,83 @@ URL: https://www.youtube.com/watch?v=R0n6Ny588dk ## Summary -In this YouTube video, the speaker discusses the Spring Boot starter for OpenTelemetry, focusing on its development, features, and practical applications. The speaker, who has extensive experience in Java development and OpenTelemetry, provides an overview of the current state of OpenTelemetry in Java and the motivations behind creating the Spring Boot starter. He compares it to existing solutions like Micrometer and highlights the challenges in achieving seamless observability integration. Throughout the video, he emphasizes the ease of use and native experience that the Spring Boot starter aims to provide. The talk includes a live demo showcasing the setup and functionality of the starter, with opportunities for audience questions interspersed. Key topics include instrumentation, API stability, and the importance of semantic conventions in observability. +In this YouTube video, the speaker, who has extensive experience in Java development and is currently an approver for OpenTelemetry Java instrumentation at Grafana, discusses the Spring Boot starter for OpenTelemetry. The presentation covers the current state of OpenTelemetry as of 2025, including its significant advancements in Java and comparisons with existing solutions like Micrometer. The speaker emphasizes the importance of a seamless integration experience for Spring Boot users and discusses the challenges of matching the large number of libraries supported by OpenTelemetry. The video features a demo showcasing how to implement the Spring Boot starter, including configuring the application and observing telemetry data. The speaker addresses various technical questions from the audience regarding context propagation, service naming, and instrumentation challenges, concluding the session with a Q&A segment. -## Transcript +## Chapters -# Spring Starter for OpenTelemetry +00:00:00 Introductions and background of the speaker +00:03:00 Overview of OpenTelemetry and its state in 2025 +00:05:30 Motivation for building a Spring Boot starter for OpenTelemetry +00:10:15 Comparison of Spring Boot starter with Micrometer +00:15:00 Explanation of how OpenTelemetry works, focusing on Java +00:20:00 Discussion of semantic conventions and their importance +00:25:00 Introduction to the demo environment setup +00:30:00 Starting the demo application and observability stack +00:35:00 Demonstrating data collection and trace visibility in Grafana +00:40:00 Q&A session on context propagation and service naming issues +00:45:00 Conclusion and final thoughts on OpenTelemetry and Spring Boot integration -All right. I want to talk about the Spring Starter for OpenTelemetry. What I'll try to do is demonstrate how it works; it just works, but except when it doesn't—thanks to the demo gods, and because it’s Spring Boot. +# Spring Boot Starter for OpenTelemetry ## Introduction +Hello everyone! Today, I want to share my insights about the Spring starter for OpenTelemetry. The aim is to demonstrate how it works, and I'm going to cover a few key points. Just a heads up, the demo gods might play a role since it’s all about Spring Boot! -About myself: 20 years ago, I started my first job as a Java developer in this building. I realized today that it wasn't the first location, but it was the first company, and we moved here from a different location in Berlin. I worked here for about three years. Ten years later, I started using Spring Boot. I still don't know everything about it. Five years ago, I started working with OpenTelemetry, and I definitely know that I will never know everything about it. Two years ago, I started working for Grafana, and now I'm an approver for OpenTelemetry Java instrumentation—the stuff that sends all the data, which you will see later. I also work on some other areas in OpenTelemetry, including the operator, collector, and specification. +## About Me +A bit about myself: I began my career as a Java developer right here in this building 20 years ago. While it wasn’t the original location, it was the first company I worked for. After about three years, I moved on. Ten years later, I started using Spring Boot. I still don’t know everything about it! Fast forward five years, I began working with OpenTelemetry and realized I would never know everything there is to know about it. Two years ago, I joined Grafana, where I'm currently an approver for OpenTelemetry Java instrumentation, which involves sending all the data you’ll see later. I also work on other areas in OpenTelemetry, such as the operator, collector, and specifications. -## Overview of OpenTelemetry in 2025 +## Overview +I want to provide you with a quick overview of the state of OpenTelemetry in 2025, highlight what’s new, and discuss the motivation behind building a Spring Boot starter in the first place. We’ll also compare it briefly to other solutions in Spring Boot, namely Micrometer. Most of our time will be spent on the demo to show you how easy it is to use, and I’ll leave some time for Q&A, but feel free to ask questions anytime. -I want to give you a quick overview of the state of OpenTelemetry in 2025, highlighting what’s new. Then, I’ll discuss the motivation behind building a Spring Boot starter in the first place. I also want to compare it quickly to another solution in Spring Boot called Micrometer. I plan to spend most of the time on the demo to actually show you that it is easy, leaving time for Q&A. But feel free to ask questions in between. I hope we have enough time; if we don’t, please ask before I try to finish the demo, as it may not work. +### State of OpenTelemetry +As you can see, there are many languages supported in OpenTelemetry, and most have moved to stable releases for traces, logs, and metrics. This is significant progress from last year. We’re focusing on Java, PHP, .NET, and C++. Profiling is a new topic that’s actively being developed and might be included in the future. -As you can see, we have quite a lot of languages in OpenTelemetry, and many of them have moved to stable for traces, logs, and metrics. I think that's quite a significant change from last year. Java, PHP, .NET, and C++ are included. For some reason, profiling is not listed here, but this is a new topic that is actively being developed, and I guess in a year or two this will also be listed. +#### Java and OpenTelemetry +Java is one of the more mature areas in OpenTelemetry, likely due to the prevalence of e-commerce and business applications written in Java. There’s a high demand for libraries, with over 100 supported libraries available. Java's compatibility with older versions is possible because of a technology called bytecode manipulation, allowing software to modify methods and classes at runtime. -## State of OpenTelemetry in Java +### Why Build a Spring Boot Starter? +The question arises: Why did we spend significant time last year creating a Spring Boot starter? One reason is that we can’t match the large number of libraries because we cannot alter existing code, and it would take a lot of time to reach the same level. As a Spring Boot user, I want a native Spring Boot experience. It should be straightforward to add a library as a dependency without any special steps, similar to adding a database. -Java is one of the more mature areas in OpenTelemetry, probably because many e-commerce and business applications are written in Java. There's a lot of demand, and I counted over 100 supported libraries, which are also compatible with older versions of Java. Java 8 was released more than ten years ago, and this is only possible because Java has a technology called byte code manipulation. This means you can write software that, at runtime, looks at all the methods and classes that are loaded and can change anything. It can change a return value and look at parameters, similar to how Python can do monkey patching. +When using a Java agent, it feels somewhat magical and detached from the integrated experience Spring Boot provides. You should be able to add properties in the `application.yaml` file easily, with names and values automatically suggested. However, if you add a Java agent, it can complicate things because adding multiple Java agents usually doesn’t work well together. -Now that I've explained how the Spring Starter works, the question is: Isn’t that good enough? Why did I and another engineer from Microsoft spend a huge amount of time last year working on making a Spring Boot starter? +### Comparing with Micrometer +Micrometer is an older solution that has been used in many applications. It added a module for exporting metrics using OTLP, but since Spring is older than OpenTelemetry, there are limitations on changing the names of their metrics. Micrometer tracing, previously known as Sleuth, is also relatively new and lacks semantic conventions, leading to inconsistencies when combining libraries. -### Limitations +### Key Concepts in OpenTelemetry +1. **Protocol**: The protocol is crucial as it ensures data is sent out correctly. +2. **Semantic Conventions**: These define names and attributes for metrics, traces, and logs. +3. **API**: Stable APIs are essential for library authors to maintain compatibility with OpenTelemetry. +4. **SDK**: The SDK implements the API and provides configuration options. +5. **Tooling**: The collector and operator are significant for managing telemetry data. -Let's start with what we cannot do or cannot yet do. We cannot match the huge number of libraries—more than 100—simply because we cannot change old code, and there's a lot of work being done. It would take a long time to have the same level of support for all libraries. +## Moving to the Demo +Before diving into the demo, are there any questions? -As a Spring Boot user, I prefer a native Spring Boot experience. This means I want to be able to select a library, add it as a dependency, and it should not require anything special. That is exactly what I wanted for observability; it should be built into the development process, much like adding a database. However, that’s not possible if you are adding a Java agent, which feels like magic. +### Q&A +One question raised was about sending Zipkin spans to the collector and whether they would show up as one trace. The response is that while both could be sent, they might not maintain the correct parent-child relationship due to the different internal mechanisms used by Spring. -As a Spring Boot user, you are also used to having a very integrated experience. This means you can take your configuration file, called `application.yaml`, and just add properties. The values and names of those properties are automatically suggested. For instance, you can easily select the service name, which we discussed a bit earlier. +## Starting the Demo +Now, let’s move on to the demo. I’m using IntelliJ and am currently in a project called `docker-hotel-lgtm`, which has examples of instrumentation and a complete observability stack based on Grafana technology. -You can also use another Java agent to do your other magic work, but adding two Java agents doesn't usually work well. Additionally, you can use a GraalVM native image that compiles your entire application into an executable, which starts faster but takes a long time to compile. I’m too impatient to show that to you, but there is a fully working example where you can try it out. +I have two build materials set up: one for OpenTelemetry and the other for Spring Boot dependencies. First, let’s start the application, and while it’s starting, I’ll also start the observability stack via Docker. -I spent a lot of time last year on this, and it became stable last year, so now I can talk about it. +### Application Setup +The observability stack consists of an application sending data to the OpenTelemetry collector, which then sends data to databases. The goal here is to create an easy trial experience, not optimized for scalability but for startup time. -## How OpenTelemetry Works +### Application Traffic +Now that the application is up, I’ll generate some traffic using a simple curl script. Let’s see if we can retrieve some data. -Before going into exactly how the Spring Starter works, here’s a recap of how OpenTelemetry itself works. This is not Java-specific or Spring-specific, but in the next slide, you will see how it relates to what I'm trying to show you. My personal preference for the order of importance is as follows: +### Analyzing Data +We have a few dice rolls recorded, and we can check the traces generated. The application is not calling any other services or databases, which is why not much data is available. -1. **Protocol**: Once you have the protocol, and your application knows data is sent out, you can switch it somewhere else without recompiling the application. -2. **Semantic Conventions**: These define the names and attributes. For example, for a metric, it would define the metric name of the duration of a request and its units. -3. **API**: This is more for library authors who want to be compatible with OpenTelemetry and do not want to spend time rebuilding their observability stack in five years. The APIs are stable. -4. **SDK**: The SDK is an implementation of the API and provides configuration options like environment variables. -5. **Tooling**: The collector and operator are also very important and helpful. +### Adding Improvements +During the demo, I asked if anyone had suggestions for improvements. The common response was to add more spans or metrics. Adding the service name correctly is also essential, and I’ll demonstrate how to do that. -## Comparison with Micrometer - -Now, I want to compare the Spring Boot Starter that I've worked on to the existing technologies in Spring Boot to explain why we built something where a similar solution already exists. - -In Micrometer, you have two things: -1. **Metrics**: This is quite old and used in many applications. Recently, a module has been added to export the metrics using OTLP. -2. **Tracing**: This is newer and was previously called Sleuth. While you can use the OpenTelemetry SDK to emit traces, it does not have semantic conventions. - -Here’s the problem: it’s not using the same API, which has created confusion for users trying to combine libraries. They often don’t know if one is using Micrometer tracing and the other is using OpenTelemetry. Sometimes it works, sometimes it doesn’t. - -I would love to have a common approach where either Micrometer APIs or OpenTelemetry APIs are used. However, this has not happened because the Spring folks have a rich tradition of maintaining their APIs, and OpenTelemetry wants to provide a consistent experience across all languages. - -## Data Flow in OpenTelemetry - -In OpenTelemetry, we have instrumentation libraries. In Spring, there are instrumentation libraries that are sometimes not maintained by OpenTelemetry but by the authors of the libraries themselves. Semantic conventions are essential for ensuring consistency. - -## Demo Preparation - -Now, I want to start the demo. If you have any questions, it probably makes sense to ask them now before we proceed. - -### Questions - -Is it possible to work with Zipkin? Yes, we have a Zipkin receiver in the collector. You can send Zipkin spans to the collector, and they will show up as one trace at the backend. However, if the Spring application does not use Zipkin, it uses something else as a common denominator for both Zipkin and OpenTelemetry. - -If you send both, they will show up, but they won’t have the correct parent-child relationship. This is because in Java, if something is asynchronous, you need an object to indicate the parent-child relationship. - -### Demo Execution - -Let me try the demo before we run out of time. - -Here, I am in a project called `docker-otel-lgtm`, which has examples of instrumentation and a complete observability stack based on Grafana technology. I will explain what I have done so far and then we will add more later. - -I have two Bill of Materials (BOMs). In Maven, this is the dependency management section. First, we have OpenTelemetry (the latest version) and Spring Boot dependencies. We want to ensure that the OpenTelemetry version is compatible with Spring Boot, and I’ve included a small controller that simply does dice rolls and has a player name. - -Starting the application now... - -While it starts, I am also starting the observability stack in a Docker container. This is a task runner using a shell script that runs Docker. - -On the left side is the application sending data to the OpenTelemetry collector, which sends the data to various databases. - -The goal of this project is to provide an easy trial experience, optimized for startup time. - -The application has started successfully. Now, I’ll generate some traffic using a curl script that pulls the endpoint. - -Let’s see if we have some data now... - -Yes, we have some dice rolls! Now let’s check the traces... - -Oh, we have an error. Let’s see what caused the error. - -### Observations - -I can see that the error occurs because the observability stack was not fully started. - -How many of you have used Spring Boot? (Audience responds.) - -How many have instrumented OpenTelemetry in any language? (More hands go up.) - -How many have used the two together? (Only one hand goes up.) - -### Feedback and Improvements - -If you see anything that could be improved, let me know. - -Do we need more spans? Yes, we could add more spans or even a database connection. - -If there’s nothing else, we can wrap up here, but if you have suggestions or requests, I’m happy to add more. - -### Conclusion +## Conclusion +To wrap up, we discussed the significance of the Spring Boot starter for OpenTelemetry, compared it to Micrometer, and walked through a demo showcasing its functionality. If you have any further questions about the implementation or concepts, feel free to ask! Thank you for your time! ## Raw YouTube Transcript -All right. Um, yeah, I want to talk about the spring starter for open telemetry. And, um, what I'll try to do is it just works, but except when it doesn't. So, that's, uh, for the demo gods and because it's spring boot. All right. Um, so about myself. Um actually what I didn't put on here but 20 years ago I started my first uh job as a Java developer in this building. Just realized today I Yeah. Yeah. Um it was not the first location but the first company and we moved here from a different location in Berlin to this one and I worked here for about three years. Um yeah 10 years later I um started using Spring Boot. Um I still don't know everything about it. Um five years ago I started uh working with hotel and I definitely know that I will never know everything about it. Um yeah and then I started working for Grafana two years ago and um now I'm an approver for hotel Java instrumentation. So the stuff that uh sends all the data but you will see that later. Um and I work a bit on some other areas in hotel um operator collector and specification. All right. Um so I want to uh give you like a quick overview of the state of hotel in 2025 what is new. Um then uh the motivation why building a spring boot startup in the first place. um also want to compare it quickly to other solutions in spring boot which is called micrometer. Um I want to uh spend most of the time in the demo to actually see um and show you that um it is easy and leave time for Q&A but uh you can also ask in between. I hope that we have enough time and if we don't then it's better if you ask before rather than I'm trying to finish the demo and it doesn't work. All right. Um so as you can see we have quite a lot of languages in hotel and um a lot of them have actually moved uh to stable stable stable for traces locks metrics. I think that's quite a bit of change for last year but I haven't recorded it. So Java which we are talking about PHP.NET and C++ and for some reason profiling is not listed here but this is like a new topic um that is uh an active development. I guess in a year or two this will also be listed here. Um and uh yeah I will not show that. Um okay state of hotel in Java. And Java is u one of the more mature areas in hotel probably because a lot of uh e-commerce and business applications are written in Java and there's a lot of demand and there are many many libraries um and uh just counted uh yesterday over 100 uh supported uh libraries are supported and it's also compatible with ancient versions of Java. So um Java 8 was released long more than 10 years ago but I did not count um and that is actually only possible because Java has a technology called byte code manipulation which means uh that uh you can write um a software that uh at runtime looks at all the methods and classes that are loaded and can basically change anything. It can change a return value and can look at parameters similar to how Python can do monkey patching. And uh now that I explained how the spring starter works, the question is uh isn't that good enough? Uh so the question is uh why uh did we or I and another engineer um from Microsoft why did we spend a huge amount of time last year working on making a spring boot startup. Um so let's start with what uh we cannot uh do or cannot yet do. uh we cannot match the huge number of libraries um more than 100 um simply because uh we cannot change old code and also because uh there's a lot of work being done um and u it would take a lot of time to have the same level and um then the question is what we can do and um as a spring boot user um I like to have a native ative spring boot experience. That means I can just select um a library add it as a dependency and it does is not anything special and that is exactly what I wanted to have for observability. It should not be anything special. Um it should be built into the development like you have a database. Well, it's probably kind of hard because without a database, you could not do anything. You could not serve customer requests. But it should be as close as possible to just adding a database. Um, and that is not possible. If you are adding a Java agent, then it's kind of this magic. Um, and as a spring boot user, you also are used to uh have a very integrated experience. That means you can take your configuration file called application YAML and you can just add properties and the the values and um names of those properties are automatically suggested. So you can basically um just select um service name because we we talked about service name a bit before and hopefully I can show you that this is very easy. Um, and then you can also uh use another Java agent uh to do your other magic work because adding two Java agents uh doesn't usually work well. And you can also use a Growlv VM native image which compiles your entire application into an executable which starts faster but it takes ages to compile. So I'm too impatient to show to that to you. But there is a full working example that uh where you can try that out. Yeah. Um and as I said I spent a lot of time last year and it also became stable last year. Um now I can talk about it. So um before going into exactly how the spring starter works um this is a recap of uh howl itself works. Um so this is not uh Java specific and not spring specific but in the next slide you will see how it relates to uh what I'm trying to show you. So um the order is uh my personal preference um and from most important to least important for for me the most important one is the protocol because once you have that protocol and you have your application you know data is sent out I can switch it somewhere else I don't have to recompile the application even if I don't have any developers and as we heard in the previous post even if the data is kind of crappy you can use AI I to make it better but you you need to have the data. Um yeah the next one is uh semantic conventions. Um because uh semantic conventions define the names and attributes. So for a metric it would be um the metric name of uh the duration of a request for example um and also the units. So metrics and traces for locks. This is a there is a bit of semantic conventions but not much. Third one is a API. API is more for uh library authors uh who want to uh be compatible with hotel and uh they don't want to uh spend time in five years uh rebuilding their observability stack and for that uh the APIs are stable and in hotel they are stable as you as you have seen in like here what it means stable is means the AP among others but one is that the AP API is stable. Um and one uh special thing is that for tracing the API is also important so that different uh libraries in the same process um know which uh parent span belongs to which child span because uh there are two cases. The easy cases you have a thread in Java and um the thread uh is for a request and the thing that is first in the thread is done first. This is a parent and the second one is the child. But sometimes you have um something that is asynchronous because it has an executive framework or something like that and then you need to have uh some kind of object that says this is a parent that is a child and that is what the API is for. Um lastly um SDK. The SDK is an implementation of the API and it also gives you configuration options for example with environment variables. The well the last one is not the least important one. It's just more like at the side the the tooling the collector and operator is actually very important and very helpful. All right. So now um I want uh to compare u the the the spring boot starter that I've worked on uh to the existing uh technologies in spring boot uh to to explain a little bit why we built something where kind of uh something similar is already there. So in micrometer um you have two things metrics is is quite old um and u used in many applications as far as I know um and more recently um a module has been added where you can um export the metrics using OTLP that's why OTLP the first one um but um as spring boot is older than hotel um they cannot just change the names um of their metrics or I mean they could but uh for spring uh they hold it very dear to uh not change anything for old applications because there are many old spring applications um um so changes are only in major versions and it takes a very long time and that's why they did not add yet add support for semantic conventions Um yeah but uh in in our in our spring starter so our when I say our it's not graphana where I work for but it's it's hotel as because um this is something that I worked on just as an hotel uh contributor um you can import those metrics because there are many applications and libraries that emit those metrics and in some cases can be useful to to use those micrometer tracing is The second part that is a bit newer. It was uh called something different uh slleuth. I think before it was called micrometer tracing but it's still newer than metrics. Um and in tracing you can use the hotel SDK um to emit um traces but it also does not have the semantic conventions even though it's newer. And um here you have the problem that it is not using the same API. Um that's why you have this problem here on the right side. Um and that has actually created um some uh questions and probably even more exploding heads that I'm not aware of for users who are trying to uh combine um some library with some other library. they usually don't know that one is using micrometer tracing and the other one is uh using hotel and uh sometimes works sometimes doesn't and sometimes works kind of um um yeah that's basically it um I would love to have like a a common um approach where uh there would be only either uh the micrometer APIs or the hotel APIs but this has not happened um because uh yeah this the spring uh folks have a rich tradition and they they know when they create an API they also want to keep it and u on the other hand the hotel folks can also or don't want to say oh we just take another API because it is for many languages and um picking a different API makes it very difficult for hotel users who are who actually love to have the same experience across all languages. All right. Um and this has been a lang a question that has uh been asked many times but so far uh nobody has really uh gave an answer to this. All right. Um this is basically what I said just more in a data flow kind of way. Um so in hotel we have instrumentation libraries um and in spring um there are instrumentation libraries in spring they are called native because they are not maintained by hotel and they are not repositories um but they are done by the the authors of different of the library itself. Um in spring this is sometimes also not uh they are also sometimes doing it for other libraries. [Music] Um and um yeah semantic conventions are I mentioned semantic conventions and later we will hopefully see why this is important. All right. Um that's actually it. Um, now I want to start the demo, but if you have any questions, um, it probably makes sense to ask them now before, uh, we go to the demo. All right. So, is used to work with um, zip key, right? Yes. And we have a zip key receiver in the collector. So, would it work to just send zip key to the collector and like does it work if I send zipin spans to the collector and hotel from my Python application to the collector and do they show up as one trace at the back end? I mean, I guess the question is on the context propagation side of things, right? Um, you could send both and they would both show up, but they would not have the correct parent child relationship because um internally Spring does not use Zipkin. uses something um else which is like a common denominator of both Zipkin and uh hotel because those are the the two targets for micrometer tracing. Um if if they would say okay we just want to optimize for hotel then it would work. Doesn't that work with the B3 headers? Um the headers are basically where it it's between processes. Yeah. Yeah, I mean I have a Python application that is sending data to my collector and hopefully that one generated a a span that originates the Java spans. Yeah, then it would work. So if if here between parent and child you have a process boundary, then it would work. If it's in the same process, then it only works in some cases. So if it's in the same Java thread, then I think it will work. Uh but if it's in a different Java thread because you have an exeutor then it will not work because the context is actually stored somewhere else as a matter object and Java does not have a context object like go for example does is that a property of Java or spring like kind of is that could this also happen just with general Java applications because I'm just thinking like we sometimes see that there are incorrect parent child relationships in spam and we're mostly using Java but I'm not sure if they only use spring. So I'm just asking like there are there are many uh scenarios where this can happen. Um even if you just use the Java agent which is like one distribution um it usually uh is because there is some other context propagation mechanism that is not uh supported or a library version uh upgrade uh changes the version how context is propagated. that that sometimes happens. But if it's between threads but in the same process, then it couldn't be you said like if it's in the same process, it could still like lead to this mismatch. Yes. But if it's in the same process cannot be like different methods of propagation at least I understand propagation is in between process. Oh, there propagation is the term that is both used between processes and also within the same process. Between processes you have um headers uh like Jerasi said B3 headers for example that are added to HTTP and in the same process it is uh some language specific u passing. Um so in Java by default the context is stored in a thread local because uh you can store a thread local without modifying the application. You don't have to pass a parameter. Um but that only works so long as uh the entire call is in one thread and uh new applications typically don't do that because this is not very efficient. So in general there is you have to expect that there are like linkage breaking between parents and child with Java. Um no no no no um the the Java agent authors uh have support for those technologies or libraries where it's not easy. So uh Java um exeutor for example is a framework where you can pass a runnable to a different in a pool and then it will be picked up by a number of different uh threads. um and they they do round robbins so that you don't need a thread for every request and um this bite code manipulation thing of the Java agent can see when uh you put an object into this uh pool and then it will um attach some metadata into that object. So uh maybe there's a map where you can put things into and then you can add a context ID I know a thread I u trace ID and a parent um span ID and then you can uh form the correct parent child relationship. But if that uh support is missing in the Java agent and here in the spring boot starter it it would be the same. If the spring boot starter does not support this then it would not work. What is even more difficult is if um there are two different instrumentation technologies that don't use the same API then they also don't know what they should expect as a context object and that is what is making it so difficult to combine a micrometer and hotel instrumentation. This might be the beard talking, but I miss EJB context. EJB. I mean, EJB context was perfect for this kind of situation. Does EJB even still exist? I don't. Maybe it would have been the solution. How old I am. The last time I used EJB was when I was in university. And as I just said, it's more than 20 years ago. Yeah. All right. Um, let me try with uh the demo before we run out of time. How much time do we have we left? Um, it's you have Okay, good. Good. As long as you still have some energy left. So, um, yeah, this is my, uh, Intelligj. Weirdly, it does not look uh, black even though like here I have the black theme. Oh, no. Oh, okay. Let's see if it does. Yep. Oh, it's Oh, wow. Should have checked that table. You just turned it off and turn it on. Yeah, it's all you have to know as an S. Okay. Um so here I'm not just in any project but I'm in the project uh called docker hotel lgtm which both has examples of uh instrumenting and also has a complete observability stack based on graphana technology of course because I'm working there um and uh I have created a new example um just so that it to get us started a bit faster. um and going to explain you what I have already done as the first iteration and then we we'll add stuff later on. So here I have uh two bill of materials. So in um we are using Maven this is a dependency management section and first we have open telemetry this is the latest version and then we have spring boot dependencies we have uh the open telemetry first so that uh if they are um libraries with the same name the one from the first one win and since uh spring boot has a micrometer um which has the hotel exporter but in an older version I want to make sure that we have the newer version and that they are compatible and then I just have a um starter web which is from spring and then I have the open telemetry spring boot starter and then there's really nothing here I don't know why we have repackage we have a spring boot app but there's nothing interesting here and then we have a small controller that uh is just doing uh rolls and it has a player name. So, first I want to start this. And while this is starting, I'm going to switch here where I have two windows. The first one is actually uh starting the image where we have the complete observability stack. So this one is actually just a um docker um script but um just to show you what this is really doing. So mis is just a task runner. So like make file but a bit nicer. And this one is just a shell script. And the shell script is just Docker. The only thing why I don't call Docker is that I probably forget some of the ports. By the way, I did not mention the ports. Yeah, I did not mention the whole slide because it said demo. Um, this is explaining the project uh much better um than I could do. Um so on the left side is the application that we're doing that is sending all the data to the hotel collector um which is sending the data to all the databases. The profiling one is uh not used here and then we can view it with graphana and that's why we have two ports on the left and one port on the right side. Yeah. So this is just um passing um the ports and uh some people like to actually persist the data but the the goal of the project is more to have an easy try out experience. So it's not optimized for uh uh scalability but it's optimized for startup time. Did it work out? Yeah, it started in 3 seconds. That was uh the latest uh iteration that I did because uh it was taking half a minute before. And then it also has an environment file which uh it's not really needed but I set it here to um an external endpoint. So I can also view the data in graphana cloud but this is actually not needed. Um okay so back to our application as it started now. Yeah I think in the beginning it um sent some errors because the observability stack was not started and now I will also put some traffic. This is just a curl script that pulls the endpoint. Sometimes it gives an error because the error is simulated. See if we actually get some data now. Yeah, we have dice rolls. Now let's actually see if we can see some data. Yeah, we have some data. We have some traces. Are the traces? Oh, cannot scroll down. Okay, so nothing really interesting. Uh why is it not interesting? Well, because the service is not calling any other service and it is not calling a database just sometimes has errors. Do we see errors? Let's see if that at least works. So this is by the way the graphana 12 that was just released today. I hope I'm uh using that correctly. Error. Yeah, it's error. So, [Music] here we have an error. Let me see if that works. Copy that. And Does it work? Oh, I did not copy the entire thing here. There's copy. Yep, that's where the error is thrown. Okay, so that part works. Oh, what I forgot to ask. Uh, how many of you have used uh Spring Boot? Okay, at least a couple people. Um, how many have instrumented Open Telemetry in any language? Even more. Good. How many have used the two together? Oh, one. Okay, good. Okay. But uh next question is not about instrumenting. Do you see anything that that could be improved that we should make better or that you want to have added more more spans? More spans. Yeah. Um an unknown service. Exactly. Unknown service. And I promise that this should be easy to do. So let's see if we can do that. Um, where's my directory here? Very slow today. So, let me see. Sir, this [Music] Service service hotel service name. No, but it's a different one. Okay. Um, let me see if I have hotel. Oh, is it not recognizing? Maybe I not I have to reload. Maven did work. today. But the goat the goat. Yeah, I did not sacrifice a goat. So I guess I have a question. Why do you look look it up? Um the new way of doing things is using um hotel config files that can be reused across services. That is right. Yeah. would that come to to the um the screen boot starter as well like can I could I just add my hotel.l file there instead of um configuring to the partition. Yeah. So before answering it just a bit on the background um right now um in hotel the standard way of doing things is environment variables and that is uh quite limited. Uh so the like the most fancy thing is if I write uh hotel resource at oh it's even completing that but this is not spring this is u the AI so I can say service name service version something like that um and anything more complicated uh is uh specific to any language um and is currently blocked and uh what I'm unsuccessfully trying to show you is the only way where where you can currently use YAML as far as I know. Um and what Drussi is saying is that um the the plan currently is to have the same experience well not in spring syntax but in YAML or in any unstructured any um structured data um to have the same experience. Um, and for spring we would probably have a major version where we would tell you to switch from the syntax uh that I'm trying to show here to the one that is compatible. I have spent some time trying to figure out if this can be integrated with spring because here you can also use a spring expression. So in spring you have a a u language where you can reference environment variables. I don't know if I can if I get it right if that is actually correct or if this is just an AI hallucination but it's there is a spring feature where you can reference other parts in spring and you can even reference Java beans and so on. And what I want for the future is that you can also have this because you're used to it as a spring user but with the common YAML format. I'm not sure if that is possible because um this is already stretching the limits of spring um and um it will be even more so with the new format. So I hope it's possible but uh no no promises. So, okay, since I'm trying to do that here, I'll just uh look in my stash to see if uh I'm just too stupid to find um the right format. Resource attributes, hotel service name. Yeah, the autocomp completion just didn't work for some reason. It's not suggesting it. So show up as a plugin on the right hand side. There's no plugin, right? Actually this is a feature of uh the um dependency here. So it basically just has a JSON file where it lists all the properties and it yeah should work. Let's see if it's uh if it's working. If we restart the service um so here tracing is always uh the tracing locks is the fastest to show because it's processed right away. metrics is ingested every minute. Um you can change it to more often but um for this demo it's not necessary. So see the last one. No, it still has unknown service. Well, what I had before is that when I started in spring, it's not working. But when I use maven to start it then it is working. So that can can be tricky. Let me see if that is the case. Oh, it's still unknown service. They look for unknown service. I think that's too old by now. That's still a node price that you were looking at. Can you filter by service? It's it should it actually says service name here. So, it's automatically grouping by service name. That is not the problem. I'm probably doing something wrong because I put it in a wrong directory or something. something that is uh a typical spring problem that uh that happens uh every time when I try to do something new. Okay. Um I'm just going to delete that file and apply the stash as I had it before. Oh yeah, it's in a different directory. That's That's why my error. Yeah. Okay. Good that I had the stash. Let me see if I can also type service. Yep. Now it works. Okay. At least that worked. Okay. But now I made the mistake that I started it here and also in the other one. It's probably saying that the port is already used. Started it here and no seems that it is loaded. Okay. See if it works better here. Here. Here it is. Okay. Here's rolled dice. Okay. Then let's just look at that instead. So here now I can show you where the service name is. Here is service name and then you also have a bunch of stuff that we did not specify and that is because there are resource detectors uh so that you don't have to do all the work yourself. So for example, you have the process ID that that was taken from the operating system and and you can also see that it was created by the spring boot starter. So if you're reporting a buck, then it's easier for the authors to find where this is coming from. And you have also this uh correlation that you have between some metric types. You can actually see the trace that was the trace that that's that uh started before the lock message was emitted. Okay. Um now that we have that al let's look at some metrics. Uh so there are um there is this duration metric that I talked about before. Um and that has the same name across all languages. And that is really useful for having an APM system where you can uh see what uh what the request rates um for all of your services are. Um here this is the open source version of Graphana. So you don't have the APM tool here. All right. Um I would have uh some things that we could do. We could add a manual metric or um a so an additional um metric where you can count the number of requests or players. Um we can add more um spans. I heard that before. Or we can add a database or we do nothing of it and we just leave more time for questions. [Music] So if you don't have anything that you want then I think we can just cut it here. But if you want something then I will add it. Yeah. I just have quick question. Why service main not on the resource attribute? Oh yeah that's a good one. Um let's I think we saw it here. Um that is uh the visualization of uh Loki. Loki shows everything flat and it's actually you're not the first one to notice that it it would be more useful if that was structured in traces. It is structured. Yeah. Yeah. It's a good point. I think it would be better. And now it's also showing up here for some reason didn't before. So here it is in resource attributes and it's down there. It's there. Service name, right? Yeah, here it is. But in the configuration you have to add it to a different place or is that like kind of free? You can Oh, service name is special. Uh in open telemetry service name is special. Um it internally it translates into resource attribute but it is like the single one that is required by every every telemetry data that I could also add it here by the way. Yeah I also find that confusing that there are two ways and I just had a customer call today where we found out that in some cases it does matter where you put it even though it should not. Yeah. Yeah, I mean I think yeah it's an implementation detail. Um basically service name is the only required attribute for open telemetry and it it is top level. An implementation detail is it is a resource attribute and so that's why sometimes it works sometimes it doesn't. I mean in theory it should only work if it is at top level right because if you add it as resource attribute then it's probably not recognized at the top level. Um confusion in OTLP it it is only resource attributes so it's only within the resource attributes that is a special one called service name and that's it uh so that's why some if you specify environment variable with the service name it it might work but then depending on the library um if you don't have the proper service name environment variable it overrides the the one that you would override with the resource attributes with a default one so that's why sometimes it doesn't work sometimes it So if as a user you should only use at the top level not at the resource attributes. So when I see in graphana on the resource attributes that's just the dual thing basically. Yeah. Is that really true? It is how it is internally uh stored. Um so in graphana you see in two places. One is uh if you go back there to to the screen uh you see the the at this um here. Yeah, if you go up there you see the road dice uh up there at the very like the trace summary. Um even more here service even more so perhaps even more. So no I cannot go up more. You can go here. So here is a service name. So that's okay. Yeah. Yeah. Got it. So but uh internally guess the service name that is within resources. So in graphana I know if you go like to the traql um quer so instead of using traceql itself it has this kind of building query by the same then there's like the top two or so. Yeah exactly. So if you go to search now not the trace. So service name, span name and status are like the special one. Yes. Well um for open telemetry only the service name is special. Um the folks at tempo they said you know span name is interesting to have it as part of the trace syntax and so it's there. Um but from the hotel perspective the only thing that we require is a service name. Even even then we don't throw an error. uh the specification says that if a service name is not specified the unknown service should be used which is what we've seen before right so the spring boot application was working without the service name and it was showing unknown service but it is a it is against this spec it should resources then in the application yo you should not put it under resources no no it's it's perfectly fine it works yeah but as a user you should not as a user you should pro probably not do that Yeah, got it. Okay. I mean it might work for spring but it might not work for only lambdas. I mean I think perp spec it should work but the case I had today was where it actually did not work. They were putting it in environment variable but it's basically the same. Most people I think most authors expected in service name that's why it's safer to put it in there. Yeah. This is also just my last obligatory slide. That's why I put it on here. Cool. All right. Thanks for your time. +All right. Um, yeah, I want to talk about the spring starter for open telemetry. And, um, what I'll try to do is it just works, but except when it doesn't. So, that's, uh, for the demo gods and because it's spring boot. All right. Um, so about myself. Um actually what I didn't put on here but 20 years ago I started my first uh job as a Java developer in this building. Just realized today I Yeah. Yeah. Um it was not the first location but the first company and we moved here from a different location in Berlin to this one and I worked here for about three years. Um yeah 10 years later I um started using Spring Boot. Um I still don't know everything about it. Um five years ago I started uh working with hotel and I definitely know that I will never know everything about it. Um yeah and then I started working for Grafana two years ago and um now I'm an approver for hotel Java instrumentation. So the stuff that uh sends all the data but you will see that later. Um and I work a bit on some other areas in hotel um operator collector and specification. All right. Um so I want to uh give you like a quick overview of the state of hotel in 2025 what is new. Um then uh the motivation why building a spring boot startup in the first place. um also want to compare it quickly to other solutions in spring boot which is called micrometer. Um I want to uh spend most of the time in the demo to actually see um and show you that um it is easy and leave time for Q&A but uh you can also ask in between. I hope that we have enough time and if we don't then it's better if you ask before rather than I'm trying to finish the demo and it doesn't work. All right. Um so as you can see we have quite a lot of languages in hotel and um a lot of them have actually moved uh to stable stable stable for traces locks metrics. I think that's quite a bit of change for last year but I haven't recorded it. So Java which we are talking about PHP.NET and C++ and for some reason profiling is not listed here but this is like a new topic um that is uh an active development. I guess in a year or two this will also be listed here. Um and uh yeah I will not show that. Um okay state of hotel in Java. And Java is u one of the more mature areas in hotel probably because a lot of uh e-commerce and business applications are written in Java and there's a lot of demand and there are many many libraries um and uh just counted uh yesterday over 100 uh supported uh libraries are supported and it's also compatible with ancient versions of Java. So um Java 8 was released long more than 10 years ago but I did not count um and that is actually only possible because Java has a technology called byte code manipulation which means uh that uh you can write um a software that uh at runtime looks at all the methods and classes that are loaded and can basically change anything. It can change a return value and can look at parameters similar to how Python can do monkey patching. And uh now that I explained how the spring starter works, the question is uh isn't that good enough? Uh so the question is uh why uh did we or I and another engineer um from Microsoft why did we spend a huge amount of time last year working on making a spring boot startup. Um so let's start with what uh we cannot uh do or cannot yet do. uh we cannot match the huge number of libraries um more than 100 um simply because uh we cannot change old code and also because uh there's a lot of work being done um and u it would take a lot of time to have the same level and um then the question is what we can do and um as a spring boot user um I like to have a native ative spring boot experience. That means I can just select um a library add it as a dependency and it does is not anything special and that is exactly what I wanted to have for observability. It should not be anything special. Um it should be built into the development like you have a database. Well, it's probably kind of hard because without a database, you could not do anything. You could not serve customer requests. But it should be as close as possible to just adding a database. Um, and that is not possible. If you are adding a Java agent, then it's kind of this magic. Um, and as a spring boot user, you also are used to uh have a very integrated experience. That means you can take your configuration file called application YAML and you can just add properties and the the values and um names of those properties are automatically suggested. So you can basically um just select um service name because we we talked about service name a bit before and hopefully I can show you that this is very easy. Um, and then you can also uh use another Java agent uh to do your other magic work because adding two Java agents uh doesn't usually work well. And you can also use a Growlv VM native image which compiles your entire application into an executable which starts faster but it takes ages to compile. So I'm too impatient to show to that to you. But there is a full working example that uh where you can try that out. Yeah. Um and as I said I spent a lot of time last year and it also became stable last year. Um now I can talk about it. So um before going into exactly how the spring starter works um this is a recap of uh howl itself works. Um so this is not uh Java specific and not spring specific but in the next slide you will see how it relates to uh what I'm trying to show you. So um the order is uh my personal preference um and from most important to least important for for me the most important one is the protocol because once you have that protocol and you have your application you know data is sent out I can switch it somewhere else I don't have to recompile the application even if I don't have any developers and as we heard in the previous post even if the data is kind of crappy you can use AI I to make it better but you you need to have the data. Um yeah the next one is uh semantic conventions. Um because uh semantic conventions define the names and attributes. So for a metric it would be um the metric name of uh the duration of a request for example um and also the units. So metrics and traces for locks. This is a there is a bit of semantic conventions but not much. Third one is a API. API is more for uh library authors uh who want to uh be compatible with hotel and uh they don't want to uh spend time in five years uh rebuilding their observability stack and for that uh the APIs are stable and in hotel they are stable as you as you have seen in like here what it means stable is means the AP among others but one is that the AP API is stable. Um and one uh special thing is that for tracing the API is also important so that different uh libraries in the same process um know which uh parent span belongs to which child span because uh there are two cases. The easy cases you have a thread in Java and um the thread uh is for a request and the thing that is first in the thread is done first. This is a parent and the second one is the child. But sometimes you have um something that is asynchronous because it has an executive framework or something like that and then you need to have uh some kind of object that says this is a parent that is a child and that is what the API is for. Um lastly um SDK. The SDK is an implementation of the API and it also gives you configuration options for example with environment variables. The well the last one is not the least important one. It's just more like at the side the the tooling the collector and operator is actually very important and very helpful. All right. So now um I want uh to compare u the the the spring boot starter that I've worked on uh to the existing uh technologies in spring boot uh to to explain a little bit why we built something where kind of uh something similar is already there. So in micrometer um you have two things metrics is is quite old um and u used in many applications as far as I know um and more recently um a module has been added where you can um export the metrics using OTLP that's why OTLP the first one um but um as spring boot is older than hotel um they cannot just change the names um of their metrics or I mean they could but uh for spring uh they hold it very dear to uh not change anything for old applications because there are many old spring applications um um so changes are only in major versions and it takes a very long time and that's why they did not add yet add support for semantic conventions Um yeah but uh in in our in our spring starter so our when I say our it's not graphana where I work for but it's it's hotel as because um this is something that I worked on just as an hotel uh contributor um you can import those metrics because there are many applications and libraries that emit those metrics and in some cases can be useful to to use those micrometer tracing is The second part that is a bit newer. It was uh called something different uh slleuth. I think before it was called micrometer tracing but it's still newer than metrics. Um and in tracing you can use the hotel SDK um to emit um traces but it also does not have the semantic conventions even though it's newer. And um here you have the problem that it is not using the same API. Um that's why you have this problem here on the right side. Um and that has actually created um some uh questions and probably even more exploding heads that I'm not aware of for users who are trying to uh combine um some library with some other library. they usually don't know that one is using micrometer tracing and the other one is uh using hotel and uh sometimes works sometimes doesn't and sometimes works kind of um um yeah that's basically it um I would love to have like a a common um approach where uh there would be only either uh the micrometer APIs or the hotel APIs but this has not happened um because uh yeah this the spring uh folks have a rich tradition and they they know when they create an API they also want to keep it and u on the other hand the hotel folks can also or don't want to say oh we just take another API because it is for many languages and um picking a different API makes it very difficult for hotel users who are who actually love to have the same experience across all languages. All right. Um and this has been a lang a question that has uh been asked many times but so far uh nobody has really uh gave an answer to this. All right. Um this is basically what I said just more in a data flow kind of way. Um so in hotel we have instrumentation libraries um and in spring um there are instrumentation libraries in spring they are called native because they are not maintained by hotel and they are not repositories um but they are done by the the authors of different of the library itself. Um in spring this is sometimes also not uh they are also sometimes doing it for other libraries. Um and um yeah semantic conventions are I mentioned semantic conventions and later we will hopefully see why this is important. All right. Um that's actually it. Um, now I want to start the demo, but if you have any questions, um, it probably makes sense to ask them now before, uh, we go to the demo. All right. So, is used to work with um, zip key, right? Yes. And we have a zip key receiver in the collector. So, would it work to just send zip key to the collector and like does it work if I send zipin spans to the collector and hotel from my Python application to the collector and do they show up as one trace at the back end? I mean, I guess the question is on the context propagation side of things, right? Um, you could send both and they would both show up, but they would not have the correct parent child relationship because um internally Spring does not use Zipkin. uses something um else which is like a common denominator of both Zipkin and uh hotel because those are the the two targets for micrometer tracing. Um if if they would say okay we just want to optimize for hotel then it would work. Doesn't that work with the B3 headers? Um the headers are basically where it it's between processes. Yeah. Yeah, I mean I have a Python application that is sending data to my collector and hopefully that one generated a a span that originates the Java spans. Yeah, then it would work. So if if here between parent and child you have a process boundary, then it would work. If it's in the same process, then it only works in some cases. So if it's in the same Java thread, then I think it will work. Uh but if it's in a different Java thread because you have an exeutor then it will not work because the context is actually stored somewhere else as a matter object and Java does not have a context object like go for example does is that a property of Java or spring like kind of is that could this also happen just with general Java applications because I'm just thinking like we sometimes see that there are incorrect parent child relationships in spam and we're mostly using Java but I'm not sure if they only use spring. So I'm just asking like there are there are many uh scenarios where this can happen. Um even if you just use the Java agent which is like one distribution um it usually uh is because there is some other context propagation mechanism that is not uh supported or a library version uh upgrade uh changes the version how context is propagated. that that sometimes happens. But if it's between threads but in the same process, then it couldn't be you said like if it's in the same process, it could still like lead to this mismatch. Yes. But if it's in the same process cannot be like different methods of propagation at least I understand propagation is in between process. Oh, there propagation is the term that is both used between processes and also within the same process. Between processes you have um headers uh like Jerasi said B3 headers for example that are added to HTTP and in the same process it is uh some language specific u passing. Um so in Java by default the context is stored in a thread local because uh you can store a thread local without modifying the application. You don't have to pass a parameter. Um but that only works so long as uh the entire call is in one thread and uh new applications typically don't do that because this is not very efficient. So in general there is you have to expect that there are like linkage breaking between parents and child with Java. Um no no no no um the the Java agent authors uh have support for those technologies or libraries where it's not easy. So uh Java um exeutor for example is a framework where you can pass a runnable to a different in a pool and then it will be picked up by a number of different uh threads. um and they they do round robbins so that you don't need a thread for every request and um this bite code manipulation thing of the Java agent can see when uh you put an object into this uh pool and then it will um attach some metadata into that object. So uh maybe there's a map where you can put things into and then you can add a context ID I know a thread I u trace ID and a parent um span ID and then you can uh form the correct parent child relationship. But if that uh support is missing in the Java agent and here in the spring boot starter it it would be the same. If the spring boot starter does not support this then it would not work. What is even more difficult is if um there are two different instrumentation technologies that don't use the same API then they also don't know what they should expect as a context object and that is what is making it so difficult to combine a micrometer and hotel instrumentation. This might be the beard talking, but I miss EJB context. EJB. I mean, EJB context was perfect for this kind of situation. Does EJB even still exist? I don't. Maybe it would have been the solution. How old I am. The last time I used EJB was when I was in university. And as I just said, it's more than 20 years ago. Yeah. All right. Um, let me try with uh the demo before we run out of time. How much time do we have we left? Um, it's you have Okay, good. Good. As long as you still have some energy left. So, um, yeah, this is my, uh, Intelligj. Weirdly, it does not look uh, black even though like here I have the black theme. Oh, no. Oh, okay. Let's see if it does. Yep. Oh, it's Oh, wow. Should have checked that table. You just turned it off and turn it on. Yeah, it's all you have to know as an S. Okay. Um so here I'm not just in any project but I'm in the project uh called docker hotel lgtm which both has examples of uh instrumenting and also has a complete observability stack based on graphana technology of course because I'm working there um and uh I have created a new example um just so that it to get us started a bit faster. um and going to explain you what I have already done as the first iteration and then we we'll add stuff later on. So here I have uh two bill of materials. So in um we are using Maven this is a dependency management section and first we have open telemetry this is the latest version and then we have spring boot dependencies we have uh the open telemetry first so that uh if they are um libraries with the same name the one from the first one win and since uh spring boot has a micrometer um which has the hotel exporter but in an older version I want to make sure that we have the newer version and that they are compatible and then I just have a um starter web which is from spring and then I have the open telemetry spring boot starter and then there's really nothing here I don't know why we have repackage we have a spring boot app but there's nothing interesting here and then we have a small controller that uh is just doing uh rolls and it has a player name. So, first I want to start this. And while this is starting, I'm going to switch here where I have two windows. The first one is actually uh starting the image where we have the complete observability stack. So this one is actually just a um docker um script but um just to show you what this is really doing. So mis is just a task runner. So like make file but a bit nicer. And this one is just a shell script. And the shell script is just Docker. The only thing why I don't call Docker is that I probably forget some of the ports. By the way, I did not mention the ports. Yeah, I did not mention the whole slide because it said demo. Um, this is explaining the project uh much better um than I could do. Um so on the left side is the application that we're doing that is sending all the data to the hotel collector um which is sending the data to all the databases. The profiling one is uh not used here and then we can view it with graphana and that's why we have two ports on the left and one port on the right side. Yeah. So this is just um passing um the ports and uh some people like to actually persist the data but the the goal of the project is more to have an easy try out experience. So it's not optimized for uh uh scalability but it's optimized for startup time. Did it work out? Yeah, it started in 3 seconds. That was uh the latest uh iteration that I did because uh it was taking half a minute before. And then it also has an environment file which uh it's not really needed but I set it here to um an external endpoint. So I can also view the data in graphana cloud but this is actually not needed. Um okay so back to our application as it started now. Yeah I think in the beginning it um sent some errors because the observability stack was not started and now I will also put some traffic. This is just a curl script that pulls the endpoint. Sometimes it gives an error because the error is simulated. See if we actually get some data now. Yeah, we have dice rolls. Now let's actually see if we can see some data. Yeah, we have some data. We have some traces. Are the traces? Oh, cannot scroll down. Okay, so nothing really interesting. Uh why is it not interesting? Well, because the service is not calling any other service and it is not calling a database just sometimes has errors. Do we see errors? Let's see if that at least works. So this is by the way the graphana 12 that was just released today. I hope I'm uh using that correctly. Error. Yeah, it's error. So, here we have an error. Let me see if that works. Copy that. And Does it work? Oh, I did not copy the entire thing here. There's copy. Yep, that's where the error is thrown. Okay, so that part works. Oh, what I forgot to ask. Uh, how many of you have used uh Spring Boot? Okay, at least a couple people. Um, how many have instrumented Open Telemetry in any language? Even more. Good. How many have used the two together? Oh, one. Okay, good. Okay. But uh next question is not about instrumenting. Do you see anything that that could be improved that we should make better or that you want to have added more more spans? More spans. Yeah. Um an unknown service. Exactly. Unknown service. And I promise that this should be easy to do. So let's see if we can do that. Um, where's my directory here? Very slow today. So, let me see. Sir, this Service service hotel service name. No, but it's a different one. Okay. Um, let me see if I have hotel. Oh, is it not recognizing? Maybe I not I have to reload. Maven did work. today. But the goat the goat. Yeah, I did not sacrifice a goat. So I guess I have a question. Why do you look look it up? Um the new way of doing things is using um hotel config files that can be reused across services. That is right. Yeah. would that come to to the um the screen boot starter as well like can I could I just add my hotel.l file there instead of um configuring to the partition. Yeah. So before answering it just a bit on the background um right now um in hotel the standard way of doing things is environment variables and that is uh quite limited. Uh so the like the most fancy thing is if I write uh hotel resource at oh it's even completing that but this is not spring this is u the AI so I can say service name service version something like that um and anything more complicated uh is uh specific to any language um and is currently blocked and uh what I'm unsuccessfully trying to show you is the only way where where you can currently use YAML as far as I know. Um and what Drussi is saying is that um the the plan currently is to have the same experience well not in spring syntax but in YAML or in any unstructured any um structured data um to have the same experience. Um, and for spring we would probably have a major version where we would tell you to switch from the syntax uh that I'm trying to show here to the one that is compatible. I have spent some time trying to figure out if this can be integrated with spring because here you can also use a spring expression. So in spring you have a a u language where you can reference environment variables. I don't know if I can if I get it right if that is actually correct or if this is just an AI hallucination but it's there is a spring feature where you can reference other parts in spring and you can even reference Java beans and so on. And what I want for the future is that you can also have this because you're used to it as a spring user but with the common YAML format. I'm not sure if that is possible because um this is already stretching the limits of spring um and um it will be even more so with the new format. So I hope it's possible but uh no no promises. So, okay, since I'm trying to do that here, I'll just uh look in my stash to see if uh I'm just too stupid to find um the right format. Resource attributes, hotel service name. Yeah, the autocomp completion just didn't work for some reason. It's not suggesting it. So show up as a plugin on the right hand side. There's no plugin, right? Actually this is a feature of uh the um dependency here. So it basically just has a JSON file where it lists all the properties and it yeah should work. Let's see if it's uh if it's working. If we restart the service um so here tracing is always uh the tracing locks is the fastest to show because it's processed right away. metrics is ingested every minute. Um you can change it to more often but um for this demo it's not necessary. So see the last one. No, it still has unknown service. Well, what I had before is that when I started in spring, it's not working. But when I use maven to start it then it is working. So that can can be tricky. Let me see if that is the case. Oh, it's still unknown service. They look for unknown service. I think that's too old by now. That's still a node price that you were looking at. Can you filter by service? It's it should it actually says service name here. So, it's automatically grouping by service name. That is not the problem. I'm probably doing something wrong because I put it in a wrong directory or something. something that is uh a typical spring problem that uh that happens uh every time when I try to do something new. Okay. Um I'm just going to delete that file and apply the stash as I had it before. Oh yeah, it's in a different directory. That's That's why my error. Yeah. Okay. Good that I had the stash. Let me see if I can also type service. Yep. Now it works. Okay. At least that worked. Okay. But now I made the mistake that I started it here and also in the other one. It's probably saying that the port is already used. Started it here and no seems that it is loaded. Okay. See if it works better here. Here. Here it is. Okay. Here's rolled dice. Okay. Then let's just look at that instead. So here now I can show you where the service name is. Here is service name and then you also have a bunch of stuff that we did not specify and that is because there are resource detectors uh so that you don't have to do all the work yourself. So for example, you have the process ID that that was taken from the operating system and and you can also see that it was created by the spring boot starter. So if you're reporting a buck, then it's easier for the authors to find where this is coming from. And you have also this uh correlation that you have between some metric types. You can actually see the trace that was the trace that that's that uh started before the lock message was emitted. Okay. Um now that we have that al let's look at some metrics. Uh so there are um there is this duration metric that I talked about before. Um and that has the same name across all languages. And that is really useful for having an APM system where you can uh see what uh what the request rates um for all of your services are. Um here this is the open source version of Graphana. So you don't have the APM tool here. All right. Um I would have uh some things that we could do. We could add a manual metric or um a so an additional um metric where you can count the number of requests or players. Um we can add more um spans. I heard that before. Or we can add a database or we do nothing of it and we just leave more time for questions. So if you don't have anything that you want then I think we can just cut it here. But if you want something then I will add it. Yeah. I just have quick question. Why service main not on the resource attribute? Oh yeah that's a good one. Um let's I think we saw it here. Um that is uh the visualization of uh Loki. Loki shows everything flat and it's actually you're not the first one to notice that it it would be more useful if that was structured in traces. It is structured. Yeah. Yeah. It's a good point. I think it would be better. And now it's also showing up here for some reason didn't before. So here it is in resource attributes and it's down there. It's there. Service name, right? Yeah, here it is. But in the configuration you have to add it to a different place or is that like kind of free? You can Oh, service name is special. Uh in open telemetry service name is special. Um it internally it translates into resource attribute but it is like the single one that is required by every every telemetry data that I could also add it here by the way. Yeah I also find that confusing that there are two ways and I just had a customer call today where we found out that in some cases it does matter where you put it even though it should not. Yeah. Yeah, I mean I think yeah it's an implementation detail. Um basically service name is the only required attribute for open telemetry and it it is top level. An implementation detail is it is a resource attribute and so that's why sometimes it works sometimes it doesn't. I mean in theory it should only work if it is at top level right because if you add it as resource attribute then it's probably not recognized at the top level. Um confusion in OTLP it it is only resource attributes so it's only within the resource attributes that is a special one called service name and that's it uh so that's why some if you specify environment variable with the service name it it might work but then depending on the library um if you don't have the proper service name environment variable it overrides the the one that you would override with the resource attributes with a default one so that's why sometimes it doesn't work sometimes it So if as a user you should only use at the top level not at the resource attributes. So when I see in graphana on the resource attributes that's just the dual thing basically. Yeah. Is that really true? It is how it is internally uh stored. Um so in graphana you see in two places. One is uh if you go back there to to the screen uh you see the the at this um here. Yeah, if you go up there you see the road dice uh up there at the very like the trace summary. Um even more here service even more so perhaps even more. So no I cannot go up more. You can go here. So here is a service name. So that's okay. Yeah. Yeah. Got it. So but uh internally guess the service name that is within resources. So in graphana I know if you go like to the traql um quer so instead of using traceql itself it has this kind of building query by the same then there's like the top two or so. Yeah exactly. So if you go to search now not the trace. So service name, span name and status are like the special one. Yes. Well um for open telemetry only the service name is special. Um the folks at tempo they said you know span name is interesting to have it as part of the trace syntax and so it's there. Um but from the hotel perspective the only thing that we require is a service name. Even even then we don't throw an error. uh the specification says that if a service name is not specified the unknown service should be used which is what we've seen before right so the spring boot application was working without the service name and it was showing unknown service but it is a it is against this spec it should resources then in the application yo you should not put it under resources no no it's it's perfectly fine it works yeah but as a user you should not as a user you should pro probably not do that Yeah, got it. Okay. I mean it might work for spring but it might not work for only lambdas. I mean I think perp spec it should work but the case I had today was where it actually did not work. They were putting it in environment variable but it's basically the same. Most people I think most authors expected in service name that's why it's safer to put it in there. Yeah. This is also just my last obligatory slide. That's why I put it on here. Cool. All right. Thanks for your time. diff --git a/video-transcripts/transcripts/2025-05-07T18:48:36Z-otel-night-berlin-v2025-05-leveraging-ai-for-opentelemetry-data.md b/video-transcripts/transcripts/2025-05-07T18:48:36Z-otel-night-berlin-v2025-05-leveraging-ai-for-opentelemetry-data.md index bc5e18a..67e477a 100644 --- a/video-transcripts/transcripts/2025-05-07T18:48:36Z-otel-night-berlin-v2025-05-leveraging-ai-for-opentelemetry-data.md +++ b/video-transcripts/transcripts/2025-05-07T18:48:36Z-otel-night-berlin-v2025-05-leveraging-ai-for-opentelemetry-data.md @@ -10,89 +10,103 @@ URL: https://www.youtube.com/watch?v=8JlFuGTDCXQ ## Summary -In this video, Lariel, a principal AI engineer at Dash Zero, shares insights into applying AI to open telemetry data, focusing on a case study of their log AI. Lariel discusses the challenges of making sense of unstructured logs, which often appear messy and unhelpful compared to the ideal structured formats presented in demo datasets. She outlines the development of their log AI system, which aims to automate log parsing and abstraction without needing manual regular expressions. The solution combines various techniques, including resource clustering and prompt engineering with language models, to effectively identify and parse log patterns. Lariel emphasizes the importance of evaluation, achieving a 98% success rate in log parsing accuracy, and highlights the potential applications of AI in observability, including query generation and trace analysis. The session concludes with key takeaways about the cost of AI in production, the use of traditional machine learning in conjunction with language models, and the significance of context in AI prompts. +In this YouTube video, Lariel, a principal AI engineer at Dash Zero, shares insights into applying AI to open telemetry data, focusing on a case study about log processing. Lariel discusses the challenges of parsing unstructured logs and the limitations of existing models, such as the drain model and large language models (LLMs). The presentation outlines a novel approach combining various techniques to efficiently predict log formats and patterns without requiring extensive human intervention. Key points include the importance of resource clustering, the integration of traditional machine learning with LLMs, and the evaluation of the system’s performance, which demonstrated a 98% success rate in log parsing. Lariel highlights the significance of context and semantic conventions in improving AI predictions. The video concludes with insights into future developments and the potential for further collaboration in the observability space. -# AI Application in Open Telemetry Data +## Chapters -## Introduction +Sure! Here are the key moments from the livestream with their corresponding timestamps: -Hello everyone, I'm Lariel. Today, I will share my experience applying AI to open telemetry data. +00:00:00 Introductions and background of Lariel +00:02:15 Overview of Dash Zero and its focus on observability +00:04:00 Introduction to the case study on log AI +00:05:30 Challenges with unstructured logs in real-world applications +00:08:00 Limitations of existing models for log parsing +00:11:45 Exploring alternatives like LLMs and deep learning for log parsing +00:15:00 Explanation of the combined approach to log parsing +00:18:30 Overview of the log parsing pipeline and resource clustering +00:23:00 Evaluation of the log AI model and its success rates +00:27:00 Demonstration of results and visualization in the UI +00:30:30 Key takeaways and additional experiments with AI agents -### About Me +Feel free to ask if you need any further assistance! -I’m from Brazil and moved to Berlin a couple of years ago. Currently, I work as a Principal AI Engineer at Dash Zero. Before that, I played various roles in data and AI, from data engineering to data science and ML Ops. My CV is quite diverse. +# AI and Open Telemetry Data: A Case Study on Log AI -For those who might not have heard of Dash Zero, we are an open telemetry native SaaS solution for observability. We work with metrics, traces, and logs, providing dashboarding and alerting services. Our platform is built on open standards like Prometheus and Prometheus Remote Storage, and we're developing innovative AI capabilities to help users derive more value from their data. +**Introduction** -## Main Subject: Case Study on Log AI +Hello everyone, I'm Lariel. Today, I'm going to share my experience in applying AI to open telemetry data. A brief introduction about myself: I'm from Brazil and moved to Berlin a couple of years ago. Currently, I work as a Principal AI Engineer at Dash Zero. Before that, I held various roles in data and AI, ranging from data engineering to data science and ML Ops. My CV looks a bit chaotic nowadays! -The main focus of today’s talk is a case study on our Log AI and how to make sense of unstructured logs. +For those who haven't heard of Dash Zero, we are an open telemetry native SaaS solution for observability, working with metrics, traces, and logs. We provide dashboarding and alerting services, built on open standards like Prometheus and Persis. We're also developing several AI capabilities to help our users derive more value from the data they send to us. -### The Challenge with Logs +**Main Subject: Log AI Case Study** -Many of you might have experienced disappointment when using open telemetry demo datasets. The logs initially seem perfect—shipped via OTLP and formatted as JSON with ample useful attributes. However, in reality, most logs look far less organized. For instance, when working with Kubernetes, logs collected from different nodes might appear as raw strings with basic attributes, making them challenging to interpret. +The main subject of today's talk is a case study on our Log AI, focusing on how to make sense of unstructured logs. -There is often hidden information in logs, such as module names, durations, timestamps, and severity levels. The motivation behind our Log AI is to parse and abstract logs at scale for any application without needing human intervention or custom regular expressions. +Many of you may have experienced disappointment when using open telemetry demo datasets. Initially, the logs look beautiful, shipped via OTLP with useful attributes and formatted as JSON, which makes them easy to work with. However, in reality, most people's logs look quite different. For instance, when running applications on Kubernetes, logs collected from each node often result in raw string bodies with minimal useful attributes. While there is information embedded in these logs, such as module names, durations, timestamps, and severity levels, the challenge is finding a way to harness that information effectively. -### Our Approach to Log Parsing +This is the motivation behind our Log AI. We aim to perform log parsing and abstraction at scale for any logs from any application without requiring human intervention or custom regular expressions. For example, when parsing logs, we want to separate the prefix (containing timestamps, host, and severity) from the message itself. Within the messages, there are patterns that we want to identify without manual regex. -We aim to separate prefixes from messages in logs. For example, consider a log entry with a prefix containing a timestamp, host, and severity level. The messages may follow different patterns, such as “ad lookup failed” or “displayed this ad to this user.” Our goal is to identify structured fields for querying without manually writing regular expressions for every log format. +**Challenges and Alternatives** -Initially, we explored available options for log parsing, including the Drain model, which is a well-known NLP model. However, it has significant limitations, such as requiring pre-processed logs and custom rules for each log source. As a result, it didn’t meet our needs. +We explored various methods to tackle these challenges. One of the most well-known is the Drain model, which comes from the family of natural language processing models that analyze word frequency. However, these algorithms have limitations: -### Alternative Solutions +1. They assume that logs are already prepared—meaning the prefix is separated from the message—requiring custom regular expressions for each log source. +2. They are highly dependent on pre-processing and manual tokenization rules. -Another option is to use large language models (LLMs) to parse logs. While LLMs can accurately parse logs on the first try, they are costly and inefficient when processing billions of logs daily from thousands of applications. +The reality is that these models cannot be applied to every log uniformly and expect good results. -We also considered deep learning models trained on labeled data to predict log patterns. However, the lack of a comprehensive labeled dataset across our diverse customer base made this approach impractical. Additionally, benchmarks for these models often indicate they only perform well on familiar log patterns, limiting their generalizability. +Another option is using large language models (LLMs) to parse logs. While they can produce impressive results, processing billions of logs daily across numerous customers would be very costly and slow. -### Our Solution +We also considered deep learning models trained to predict log patterns. However, this approach requires substantial labeled data—logs with known patterns—which we cannot obtain for all our customers. Additionally, benchmarked models often suffer from data contamination, meaning they perform well on familiar patterns but fail to generalize to new ones. -To address these challenges, we combined different approaches to leverage their strengths while mitigating limitations. +**Our Solution** -1. **Defining Instance Granularity**: We defined the right level of granularity for predicting log formats. Instead of treating each Kubernetes pod as a separate resource, which would create inefficiencies, we implemented resource clustering rules to group similar resources, allowing us to cache and reuse predicted log formats. +After analyzing various options, we decided to combine different approaches to address the limitations of each. -2. **Processing Pipeline**: Our pipeline begins by clustering logs by resource attributes. We then employ heuristics and prompt engineering to identify log prefixes and messages, extracting important fields like log severity. After parsing, we use traditional word frequency models to cluster log messages by structure before leveraging LLMs to refine variable names and types. +First, we established the right level of granularity for predicting log formats and patterns. In open telemetry data, the schema is highly nested, making it challenging to define an instance. We initially attempted to predict log formats for every resource but realized this was inefficient. Instead, we implemented resource clustering rules that allow us to cache and reuse predicted log formats across similar resources. -3. **Evaluation**: Before deploying our Log AI in production, we created an evaluation dataset combining community and proprietary logs to test our model’s performance across various use cases. We aimed for graceful degradation, meaning that while the model doesn’t need to work perfectly all the time, it should provide accurate information when it does function. +Next, we built a pipeline that starts by clustering logs based on resource attributes. We then use a combination of heuristics and prompt engineering to identify log prefixes and messages, extracting valuable parts like log severity. After parsing, we apply traditional word frequency models, such as Drain, to cluster log messages with similar structures. -### Results +At this stage, we inject the outputs from the Drain model into the prompts for a language model to identify final variables and assign appropriate names and types. Including semantic convention information and resource attributes significantly improves the model's ability to generate meaningful variable names. -Our evaluation revealed a 98% success rate in log parsing, with 100% accuracy in log severity among successfully parsed cases. The patterns we extracted covered an average of 85% of each application’s logs, successfully abstracting most of the available information. +Finally, during ingestion, our open telemetry collector matches logs against cached patterns and attaches relevant attributes to the identified variables. -### User Interface and Visualization +**Evaluation** -When integrated into our UI, users can visualize log severities over time, with clear distinctions between known and unknown severities. We also provide dedicated visualizations for log patterns, allowing users to see the structure of log messages and relevant variables. For example, a pattern might show a product ID lookup failure, which can trigger alerts based on frequency. +Before deploying this solution across all logs and customers, we needed to evaluate its effectiveness. We created a diverse evaluation dataset combining community and proprietary logs to stress-test the model and confirm its performance across different edge cases. Our goal was to ensure the model exhibited *graceful degradation*, meaning it doesn't need to work perfectly all the time, but when it does, it must produce accurate information without side effects. -## Other Experiments +We achieved a 98% success rate in log parsing, with 100% accuracy in identifying log severity among successfully parsed logs. Notably, our patterns covered an average of 85% of each application's logs, indicating our ability to abstract most available information. -We’re also experimenting with AI agents for writing and debugging Prometheus query language expressions and diagnosing failures in alerts. By providing agents with semantic conventions and resource attributes, we improve the accuracy of generated queries. Additionally, our Trace AI groups duplicate spans by attribute similarity and generates meaningful trace names and descriptions. +**Results and User Interface** -## Key Takeaways +Now, let me briefly share what the results look like in our user interface. -1. **Cost of AI in Production**: Applying AI at scale can be costly. However, leveraging attributes in the open telemetry schema can help cluster similar resources, enabling caching and reuse of AI predictions. +- We visualize log severities over time, with the introduction of our log processor leading to colorful histograms that reflect various log severities. +- We also developed a dedicated visualization for log patterns, allowing users to see variable names and values in structured data, which can be referenced in queries and filters. -2. **Combining Techniques**: While LLMs are powerful, they can be expensive. It’s essential to balance traditional machine learning techniques with LLMs for optimal results. +For instance, if we want to create an alert based on log frequency using extracted patterns, we can do so without requiring additional metrics or writing custom regular expressions. -3. **Importance of Evaluation**: Evaluation is crucial, even for simpler LLM applications. By modeling and measuring expected behaviors, we can quantify limitations before production deployment. +**Conclusion and Key Takeaways** -4. **Contextualization for LLMs**: Providing context using resource attributes and semantic conventions enhances the performance of LLMs, leading to better results. +In summary, this case study on Log AI demonstrates our approach to making unstructured logs actionable. -## Conclusion +Here are four key takeaways: -Thank you for your attention. If you’d like to connect with us, please follow us on LinkedIn, visit our blog, or check out our Code Red podcast on Spotify and Apple. We also have open positions for an AI engineer and a product manager with a background in observability. +1. **Cost Management**: Applying AI at scale can become expensive quickly. However, leveraging attributes in the open telemetry schema allows for clustering similar resources, making predictions efficient. + +2. **Balancing Techniques**: While LLMs are powerful, they are also costly. Combining traditional machine learning techniques with LLMs can yield better results. -### Q&A Session +3. **Importance of Evaluation**: Regardless of whether you're building a simple LLM application or a complex model, it's crucial to evaluate expected behaviors and measure performance before deploying. -**Question:** How do you handle logs with no severity level? -**Answer:** We only measure accuracy where there is a ground truth. If a log lacks a clear severity, we do not assign one, avoiding false positives. +4. **Context is Key**: Always utilize resource attributes and semantic conventions to contextualize prompts for better results with language models. -**Question:** Can customers customize semantic conventions? -**Answer:** While we use a standardized workflow for all customers, we can improve contextualization by incorporating customer-specific semantic conventions. +Thank you for your attention! If you wish to connect with us, please follow us on LinkedIn. We also have a blog on our website, a podcast on Spotify and Apple, and several open positions, including roles for AI engineers and product managers with a background in observability. -Feel free to ask more questions! Thank you! +**Q&A Session** + +Now, I would like to open the floor for any questions you may have! ## Raw YouTube Transcript -All right. So, hello everyone. I'm Lariel. Uh, today I'm going to share a little bit of my experience in applying AI to open telemetry data. So, short intro about myself. I'm from Brazil, moved to Berlin a couple years ago. Nowadays, I work as principal AI engineer at Dash Zero with these guys here. Uh, before that, I had been around uh playing different roles in data and AI. So ranging from data engineering to data scientist, ML ops. So my CV kind of looks like a mess nowadays. Um also for those of you who have not heard of D-Zero, just quick introduction, we are an open telemetry native SAS solution for observability. So we work with matrix, traces, logs, we do dashboarding, alerting, we're built on open standards like Prometheus and Persis. And obviously we are building lots of cool AI capabilities to help our users get more value from the data that they send to us. And that brings me to the subject of today's talk. Actually the main subject which is a case study on our log AI. So how to make sense of unstructured logs. Okay. Uh many of you probably had a similar disappointment at some point. Okay, you plug in the open telemetry demo data set and then the logs look beautiful. The logs are shipped via OTLP. They have plenty of useful attributes. They are uh formatted as JSON. So they're very easy to work with, right? But in reality, most people's logs look like this. So you have some something running on on Kubernetes and you have those uh file listeners that get uh the logs from every node and in the end uh the result is something like this. So your log has some uh raw string body and some attributes saying what file it was collected from and that not very useful but you know that there is information there like if you look into the log there is something that looks like a module name something that looks like a duration a time stamp a severity at debug level and how can we harness that information so that's the motivation behind behind uh the log AI that's the thing we we developed. So we want to do log parsing and log abstraction at scale for any logs of any application without a human in the loop. So without having to write custom regular expressions. For example, if I have an application writing logs like this, I want to parse them by separating this prefix from the message. So I have in the prefix time stamp host a severity text and we can see that the messages follow different patterns right I have some ad lookup failed displayed this ad to this user. So there are basically two different patterns of messages log messages that my application is writing and there are some structured fields that I would like to be able to query right there is some add ID in the middle there is some user ID um and we want to abstract that in the form of a pattern without writing a regular expression manually and we want to do that for every log of every application that's the challenge and the motivation behind our log AI. Uh so first we started looking into the available options to do that. Now the most famous one is the drain model that some of you might have uh seen already. So drain comes from this family of natural language processing uh models that work with uh word frequency. So they basically uh compare the parts of the text that change frequently with the parts that are often uh the same. So they can identify where the variables are and what the patterns should be. Uh but those algorithms have plenty of limitations. First one, they assume that the log is already prepared. So you have already separated that prefix from the the message that can change from one pattern to another. And for that you need custom regular expressions written by hand for each log source. That is already a deal breakaker. And secondly, those algorithms are very dependent on pre-processing. So there is this paper called pre-processing is all you need where they describe this manual iterative process of coming up with tokenization rules, um variable identification rules, uh text cleaning rules for each of your applications. And unless you do that, you cannot expect these models to produce any decent results for your custom logs. Believe me or not, the screenshot here on the left side is real code from the repository where they have the benchmarks for this models. So you can see that for each log data set that they are benchmarking on they have hardcoded rules otherwise the model doesn't work. And yeah that's a real deal breaker for us. We cannot just apply this drain model to every log and expect it to to work right. Um, okay. So, we started looking into other alternatives. What's very popular nowadays is just, yeah, throwing things at a large language model. So, if you ask GPT to parse a log, it's going to get everything right on the first try. You get the fields really nice. The problem is with hundreds of customers uh instrumenting thousands of applications and sending billions of logs every day, the using an LLM for everything would be very uh cost ineffective and slow. So what do we do? Yet another option is like this paper suggests using a deep learning model which is trained to predict the pattern on every log. But in order to train a model in the first place, you need to have lots of labeled data. So logs where you already know what the correct pattern should be. So you can teach the model how to infer that. And we cannot possibly have such a data set for all of our customers. Yet another problem with these uh models is that although they have a nice benchmarks, those can be interpreted as a form of data contamination because uh the the logs used to benchmark the model. They were produced from the same applications with the same templates as the logs that were used to train the model in the first place. which indicates that the model can actually only identify patterns that it is already familiar with or parse logs of applications that it is already familiar with. It doesn't generalize for any logs of any applications. Uh so yeah, this was the status quo. Plenty of challenges, different options, each one with their limitations. So how we solved this in the end was no magic at all. We actually just combined different approaches trying to balance out the limitations of some with the advantages of others. So first thing we did was figure out the right uh level of granularity for predicting log formats and patterns. So um I don't know if anyone here has already worked with predictive AI uh in in production but you usually have the concept of an instance. For example, if you are doing customer churn prediction, then every customer is an instance. If you're doing product sales forecast, then every product is an instance. It's quite straightforward. When it comes to open telemetry data, it's quite challenging because uh there are different levels of granularity. The schema is very nested. Um so what is an instance? We started with the concept of the open telemetry resource. So we tried to predict the log format and patterns for every resource but uh it wasn't efficient at all because if you think for example of a Kubernetes deployment with autoscale it is creating new pods and killing old pods all the time and every new pod has different resource attributes. So it's treated as a new resource. If we would predict log formats and patterns for every resource, our model would be working all the time, which would not be efficient at all. So instead of doing this, we came up with resource clustering rules that we apply to resource attributes of different resource types. This allows us to scope and cache and reuse the predicted log formats and log patterns between different resources that belong in the same group. And then to make this work in production, we also came up with a recommended configuration for the open telemetry collector that guarantees that the logs that we get from our customers always contain all the resource attributes that are relevant for our resource clustering rules. So all right, we have now a concept of an instance that we want to make predictions for. Next step okay how do we predict the the patterns. So uh we came up with this pipeline where we start with all the logs and then we first clusterize by resource attributes like as I said before. Then we focus on parsing. So we use a combination of eristics and prompt engineering to identify uh what is the the prefix, what is the message, what parts of the prefix are worth extracting like the log severity and we come up with log part log formats for each uh resource cluster. And after the logs are parsed, we apply those traditional uh word frequency models like drain in order to clusterize the log messages by structure. So every log message that has kind of the same structure goes into the same cluster. And then looking at each log cluster, we take the output of the drain model and inject it into the prompt uh for a language model in order to identify the final variables and also give names and types to the variables. At this stage, we observe that including semantic convention information and resource attributes in the prompt helps a lot and getting variables names and types that make sense for each application. Then in the end we just apply some eristics to optimize the patterns that we get so they match as best as possible the respective log cluster. Um then at ingestion time it's actually really straightforward in our uh open telemetry collector the processor um is going to match the logs against patterns from the respective cluster and attach the attributes with the variables that that it finds. So yeah, this uh overview of our solution. But before I show you what the results look like in our beautiful UI, I'm just going to have a quick word about evaluation. So this is really important before we applied this to every log of every customer all the time in in production. We needed to build some confidence to know that it would produce the expected results. So we came up with an evaluation data set that combines um logs from community data sets with logs from our own proprietary data. Uh the focus was not to have too many logs or quantity but rather to have diversity of logs so we could uh really stress the the model and confirm that it was working as expected for many different edge cases. And our main goal with this evaluation was to confirm that our approach had this uh kind of builtin graceful degradation. What does that mean? Means that the model doesn't need to work all the times. It doesn't need to work for all the every of log from the billions of logs that we are ingesting. But whenever it works, it needs to produce correct information. when it doesn't work then it cannot produce any unwanted side effects. So graceful degradation and that's basically what we observed in the evaluation. So we had a 98 uh success like 98% success rate at the log parsing step with 100% uh log severity accuracy between among the cases that are successfully parsed. So the model almost always understands the log format and when it does the log severity that it extracts is always correct. When it does not then it doesn't extract anything and the the log stays as it was before. Besides that the patterns that we extracted for each uh applications logs they cover an average of 85% of that application's logs. That means we successfully abstract almost all the information that is available to be abstracted. So yeah. Oh, sorry. Ask a question right at the end. Uh let's save the questions for for the end if you don't mind. Don't forget it. Uh okay. I'm just quickly going to show what the results look like in our UI. So this is a histogram of uh log severities over time. The gray uh bars are logs with unknown severity. And then uh as soon as we plug in the new uh log processor that applies the log formats and patterns, then we start having colorful logs in the histogram indicating the information and warning and error logs. Also, we observe a couple examples here of uh errors and warnings that would have passed by unnoticed if it wasn't for the the correct log parsing. What else? Um we come up with this dedicated uh visualization for the patterns. So this is data from the open telemetry demo and then uh we have uh different patterns where the variable is in the middle. things that say for example um targeted ad request received for add category and then add category is is a variable or a method name called with user ID and then the method name and the user ID are variables. Um and if we open one log record like this one with product ID name then uh we know which pattern it matches and we have the product ID and product name as dedicated variables. This is structured data that can be referenced in queries in filters in group by expressions also in Prometheus query language. So uh example here let's say if I want to create an alert based on log frequency using the patterns. So I have some logs that say product ID lookup failed and then it has some ID in the end. This one matches a pattern and the product ID is a variable. So I can create a log frequency alert with the filter saying that the pattern should be that one and grouping by the product ID field that comes from the the semistructure text. Right? So this is the Prometheus query language expression to uh alert on that log frequency and then the result is when uh I'm having too many lookup errors for that product I get a nice u failed check with the the custom message up here that says lookup failures increasing for product with the ID of of the product that so basically alerting based on semistructure information that was abstracted from from logs. This happens without any metrics, without needing an extra metric, without c needing uh to write any regular expression, without depending on on traces and anything else, just based on on the locks. U yeah, this is really cool. So yeah, this was the case study on the log AI that we have been uh developing. And I'm just going to quickly comment on a couple other experiments that we have been running. So uh we have been experimenting with AI agents to write and debug Prometheus query language expressions and also to diagnose and find the root cause of failed checks and alerts. uh the agent that we work with, it has access to a model context protocol, MCP server that allows it to browse through every metric and log and trace kind of like the same way that a human would do in our UI. Um in this context we also observe that if we feed the agent with information about semantic conventions and um resource attributes then it gets way better at writing correct queries in the first try. So it doesn't take so many iterations to figure out the correct metric names, the relevant attributes and so on because it builds on top of the semantic conventions. And yet another use case is our trace AI where we group uh duplicated spans in our trace explorer uh by their attribute similarity and we also give all we generate uh trace names and trace descriptions to clusters of similar traces. In this context, we also observe that giving uh semantic convention information to the the prompt increases the quality of the the names and the descriptions that we get. And also when clusterizing the traces by uh their semantica embeddings, if we include the resource attributes in the embedding, we get way more meaningful embeddings. Therefore, a better clustering of similar traces. So yeah, those were just a couple other examples of use cases that we have been working on. So before we go on to the questions, I just wanted to leave you with four key takeaways. So first yeah, applying AI at production at scale can be expensive. It can get expensive very quickly. But remember that in the open telemetry schema we have plenty of attributes that you can use to clusterize similar resources. This way you can scope cache and reuse AI predictions between different resources. Second LLMs are cool but they are expensive. Um, so we always try to see what we can do with traditional machine learning techniques or how we can combine them with LLMs to get the best of both worlds. Third, evaluation is very important. Even if you're building a simple um LLM application and you're not really training or fine-tuning any model, you can always try to model the expected behavior of your application and measure how often and how close it gets to the expected results. So you kind of quantify the limitations and the risks before shipping it to production. And lastly, uh, context is everything when it comes to LLMs. So, always benefit from resource attributes from the rich open telemetry schema and the semantic conventions so you contextualize your prompts and get better results from language models. Yeah, that was it. Uh, if you want to connect with us, uh, please follow- on linkading. We also have a a blog on our website. We're always posting some cool stuff. We have the Code Red uh podcast on Spotify and Apple. And we have a couple open positions in our careers website too, including one for another AI engineer and for a product manager with a background in observability. That was it. Uh thanks for your attention. I'm sorry. Do you remember your question? Yes. I think it was about uh the lock levels. Um you had 98% with lock levels, but there are some locks that do not have a lock level. How is that working out? We only measure the accuracy where there is a ground truth. So for a a log that uh really doesn't make sense to assign a level to it doesn't have the level even in an unstructured way or in the form of a status code or anything then um we do not assign it a log level. Actually in that case if um we would assign a log level let's say error warning or anything to a log that doesn't contain any textual information that references that it would count as a false positive and we have a separate metric for measuring how often that occurs. It's the specificity metric. So it's how often we avoid uh false positive and that one is maximized as well. uh but it wasn't on the slide. Okay. I think there's even a unspecified level for open country right there is uh yes but it is the default or at least when we ingest if that field is not set we set it to unspecified and then after the AI runs if we don't identify a an explicit level it remains at unidentified. Any other questions you there? I didn't get this specifically about the semantic conventions and resource. Did you use them to insert resource attributes and semantic conventions into logs that don't have it or what was the No. Uh the the thing is whenever we get uh any signal that has u attributes or like a span name, a metric name that matches the latest semantic conventions, then we have access to the documentation of that metric or attribute and we include that uh documentation in the prompt. So the model uh has context of what that metric means in which uh system it belongs. So it is able to for example give uh an appropriate name to to a variable or give a a better description to a trace. So that's how we work with semantic conventions for prompt contextualization. That's very interesting. I was thinking it would be also nice to um have like these kind of resource attributes and convention are sometimes not like the values are sometimes not correct especially if you have your own semantic conventions on top of hotel like business relevant semantic conventions and um if you could use AI to rectify basically um the discrepancies in what the users are putting in because they usually they put in a lot of stuff and a lot of different stuff and even if you have like written it down please use these values they make up their own values and their own writing systems And then instead of going there and telling them every time no please lower case not uppercase or something along the lines if we could use like kind of AI to understand the not sure if AI is really necessary. Sometimes regular expressions do the trick but sometimes they don't. So yeah that that can actually be a challenge. Every case where I I mentioned that we employed so many conventions it was the official one ones like from the the open source repositories theoretically and theoretically yes yeah sorry I have a follow-up question on this so a question I asked myself uh I don't know how dero works so as an as an obsability platform but my assumption is that the log AI feature is the same for every customer because you used something like yeah now someone is turning around if you're not so uh you found an approach how to leverage different kind of AI technologies to have lock AI enabled and it's the same for every customer so do I train this follow-up question on your so with my own semantics as my my own and everyone has something like a slightly different uh implemented AI approach so that's a that's a good question so uh the approach right now is the same for everyone But we work on a multi-tenant way. So uh even if two customers have let's say Oracle databases, we don't mix like the the log formats and patterns of one customer with the other. So all of your data will be processed with formats and uh patterns that we inferred from your data only. Uh it is but it's always pre-trained. Yes. uh we are not uh customizing by uh customer although not right now maybe in the future. Yeah. Yeah. Thank you very much. There are plans on open sourcing any of that. I suppose that I mean I love the log AI part but I suppose the the Prometheus part the Prometheus would also to have that kind of feature and I think I heard other companies doing the same right? Uh I think I heard some other companies doing something very similar. I see a lot of potential collaboration like opportunities there. Yeah, there could be uh yeah uh regarding any kind of open source you know need to talk to our CTO uh but uh what we are planning to do is to release an MCP uh so anyone can connect their agents to -0 data if they want to over there. So here to me so you apply a model to every single log. Do you use an LM? No. Okay. So how do we apply? Okay. uh we do it in a batch where the logs are already clustered by by resource and then they are clustered by structure and then within each log cluster we identify the pattern with the LLM and then we store that pattern then during ingestion time we don't invoke the LLM at all we just apply the patterns that we have already cached so by pattern You mean regular expression? Yes. No. Um, you can try that. Uh, maybe the newer ones will succeed for some cases, but LLMs are usually bad at writing regular expressions on their first try. We actually uh use them to infer parts of the the log, extract variables, uh get some insights on the log structure and then we apply aristics to these results to compose the regular expression ourselves. Yeah, I hope that rest on the data at rest. Yes, it was also part of the processor which was done on the fly. On the fly during ingestion, we apply log formats and patterns that have already been cached. So there is like this uh pattern identification phase and there is the production time which is exploiting the patterns that have already been cached. Question. So um at what frequency do you refresh the cache? That's uh in our documentation uh it should be every two hours. Why should it's in beta phase? I'm working on it but yeah um it's meant to be every every two hours. There is obviously a quot per uh like rate limits per per customer. So if you're sending logs uh of different formats and different pattern all the time like generating random formats just to make us spend money it's not not going to work. So refreshes for when patterns change but actually it's only needed when it changes. Sorry you said like the cash refresh the cache every two hours. Yes. And so that cache is for you're caching the lock patterns or Yes. So when the lock pattern changes the cache needs to be refreshed too. Yes. So then yeah and it would be good if you could um do this like on demand. It is on demand every two hours. Okay. Seriously that's that wasn't a joke. I mean we if they change we realize that they changed at latest to ours after and then update them. Uh I mean one very important aspect of dash series is that we do not sell AI capabilities as separate modules. We give them kind of for free. You only pay for the data that you send and every capability all the cool stuff in the UI you get out of box. So whenever we implement something like this, we have to establish rate limits uh and uh yeah just optimize it as best as we can to make it uh cost effective. But if I as a customer know that my won't change let's say or I know when it will change is there a possibility to change that to tell zero that now there's a new pattern coming uh right now uh Right now the way to do that would be you you message our customer success people and then they they would ping me but we wouldn't need to do anything if they change and at latest two hours later they would I mean the pipeline would realize exactly. So what it means is um within the first two hours I might not recognize all of the new fuse or all of the new patterns that that are flowing through the pipeline. Yeah. After two hours it's going to be recognized and the bar. So that graph they've shown with the gray bars that could we could see a difference there for the first two hours and then it goes back to the same level it exactly. That's very cool. Wow. Go on. So one question is uh so you mentioned that you have some kind of prompt engineering in the pipeline detecting the patterns. Uh do customers somehow can can they involve are they involved in this process? Can they change something like for example adding their own smart? Uh well we use the same um workflow for every customer but oh something is falling over there. [Laughter] monitoring. Yeah, we use the same workflow for every customer, but uh the more information we have for each customer, the better we could uh contextualize the prompt and maybe having custom semantic conventions for each customer's domain would be something to to help improve to the prompt. Yeah, that's something we could we could build. And a second question, how do you monitor like end to end for example how many how much money you spent on how much tokens and so on well we use -0 to monitor- zero right uh so um we also use the community uh libraries for instrumenting LLM applications the ones from the uh open Python open telemetry contrib uh so from those we get LLM traces and a cost uh for every LLM request that we make the costs are also u like contextualized with the price per token of each uh vendor and model variant and we have a nice um LLM trace view in our UI where we can uh debug LM LLM interactions like with what tools were called and so on. Folks, if anyone has other questions, we can chat uh later. Uh yeah, here around in the kitchen. And uh yeah, I'm going to give into the the next and thanks for having me. +All right. So, hello everyone. I'm Lariel. Uh, today I'm going to share a little bit of my experience in applying AI to open telemetry data. So, short intro about myself. I'm from Brazil, moved to Berlin a couple years ago. Nowadays, I work as principal AI engineer at Dash Zero with these guys here. Uh, before that, I had been around uh playing different roles in data and AI. So ranging from data engineering to data scientist, ML ops. So my CV kind of looks like a mess nowadays. Um also for those of you who have not heard of D-Zero, just quick introduction, we are an open telemetry native SAS solution for observability. So we work with matrix, traces, logs, we do dashboarding, alerting, we're built on open standards like Prometheus and Persis. And obviously we are building lots of cool AI capabilities to help our users get more value from the data that they send to us. And that brings me to the subject of today's talk. Actually the main subject which is a case study on our log AI. So how to make sense of unstructured logs. Okay. Uh many of you probably had a similar disappointment at some point. Okay, you plug in the open telemetry demo data set and then the logs look beautiful. The logs are shipped via OTLP. They have plenty of useful attributes. They are uh formatted as JSON. So they're very easy to work with, right? But in reality, most people's logs look like this. So you have some something running on on Kubernetes and you have those uh file listeners that get uh the logs from every node and in the end uh the result is something like this. So your log has some uh raw string body and some attributes saying what file it was collected from and that not very useful but you know that there is information there like if you look into the log there is something that looks like a module name something that looks like a duration a time stamp a severity at debug level and how can we harness that information so that's the motivation behind behind uh the log AI that's the thing we we developed. So we want to do log parsing and log abstraction at scale for any logs of any application without a human in the loop. So without having to write custom regular expressions. For example, if I have an application writing logs like this, I want to parse them by separating this prefix from the message. So I have in the prefix time stamp host a severity text and we can see that the messages follow different patterns right I have some ad lookup failed displayed this ad to this user. So there are basically two different patterns of messages log messages that my application is writing and there are some structured fields that I would like to be able to query right there is some add ID in the middle there is some user ID um and we want to abstract that in the form of a pattern without writing a regular expression manually and we want to do that for every log of every application that's the challenge and the motivation behind our log AI. Uh so first we started looking into the available options to do that. Now the most famous one is the drain model that some of you might have uh seen already. So drain comes from this family of natural language processing uh models that work with uh word frequency. So they basically uh compare the parts of the text that change frequently with the parts that are often uh the same. So they can identify where the variables are and what the patterns should be. Uh but those algorithms have plenty of limitations. First one, they assume that the log is already prepared. So you have already separated that prefix from the the message that can change from one pattern to another. And for that you need custom regular expressions written by hand for each log source. That is already a deal breakaker. And secondly, those algorithms are very dependent on pre-processing. So there is this paper called pre-processing is all you need where they describe this manual iterative process of coming up with tokenization rules, um variable identification rules, uh text cleaning rules for each of your applications. And unless you do that, you cannot expect these models to produce any decent results for your custom logs. Believe me or not, the screenshot here on the left side is real code from the repository where they have the benchmarks for this models. So you can see that for each log data set that they are benchmarking on they have hardcoded rules otherwise the model doesn't work. And yeah that's a real deal breaker for us. We cannot just apply this drain model to every log and expect it to to work right. Um, okay. So, we started looking into other alternatives. What's very popular nowadays is just, yeah, throwing things at a large language model. So, if you ask GPT to parse a log, it's going to get everything right on the first try. You get the fields really nice. The problem is with hundreds of customers uh instrumenting thousands of applications and sending billions of logs every day, the using an LLM for everything would be very uh cost ineffective and slow. So what do we do? Yet another option is like this paper suggests using a deep learning model which is trained to predict the pattern on every log. But in order to train a model in the first place, you need to have lots of labeled data. So logs where you already know what the correct pattern should be. So you can teach the model how to infer that. And we cannot possibly have such a data set for all of our customers. Yet another problem with these uh models is that although they have a nice benchmarks, those can be interpreted as a form of data contamination because uh the the logs used to benchmark the model. They were produced from the same applications with the same templates as the logs that were used to train the model in the first place. which indicates that the model can actually only identify patterns that it is already familiar with or parse logs of applications that it is already familiar with. It doesn't generalize for any logs of any applications. Uh so yeah, this was the status quo. Plenty of challenges, different options, each one with their limitations. So how we solved this in the end was no magic at all. We actually just combined different approaches trying to balance out the limitations of some with the advantages of others. So first thing we did was figure out the right uh level of granularity for predicting log formats and patterns. So um I don't know if anyone here has already worked with predictive AI uh in in production but you usually have the concept of an instance. For example, if you are doing customer churn prediction, then every customer is an instance. If you're doing product sales forecast, then every product is an instance. It's quite straightforward. When it comes to open telemetry data, it's quite challenging because uh there are different levels of granularity. The schema is very nested. Um so what is an instance? We started with the concept of the open telemetry resource. So we tried to predict the log format and patterns for every resource but uh it wasn't efficient at all because if you think for example of a Kubernetes deployment with autoscale it is creating new pods and killing old pods all the time and every new pod has different resource attributes. So it's treated as a new resource. If we would predict log formats and patterns for every resource, our model would be working all the time, which would not be efficient at all. So instead of doing this, we came up with resource clustering rules that we apply to resource attributes of different resource types. This allows us to scope and cache and reuse the predicted log formats and log patterns between different resources that belong in the same group. And then to make this work in production, we also came up with a recommended configuration for the open telemetry collector that guarantees that the logs that we get from our customers always contain all the resource attributes that are relevant for our resource clustering rules. So all right, we have now a concept of an instance that we want to make predictions for. Next step okay how do we predict the the patterns. So uh we came up with this pipeline where we start with all the logs and then we first clusterize by resource attributes like as I said before. Then we focus on parsing. So we use a combination of eristics and prompt engineering to identify uh what is the the prefix, what is the message, what parts of the prefix are worth extracting like the log severity and we come up with log part log formats for each uh resource cluster. And after the logs are parsed, we apply those traditional uh word frequency models like drain in order to clusterize the log messages by structure. So every log message that has kind of the same structure goes into the same cluster. And then looking at each log cluster, we take the output of the drain model and inject it into the prompt uh for a language model in order to identify the final variables and also give names and types to the variables. At this stage, we observe that including semantic convention information and resource attributes in the prompt helps a lot and getting variables names and types that make sense for each application. Then in the end we just apply some eristics to optimize the patterns that we get so they match as best as possible the respective log cluster. Um then at ingestion time it's actually really straightforward in our uh open telemetry collector the processor um is going to match the logs against patterns from the respective cluster and attach the attributes with the variables that that it finds. So yeah, this uh overview of our solution. But before I show you what the results look like in our beautiful UI, I'm just going to have a quick word about evaluation. So this is really important before we applied this to every log of every customer all the time in in production. We needed to build some confidence to know that it would produce the expected results. So we came up with an evaluation data set that combines um logs from community data sets with logs from our own proprietary data. Uh the focus was not to have too many logs or quantity but rather to have diversity of logs so we could uh really stress the the model and confirm that it was working as expected for many different edge cases. And our main goal with this evaluation was to confirm that our approach had this uh kind of builtin graceful degradation. What does that mean? Means that the model doesn't need to work all the times. It doesn't need to work for all the every of log from the billions of logs that we are ingesting. But whenever it works, it needs to produce correct information. when it doesn't work then it cannot produce any unwanted side effects. So graceful degradation and that's basically what we observed in the evaluation. So we had a 98 uh success like 98% success rate at the log parsing step with 100% uh log severity accuracy between among the cases that are successfully parsed. So the model almost always understands the log format and when it does the log severity that it extracts is always correct. When it does not then it doesn't extract anything and the the log stays as it was before. Besides that the patterns that we extracted for each uh applications logs they cover an average of 85% of that application's logs. That means we successfully abstract almost all the information that is available to be abstracted. So yeah. Oh, sorry. Ask a question right at the end. Uh let's save the questions for for the end if you don't mind. Don't forget it. Uh okay. I'm just quickly going to show what the results look like in our UI. So this is a histogram of uh log severities over time. The gray uh bars are logs with unknown severity. And then uh as soon as we plug in the new uh log processor that applies the log formats and patterns, then we start having colorful logs in the histogram indicating the information and warning and error logs. Also, we observe a couple examples here of uh errors and warnings that would have passed by unnoticed if it wasn't for the the correct log parsing. What else? Um we come up with this dedicated uh visualization for the patterns. So this is data from the open telemetry demo and then uh we have uh different patterns where the variable is in the middle. things that say for example um targeted ad request received for add category and then add category is is a variable or a method name called with user ID and then the method name and the user ID are variables. Um and if we open one log record like this one with product ID name then uh we know which pattern it matches and we have the product ID and product name as dedicated variables. This is structured data that can be referenced in queries in filters in group by expressions also in Prometheus query language. So uh example here let's say if I want to create an alert based on log frequency using the patterns. So I have some logs that say product ID lookup failed and then it has some ID in the end. This one matches a pattern and the product ID is a variable. So I can create a log frequency alert with the filter saying that the pattern should be that one and grouping by the product ID field that comes from the the semistructure text. Right? So this is the Prometheus query language expression to uh alert on that log frequency and then the result is when uh I'm having too many lookup errors for that product I get a nice u failed check with the the custom message up here that says lookup failures increasing for product with the ID of of the product that so basically alerting based on semistructure information that was abstracted from from logs. This happens without any metrics, without needing an extra metric, without c needing uh to write any regular expression, without depending on on traces and anything else, just based on on the locks. U yeah, this is really cool. So yeah, this was the case study on the log AI that we have been uh developing. And I'm just going to quickly comment on a couple other experiments that we have been running. So uh we have been experimenting with AI agents to write and debug Prometheus query language expressions and also to diagnose and find the root cause of failed checks and alerts. uh the agent that we work with, it has access to a model context protocol, MCP server that allows it to browse through every metric and log and trace kind of like the same way that a human would do in our UI. Um in this context we also observe that if we feed the agent with information about semantic conventions and um resource attributes then it gets way better at writing correct queries in the first try. So it doesn't take so many iterations to figure out the correct metric names, the relevant attributes and so on because it builds on top of the semantic conventions. And yet another use case is our trace AI where we group uh duplicated spans in our trace explorer uh by their attribute similarity and we also give all we generate uh trace names and trace descriptions to clusters of similar traces. In this context, we also observe that giving uh semantic convention information to the the prompt increases the quality of the the names and the descriptions that we get. And also when clusterizing the traces by uh their semantica embeddings, if we include the resource attributes in the embedding, we get way more meaningful embeddings. Therefore, a better clustering of similar traces. So yeah, those were just a couple other examples of use cases that we have been working on. So before we go on to the questions, I just wanted to leave you with four key takeaways. So first yeah, applying AI at production at scale can be expensive. It can get expensive very quickly. But remember that in the open telemetry schema we have plenty of attributes that you can use to clusterize similar resources. This way you can scope cache and reuse AI predictions between different resources. Second LLMs are cool but they are expensive. Um, so we always try to see what we can do with traditional machine learning techniques or how we can combine them with LLMs to get the best of both worlds. Third, evaluation is very important. Even if you're building a simple um LLM application and you're not really training or fine-tuning any model, you can always try to model the expected behavior of your application and measure how often and how close it gets to the expected results. So you kind of quantify the limitations and the risks before shipping it to production. And lastly, uh, context is everything when it comes to LLMs. So, always benefit from resource attributes from the rich open telemetry schema and the semantic conventions so you contextualize your prompts and get better results from language models. Yeah, that was it. Uh, if you want to connect with us, uh, please follow- on linkading. We also have a a blog on our website. We're always posting some cool stuff. We have the Code Red uh podcast on Spotify and Apple. And we have a couple open positions in our careers website too, including one for another AI engineer and for a product manager with a background in observability. That was it. Uh thanks for your attention. I'm sorry. Do you remember your question? Yes. I think it was about uh the lock levels. Um you had 98% with lock levels, but there are some locks that do not have a lock level. How is that working out? We only measure the accuracy where there is a ground truth. So for a a log that uh really doesn't make sense to assign a level to it doesn't have the level even in an unstructured way or in the form of a status code or anything then um we do not assign it a log level. Actually in that case if um we would assign a log level let's say error warning or anything to a log that doesn't contain any textual information that references that it would count as a false positive and we have a separate metric for measuring how often that occurs. It's the specificity metric. So it's how often we avoid uh false positive and that one is maximized as well. uh but it wasn't on the slide. Okay. I think there's even a unspecified level for open country right there is uh yes but it is the default or at least when we ingest if that field is not set we set it to unspecified and then after the AI runs if we don't identify a an explicit level it remains at unidentified. Any other questions you there? I didn't get this specifically about the semantic conventions and resource. Did you use them to insert resource attributes and semantic conventions into logs that don't have it or what was the No. Uh the the thing is whenever we get uh any signal that has u attributes or like a span name, a metric name that matches the latest semantic conventions, then we have access to the documentation of that metric or attribute and we include that uh documentation in the prompt. So the model uh has context of what that metric means in which uh system it belongs. So it is able to for example give uh an appropriate name to to a variable or give a a better description to a trace. So that's how we work with semantic conventions for prompt contextualization. That's very interesting. I was thinking it would be also nice to um have like these kind of resource attributes and convention are sometimes not like the values are sometimes not correct especially if you have your own semantic conventions on top of hotel like business relevant semantic conventions and um if you could use AI to rectify basically um the discrepancies in what the users are putting in because they usually they put in a lot of stuff and a lot of different stuff and even if you have like written it down please use these values they make up their own values and their own writing systems And then instead of going there and telling them every time no please lower case not uppercase or something along the lines if we could use like kind of AI to understand the not sure if AI is really necessary. Sometimes regular expressions do the trick but sometimes they don't. So yeah that that can actually be a challenge. Every case where I I mentioned that we employed so many conventions it was the official one ones like from the the open source repositories theoretically and theoretically yes yeah sorry I have a follow-up question on this so a question I asked myself uh I don't know how dero works so as an as an obsability platform but my assumption is that the log AI feature is the same for every customer because you used something like yeah now someone is turning around if you're not so uh you found an approach how to leverage different kind of AI technologies to have lock AI enabled and it's the same for every customer so do I train this follow-up question on your so with my own semantics as my my own and everyone has something like a slightly different uh implemented AI approach so that's a that's a good question so uh the approach right now is the same for everyone But we work on a multi-tenant way. So uh even if two customers have let's say Oracle databases, we don't mix like the the log formats and patterns of one customer with the other. So all of your data will be processed with formats and uh patterns that we inferred from your data only. Uh it is but it's always pre-trained. Yes. uh we are not uh customizing by uh customer although not right now maybe in the future. Yeah. Yeah. Thank you very much. There are plans on open sourcing any of that. I suppose that I mean I love the log AI part but I suppose the the Prometheus part the Prometheus would also to have that kind of feature and I think I heard other companies doing the same right? Uh I think I heard some other companies doing something very similar. I see a lot of potential collaboration like opportunities there. Yeah, there could be uh yeah uh regarding any kind of open source you know need to talk to our CTO uh but uh what we are planning to do is to release an MCP uh so anyone can connect their agents to -0 data if they want to over there. So here to me so you apply a model to every single log. Do you use an LM? No. Okay. So how do we apply? Okay. uh we do it in a batch where the logs are already clustered by by resource and then they are clustered by structure and then within each log cluster we identify the pattern with the LLM and then we store that pattern then during ingestion time we don't invoke the LLM at all we just apply the patterns that we have already cached so by pattern You mean regular expression? Yes. No. Um, you can try that. Uh, maybe the newer ones will succeed for some cases, but LLMs are usually bad at writing regular expressions on their first try. We actually uh use them to infer parts of the the log, extract variables, uh get some insights on the log structure and then we apply aristics to these results to compose the regular expression ourselves. Yeah, I hope that rest on the data at rest. Yes, it was also part of the processor which was done on the fly. On the fly during ingestion, we apply log formats and patterns that have already been cached. So there is like this uh pattern identification phase and there is the production time which is exploiting the patterns that have already been cached. Question. So um at what frequency do you refresh the cache? That's uh in our documentation uh it should be every two hours. Why should it's in beta phase? I'm working on it but yeah um it's meant to be every every two hours. There is obviously a quot per uh like rate limits per per customer. So if you're sending logs uh of different formats and different pattern all the time like generating random formats just to make us spend money it's not not going to work. So refreshes for when patterns change but actually it's only needed when it changes. Sorry you said like the cash refresh the cache every two hours. Yes. And so that cache is for you're caching the lock patterns or Yes. So when the lock pattern changes the cache needs to be refreshed too. Yes. So then yeah and it would be good if you could um do this like on demand. It is on demand every two hours. Okay. Seriously that's that wasn't a joke. I mean we if they change we realize that they changed at latest to ours after and then update them. Uh I mean one very important aspect of dash series is that we do not sell AI capabilities as separate modules. We give them kind of for free. You only pay for the data that you send and every capability all the cool stuff in the UI you get out of box. So whenever we implement something like this, we have to establish rate limits uh and uh yeah just optimize it as best as we can to make it uh cost effective. But if I as a customer know that my won't change let's say or I know when it will change is there a possibility to change that to tell zero that now there's a new pattern coming uh right now uh Right now the way to do that would be you you message our customer success people and then they they would ping me but we wouldn't need to do anything if they change and at latest two hours later they would I mean the pipeline would realize exactly. So what it means is um within the first two hours I might not recognize all of the new fuse or all of the new patterns that that are flowing through the pipeline. Yeah. After two hours it's going to be recognized and the bar. So that graph they've shown with the gray bars that could we could see a difference there for the first two hours and then it goes back to the same level it exactly. That's very cool. Wow. Go on. So one question is uh so you mentioned that you have some kind of prompt engineering in the pipeline detecting the patterns. Uh do customers somehow can can they involve are they involved in this process? Can they change something like for example adding their own smart? Uh well we use the same um workflow for every customer but oh something is falling over there. monitoring. Yeah, we use the same workflow for every customer, but uh the more information we have for each customer, the better we could uh contextualize the prompt and maybe having custom semantic conventions for each customer's domain would be something to to help improve to the prompt. Yeah, that's something we could we could build. And a second question, how do you monitor like end to end for example how many how much money you spent on how much tokens and so on well we use -0 to monitor- zero right uh so um we also use the community uh libraries for instrumenting LLM applications the ones from the uh open Python open telemetry contrib uh so from those we get LLM traces and a cost uh for every LLM request that we make the costs are also u like contextualized with the price per token of each uh vendor and model variant and we have a nice um LLM trace view in our UI where we can uh debug LM LLM interactions like with what tools were called and so on. Folks, if anyone has other questions, we can chat uh later. Uh yeah, here around in the kitchen. And uh yeah, I'm going to give into the the next and thanks for having me. diff --git a/video-transcripts/transcripts/2025-06-11T05:54:28Z-otel-me-with-oluwatomisin-taiwo-and-andrei-morozov.md b/video-transcripts/transcripts/2025-06-11T05:54:28Z-otel-me-with-oluwatomisin-taiwo-and-andrei-morozov.md new file mode 100644 index 0000000..563552e --- /dev/null +++ b/video-transcripts/transcripts/2025-06-11T05:54:28Z-otel-me-with-oluwatomisin-taiwo-and-andrei-morozov.md @@ -0,0 +1,116 @@ +# OTel Me...with Oluwatomisin Taiwo and Andrei Morozov + +Published on 2025-06-11T05:54:28Z + +## Description + +oin us for the next edition of OTel Me. This time, we'll hear from Oluwatomisin Taiwo and Andrei Morozov of Compass Digital as ... + +URL: https://www.youtube.com/watch?v=tTCuTAPE5aQ + +## Summary + +In this episode of Hotel Me, hosts Adriana Vila and Andre Kapolski discuss observability practices at Compass Digital with guests Tommy and Andre M. from the S3 team. They delve into the roles of observability in ensuring system reliability and scalability, touching on the use of AWS services like Lambda and ECS, and the integration of OpenTelemetry for distributed tracing and performance monitoring. Key challenges addressed include context propagation in microservices, manual instrumentation for business logic, and the need for effective log management. The conversation highlights the importance of community support, with participants sharing their experiences and encouraging more complex examples in OpenTelemetry resources. They also discuss future expectations for OpenTelemetry, emphasizing the potential for enhanced data insights and improved developer experiences. The session concludes with an invitation for audience engagement and a reminder of upcoming community events. + +## Chapters + +Sure! Here are the key moments from the livestream along with their timestamps: + +00:00:00 Introductions by Adriana Vila and Andre Kapolski +00:02:00 Guests Introduce Themselves: Tommy and Andre M +00:04:30 Discussion on Roles at Compass Digital +00:06:15 Overview of System Architecture and Technologies Used +00:10:00 Challenges Faced in Achieving Observability +00:13:45 Insights on System Instrumentation Techniques +00:18:30 Discussion on OpenTelemetry Setup and Configuration +00:25:00 Audience Questions on Metrics and Logging +00:30:00 Collecting and Managing Observability Data +00:35:00 Future of OpenTelemetry in Their Workflow +00:40:00 Closing Remarks and Upcoming Events + +Feel free to ask if you need further details! + +# Hotel Me Session Transcript + +**Adriana:** Heat. Heat. Hello everyone and welcome to our latest Hotel Me. I'm super excited for those who are able to join, and for those who are not, we do have this recording available after the fact, both on LinkedIn and YouTube. My name is Adriana Vila, and I am one of the maintainers of the Hotel End User SIG. I am happy to introduce my co-presenter today, Andre. + +**Andre:** Hi everyone! Yeah, awesome to be here. My name is Andre Kapolski, and I'm a fairly recent contributor to the End User SIG. I'm super excited to hear more from our guests today. + +**Adriana:** I think this is a perfect segue as we bring on our guests. Folks in the chat, please feel free to say where you're watching from. + +**Andre:** I can start. I'm joining from the Czech Republic, specifically from Brno. + +**Adriana:** Oh, that’s right! I'm in Toronto, Canada, so it's I guess 1:00 PM for me and evening for you, right? + +**Andre:** That's correct! + +**Adriana:** Awesome! All right, let's bring our guests on. Hello everyone! + +**Tommy:** Hello! + +**Andre M:** Hey. + +**Adriana:** Why don't you both introduce yourselves? Let's start with Tommy. + +**Tommy:** My name is Tommy. I'm an SRE here at Compass Digital, and my primary responsibility is ensuring the reliability, scalability, and observability of our web platform. I work closely with the engineering team, product teams, and test engineers to ensure that our systems are robust and we have actionable insights into what's going on within our services. I'm calling from Cambridge, Ontario, Canada. + +**Adriana:** Awesome! We're practically neighbors, just a few hours away from each other. + +**Andre M:** Hi, I'm Andre M, from Bonneville, Durham region, also in Ontario. I've been in this industry for 11 years, always interested in tech since I was a kid. I work with Tommy on the same SRE team at Compass Digital and have similar responsibilities focused on observability and on-call incidents for our systems. + +**Adriana:** Well, we're super excited to have you both on! Just a reminder for folks watching: if you have questions along the way, please feel free to post them in the chat. We will also take questions at the end. + +**Andre:** Let's get started! You both talked a little bit about your roles; could you talk a bit more about your respective roles at Compass Digital in more detail? + +**Tommy:** Sure! Most of my role is focused on ensuring the reliability and scalability of our systems. Recently, I've been working on one of our products, which is E-Club. I'm helping to get observability into what they're doing there. I work with AWS SDK, CDK, Terraform, and PagerDuty. I also help with observability backend data tracing, where we send all our OpenTelemetry data. + +**Andre M:** When I started with Compass about five years ago, I began as a senior software engineer. My background is primarily in software engineering, but I've developed an interest in ops and infrastructure. My focus is on developer experience and platform engineering. Recently, I've been working on understanding systems and promoting observability, especially given our microservices architecture. + +**Adriana:** Folks, can you tell us a bit more about the system you are operating? What is its architecture and what programming languages are used? + +**Tommy:** We have two products, but I’ll focus on E-Club. Our main infrastructure is about 95% Lambda functions. We have over 300 Lambdas in our ecosystem, utilizing a hybrid of DynamoDB and Aurora RDS, and we're starting to use Fargate services. + +**Andre M:** On the E-Club side, we have a blend of Python and Django on the backend and JavaScript/TypeScript on the frontend. Everything runs on AWS ECS, which gives us flexibility and consistency across environments. Our CI/CD pipelines automate the process of building Docker images and deploying them onto ECS. + +**Adriana:** Thank you! As a follow-up, what were the top three problems that compelled you to implement observability? + +**Tommy:** Observability in a distributed system is never fully solved. One major challenge was distributed tracing. A single user request can touch many services, and without distributed tracing, reconstructing the journey of a request is nearly impossible, especially during issues. Another challenge was performance monitoring. For instance, we needed to know not just if a task succeeded, but how long it took and what resources were consumed. Finally, error tracking was crucial for us to correlate errors with logs and traces. + +**Andre M:** For our end, context propagation was a primary concern due to our microservices and distributed nature. Additionally, we needed to provide developers with the ability to create their own alerts via Terraform instead of relying on a single team. + +**Adriana:** Can you elaborate on the instrumentation of your systems and what types of instrumentation you are using? + +**Tommy:** We use an agent for instrumentation on the E-Club side. It works well with our infrastructure, especially with Lambdas using Lambda layers. For ECS, we utilize AWS FireLens for log management, redirecting logs to a sidecar that handles exporting. + +**Andre M:** Our platform uses a vendor agent that integrates seamlessly with our Lambda functions. We manage our ECS Fargate tasks using Terraform, which allows us to customize configurations and manage the collectors effectively. + +**Adriana:** Who is responsible for instrumentation in your teams? + +**Tommy:** We're trying to build a pre-packaged solution for developers to leverage for their own instrumentation. For now, most of the initial instrumentation was done manually for critical code paths. + +**Andre M:** We’re evolving our processes and looking to provide more out-of-the-box solutions. The goal is to empower developers to manage their own instrumentation. + +**Adriana:** Do you have any current challenges with OpenTelemetry that you think could be improved? + +**Tommy:** One major challenge was the initial setup and ensuring context propagation across services. The API can be overwhelming, and more complex examples would help us navigate this better. + +**Andre M:** I've mostly been learning through experience, and I agree that more real-world examples would be beneficial. + +**Adriana:** How do you see OpenTelemetry evolving in your workflow over the next year? + +**Tommy:** I believe the real value lies in the data we extract. We want to apply user journeys and business events to gain insights. + +**Andre M:** I see OpenTelemetry helping us mature our observability processes further, especially around logs and complex setups. More out-of-the-box integrations would simplify our configurations. + +**Adriana:** Thank you both for sharing your insights today! Before we wrap up, do you have any final thoughts or shoutouts? + +**Tommy:** Just a big thank you for having us! + +**Andre M:** Same here! It's great to share our experiences. + +**Adriana:** Thank you to everyone who attended! The recording will be available on LinkedIn and YouTube. Be sure to check out the Hotel End User SIG group on CNCF Slack to share your stories and join us for our meetings every two weeks. Thanks again, and see you next time! + +## Raw YouTube Transcript + +Heat. Heat. Hello everyone and welcome to our latest hotel me. super excited for the those who are able to join and for those who are not we do have the uh this recording available after the fact both on LinkedIn and YouTube. My name is Adriana Vila and I am one of the maintainers of the hotel enduser SIG and I am happy to introduce my uh co- uh presenter today Andre. Hi everyone and yeah awesome to be here. My name is Andre Kapolski and I'm a fairly recent uh contributor to and end user sik. So yeah, I'm super excited to hear more from our our guests today. Yeah, and I think uh this is a perfect segue as we bring on our guests. And as we do um folks on the chat, please feel free to uh say where you're uh where you're watching from. Yeah. I can perhaps start. I'm watching from or I'm joining from the Czech Republic uh and in in EU uh and specifically in Berno. Adriana, what about you? Where I I don't think you mentioned it, have you? Oh, yeah. That's right. I'm in Toronto, Canada, so it's I guess 1:00 for me and I guess it's evening for you, right? You're probably 7 o'clock in the That's correct. That's correct. Yeah. Awesome. Awesome. All right. Uh I guess let's bring our guests on. Hello everyone. Hello. Hey. So, uh why don't uh you both introduce yourselves? Let's start with Tommy. My name is Tommy. Uh I'm an S sur here at Compass Digital and uh my primary responsibility essentially is uh ensuring the reliability, scalability and observability of our web platform. uh I work closely with the engineering team, product teams, test engineers to ensure that our systems are robust and we have actionable insights into what's going on within uh our services. So yeah, that's me. And where are you uh where are you calling? Oh yeah, I'm calling from Cambridge, Ontario in Canada. Awesome. We're we're like practically neighbors a few a few hours away from each other. Oh yeah. Awesome. Okay. And and then we have another Andre today. Um, hi. I'm out from Boneville Durm region. So, so same type of area. And uh, for me, I've been in this industry for 11 years, but always been since I was a kid and into the tech stuff. So uh I work with Tommy and the same team S3 at Compass uh digital here and uh very similar responsibilities across observability to um on call PI all the type of stuff for uh for our systems. Awesome. Uh well we're super excited to have you both on and just a reminder for folks watching if you do have questions along the way please feel free to post them on the chat. Um and also we will be taking questions at the end um if uh if anyone has questions. So I guess uh let us get started. So I know you guys both uh talked a little bit about um you know g gave your intros. Maybe you could talk a little bit about uh your respective roles at uh at Compass Digital in a little bit more detail. Sounds good. I'll start. Okay. Yeah, sorry. Uh I I'll go first then. Um as I said, um most of my roles primarily focused um on ensuring the reliability and scalability of our systems. Uh very recently or for the most part that I've been here, I've been working on uh one of our products. It's a company I actually bought over is e-club um to help get you know observability into what they're doing there. Um and largely you know I work with things like uh AWS SDK uh the CDK TF rather and uh PA duty um we have observability back end data trace where we send all our open telemetry stuff to um so uh pretty much just S sur uh there's not nothing too crazy happening on there but uh I believe as we go um along on this podcast I'll get more um nuanced into what I do and how your architecture is structured and the stack there. Sounds good. And uh Andre M uh if you could elaborate a little bit on on your Yeah, certainly. So when I started with Compass about 5 years now, um I started off as a senior software engineer. So that's my background in software engineering less so on the ops and infra side but that's uh kind of been a new love with terraform and um services but um primarily about the developer experience the platform engineering aspect of it and more recently angle it's more about understanding the uh system that you can worry about uh and they'll tell us that linkability how potentially across that and give us the insights with our platform and a vendor that we utilize that with. So just to give you some pretty cool things is like identifying any services that are you know looping into each other for instance uh via API rather than just like calling it internally with a function. So it's more about just like setting up these guard rails um and looking into the optimization of our system. And the other aspect is because we are a microser shop uh it's uh it's been a challenge for us to get that context propagation and seeing all the services all together. Um and now we're able to kind of have that full aspect all together and actually run real um I guess business impacts and analysis onto that. Um yeah that's primarily an area I've been looking at more or less. Cool. Uh folks uh can you to start with can you tell us a bit more about about the system that you are operating? What is it what is its architecture? What uh programming languages are used and yeah u yeah how how it is deployed and and so on. Okay. So we have two products. So I'll talk about the one that I primarily focus on. So for us our main infrastructure and architecture all runs the 90 95% is all lambdas. We have about 300 plus lambdas in our ecosystem. We run hybrid of dynamotp with Aurora RDS and that's the exact thing progress and we're slowly getting into argate services. Um so that's a little bit different model but uh that's the primary architecture of our ecosystem you know the standard services like S3 HP gateway just standard service stuff that as ecosystem provides Tommy. Oh yeah. Uh on the E-Club side, what we have is a blend of Python and Django on the back end. Um and then JavaScript, TypeScript on the front end and we run everything on that side on AWS ECS which uh gives us that flexibility and consistency across all our environment. Okay, thank you. Um so our CI/CD pipelines automate the process of building those Docker images and then deploying them onto ECS so we can roll out those changes quickly and safely. uh but to you know do a quick walk through uh across this little diagram that I have here. Uh so imagine a user interacting with application. What they hit first is the load balancers that we have here and then uh it distributes traffic to the web service that we have on the back end which runs ECS and instrumented with the open telemetry uh SDK. Now as this web service this web service or web services begin to process this request uh what it does is to offload uh some heavy tasks or async tasks to this celery worker down here which also runs on ECS and is also instrumented with uh open telemetry SDK and uh thanks to the context oops typo context propagation here uh the trace context can flow seamlessly between the web service uh and the worker and um with the introduction of the open telemetry SDK now we can follow a request end to end from um inception to the end right and um both these services that's the web service and celery worker the emit traces logs um are metrics um and we sample them make sure they um they they have high cardality and they are batched also for efficiency and all these data that you know I'm talking about ascent to open telemetry collectors that run as a sidecar on um ECS to send this data to our obserility back end. So this is uh pretty much in a nutshell what the uh architecture on the e-club side of uh what we are doing looks like. Awesome. Thanks for thanks for that. Um as a as a follow-up question, you know, given given your your your system architecture, um what was what what were the top three problems that you were facing that kind of compelled you to um to go like you know to say hey I need observability. Oh yeah. Uh sure. uh I'm sure you know and uh probably everyone listening knows that observability in a distributed system is never a solved problem and uh some of the nuance challenges that we faced was again as I said distributed tracing um with microservices a single user request can touch so many services and background jobs and uh without distributed tracing it's almost impossible to reconstruct that full journey of a request from the user's end to you know the the response back to the user especially when something goes wrong right so for instance if a user says that they had a slow user slow order placement rather um we need to trace our request from the web front end the back end the salary workers and even to third party APIs and the introduction of open telemetry helps us stitch all these things together right and also have consistent context propagation um across all these boundaries and um one of the things that we were also looking at was performance monitoring because we rely on celery heavily on E-club for background processing. However, uh these things can be black boxes if you don't instrument them properly. So, we need to know not just if a task has succeeded but how long it took uh what resources consumed and whether it's causing bottlenecks along the way. um introducing celery with uh sorry instrumenting celery with open telemetry um required us to go you know with out ofthebox solutions uh again thanks for that um and to to be able to add custom spans and metrics to these things and um also error tracking um you know errors can manifest in logs traces or metrics um wherever and um the most important thing um for us at the time was to be able to correlate these things together so that's having your trace ID and span ID and logs and uh being able to look at that trace ID in the log and correlate it with an actual trace. Right? So a spike in error log sometimes might cor correspond rather with a specific trace or a drop in a metric that probably we're tracking. Right? So what we've done is we've worked to ensure that uh our telemetry includes enough context like as I said the trace ID request ID so we can pivot between the logs traces and the metrics um seamlessly. So um yeah that's what got us to where we are now. Awesome. Um Andre uh do you have other Andre uh do you have any anything else to add? Uh so for our end I think the primary problem again really was that context propagation because again the microser and distributed um nature of that. I think the other part that we weren't even really aware about is just how the system really behaves. So a big part of it is uh tying back into alerting. So depending on what vendor you use it might have automatic alerting but being able to provide that to our developers in our case uh via Terraform. So developers can open up PR, create their own alerts, and now we have that across our ecosystems has really changed a lot of um how we're able to kind of empower the different developer teams to just take control of their own alerting and systems rather than um you know just one team kind of owning that. Uh can you tell us a bit more about how are your systems instrumented like what yeah what types of instrumentation are you using? Certainly. Uh so on uh our platform uh currently for us we do use our vendor uh agent. Uh so in our end um the agent's really nice for the most part. Uh it uh works seamlessly with both of our infrastructure pieces of for lambda. Lambda has the concept of a lambda layer where you can basically put on something. So their agent just goes onto that kind of bolts right on and works essentially right out of the box. Uh and the second part to that that we had to play around with a little bit was our ECS Fargate. Uh there is a solution from Adabus called uh adabus fire lens which allows you to kind of have your task um logs uh standard error and center out to basically be sent to a specific um sidecar where then that sidecar will run something like fluent bit or um they cubit uh and essentially take your logs and ship them to where you need them. Yeah, that makes sense. Uh I would ask a followup right away. Uh so are you using hotel out auto out auto out auto out auto out auto out auto out auto out auto out auto out auto out auto instrumentation or are you manually instrumenting your your services? So for us so because we do have the 300 lambdas each one need we basically have like a core library that all these lambdas will all utilize very thin layer but in there we do have um hotel instrumentation for the Node.js JS I believe it's just the auto instrument or I think we might uh we might have changed that for the finetuner because it needs to be small uh because lamp is you want to keep them really tiny. So I think we we do have the um uh the custom one where it's not the auto instrument and uh yeah how was how was the how was your experience with with doing that so far? Uh I haven't heard any um issues. So I didn't do that one. It was actually our coworker Matt who did that one, but I didn't see any too many I didn't see any issues with that. PR went through pretty smoothly and uh the prim primary purpose was we use our own custom logger uh one that isn't supported by um hotel or vendor. So we had to use that to kind of pull in the trace ID from the um headers and everything else and be able to actually populate the spans that we require. Cool. So, so to clarify then in in a nutshell, you you basically have like a wrapper around hotel from what it sounds like. Uh kind of it's um I guess so. Yeah. Yeah. I guess so. Yeah. Cool. Cool. And um question for you around uh just to keep on the instrumentation thread um who is responsible for instrumenting? Well, that's that's an interesting journey. So we actually do have a new project upcoming. So the way our system has kind of evolved um with Terraform in particular is you know we're IA first for no matter what we do infrastructure as code and a big part of that is we use the cloud development kit extension on Terraform. So it gives us the ability to kind of have this interherence model and allows us to build these L1 L2 constructs. Um so if anyone's familiar with that it's a very useful way of uh I guess building abstractions for your infrastructure and enabling teams to kind of um handle that as well. So uh in our case what we're trying to build right now is kind of an L3 construct which is like an application service um level where we can give it to developers and they can essentially have all the nuts and bolts kind of already instrumented out of the box uh with the right permissions and the right access and the right agents and everything else. Um, and we kind of support this golden path for them. Awesome. Um, Tommy, do you have anything else that you want to add? Um, no, not really. We, as I said, what I think we're trying to get to a point where all of these things are, would I say, prepackaged for the devs to be able to um to use them, to leverage them, to do their own instrumentation on their own. And um I would say for the most part we are automating um many of the um many of our collector setup or the open telemetry setup. But uh I think at the beginning what we did especially on the e-club side was to auto instrument so we can have quick wins and and move quickly. But um when we're getting down to things like business logic um salary tasks and things that were unique to the e-club side, we manually created spans along some critical code paths um and added some custom attributes here and there so we can um get more than just what open telemetry offers out of the box. So I think um for e-club it's just a little slightly different from u what we have on the centric side. Nice. Nice. That's really cool because I think that's a path that a lot of organizations start with anyway. Like the auto instrumentation is that quick win and then it's like, oh yeah, we're we're lacking a little bit more. Let's go with the manual instrumentation. That's awesome. Um, we do have a question from uh someone in the audience from Buddha. Um, hi Buddha, nice to have you join us. So, uh, Buddha asks, are you currently using all hotel signals, logs, metrics, traces, profiles or a combination of them? Uh so we are using logs, metrics and traces. Uh so logs are coming through depending on uh which infrastructure piece. So if it's Fargate, it'll come through the AWS fire lens and fluent bit solution and then be sent to our agent sorry to our vendor. Otherwise if it's uh lambda it come straight from the um lambda layer agent um with the um yeah with the lamb layer. And then for metrics uh that comes in through a different connector with our vendor. they have a solution we install in our uh cloud environment and then it kind of pulls all the metrics over to that. Now profiles is interesting. We don't use profiles but uh the use case I think for profiles is enabling maybe uh common attributes or standardization uh and so we handle that through again our vendor has a solution out of the box uh that allows us to kind of set the schema and richer data as it gets ingested. Great. Um, and then another follow-up question from Buddha. And by the way, actually before I I asked that, um, there was a question from, uh, Manish, um, who's asking if we are recording the session and yes, we are recording the session. So, it should be available on on LinkedIn and, um, YouTube on demand after the fact. So, um, great question. So, uh, the follow-up question from Buddha was, uh, was this how you started, uh, or slowly built up to it? Um, who would like to take that one on? Um, was that one for like the terraform aspect or just in general? I I'm assuming it was for the instrumentation in general, I would say. Yeah. Uh, at least one for Tommy. Tommy had a fun time doing that. Okay. Well, um, this was not how we started. And for context, I I joined about 10 months ago. And, um, yeah, as time I joined, we did not have open telemetry setup anywhere, at least best of my knowledge. And um I know that some of yeah a couple of the motivations um for adopting open telemetry on our side was um standardization um to because we had like a patchwork of of monitoring tools like um logs metrics everything scattered all over the place but open telemetry gave us a way to now unify all these and to reduce uh cognitive load and then the interaction overhead that comes with that. Um and uh additionally we wanted to be vendor neutral you know um using op telemetry allows us not to be locked to a single vendor so I say um I know I'm getting into the motivations for why we adopted it but it also cuts back into um the fact that we did not start out that way slowly built into it and these were some of the decisions that drove um us to using of telemetry Uh yeah, I hope that answer question. I hope so as well. How long did it take you to to get into the point where you are at currently? Uh I'd say we're still we're still getting there, right? Um and if I remember correctly, we started somewhere around September. Andre, keep me honest here. And I think by March, we closed the chapter on that one. um from I would say from conceptualization but I say from implementation to when we thought that we were in a good place so roughly seven months right um trying out stuff and uh yeah I think we're in a good place now all righty uh yeah we have a we have a question in chat from Manish uh when we talk of open telemetry data there is always a huge amount of data how are you optimizing your storage and managing doing it. Uh, okay. I I I'll take this one a bit and then I'll let Andre finish off cuz he's the geek with that. Um, one of the things we do is sampling, of course. Uh, and I know when we query for stuff, we do a sampling ratio of I think 1 to 1,000 for traces at least on the E- Club side. Um, this is so uh this is tuned in this way to balance cost, performance and and visibility. And of course we monitor the effectiveness of this sampling setting and adjust it dynamically during incidents or specific services um that require deeper analysis. And for logs for querying um and most of the other things we do with logs we enforce a scan limit of 500 GB per query to keep the back end performant and cost cost effective. Um but I know again Android geeks out on on things like this. So I think you'll be able to give uh a better response. Technically we got a really big budget for our vendors. So we actually have uh zero sampling at the moment. So we just we grab it all. Um we're just trying to understand from that point and uh slim it down and um you know the first exercise that we we kind of ran through was grab all the logs kind of run like a pivot on it like the error logs do I count on uh you know sorry just general logs and are there any useless logs that are being constantly created by the system or just noise and can we remove that and the answer was yes so I think we had something like two billion three billion logs coming in that are just like success that we just didn't need to have in place for it. Um, so yeah, we could been cleaning up the system a lot. Uh, we go through all our pods and kind of hand out tickets to them as we find through the investigation and be like, "Yeah, let's get rid of this or uh you make it a debug log potentially if you guys actually need it, enable it during your debug sessions." Otherwise, let's not try to have too much noise uh into our ecosystem. I will say though that one caveat is because we are running Lambda that that one agent and Cloudatch requires certain logs to come out. So we can't get away from certain noise like right now I think we're ingesting 54 million logs a day um that we just we can't get away from and they're just noise at the moment. So uh another motivation to move to Fargate where we have a bit more control over system as we uh adapt a bit more of the observability ecosystem. Cool. Cool. Uh yeah Manish if you have any follow-ups feel free to feel free to post them in chat and I would continue with the next question about collectors can you folks tell us a bit about the way how you set up your collector or collectors what is the distribution and yeah what is the architecture overall super curious about that okay well I I'll begin again with the e-club side and again for context um as I said I I handle most of the e-lub side does pick stuff um concentric. Um and for E- Club, what we do is try to make our collector um environment aware. Um so in Sandbox, cuz what we have is Sandbox, staging and production on E- Club. So on Sandbox, we sample a little more aggressively to capture as much data as possible for debugging especially for me, right? To really know what's going on there and um also help the devs um really debug um stuff that they have going on there. But in production what we do is a little more probabilistic. I hope I got that right. To balance visibility with cost, right? And um what we do is we dynamically also adjust that sampling ratio um if there's an incident and then we temporarily increase that to capture more data. Um and again the collector definitely does all the batching filtering um and exporting telemetry to the back end our vendor. And uh of course this is done in a yaml config to define our pipeline for traces for metrics and log and to also set the environment specific parameters like service name the host type and then all the open telemetry options we want to set um on that. So that's what we have for our collector on club and uh as I said at the beginning it runs as a sidec car through the services that celery the um the jungle application also. So as a followup um on your um collector setup so um do you have um so I guess two two questions. Do you have a way to manage like a fleet of your your because it sounds like you have a number of collectors. Do you have a way to manage them effectively? Like do you use op amp for that or do you use like kind of a homegrown thing? And also do you do you use like a collector gateway um to like funnel all of your collectors into like a central gateway before pushing that off to your your uh observability? To be honest, not really. uh we just manage the fleet using ECS as I said and then um we run them outside car to our standalone services and the um environment variables that I said help us customize those things I said earlier which are the sampling rates um the exporter and the service names so we don't really have uh we don't really manage the fleets independently um or independent of our services we just run them as a sidecar and lets us do the rest for us Um, so I I think that's pretty much it. But we we also monitor the performance, right? Definitely. Um, things like the Q length and then we have some drop spans along the way. So we can scale them horizontally if we do need to. Um, but for um high throughput services like the I can't remember the name, one of our of our celery I think it's celery flour, right? Um we have of course a higher instance for that to manage um the size and scale of how things can be and to avoid bottlenecks. Um so yeah so do so just to clarify do all your um site car collectors then do each of them go directly to your back end? Oh okay got it. And it sounds like uh Andre M had a a screen share for us. Yeah. So um and just it's a visual essentially the same idea that um Tommy was walking through there. Um before we get started though, shout out to Sky Draw for anyone using that. The best platform for uh brainstorming ideas and a little whiteboard place. Uh but uh essentially the ECS Fargate again I was mentioning the ads fire lens configuration for us. um the management all of it comes back to Terraform. So that allows us to easily manage our collectors because even the configurations everything in code. So it makes that part a little bit easier. um if that's what you're trying to guess prelude to uh our vendor does have that uh instance within our um account but we don't utilize that as we found that uh sometimes the logs are being dropped either we underprovisioned it and we found that if we just directly send them straight to the vendor there wasn't any issues that uh we've noticed aside us losing some of the benefits of that additional collecting uh if we have to do some data masking or dropping those logs prior to um being sent and ingested by the vendor there. So, um definitely a area we're trying to balance. Um but, uh the other part is obviously the lambda, which again is super straightforward. It's just straight from the the lambda and straight to the vendor again um without any um additional pieces there. So, that may be an area now that we're talking here um area for improvement um because if we have those 54 million logs that are just noise, we might benefit by going to that uh filtering agent first. So very useful. Cool. Thanks for sharing. Yeah. Um you already mentioned uh this like areas for improvement and I'm wondering uh do you have any challenges with open telemetry currently and any any thing that uh perhaps the project could could do better to to serve you as a as as its user? Well, I'll say for me one of the biggest challenges was just initial setup, right? Uh configuring the connector and ensuring context propagation across all the services which required me to do you know manual instrumentation and uh of course there's a learning curve uh telemetry is powerful uh but again the API office is just so large and uh of course with in terms of best practices we are still evolving. Uh, one of the things that I'll probably like to see, um, and I think I'm speaking to the community when I say this is, uh, maybe more complex example project, speaking to myself too, of course, putting it out there. Um, but I would say the community has been has been really helpful, right? There there were so many things that I benefited from from GitHub mainly and then uh, a bunch of articles that people have written on medium, right? So, um, for me it's just more real world examples, more complex setups like probably what we have. Uh, maybe, you know, somebody just doing something that that big and putting out there for anybody to be able to use, but it say for me that's just been the the challenge. Um, not nothing too crazy. Andre, what about you? Do you have anything in mind? Um, not so much I can give feedback to really improve. I mean the whole uh time has just been learning learning learning right there's just a lot um I mean it's first exposure to it really for me it's not even that I had experience with other observability tooling it's just like straight to our vendor and then also hotel so just learning the best of the best right off the bat basically right uh and then learn the history so for me it's just it's been a constant um uphill battle to learn all the different intricacies um probably just want to echo Tommy's um message just more examples right being able pull things down, play with it. Uh getting into your hands really quickly. That's that's usually the best place to uh to learn. Yeah, I I totally agree. I think uh I think we're at we're open telemetry has been around long enough now that I I think many organizations are kind of past that 101 phase and are are eager to get into like the the more gnarly use cases. And and that's why, you know, we really appreciate folks like you jumping on onto these Hotel Me sessions talking about like your mature hotel setups because it really really helps others in the community. Um so we definitely appreciate that. Um switching gears a little bit. Um we were wondering um if you could uh if you have had any like interactions with the um hotel community um things around you know like you know if you got stucker in on on a particular issue like how how did you go about it if you got stuck like did you did you uh go on Slack or did you like Google stuff? Um how how how did that work for you? Uh I remember the first thing it was for E- Club. Um when we were trying to instrument it originally it was uh there's an extension we use for is it parallel processing Tommy? Do you remember that one? It's not uh W's wake there's another one. Well it's a Django app that we run. Uh and the problem with that is that uh there's an extension we use uh I think it optimizes performance and the problem with that is that it somehow was fighting hotel. Um yeah uh um I really can't remember the name but then I I I know they were fighting for the um thread the threading running in the background. Uh I think that that was one of the challenges that we had in the beginning. I really can't remember what specifically was but uh I know that was a big uphill battle back then. Yeah. So we had a open I think issue on GitHub with that. Um but uh luckily our vendor was able to um get their agent working so um it worked for that environment but I think um I think it's issue is still open but wasn't sure if it was like upstream with Django or was it so much with hotel itself. Cool. Cool. Um yeah thanks for that. Um, as a followup, um, have you gotten to the point where you have made any contributions to hotel, um, or, uh, or planning to make any contributions? 100%. Uh, nothing yet in terms of contribution uh, commit, but 100% we'll probably be looking at something uh, within the ecosystem of um, probably node or go, one of those two. Awesome. Um, final thing. So, I I think we've we've covered I think most of the um most of the the the base questions. We do have a a cool audience question that came in um from Esther. Uh it says, "How do you see open telemetry evolving in your workflow over the next year?" Really great question. H that is a it's a tough question. So well because I mean open telemetry I mean it's more about is it so much like the framework itself or is more about the data we're getting out of it because for me I think the the value I mean both both are great right because one gives us a standardized um tooling and framework to be able to switch vendors or anything else we have the capability now but for me I think the real value comes into what type of data do we really pull out of that open telemetry uh and what can we really get from it um I think that's where real value is. So being able to now take that data and apply user journeys and business events like those things I think are where um our organization can really uh I guess expand or benefit from for the most part. Mhm. Tom, do you have anything to add? uh not much but uh I think just like open test metro also keep maturing in our observability journey especially around logs and for complex setups like accelerary um or serless that's with lambda because I know there were there were also some battles fought there uh I also think we are probably going to have more out of the box integration um and tools to simplify the configuration for our collector right so I mean the ecosystem is growing so fast. So, um I would not be surprised to see us adopting it um a little bit more. Um as Andre said, we are growing every day. We are learning every day. Um but I think we'll be leaning more towards probably much better or more standard industry practices when it comes to um adopting hotel and uh using it to really observe our system. So uh that's what I see happening over the next year but who knows. Sounds good. Sounds good. Uh yeah we have more audience questions. Uh today we just we just uh talked over at the end with Adrian that we have really really engaged audience. So that's amazing. Uh so Manish is asking how are you doing the instrumentation of legacy systems? Is anything like that happening in like do do you have a use case for that? um think we have some older.net projects but it seems as if um our vendor supports out of the box and anything I think we even with that solution the vendor allows us to still have it in hotel format as it comes through. So um not really a concern for on our side from what I've seen so far. All righty. All righty. Uh and uh one more from Buddha. Uh how do you report on business value metrics for decision makers? What metrics do you report on? Yeah, I know I think Tommy you mentioned it that for that business logic you had to do some manual instrumentation. Can you tell us a bit more about that? Yeah. So, as I said earlier, there were um certain business logic that we needed to especially on the salary side cuz that's where we upload um stuff to um to really see how um data was flowing through that and then report on I think specifically it was it was a reporting service or I think an order service at the time. Um so what we got out of that really was just the rise and fall or the spike of um the would I say the others process within that within a certain period or within a certain trace. Um but you know reporting back to business people I think that one comes more out of the box with our um vendor and um we as Andre said build stuff that aggregates the amount of logs that we use um that translate into how much we pay for these things even though we have a big budget you know we we still don't want to be we still don't want to end up overshooting that um but I think again Andre would have more context into this cuz he builds most of the stuff that I report back to business leaders and that that's something he's so in love with. So I I would allow him take the floor on that. So I think for me the biggest one at least it's been months in development is um the most important key metric I've come to love recently is um users. So what is a single user doing across a system and aggregate that over a certain period of time. So um that alone allowed us to find a particular leak where we had a single user was a bug in our client uh and within our service uh that wasn't like a batch request where this one user within a short period of time was invoking over 200,000 requests into our system and with lambda pay as you go it's not a good uh not a good model there. So without this observability in place, we wouldn't even know that there is, you know, one single user somehow invoking 200,000 requests within like a six-hour period. Um, so these tips, it's small little things uh where now we can focus on a single user or aggregate it across a grouping uh and be able to find behaviors and it's really good when you throw into like a time series or flame graph and you can see these kind of like fat spots where you can just visually identify something doesn't look good here. Investigate That's great. Um, thank you so much um both of you, Andre and and Tommy for uh for joining us and and thank you to my co-host, the other Andre um for joining uh and um and also this is uh Andre K's first uh first time on on a stream. So like big uh big kudos. Um, awesome job as as co-hosts. So, I want to give a shout out. Um, and you know, as always, we appreciate all the stories that we hear from our community members showing how they use open telemetry out in the wild. It really helps us. Um, and it helps show folks in the community like, you know, open telemetry is here to stay. We are we've got some like gnarly use cases of it being used out in the wild and it's it's great to hear those stories. Um, coming up next, stay tuned. We should have uh another Hotel Me planned, I think, in July. So, stay tuned on the hotel socials for that. Um, in the meantime, um, coming up at the end of this month, we have Hotel Community Day. Um, it's part of, uh, Open Observability Con. So, it's it's, uh, combined with Hotel Community Day. So that's happening um on the heels of Open Source Summit in uh Denver. Open Source Summit North America in Denver. So if you're around for Open Source Summit, check out Hotel Community Day and Open Observability Con that are happening together. And I think we've got a couple of CubeCons also um coming up. We've got I think I want to say this week is CubeCon um China. And then next week, I'm actually jetting off to Japan at the end of this week for CubeCon Japan. Um, and it's the first CubeCon Japan. So, very exciting stuff. So, if if anyone's around um for CubeCon Japan, come say hello. We'd love to uh would love to to meet you and talk. Um, and yeah, I think that is a wrap. And again for uh for those who attended tell your friends if they missed it the recording will be available um both on LinkedIn YouTube also check out the hotel end user s also have a um uh a group on uh on CNCF Slack we are called um I think we're called-ig- user so come share your stories on the hotel and user sig um chat and also if if you'd love to join us for one of our meetings. We meet every two weeks, so I think our next one is next week. So, yeah, thank you everyone once again for for joining and we'll see you next time. Thanks so much for having us. + diff --git a/video-transcripts/transcripts/2025-07-16T20:50:15Z-otel-in-practice-alibabas-opentelemetry-journey.md b/video-transcripts/transcripts/2025-07-16T20:50:15Z-otel-in-practice-alibabas-opentelemetry-journey.md new file mode 100644 index 0000000..04d1a32 --- /dev/null +++ b/video-transcripts/transcripts/2025-07-16T20:50:15Z-otel-in-practice-alibabas-opentelemetry-journey.md @@ -0,0 +1,132 @@ +# OTel in Practice: Alibaba's OpenTelemetry Journey + +Published on 2025-07-16T20:50:15Z + +## Description + +Join us for a new session of OTel in Practice with Huxing Zhang, Staff Engineer at Alibaba and OTel Go Compile-Time ... + +URL: https://www.youtube.com/watch?v=fgbB0HhVBq8 + +## Summary + +In this episode of "Hotel in Practice," Dan hosts Hushing Jang and Steve Ra from Alibaba to discuss the adoption of OpenTelemetry within their organization. Hushing and Steve share their experiences transitioning from Alibaba's internal instrumentation to OpenTelemetry, detailing the challenges they faced and the enhancements they made during the migration. They explain how Alibaba has developed Java and Go instrumentation, focusing on automatic context propagation and the integration of profiling capabilities. The discussion also touches on Alibaba's commitment to contributing back to the OpenTelemetry community and promoting its usage in the Asia-Pacific region. The video concludes with a Q&A session, addressing the ease of migration for teams and encouraging best practices for context propagation. + +## Chapters + +Here are the key moments from the livestream along with their timestamps: + +00:00:00 Introductions and overview of the session +00:02:45 Introduction of Hushing Jang and Steve Ra from Alibaba +00:05:30 Background on Alibaba's observability capabilities +00:09:15 Discussion on the migration from Pinpoint to OpenTelemetry +00:12:00 Enhancements made during migration to OpenTelemetry +00:17:30 Introduction of Alibaba's Golang and Python instrumentation +00:22:00 Explanation of compile-time instrumentation approach +00:27:45 Use cases for OpenTelemetry in AI applications +00:33:15 Challenges of context propagation and solutions +00:38:00 Discussion on internal adoption and user support for migration +00:43:00 Closing remarks and invitation for further conversation + +These timestamps provide a clear overview of the key topics discussed during the livestream. + +# OpenTelemetry Adoption at Alibaba + +Good morning, good afternoon, and good evening, whenever and wherever you're watching this. Welcome to a new edition of *Hotel in Practice*, where industry experts, end users, and contributors discuss specific areas of OpenTelemetry and how it can benefit, or has already benefited, end users. + +My name is Dan, and I'm a member of the End User Special Interest Group and the Governance Committee. I'm excited today to have Hushing and Steve here to talk about the adoption of OpenTelemetry within Alibaba. I'm connecting from Edinburgh in Scotland, where you can see the hills in the background. If you're watching live on YouTube or LinkedIn, please say hi in the chat and share where you're connecting from. + +Before we get started, I want to cover a couple of housekeeping items. During the presentation, please drop your questions in the chat. While we'll do our best to answer all of them, I can't promise we'll get to every question. If your question remains unanswered or you'd like to continue the conversation, please visit the End User SIG resources linked in the chat. You can also find your way to Slack, where we have a large community of end users, including Hushing and Steve, who will be happy to share their insights. + +Please remember to keep your questions relevant to the topic of the presentation. With that said, let's get started and welcome Hushing Jang, a Staff Engineer at Alibaba and Hotel Compile Instrumentation Maintainer. Hi Hushing, can you tell us a bit more about what you do? + +### Introduction of Speakers + +**Hushing:** Hello Dan, very nice to meet you! I’m Hushing from Alibaba. I work as an observability engineer in Alibaba Cloud, focusing on client instrumentations. We work closely with the OpenTelemetry community, developing various language instrumentations and collectors, providing services for both internal and external clients. + +**Dan:** Thanks, that’s awesome! Also, welcome Steve Ra, Senior Software Engineer at Alibaba and Hotel Java Instrumentation Approver. Hi Steve! + +**Steve:** Hi Dan, I'm Steve, also from Alibaba Cloud. I work in the observability team and participate in Java agent development. I'm responsible for profiling in our team and providing out-of-the-box profiling capabilities for our users. + +**Dan:** Great, welcome both! I’m excited to hear more about how you approach OpenTelemetry at Alibaba and how you became major contributors. Now that we’re all set, I understand you have a presentation to guide us through. Please take it away! + +### Presentation by Hushing and Steve + +**Hushing:** Hello everyone, we are Steve and Hushing from Alibaba, and we are glad to share our experience with OpenTelemetry adoption. Today, we will explain our talk in five parts. I will introduce the first two parts, and the remaining will be presented by my colleague. + +#### Background of OpenTelemetry Adoption + +In 2013, we provided observability capabilities for our internal users through manual instrumentation, integrating this ability into our internal software frameworks. We also designed a trace propagation protocol called Eagle I. In 2017, we developed the Alibaba Java agent based on the Pinpoint Java agent. After several years of usage, we encountered some issues. For example, Pinpoint could not verify the supporting version of a framework; if a framework released a new version, the instrumentation might not take effect. We also found that Pinpoint did not support scenarios like synchronous trace propagation automatically. + +From 2020 to 2023, we researched alternative solutions and found OpenTelemetry to be a robust ecosystem with reach semantic conventions, representing a standard in observability. After several internal discussions, we decided to migrate our Alibaba Java agent from Pinpoint to OpenTelemetry. After months of development, we launched our Alibaba Java agent based on OpenTelemetry in 2024, along with our Golang and Python instrumentations, all based on OpenTelemetry's semantic conventions and SDK. Currently, we fully embrace OpenTelemetry. + +#### Enhancements Made During Migration + +During our migration, we made significant enhancements to the OpenTelemetry Java agent. The first enhancement was instrumentation. We discovered that OpenTelemetry was not widely used in China because some widely-used frameworks were not supported by the OpenTelemetry Java agent. Therefore, we implemented support for these frameworks based on the OpenTelemetry architecture. + +The second enhancement was profiling. Profiling is a powerful tool that helps us inspect the behavioral performance of our applications at runtime, allowing us to identify which code is responsible for spikes in CPU and memory usage. We had provided this capability for several years, and its migration was crucial for us. + +Other enhancements included abundant sampling, additional metrics, and popular debugging tools within China. Moreover, we not only supported observability through the Java agent but also contributed to microservices governance, such as traffic control and security management, through this Java agent. + +To achieve our migration goals, we devised a two-step solution. First, we forked a copy of the Java agent repository and quickly implemented our commercial enhancements. This process was challenging due to the many commercial features involved. We had to address numerous issues during migration, particularly those related to the upstream, and contributed fixes back to the upstream community. After about six months, we launched the first release of our Alibaba Java agent based on OpenTelemetry. + +Following the initial release, we recognized that it was not enough to simply migrate; we also wanted to leverage OpenTelemetry's strong ecosystem. We encouraged users to adopt our initial version while inviting developers to design extensions and migrate our commercial features to OpenTelemetry. This was a time-consuming process but essential for decoupling our commercial code from the open-source code. + +#### Benefits of Migrating to OpenTelemetry + +After migrating to OpenTelemetry, we experienced several benefits. Firstly, the OpenTelemetry semantic conventions became the foundation of our observability efforts. By adopting OpenTelemetry, users could independently access materials and documentation, which significantly reduced their learning burden. + +Secondly, the good architectural design and better scalability of OpenTelemetry instrumentation made it easier for us to promote our Java agent. Previously, if a user employed a framework that we did not support, they would create issues asking for support. Now, we can inform them that many extensions are available for them to use independently. + +**Steve:** Thank you, Hushing. I will now introduce the remaining parts of our presentation. + +### Compile-Time Instrumentation Approach + +In 2023, we sought a solution for Go applications that could simplify instrumentation. Initially, we explored the eBPF-based approach provided by the OpenTelemetry community, but encountered limitations. Eventually, we developed a compile-time instrumentation method, which hooks into the Go compile command, ensuring no modifications are required in the user's code. + +#### Highlights of Our Approach + +1. **Asynchronous Context Propagation:** This feature automatically propagates context, even when users do not pass the object correctly, addressing common issues with incomplete traces. + +2. **Compatibility with Old SDK:** Users who have previously used manual instrumentation can continue to work seamlessly with our approach, allowing for a smooth transition. + +3. **Custom Extensions:** Users can instrument any line of code in their applications, enabling customizations for traffic management and security. + +### Adoption Cases + +We've seen many adoption cases within Alibaba. For instance, our cloud service's data flow uses OpenTelemetry to ensure that all key services embrace the W3 tracing model by default. The AI Studio utilizes Java and Python instrumentation for end-to-end observability, while services like the Elastic Cloud Computing Service and Video on Demand services have adopted OpenTelemetry for runtime monitoring and profiling. + +### Contributions Back to the Community + +Beyond our internal adoption, we strive to contribute enhancements and extensions back to the OpenTelemetry community. We've organized friendly meetings for the APAC region and engaged with conferences like KubeCon. In 2024, we decided to donate our Go compile-time instrumentation project to the community, forming a special interest group with collaborators from DataDog and Quesma to build this project from scratch. + +### Future Work + +Looking ahead, we plan to implement a proxy agent mode for our Java instrumentation to ensure all extension points can be merged into the upstream. We are also working towards the first release of our Go compile-time instrumentation and will support additional signals, including metrics and profiles. + +### Conclusion + +Thank you for your attention, and we appreciate the opportunity to share our journey with OpenTelemetry at Alibaba! + +--- + +**Dan:** Thank you, Hushing and Steve! That was an insightful presentation. If anyone has questions, feel free to drop them in the chat. + +**Audience Interaction** + +One question I have is about the ease of migration within Alibaba. How did you facilitate the transition to OpenTelemetry? Did you use common libraries or migration scripts? + +**Steve:** Migrating to OpenTelemetry wasn't easy. We wrote internal articles sharing our practices and benefits. We also aimed to minimize the effort for users by integrating our legacy project with the new architecture, ensuring a seamless transition. + +**Dan:** It sounds like you really focused on making it easy for your teams to adopt! + +**Steve:** Exactly! We aimed to take on the burden of migration rather than placing it on our users. + +**Dan:** Great approach! + +If there are no further questions, I want to thank Hushing and Steve again for joining us today. If you'd like to continue the conversation, please check out the End User SIG resources at OpenTelemetry.io. Thank you all for watching, and see you next time! + +## Raw YouTube Transcript + +Good morning, good afternoon and good evening uh whenever you're watching this or wherever you're watching this. Welcome to a new edition of uh hotel in practice where industry experts end users and contributors tell us about a specific area of open telemetry and how it can benefit or it has already benefited end users. My name is Dan. I'm a member of the end user special interest group and I'm also a member of the governance committee and I'm very excited today to have Hushing and Steve uh to talk about adoption of open telemetry within Alibaba. I'm connecting from Edinburgh in Scotland. You can see the hills here. And if you're watching live on YouTube or LinkedIn, please say hi in the chat and tell us uh where you're watching from or where you're connecting from. Before we get started, I just wanted to cover a couple of uh housekeeping items. Uh during the presentation, please drop your questions in the chat and uh although we'll make our best to to get to all of your questions, I can't promise that we'll get to all of them. But fear not, if that's the case, if we can get to all of them um and your question is not answered or you want to continue the conversation, you can go to the end user s um and then we've got resources in this link and you can uh you can continue the conversation there. You find your way to Slack where we have a big community of end users including Hushing and Steve and we will be happy to go and um provide you know provide um some of their insights there. Also, please remember to keep the questions to date and topic of the presentation. And yeah, so with that out of the way, let's get started and welcome Hushing Jang, staff engineer at Alibaba and hotel compile instrumentation maintainer. Hi Hushin, can you tell us a bit more about what you do? Hello Dan. Uh, very nice to meet you and uh, hello. I'm Hushing. I'm from Alibaba. Yeah, I'm working on as a observability engineer in Alibaba cloud and we are also working on the client instrumentations. So we work very closely with the open telemetry community. We we're working on different kinds of language instrumentations collectors and something like that also provide service both for internal and external clients customers. Thanks. That's awesome. and also welcome Steve Steve Steve Ra senior software engineer at Alibaba and hotel Java instrumentation approver. Hi Steve. Yeah. Hi Dan. Hi I am. I'm Steve. I'm also from Alibaba cloud. I'm work in observability uh team in Alibaba and I'm uh participate in uh Java um agent and I also responsible for profiling uh in our team and provide out ofbox uh profiling ability for our users. Thank you. Nice. Well, welcome both. Um I'm really exciting to hear more about you know how you're approaching open telemetry in Alibaba and you know how you ended up contributing as well being major contributors to open telemetry. So, um, now that we're all set, um, you've got a presentation and you'll you'll guide us through it. I'm going to move out and then, uh, please take it away. Okay. Uh, hello everyone. We are Steve and Husinang from Alibaba and we are very glad to uh, share our talk Alibaba practice experience. Today we will explain our talk from following five parts and I will introduce the uh first two parts and the remaining are giving to my colleague. Uh the reason for choosing hotel. Uh let me briefly introduce our background in uh 13. We have provide uh observability capability for our internal users by uh manual instrumentation and we uh provide this ability into the our internal uh software frameworks and we also designed a set of trace propagation protocol called Eagle I and in 2017 We a developer of Alibaba Java agent uh based on pinpoint a Java agent. After several years of used, we found some problems such as ppoint cannot support uh verify the uh supporting version of a framework. For example, if a framework released a new version and instrumentation uh may not take effect and we can perceive this uh immediately. Uh another thing sounds like uh pinpoint cannot support very well in some scenarios such as um yeah trace propagation synchronously. uh it doesn't support it automatically and there also have some other reason due to the limited time I I cannot uh list them one by one. So from uh 2020 to uh 2023 and we uh research some other solution um from open source at the time we found open telemetry is a a very strong ecosystem. uh it has uh reach uh hotel uh semantic convention and it represent uh the factor standard in observability. Uh after several internal discussion we made up the uh mind to migrate our Alibaba Java agent from pinpoint to over telemetry. uh after several uh months of development and we launched our Alibaba Java agent based on of telemetry in 2024. We also launched our Golong and Python instrumentation based on old semantic convention or SDK. Currently we embraced of telemetry completely. Yeah. after introduce our background uh yeah during our migration we really uh did a lot of enhancements um on hotel Java agent. Uh the first one is instrumentation. At the time we found uh of telemetry not widely used in China because there are some uh instrument uh framework they are not support um from uh hotel Java agent but they are widely used in uh China. We have supporting them before. So we need to implement them uh based on uh hotel Java agent architecture. And the second point is profiles. As we know profile is very powerful too. Uh it can help us to inspect the behavioral performance of our application at runtime. Um it can help us to know which code is responsible for the spikers. um in CPU uh memory usage we have supported provide uh uh for several years. So uh migr migration of this feature is also very important for us other abilities such as abundant uh sampling or more matrix and we also need to uh debark um a very popular uh debugging to ours in China. Last but not least, uh we not only uh support observability uh by Java agent, we also uh support microservices uh governance such as traffic control or security management by this Java agent. So uh how to compel them in uh hotel Java agent u architecture? It's a yeah it's a tosser we need to solve uh yeah introduced the enhancement we need to do and um u due to the limited time at the time and how to migrate them uh it's a challenging for us in fact at the time we don't very uh for many with telemetry and uh in order to uh achieve this goal we We come up with a two-step uh solution to solve this question. First one um yeah we forked a copy uh Java agent repository and we pro promptly implement uh all our commercial enhancements on the Java agent. Yeah, in this process is not very easy because there are lot of commercial uh features. Um uh if we migrate to the um of telemetry there are a lot of issue or problems and due to the rich uh business scenarios and um yeah during migration we also found some problems and we need to fix them and some of them are uh strongly related to the upstream we also uh contributed them to the upstream. uh after about six month from research to uh to launch uh the first release of uh our Alibaba Java agent based on open telemetry. uh after finish this uh um phase and we uh know that it's not enough because we um yeah we know the biggest charm of open telemetry is it stronger ecosystem uh if we combat our uh code compared with open source yeah uh it uh it's challenging for us to uh upstream some to merge some upstream update to our uh Alibaba Java agent. So we uh on the one hand we promote uh our user to use our first um first uh version and then we also invite some uh developers to uh yeah to design some extension or use some extension from open source to u migrate our commercial feature and this process is time consuming because we need to design some extension based on Java agent and we also need to um contribute them to the upstream and u the other things we are having have been doing uh currently and u yeah after this step we can uh yeah we can see the uh open source hotel Java agent just like a dependency for us and we can decoper our commercial code and the uh open source code and we can merge upstream easily. Yeah, this is our u whole approach to achieve migration. Um after migration uh to open telemetry, we rarely get a lot of things from open open source and uh yeah we also get something um we uh doesn't anticipate at the beginning. Uh the first one I want to share is um yeah hotel semantic convention. Yeah, that I guess that is the soul of the open telemetry and when we migrate to open telemetry and we tell our user we achieve this Java agent based on open telemetry and um they can um do a lot of things by themsel such as they can uh yeah search some materials there some documentary by themsel and they uh can learn how to use how to uh use some extension to write something by themsel and they can debug some problem by themsel. I I think uh yeah that is very um yeah good point for us to uh embracing of telemetry and our user uh can do a lot of things by themsel uh and uh it's rarely reduce users learning burden. Yeah, because they just need to learn um the standards the semantic convention or uh implementation of optometry and they can um yeah they can uh do a lot of things by themsel. Yeah, this is the first point and the second point uh good architectural design and yeah better uh scalability of hotel uh instrumentation that it's very uh very useful for us to promote our users uh to use our um Java agent. Uh how to understand this? For example, uh before Java uh hotel Java Asian and um if a user use a uh framework that is not widely used and uh we don't support that framework in uh from our Java agent and they will create a issue for us and ask we to uh schedule resource to support this framework. But sometime uh we take the cost and um yeah take the energy into consideration we are reluctant to support that. Uh so how to communicate with our user that is a a question at that time. Uh now day uh this is not a a big question. Yeah, because we can tell our user there are a lot of extension uh you can use this extension to support your own framework and you can do it by yourself and a lot of users um yeah they are very interesting uh about this because they can extend a lot of things by themsel uh don't light on ours and uh yeah it's only but I was uh after things I I want to share today and remaining time to introduce something uh from other aspect. Thank you. Thank you Steve. Uh I will I next to introduce Hello. Can you hear me very well? Yeah. Okay. Sorry. Uh there's some um network uh issues. Maybe I just lost the connections. I I will go into for the remaining part. Yes. Um actually this uh solution has been started in 2023 and at that time we want to looking for a solution that uh can be easily for go applications for the to do instrumentations. Actually we when we are um we're trying we're trying the EVPF based approach that is already been provided by the open telemetry community but uh it turns out that there's something uh limit some limitations there and we uh finally we have figured it out the way that to do the instrumentations at to satisfy our needs that is uh in a compile time instrumentation If if you're looking at the figure in the right there's a approach that's the the in injection has been happened in the middle of the compilation process actually we hooked the golan compile command and to do to make sure that can be done for users there's no you modifications to users code that's a key feature that we want to achieve and the Next I want to uh address several highlights of the this approach. First is the async context propagations. Actually the users uh we we encourage them to follow the guidelines but actually not all the users follow the rules. So because they do not pass the sometimes they do not pass the context object correctly the the traces may break break and become incomplete. So this feature can actually can do the context propagation automatically for us even when we are not passing the object correctly. The second one is the actually compatible with the old SDK. That means that users have used have used the manual instrumentations they can work well with our approach. So the the traces can be merged. And the last one is the the I think the most powerful features that we have done is kind of like customer extension. And that means that you can instrument any code into any point any line of that applications and this allows us to to solve some special requirements from users. Then they can do some uh customizations like traffic management and uh security. Yeah, please. Yeah, that's what I I think is very important. And uh next uh we will do we'll introduce some a practice that we have in the LM of observability as we know that the LM has been uh popular recent rec recently especially this year a lot of AI agent has become uh active and there's a typical use cases that uh we found in the use cases is a user send a request to the API gateway and the API gateway sent to the ALM applications which is written in maybe Python, Java or Golan and the uh application will call different kind of models uh for example deepseek queen or open AI and there's actually in in uh we we have adopt uh like AI gateway in the middle and which can do a lot of things for us for example it can do unified uh access and do token token limitations, token cache and or stuff like that and so we have adopted the uh open source project called high highress that can do it very well for us. So actually u we when we are building the um observability across the different kind of uh components we make sure we're following this latest JNI semantic convention and the uh because the convention is still in development we were trying to keep the pace with them as much as possible and we actually we have done some instrumentations based on this semantic convention. Uh this includes major AI agent framework uh like uh high code frameworks like agent scope, agono, lang, spring, Alibaba and etc. And uh for low code platform like DIY is very is a very popular open source project to build AI agents with low code. And we also do some instrumentation for the official MCP clients. uh for example the we want we want to capture the MCP2 course we want to capture the M32 responses and to make sure our observability is end to end so we want to put our instrumentations into the internal models models that brings up with ser serving framework like VM or SGAN is actually they are a Python process running there. So we can put our Python instrumentations in into that Python process to capture traces and key metrics like TTFT time to first token and TBOT uh time per output token which is very important uh metrics for us to to identify bottlenecks of LM um serving issues. Yes. Ne next please. Yeah, talking about the adoption cases. Actually we have uh uh many adoption cases. I want today I want to introduce uh several of them. I divided into two parts. The first part is the cloud service data flow because when users they are using color service make sure all the uh services that in the key key uh key positions of the application they embrace the official W3 tracing gole by default. That means that balancer way or uh they have support the W3 W3 protocol by default will uh have the me using they're using open telemetry the async uh messagings like how how we how a consumer consumer is uh consuming the messages well and to measure the latency or the identifying bottlenecks they are using open telemetry to to do that and for the AI studio they just I mentioned in the previous slides and they are using actually Java and Python hotel instrumentation for end to end observability and the next part two is the cloud service management console. Actually a lot of management console service has embraced the open telemetry as well. Uh for example the elastic cloud computing service they using hotel uh Java instrumentation quite a lot and they extend the make some extensions to that and they can they will they want to do uh like ser request based traffic routing. They want to route the service to different uh uh machine groups to Yeah. And then they actually done a lot of things be beyond the instrumentations. Yeah. The video on demand service also they are running large large amount of clusters written in Golan and they have adopt the coline compile time instrumentation and they uh mostly for the uh runtime monitoring. Yeah, they want to monitoring the runtime metrics also pro providing data and another case is the institution uh in uh detection service actually they are there's a agent running on the host they're written in go and they want before open telemetry they have nothing to do with the observability because they lack of techniques lack of tool tools to do that. Now they can use hotel go compile time instrumentation for their runtime and monitoring and pro profiling. And the last we as uh our own service uh cloud monitoring service actually also uh implement self observatory with Java instrumentation. We do a lot of things with Java instrumentation to do traces and profilings. Yes. And this is good example of dog fooding, right? Yeah. Next, please. Yes. uh be uh uh besides our internal adoption, we also trying to contribute this enhancement or extensions back to the uh Otto community. uh we actually have u mentioned in before a lot of instrumentations and uh some of them has been contribute and some are also on the way of contributing back. Yes. Um we also helped uh to promote open telemetry in the APAC area. We help to organize some meetings that is friendly to the APAC Asia-Pacific uh areas like Java go and Jingi meetings and we also um engaged with the uh conference like KubeCon and hotel community and we al we're also organizing uh events like KCD and to help to promote the open telemetry in Asia. Pacific areas. Yes. And uh in 2024 is we when we uh have uh decided to donate the this Goline compile time instrumentations to the hotel community and uh after a long discussion with the community and we also found that data dog has showed a very uh strong interest in this project and they actually they also uh submit a proposal So to donate their uh oran framework is also a compile time instrumentation project to open telemetry. So we decide that we do not we we we do not depend on single one company effort from single one company but we form a sig and to build the this project from scratch. And so this SIG is co-ounded by the Alibaba data dog and the Quesma to uh we have a common effort to uh to implement this compile time instrumentation for Golan and this project is uh still this project is still in active development and we are going to we are working on the on the first release and we are working on the first MVP version of that and this year. Yes. And if you are interested, yes, please definitely check it out. And talking about the the future work, yeah, for Java instrumentations actually we we are planning to uh implement the prox proxy agent mode that that means that we want to use the Java agent as a dependency rather than booking upstream project because we do not want to maintain such kind of uh project. We want to make sure all the extension point can be uh merged into the upstream so that we can we can extend this uh project very very well. So for the go line in compile time instrumentation, yeah, we are actually we are working well with the MVP version and we're making sure we are very in the first release uh working towards the first release. We also will support more more signals including metrics and profiles. Yeah, for GI we we also want to contribute some of the instrumentations up to the community because this GI is still in active the semantic commission is still in active development. Yes, we we put some of them in our repo and we want to uh contribute in the back as well. Yeah. Yeah. Actually we also have our own make our own open telemetry distribution is called long sweet long is for chi Chinese version of uh dragon long yes yeah it's kind of distribution built on various language uh instrumentation as well as uh our collector project called the long collector. Yeah. So we want to make sure that uh all the all the enhancement we want to trying to put them uh contribute back to the hotel project as much as possible but for some reasons the the community may not accept our uh contributions they will be in our own distributions. Yeah. And uh that is all for uh the presentation and thanks for everyone for listening. Yeah, thank you. Um thank you Hushi and Steve. Uh that was great. I think um yeah feel free to drop any questions in the chat and then uh you know if if you're if you're watching live on YouTube or LinkedIn um have one which is um you know you mentioned like adoption of uh open telemetry within um within Alibaba and uh I wanted to know how easy how easy was it for teams within Alibaba to migrate to open telemetry is there anything you can share in terms of facilitating a migration maybe they use something like a you know common set of libraries or migration script now with code generation or something else. Yeah, I think it's not quite easy actually. It's not quite easy for them to migrate. We have done a lot of things uh to uh advoc advocate for that. First we wrote some like articles internals and to and to share the some of our practice how uh like a kind of team they adopt the this technology and how they benefit from that and then we put that into the internal article and to help them uh to promote this open telemetry. Secondly, we actually we we want to try to minimize the effort for the user to migrate from because we have a legacy uh uh project called called ecoi and they want they already have some instrumentations there. If we are switching to this like C architecture to the auto instrumentations they might encounter some conflicts or something like that. We we we actually we have done a lot of things uh to pro to prevent such kind of things to happen and we build build that within our agent with our instrumentations as much as possible and to make sure the the users have minimized their effort to to start up and to migrate. I think this the idea is to try to put things uh to our team rather than put the efforts to the users to make sure that simly seamlessly uh migration from the oxy architecture to the hotel based one. Yeah. Nice. So you like do food in that so to make sure that you know you adopt it internally as well. Yeah. Yeah. Yeah. Makes sense. Okay. Um I think we've got some comments uh on the chat but no question so far as I can tell. Uh I have another one as well from myself. Uh you mentioned that you run on the hotel Java agent stuff you run a fork or a distribution of a of that of that agent but in terms of API usage do internal users in Alibaba use the open telemetry API directly when instrumented applications or do you have some kind of abstraction on top or a helper library that you know that they use instead of the hotel API? Yeah. uh I I think for most the cases they we we we we would them to use the auto instrumentations as the first choice because that is uh less effort for them to start their adoption and but for some special cases uh actually we can't satisfied so we will uh encourage them to use the SDK one to add some custom uh instrumentations. For example, they want to add some custom spans or metrics. So uh when that we encourage them normally we use we use the official uh hotel SDK as much as possible because that is well documented and uh we actually we battle tested with the latest version of that SDK. So we do not want them to like build a another abstraction layer on top of that. But uh there's a a special cases that we introduced in like LLM application observabilities recently actually and they uh actually our AI studio that actually they are using the LT SDK they found it that uh they have to write a lot of code in order to add their instrumentations because they there's so much things that want to uh capture in a span like they want to capture very uh very in detail want to capture a lot of things. So they have to write a lot of instrumentations. So they ask us to we can provide actually provide an abstract layers for them to simplify their work to like to do their instrumentation. So in that case we actually work with them to provide higher level of that SDK. So they they actually that did work because the they their instrumentation effort have been like reduced and they are very happy to do that. Yes. So that's is the one case I think I can share and for most cases I think it's using uh official official OT SDK. Yeah. Yeah. And in terms of the API as well because it's got that um decoupling of the API and the implementation like you can always swap the the implementation underneath with the SDK right? Yes. Okay. I want to share another point about this question. Sure. Um yeah for our uh internal users some of them if we don't provide um auto instrumentation such as C++ application and uh yeah we encourage them to use the API just like yeah HI mentioned and another point for uh Java application um yeah some users um as we know uh if they use trace or spam uh sometime that is a monitoring like um spot uh if we just uh instrument some uh framework some key uh methods and uh if the latency um yeah belongs to users uh code yeah maybe they can get the information why the latency is so long um yeah during the two trace at this time yeah we provide out of box profiles ability we uh correlate the trace this with profile and we can uh provide uh diagnose for users to uh to understand um yeah why the latency is so long which code responsible for the latency uh so they don't need to use the uh API to instrument their own method uh manually okay yeah that makes sense um okay last question from from me And you know again the audience if you're if you're watching and want to drop a question feel free. Um both of you talked about context propagation as something that has definitely improved a lot thanks to open telemetry instrumentation and I know this from experience especially in when we're talking about async tasks. However, sometimes it's still challenging in certain scenarios to make sure that internally in a cont in a in a service that is the context is propagated uh correctly. Um, how do you encourage teams to ensure that context is propagated correctly within not just across services but also within their own services? And also like do you measure do you know they measure that in a in some way if you know if context is broken somewhere or leaking? Yeah, that's a good question. uh actually I think for our internal users it's very difficult for them to like even if you improve approve uh provide like guidelines or encourage them to do so they not they may not uh follow this rule and they there's very much opportunity for them to like to miss that or to fail to follow that. So I think to to to solve this problem is uh our team's duty to like to make sure the um all the possible possible ways of facing cross propagations has been propagated correctly by our auto instrumentations that that is way we want we solve this uh problem. So we have the popular frameworks for example in Java the thread pool or the uh some kind of similar scenarios and we do the test to make sure that uh in the in the bottom of the uh technique they can be a sync uh context can be propagated properly. Yes. And this is another uh frameworks it's very challenging for us to do. So for example the uh reactor netty or such a kind of frameworks actually they are very uh high frequency for them to like to switch in context and asynchronous asynchronous is common for them and the this for this kind of scenarios we might have something working not very well or as expect for for this case we will like to trying to work with the community or uh want to solve it. first of all to figure it out by ourselves and we as as we as at the same time we will seek help from the community and to like to trying to identify the issue and try to fix it and uh yeah this is the common cases I think and then I used to remember the day the days where you had to uh pass a context object around in function calls and it wasn't it wasn't pretty and it wasn't good either. And so yeah, I think auto instrumentation is the and you know anything that can get done transparently is also is always going to be covering more and resulting in better context propagation. Right. Yeah. As long as the users has uh stick to that uh auto automation automatic instrumentation approach they will like to ask for that. Have you satisfied this or make why why did that happen? So they want to they will come to us and looking for help rather than Yeah. do it by theirelves. Indeed. Okay. I think that's all we have time for today. Uh thank you so much for for joining us both Hushin and Steve. It was great like knowing more about you know Alibaba's approach to open telemetry and thanks for all that joined live in LinkedIn and and and YouTube and uh yeah so we've got the remember that if you want to continue the conversation we've got the enduser seg resources as part of the open telemetry.io your website and uh yeah, feel free to continue the conversation there and uh yeah, hope hope to see you hope to see you there as well and thank you again uh both and uh see you in the next one. Thank you. Byebye. + diff --git a/video-transcripts/transcripts/2025-09-25T05:32:25Z-otel-in-practice-how-we-scaled-kafkalog-ingestion-for-otel-by-150.md b/video-transcripts/transcripts/2025-09-25T05:32:25Z-otel-in-practice-how-we-scaled-kafkalog-ingestion-for-otel-by-150.md new file mode 100644 index 0000000..554727d --- /dev/null +++ b/video-transcripts/transcripts/2025-09-25T05:32:25Z-otel-in-practice-how-we-scaled-kafkalog-ingestion-for-otel-by-150.md @@ -0,0 +1,140 @@ +# OTel in Practice: How We Scaled KafkaLog Ingestion for OTel by 150% + +Published on 2025-09-25T05:32:25Z + +## Description + +Join us for what's sure to be an insightful session, as Dakota Paasman from Bindplane shares how their team solved a customer ... + +URL: https://www.youtube.com/watch?v=GrQyUPeCljo + +## Summary + +In this edition of "Hotel in Practice," host Henrik welcomed Dakota Passman from Bindplane to discuss scaling open telemetry logs with Kafka. Dakota, a software engineer at Bindplane, shared insights from a project where they helped a customer overcome performance bottlenecks while pulling events from Kafka. The customer initially faced issues with only 12,000 events per second per partition, which led to a growing backlog. Dakota highlighted key strategies that improved performance, including using a new Kafka client, optimizing log encoding, reconfiguring the data processing pipeline to prioritize batching, and switching the exporter protocol. After implementing these changes, the throughput increased significantly, demonstrating how careful configuration can enhance system performance. The session concluded with a Q&A segment where Dakota addressed audience questions related to performance tuning and configuration best practices. Viewers were encouraged to access the recording on YouTube and LinkedIn for further learning. + +## Chapters + +Here are the key moments from the livestream with timestamps: + +00:00:00 Introductions and welcome to the stream +00:01:45 Introduction of speaker Dakota Passman +00:03:10 Overview of the topic: Scaling OpenTelemetry logs with Kafka +00:05:00 Discussing performance issues with the customer's Kafka integration +00:07:30 Key takeaways for improving Kafka receiver performance +00:09:20 Explanation of the customer's original configuration and bottleneck +00:13:00 Live demo setup and initial performance metrics +00:15:30 Implementation of changes to improve performance +00:20:00 Analysis of the changes made and their impact on throughput +00:25:00 Q&A session discussing exporters and batching strategies +00:30:00 Closing remarks and information about future events and community engagement + +# Hotel in Practice: Scaling OpenTelemetry Logs with Kafka + +Hello everyone, and welcome to our latest edition of Hotel in Practice! We are so excited to have you join us today. Remember to tell your friends that if they weren't able to make it, the recordings will be available afterward. You can check out our recordings on our YouTube channel at **hotel-official** and also on the Hotel LinkedIn page. + +Today, we have a very special guest, Dakota Passman from Bindplane, who will talk about scaling OpenTelemetry logs with Kafka. Without further ado, let’s bring Dakota on! + +--- + +### Introduction + +**Dakota:** Hi! Thanks for having me. I'm Dakota, and I've been a software engineer at Bindplane for a little over two and a half years. I work on our platform, focusing a lot on the hotel project and making various contributions, especially regarding the supervisor. I'm happy to be here. + +**Host:** Excellent! If you're ready, let’s get started. For anyone interested, we will have time for questions after Dakota's presentation, so please feel free to post them in the chat. + +--- + +### Presentation Overview + +**Dakota:** Today, I want to discuss a situation we handled with a customer using the Kafka receiver at scale. They were experiencing performance issues and encountered a bottleneck while trying to pull events from Kafka. Initially, they were pulling only 12,000 events per second per partition, which was not enough to keep up. We're going to cover that bottleneck and how we helped them reach a target of 30,000 events per second (EPS). We’ll also demo those changes live in a replica environment. + +**Key Takeaways:** +1. **Don't Trust Defaults:** The Kafka receiver uses a default Kafka client that performs differently. A new client implementation added shortly before we encountered issues provided a significant improvement. + +2. **Configuration Matters:** Proper configuration of the collector can make a huge difference. This might seem obvious, but it’s easy to overlook if you’re unsure of the configuration options. + +3. **Pipeline Configuration:** The configuration of the pipeline is also crucial. It can greatly affect performance. + +4. **Understand Your Observability Goals:** This understanding helps tune the environment effectively to ensure your Kafka receiver performs well and meets your observability goals. + +--- + +### The Customer's Situation + +The customer was pulling 192,000 events per second across all their partitions, with a single topic of 16 partitions. They needed to reach 480,000 events per second to keep up with the data going into Kafka and start cutting down their backlog. They were using the OpenTelemetry collector and had many default options set, leading to rapid backlog growth. Kafka was ingesting 30,000 events per second per partition, resulting in a backlog increasing by 288,000 events per second. + +--- + +### Changes Made + +Here are some changes we implemented: + +1. **Client Upgrade:** We switched to the new FronGo client, which improved throughput by about 30%. + +2. **Log Encoding:** The receiver was initially set to use OTLP JSON, which was inefficient for their use case. Switching it to raw JSON provided a significant performance boost. + +3. **Batching:** We moved the batch processor to the start of the pipeline, allowing the receiver to push out large quantities of events consistently. + +4. **Protocol Change:** Changing the protocol from gRPC to HTTP for the exporter also improved performance significantly in their specific case. + +--- + +### Demo + +In the demo, we will use a single collector and a single partition for simplicity. Initially, we were pulling around 12,000 events per second. After applying the changes, we expect to see significant improvements. + +(At this point in the presentation, Dakota demonstrates the changes made to the configuration and shows the performance metrics.) + +--- + +### Results + +We witnessed a jump from 12,000 events per second to around 25,000-26,000 events per second after implementing the changes. This improvement allowed the customer to catch up with Kafka's ingestion rate and start reducing their backlog. + +--- + +### Conclusion + +In summary, we identified several key factors that contributed to the bottleneck: +- Back pressure from upstream components affecting the receiver. +- The performance differences between the FronGo client and the default client. +- Avoiding unnecessary encoding conversions based on telemetry goals. + +Thank you for attending this session! We have some links to share, including a blog post about this topic and a link to the CNCF Slack. + +--- + +### Q&A Session + +**Host:** Now, let’s move on to questions from the audience. + +1. **How do different exporters impact performance?** + - The impact will depend on the exporters used. For this customer, switching from gRPC to HTTP for the SecOps exporter resulted in better throughput. + +2. **What was the initial reason to put batching at the end of the pipeline?** + - This approach may have worked better in other environments, but it depends on the processors in your pipeline. + +3. **Would using a memory limiter bring extra performance?** + - I'm not familiar enough with that processor to give a definitive answer, but I’m open to looking into it. + +4. **Is the FronGo feature flag only useful for the Kafka receiver?** + - No, it's a general implementation not specific to the receiver. Other Kafka components might utilize this client as well. + +5. **Did you size the batching to improve performance?** + - Yes, we experimented with the size of batching to find what worked best for our environment. + +6. **What about resource usage? Does it have any consequences?** + - The configuration changes didn’t significantly affect resource consumption, which remained reasonable. + +--- + +### Closing Remarks + +If there are no more questions, that wraps up our session. Thank you all for attending! Remember, the recording will be available on our LinkedIn page and YouTube channel. Also, for those interested in sharing their stories with the Hotel End User SIG, please reach out to us on CNCF Slack. + +We would love to hear from you, especially if you have experiences with OpenTelemetry to share. Thank you so much, and we hope to see you at CubeCon North America! + +## Raw YouTube Transcript + +Hello everyone and welcome to our latest edition of Hotel in Practice. Um we are so excited to uh have you join here today. And remember um tell your friends for anyone who wasn't able to make it today that we will have our recordings available after the fact. Um you can go to our YouTube channel at hotel-official and you can also check out the hotel LinkedIn page. We we should have the recording available on there too. So we have a very special um hotel in practice today. We have Dakota Passman from Bindplane talking about um doing some gnarly scaling of open telemetry logs with Kafka. So without further ado, let's bring Dakota on. >> Hello. >> Welcome Dakota. >> Hi. Thanks for having me >> and to be here. >> Oh, super excited to have you here. Um, so, uh, would you like to do a, uh, a brief intro of yourself before starting? >> Yeah. Yeah. Um, I'm Dakota. I've been a software engineer at Biome Plan for a little over two and a half years now. Uh, working on our platform. Uh, focusing a lot on the hotel project. Uh, making some different contributions. Uh, focusing a lot these days on the supervisor. Um, it's a pretty cool project going on there. Um but yeah, happy to be here. >> Excellent. So if if you're ready to go, then uh let's get started. And um by the way, for anyone who is interested, um we will have some time after Dakota's presentation for questions. So please feel free to post them in the chat and we will address them after the the presentation. >> Yeah. Cool. All right. Um, so yeah, today I want to talk about a situation um, we handled with a customer of ours using the Kafka receiver at scale. Um, so yeah, um, we had to scale their environment. Uh, they were having performance issues. They're running into a bottleneck uh, trying to pull events from Kafka. um you know they were doing when we started talking to them they were pulling only 20 or 12,000 events per second um per partition in their topic um which was just not enough to keep up. Um so we're going to talk about that bottleneck, how we got them to get up to a target EPS of 30,000. Um and then we're going to demo those changes uh live in a replica environment. Um, so to start, we're going to go over some of the takeaways just so that we can keep these in mind as we're going through this, um, and see where they're coming up. Um, for starters, uh, don't trust defaults. Um, the Kafka receiver by default uses a certain Kafka client. Um, and these clients behave differently. They perform differently. Um, it's a default. However, uh shortly before we had this issue with the customer, there was a new client implementation added um that we were able to use um and using that client over the default client was a big improvement. Um secondly, uh configuration of the collector makes a huge difference. Uh, and that might seem kind of obvious, but it's sometimes easy to overlook that, especially if you're not entirely sure of what configuration options you have and what they might exactly do. Um, and something else that I think gets overlooked sometimes is configuration of the pipeline matters a lot too, especially in this situation. Um, and then finally, it's important to understand your observability goals. Um, so you can tune the environment to get it right. um and make sure that your Kafka receiver or really any receiver that you might be using is performing well um in your environment and hitting the observability goals that you're looking for. Um so yeah, to kind of set the stage a bit, so this customer of ours um they were pulling 192,000 events per second across all of their partitions. Um they had just a single topic with 16 partitions. Uh again doing roughly 12,000 events per second per partition as a sum that's 192,000 events per second. Um however, you know, they needed to really they need to be at 480,000 um events per second uh in order to keep up with the amount of data going into Kafka. Um and at that point then be able to start cutting down this backlog that they had going. Um so yeah they're using the open telemetry collector uh using the cafka receiver a lot of default options set um you know one topic 16 partitions each one's only doing 12,000 events per second. Um the backlog was growing rapidly. you know, Kafka was ingesting 30,000 events per second per partition roughly and so their backlog is growing by 288,000 events per second across all the partitions. Um, so they're quickly falling behind. Um, in this situation, it was security telemetry that they were collecting from Kafka and trying to send to a seam back end. Um, and so already they're dealing with delayed telemetry. Um, and at this uh performance, you know, this poor performance, they're at risk of starting to drop some of this telemetry, too. So, some of the changes we made. So, this is kind of like a illustration of the configuration and the pipeline that they had going. And you can see some of these changes as we moved through uh the pipeline. So first off the Kafka receiver uh using the new FronGo client which was the Kafka client I mentioned before using that implementation uh we saw a big jump you know we started we jumped up to about 30% um just using that client instead of the default um another thing the log encoding that we were pulling from Kafka that made a big um they were the receiver was configured to use OTLP JSON um which didn't really make sense in their situation um and so there's a lot of unnecessary work being done to handle OTLP JSON uh so instead we switched it to just raw JSON or normal JSON um not OTLP specifically and that was another big performance boost there. Um, one other thing, batching. Uh, we moved our batch processor to the start of the processor pipeline, uh, right after the receiver. Uh, the reason we did this, it allowed the receiver to pump out large quantities of events rather than being restricted by the data flow of the process user in front of it. So it was reliably able to push out, you know, thousands of events in a batch at once rather than maybe, you know, a couple hundred and then jump up to a couple thousand back to a couple hundred. You know, it's much more consistent performance batching right after the receiver. That's what we saw. And as a result, we're able to boost our throughput doing that. Um and then one final change we made uh this is again specific to their environment and their pipeline. Um they were sending the Zack Ops seam back seam back end and we saw that changing the protocol used from gRPC to HTTP um actually increased the performance. Um so in their specific case uh this made a big impact. We were able to get uh increased throughput doing this as well. Um the idea here is similar to the batch processor we discussed here. Um you know the exporter is just able to handle throughput better at HTTP or using HTTP instead of gRPC. Um, and this affected the receiver upstream from the exporter, allowing it to push data out a lot faster. So in this demo, um, you know, the customer, they had one topic, 16 partitions. Um, they had a single collector per partition. Um, in this demo, we're just going to be using a single collector, a single partition. Um, demoing with 16 partitions and managing collectors at that scale is difficult to do manually. Um, so we're just going to keep it simple here, just a single collector. Um, going to start the environment should be about 12,000 events per second that we're pulling. Um and then we're going to apply the changes to the environment and then we'll see um you know we'll discuss those changes more in depth and see how it's performing afterwards. Um yeah. So if I come over here and check this out. So I started this up shortly before we started here. So this green line up here, this is going to be the events per second that the Kafka topic in partition is pulling in. And we're stable at around about 26 27,000 events per second. And then down here is our single receiver, single copy receiver. Um this is using the default client. Um it's pulling in these events as OTLPJSON. Um, and batching is being done afterwards. Um, and here you can see that we're stable at 12,000 events. Now, I'm going to just quickly stop this collector and then I'm going to start it up using the new config. And then also crucially here, I'm going to start it up using this feature gate to enable the Fronco appliance. So, I'm going to start that. I'm going to let that run in the background for a little bit, let these changes trickle through. Um, and I'm going to talk about the changes we made more in depth. So on the left here, this is the first configuration that the collector was using. Um, and this is the configuration for the Kafka receiver. This is a pretty default config. Uh the key difference here again is this OTLP JSON encoding versus on the right here in the new config, we're going to be using text. Um using text is just going to be able to pull the data faster. It's not going to have to worry about um transforming or handling that data as much. Um and so it can just push the events through faster. It's not going to get bogged down trying to transform it. Um I've got some collecting the uh metrics from this receiver. Um here I've got some processors. These are define the same across the two here. This is to generate or to simulate um how moving the badge processor around in the pipeline affects the actual throughput. Um so these processes are are all the same, you know, just pretty basic adding some fields. Um, next exporters. We've got no oporter. We're not going to show um the effects of the stock ops exporter in this demo just because that's can it's a little much. Um, so we're going to just keep it simple here using the OP exporter. Um, you know, eliminate that uh lever from the situation. Um, and then we're sending our internal telemetry. And then the other important part here. So yeah, this is the initial configuration. We can see here the order we define our processors. You know, we've got our two transform processors and then the batch. Again, the improvement we saw was adding the batch processor at the start of this processors uh definition list. Um again this is so that the receiver is sending directly to the batch processor. Um it can get consistent performance consistently pushing out a high number of events um rather than being bottlenecked by the process the other processors in front of it. Um and then the other crucial thing again I pointed out when I restarted the collector I used the new feature gate um to use the frongo client instead of the default So, we'll come back here. We'll give this a refresh. And this is going to keep going. But at the moment, we can see this new collector here. This is the old one turning off. And this is the new one coming back on. And we briefly spike up. And I expect if we let this go for a little while, we'll see this um stabilize and mirror the Kafka uh Kafka line. Um let this go for a little bit. Um, but I guess yeah, just to kind of go back here and reiterate some of these points again, use the Fronco feature feature flag to use a Fronco client just performs a lot better than the default client in the Kafka receiver. uh your log encoding. Again, if you're uh handling the data unnecessarily, in this case, we're pulling data from Kafka that's not necessarily OTLP JSON, but because we have a receiver set up for that encoding, um there's unnecessary work being done to transform it into that. Um and we can just pull it in as normal JSON and handle it afterwards. Um that was a big source of improvement. Um, and then again batching early, the receiver is able to consistently push out a consistent number of of events rather than having seen that fluctuate based on what's happening upstream of it. Um, and again in a similar vein um not shown shown in the demo but for sec ops or for this exporter um tuning and configuring that affects the receiver upstream of it. Um, yeah. So, those were again some of the changes we made. We'll see if we've got some better data here. So, it's still early, but you can see how this collector is starting to stabilize kind of right around the throughput that the Kafka uh topic is getting. Um, I expect this to get, you know, closer as it comes online and starts to stabilize. But again, you can see we jumped from doing 12,000 events per second and then with all those changes we made, now we're closing in around 25 26,000 events per second. Um, drastically better improvement. And this is how we were able to help this customer, scale up their throughput so that they're able to keep up with Kafka. Um, and in this in their case, we were able to get past what Kafka was ingesting so that we're able to also start cutting down the backlog and catch up. Um, yeah. So, got refresh this a bit. Cool. So, yeah, you can see this is stabilizing. So again, why it worked? Back pressure. Back pressure is the the term for this idea I've been talking about where what's happening upstream from or downstream of the receiver is affecting the receiver. You know, it's just not able to push events through as fast because it's got to wait on the components ahead of it to process their events. Um you don't know you don't notice it until in this case the receiver starts falling behind and is affected by it. Um, again very similar to the early batching. Um, the FronGo client just performs a lot better than the default client um, in terms of pulling events from Kafka. Again, use that feature gate to enable it. Um, and then yeah, finally, avoid doing unnecessary encoding conversions. Again, this is understanding your telemetry goals in your environment. um understanding the what the data you're sending looks like so that you can uh make sure that your receivers and other components are configured properly to handle it. Um and yeah, I think that's all I have for a demo. Um we can keep checking out that graph to see how it's looking as it's coming online. Um, got a couple links here. One to the blog post that we made about this. Um, and then also the link out to the CNCF Slack. And I think that is it for my slides. Um, yeah. Cool. Sorry. mute muted. Mike, looks like we have a question uh coming in from LinkedIn. Um so, how do different exporters impact performance? >> Yeah. Um I think that's definitely going to be very dependent on the exporters being used. Um you know, it's going to be something you have to investigate per case and and again, you know, fine-tune the configuration of them. Again in this case for the customers a sec ops exporter uh using gRPC just wasn't able to keep up with the throughput we had um and in this situation HTTP was able to perform a lot better. Um so yeah it's just a matter of trial and error see what's hap what works best for your environment based on your telemetry and what your your goals are. >> Awesome. Um, looks like we have another question. Um, batch has been recommended to be the first processor of the pipeline. What was the initial reason to put it at the end of the first pipeline? >> Yeah, so the reason for putting it at the end, um, you know, I'm not entirely sure of the decision to do that or, you know, what went into that decision. Um, I think it's a perfectly valid approach. I know maybe in other environments that works better. I think it depends on the processors in your pipeline. Um you know I think some processors it be better to batch afterwards um rather than before. Um yeah again it's very dependent on you know what your pipeline looks like, what your goals are um and what you're trying to achieve. >> Awesome. Another follow-up question or another question. Uh, you did not use the memory, sorry, you did not use the memory limiter. Would that bring extra performance uh from using it if you if you use the memory limiter? >> That's a great question. Um, I can't say I'm too familiar with that processor, so I can't I don't know if I have a good answer for that. um certainly would be willing to look into it and understand it and see if that could have applied here. >> Fair enough. Um another one that we have is uh the France Go feature flag is it only useful for the Kafka receiver? >> It's a good question too. Yeah. So no. Um the Fran Go client implementation is just a general implementation not specific to the receiver. So I think some of the other Kafka components in the project um you know it's a matter of whether or not they utilize this client as well um and looking into seeing if there's a way to enable them to use this client or not. Um if not I imagine that's something that'll happen shortly is the ability to use the other Kafka components with that Fronco client. Um so yeah. >> All right. Awesome. Um, another question we have. Did you size the batching to get improvements and did you export and sorry, did you adjust the exporter batch as well? >> Yeah, that's another good question. Um, yeah, we definitely did play around with the size of the batching. Um, you know, I think in this demo we've got it at Yeah, we've got it configured like this. Um, yeah, that's definitely something that you can play around with. you know, toggle that, fine-tune it, see what works best for your environment and your throughput. Um, and yeah, what's able to get you through or get your telemetry through? Um, yeah. >> And then final question that we have here, uh, what about the resource usage? Does it have any consequences? >> Yeah, that's a great question, too. Um, yeah, I mean, definitely something you want to be aware of. um with our customer, you know, making these configuration changes. Um they didn't affect the resource consumption and usage like that, you know, like the the amount of resources the collector was using. It was still pretty um not negligible, but it didn't affect the system. It wasn't like all of a sudden we jumped from using, you know, 20% memory to 80% memory or CPU, you know, it was very reasonable. Um, so it wasn't necessarily a concern. >> Excellent. Um, do we have any other questions from the audience? Um, let me share these links in the chat as well. >> Oh, perfect. Yeah. Stay tuned. >> We need like a little drum roll. >> Yeah. >> Okay. Awesome. Henrik has shared the links. >> Perfect. >> Chat. Amazing. Um well I suppose then if we don't have any other questions I guess that is a wrap. So um short and sweet informative that's a lot that's a lot to cover in a short time and also like lots of great lots of great questions lots of great uh considerations from a from a performance side um as well. So definitely appreciate the uh the questions that we've gotten and again um tell your friends for those who are on the stream that this recording will be available after the fact both on our LinkedIn page the open telemetry LinkedIn page and on the open telemetry YouTube channel which is hotel-official. Um and then a couple of housekeeping notes um for anyone who is interested. We the CFPs are still open for uh for CubeCon and I think in the last week the CFPs have also opened for the CubeCon colloccated events. That's the CubeCon in um in Europe in Amsterdam. Not not the upcoming one. The upcoming one that's that's done. Those CFPs are closed. But if you are interested in applying to uh CubeCon EU and Amsterdam um and or to the colllocated events um these are the links and remember folks that the submission limit for CubeCon is now three um three proposals per person and that includes whether you're a uh the main submitter or a uh a co-speaker. Um and then the submission limit for the colllocated events is 10 across all coll-located events which is very cool. So uh yeah there's that going on. And then for anyone who is interested in sharing their stories with the hotel end user SIG, we love hearing from the community. So please reach out to us. You can find us in uh CNCF Slack. We have a lovely um Slack channel which is uh I believe SIG hotel end user and Henrik has been nice enough to put the uh the link to our Slack channel on there as well. Oh uh yeah, sorry I had it backwards is hotel-end user. Um and yeah, we would love to hear your stories whether it is through hotel in practice. So this type style of presentation and you know if you're if you have a presentation that you want to test out this is a great proving ground or if it's you know a talk you've given somewhere and you just want to share it with more folks this is also another great way to uh to get the the topic out there. Um Dakota you presented this at um at open source summit right the collo the observability day or the the I forget what the hotel day was that is that correct day >> um I didn't I didn't uh present anything but one of our co-workers at bonf did present a topic about the cafner receiver >> ah okay >> his presentation was yeah a deep dive into his understanding of it and some of the unique aspects of it he figured out so >> oh Nice. Very nice. That's awesome. Yeah. So, we we would love to hear topics like that. We also love to hear um so anything where you've like forot and practice, anything that you've learned cool stuff about open telemetry, we would love to hear from you. So, hit us up on our um on our Slack channel. Uh we also have hotel Q&A. So, for anyone who wants to talk about their hotel journey, um this one's an interview style um uh format. Both of these are live streams. So we would love to hear from you and we have uh recently had I think our previous hotel and practice was with folks from Alibaba in the APAC region. So we are looking for more folks in the APAC region as well who would love to share their story because you know what open telemetry CNCF it's a global undertaking. We have uh lots of love from folks all all across the globe. So um we are definitely if you're in the AP pack region and you have a story to share um we would schedule an AP pack friendly hotel and practice hotel Q&A um to to cater to the uh to the time zone. So um that is it for us. Um and hope to uh if you're going to CubeCon North America uh you'll probably see some of our crew at um at CubeCon North America. So excited to uh excited to have folks connect in person on on open telemetry. Thank you so much everyone for attending. >> Thanks + diff --git a/video-transcripts/transcripts/2025-10-22T04:08:27Z-whats-new-in-otel.md b/video-transcripts/transcripts/2025-10-22T04:08:27Z-whats-new-in-otel.md new file mode 100644 index 0000000..5da9ddf --- /dev/null +++ b/video-transcripts/transcripts/2025-10-22T04:08:27Z-whats-new-in-otel.md @@ -0,0 +1,329 @@ +# What's New in Otel? + +Published on 2025-10-22T04:08:27Z + +## Description + +Been too busy to keep up with what's going on in the OTel space? No worries--we've been there, too! That's why we've put ... + +URL: https://www.youtube.com/watch?v=NFtcp0LPtFA + +## Summary + +In this YouTube video, Bree Lee and Adriana Villa host a special edition of "Open Telemetry and User SIG Live," introducing the segment "What's New in Hotel." They discuss updates and developments in the OpenTelemetry community with guests Severin Newman and Marilia Gutierrez. Severin shares impressive statistics about contributor growth and community engagement in OpenTelemetry, highlighting the transition towards more robust configuration options via declarative configuration rather than just environment variables. Marilia provides insights into this new configuration system, explaining how it simplifies setups for various SDKs and is already being implemented in languages like Java and JavaScript. The session also features a Q&A, addressing audience questions about configuration precedence and the adoption of OpenTelemetry in various contexts. The hosts encourage community participation and outline plans for future events, including a live stream from KubeCon. + +## Chapters + +00:00:00 Introductions and welcome +00:02:30 Overview of the new segment "What's New in Hotel" +00:03:50 Introducing the first guest, Severin Newman +00:05:30 Severin discusses OpenTelemetry community numbers and growth +00:12:00 Highlighting community updates and new translations on OpenTelemetry.io +00:18:00 Discussion about the upcoming governance committee elections +00:23:00 Severin shares insights on the technical updates in OpenTelemetry +00:33:00 Marilia Gutierrez introduces declarative configuration for OpenTelemetry +00:40:00 Marilia discusses the benefits of the new configuration system +00:47:00 Q&A session with audience questions about OpenTelemetry features and trends + +# Open Telemetry and User SIG Live Transcript + +**Heat. Heat. Hello.** + +**Bree:** Hello everyone. Welcome. If you're on Pacific time like me, it's early, 8:00 AM. Still dark. Thank you for joining if you're on Pacific time. + +**Adriana:** You're welcome. + +**Bree:** Well, hi. Welcome to this special edition of Open Telemetry and User SIG Live. We're very excited to bring you a brand new segment called "What's New in Hotel." I am Bree Lee, a senior developer relations engineer at New Relic. I get to work with the lovely Adriana Villa. Would you like to introduce yourself? + +**Adriana:** Yes. My name is Adriana Villa. I work with Bree in the Hotel and User SIG, and I am a principal developer advocate at Dynatrace. Like Bree said, this is our very first "What's New in Hotel." I think this is really exciting because I don’t know about you, but I can never keep up with all the stuff going on in Hotel. So, I kind of need someone to tell me. This is a little self-serving, and we're basically bringing folks in from the Hotel community to tell us some of the cool stuff that is going on. + +**Bree:** Yes. And hello, Michael! + +**Adriana:** Hi! Yay, thanks for joining us. I know it's late-ish for you. It's 5:00 PM for you, so thank you for joining, Michael. + +**Bree:** Yes. By the way, I am coming in from Portland, Oregon. If you would like to share where you're tuning in from in the chat of whichever app you're using, we would love to see where you're from because there's a pretty good variety of time zones on this call as well. For example, Adriana is coming from Toronto. + +**Adriana:** That's right! Yeah. And you said it right; you said Toronto, not Toronto. + +**Bree:** Amazing! Yes, I'm from Canada, representing. So, I guess this is a good time to bring in our first guest. + +**Adriana:** Oh yeah, sorry! + +**Bree:** Just to give you all a sneak peek into what our new segment is going to look like, we are going to speak with Severin Newman, then Marilia Gutierrez, and finally, we are going to bring on Lisa Jung, who will host our audience Q&A at the very end. So, if you have questions, put them in the chat, and we will get to them at the end of both of the first two interviews during the audience Q&A. Hopefully, that'll make sense. Yes, like Adriana said, let's bring on Severin. + +**Severin:** Hey! + +**Bree:** Happy to be here. + +**Adriana:** Thanks for joining us! Where are you coming to us live from? + +**Severin:** I'm based in Germany, close to Nuremberg. So, central European time. + +**Bree:** So, it's evening for you. + +**Severin:** Yes, it's 5:00 PM, so it's getting dark again. Summer is officially over, unfortunately. And next week, you guys get the time change of one hour back, and then North America gets the time change of one hour back this week. It's screwing up my whole calendar. Everything is now at 5:00 PM, which should be at 6:00 PM. + +**Bree:** Well, at least we have one more week of not having to worry about that. + +**Severin:** Yeah. + +**Bree:** So, do you want to introduce yourself briefly? + +**Severin:** Yeah, sure. So, I am the head of community and developer relations at Cosley. I'm part of the Open Telemetry governance committee, and I'm also one of the co-maintainers of what we call the SIGCOMs. What we do is the website, the blog, social media channels, and everything around that. That's a very short version of what I do for the Open Telemetry community. I'm super excited to be here today to give you a first start into what's new in Open Telemetry. + +**Bree:** Amazing! So, let's get started then. Do you have some goodies that you can share with us? + +**Severin:** Yes, absolutely. I have prepared a little bit of slides—not to bore everybody with a lot of text, but to make a few of these things I wanted to share a little more visual. I have three topics to talk about: a little bit about Hotel and numbers, a little bit about the community, and then a little bit about the tech stuff. I think that's what most people are excited about. So, let me share my screen and get started with a little bit on the numbers. + +So, those numbers are for all the time, right? Since Open Telemetry was created, what is it now, six years ago, we had almost 20,000 contributors from 120 countries and almost 2,000 organizations. I looked up the numbers for the last 12 months—still 6,000 contributors, over 70 countries, and over 600 organizations. So, you see there’s a lot of velocity in our project. This is growing every year, and we see it also with the PRs and the work happening on GitHub. + +We have to add here that something like this session is not recorded by anything you see on CNCF DevStats. There’s a lot of work going on in our community that’s not tracked on those boards, unfortunately. So, a big shout-out to everybody doing that work outside of the repositories. But I still think those numbers are impressive. Since the foundation of the project, we had like 60,000 PRs reviewed, as you can see, most of them three to four times, and over half a million comments. This means there were over half a million times someone wrote something on a blog, issue, or pull request, which I think is really great. + +What’s exciting is that in the last year, 20% of those PRs have been done, meaning we are leveling up our reviews. We have more repositories where we say, "Hey, you're not done with one review; you maybe need two reviews." For example, in our Open Telemetry IO repository, we need a review from docs maintainers and from co-owners. I think this helps with the quality of the project. + +And who is using Open Telemetry? There are three groups we mention: first, the vendors or backends—companies that consume Open Telemetry data and do something with it. There are also open-source projects, including Jaeger or OpenSearch, and what we call integrations—projects that added Open Telemetry natively. This is probably a much bigger number than what we have on our website. + +Things like Kubernetes have their API server supporting Open Telemetry, or there's the PHP server called Roadrunner that has Open Telemetry. I always love mentioning that one because it’s written in Go and then does PHP. You can even do tracing from Go into PHP in this web server, which is really something I always love seeing. Oh, and also Docker! If you look at Docker BuildKit and another Docker project, you can see your containers being built with Open Telemetry. + +I think that's also very fun. And then, of course, thousands of adopters. Every time I'm in a presentation or in a room, I ask people, "Hey, who of you knows about Open Telemetry?" Most of their hands go up. Then if I say, "Hey, and who of you is using Open Telemetry?" also like 80% of the people raise their hands. Unfortunately, we don’t have that many listed on our website, so here's the first call to action: if you are an Open Telemetry adopter, if you have maybe even written a blog from your company saying, "Here’s how we use Open Telemetry," just use that link, and you can reach out, and we can add you there. + +So, that’s a little bit on Open Telemetry numbers. I think that's just a good starting point. Now, let’s talk about the community. + +**Adriana:** Yeah, I think let's dive into the next topic. + +**Severin:** Awesome! As I said, I always love to start with those numbers to emphasize how big this community has grown. And that also means there's a lot of community updates. Especially, we all know KubeCon is coming, and for whatever reasons, September, October, and November are when all those community changes happen. + +One I'd like to highlight, because I'm one of the maintainers of this project, is if you go to the OpenTelemetry.io website, there are now nine languages into which we have started to translate the OpenTelemetry website. That includes English, Bengali, Spanish, French, Japanese, Portuguese, Romanian, Ukrainian, and Chinese. I’ve written it down so I don't forget anybody. I think that’s amazing! We started with four languages a little more than a year ago, and it has grown to nine languages. This is really amazing for the community because I think it’s a great way to get started with OpenTelemetry—you can translate the things into your native language, learn about them, and then maybe move into different projects of the OpenTelemetry project with this additional understanding. + +Another big call-out: the technical committee has added two members. David Ashpole and Josh McDonald have joined the TC. David is a subject matter expert on metrics and Prometheus. He’s a Kubernetes contributor as well. Josh has stepped down from the TC for a while but is now coming back in. He’s involved in the sampling SIG and is one of the people behind OpenTelemetry Arrow and many other things. I’m probably not doing him justice by only calling out those two things, but big congratulations to both for joining the TC. We will see the impact they will have on the project very soon. + +The governance committee election is coming up next week. We just announced the candidates. If you are a member of standing, as we call it in the OpenTelemetry community, you should have a notification about being allowed to elect. If you think you’re allowed to be in that election as a voter and haven’t received your access, reach out. There’s a link for that as well. + +For people new to the OpenTelemetry project, we have a new Slack channel called OpenTelemetry New Contributors. If you want to start contributing, go over there, and you can get started or ask your questions. Say, "Hey, I picked up that issue, and I need some help," or "Hey, I want to help with documentation. Can somebody guide me?" I think that's a really great opportunity to get started. + +Another call to action: we have community awards! We started this last year, and we want to do this again. At KubeCon, we will call out a few members of the community who the community thinks deserve an award. Everybody is allowed to nominate someone, and this does not only include people contributing to the project. This also includes people promoting the project in any way. If you have a person in your company running around convincing everybody to use OpenTelemetry, you can nominate them! If you find four or five friends in your company and say, "Hey, let's nominate that person," that’s perfectly fine with us. We want to hear those stories of people promoting OpenTelemetry. + +Speaking of KubeCon, the observatory is back at KubeCon. If you want to speak with us at KubeCon, this will be my first time there, by the way, so if you want to catch up with us, you will find us there. We also have a schedule and some programs planned. You can probably tell more about that than I can. + +**Adriana:** Yes, I wanted to interject quickly. I wanted to dig into the GC elections because that's pretty much open to anyone, right? I know you said it's happening next week, but anyone who's been involved in OpenTelemetry can nominate themselves for the GC elections, correct? + +**Severin:** Sorry, can you now? I lost you for a minute. + +**Adriana:** Oh, anyone can nominate themselves for the GC elections that has been working in OpenTelemetry. Is that correct? + +**Severin:** Yes, but I think the nomination is closed since like... + +**Adriana:** Okay, okay. + +**Severin:** Yeah, yeah. + +**Adriana:** So, that's why I didn't call it the nomination. But yeah, no, fair enough. + +**Severin:** I think you don't have to be like— + +**Adriana:** So, yeah, everybody can be nominated. I think you need two supporters, and then you're in. I said it's closed for this year, but next year, we will do this once again. Maybe it would be good for anyone interested in running next year to give them a heads-up about what the process might be like. + +**Severin:** Yeah, sure. Normally, we send out a blog post, I think at the end of September or early October, where we say, "Hey, elections are coming up. If you want to run as a candidate, create a pull request against the community repository and have two people support you." If you want to do that for the first time, wait for the first people to submit their nominations or self-nominations. Let’s be honest; most people self-nominate, and there’s nothing wrong with that. I do that. Just use that as a template and think about if you would be elected in the governance committee of OpenTelemetry, which is responsible for managing the whole project, for the health of the project, and the community as a whole. What are the things you would do for the project? + +The GC is less about "I want the collector to do X, Y, and Z," or "I want to have added some specific language to the OpenTelemetry project." Of course, from a GC perspective, you can engage in those conversations with those communities, but the GC is much more around how we can grow and make our community stronger and better and how we can manage the project as a whole. That means how can we get projects added to what we are doing, how can we ensure they are sustainable, and how can we make sure our project is thriving? So yeah, next year, around the same time window, if you want to run, that’s where you should take a look. + +**Adriana:** There you go. For anyone planning for next year, keep an eye on the nominees this year. See what they’ve written for their nominations to get an idea of the types of things that get you elected. You’re on the GC, right? + +**Severin:** Yes. + +**Adriana:** So, what gets you elected? + +**Severin:** I think you show that interest in the community. At the end of the day, it’s also like... it’s very hard to say that since I’m a member of the GC myself. But before I was a member of the GC, I always looked at what those people had done before. Sure, putting in your nomination is one thing. Writing down, "Here are the things I want to do for the community" is one thing. But it's more about what have you done in the last few months to help people with that. + +**Adriana:** From a voter perspective, it definitely helps if you’ve been visible in the community. It could be in a number of ways, right? Code contributions, talks, blog posts, being active in the channels, answering questions. + +**Severin:** That’s maybe a very important point, right? You don’t have to be part of the GC to do that. We need a lot of people to do a lot of work outside of writing all the code and documentation. There are so many great things you can contribute to this project. I say it very often to people that want to break into the OpenTelemetry community or any open-source community: you already have skills. You could be a good marketer, designer, or technical writer. Think about what you bring to the table. That helps you be appreciated in the community. Don’t think, "Oh, I have to be a good Go programmer to be valuable to the OpenTelemetry community." That’s just not true. + +**Adriana:** That’s a great point! Before we transition over to Marilia, do you still have one more thing you wanted to show us quickly? + +**Severin:** Yes, I have a slide and another one. I’m not sure how we’re doing on timing. I’m talking a little long now. I still have one more, but let’s see if we can get through that. I think people are more excited about the declarative configuration part. But people should be very excited about OpenTelemetry Unplugged. This is an unconference we’re planning for happening after FOSDEM next year, the Monday after FOSDEM. If you're still around, register for this event, and you can meet with the OpenTelemetry community. There's a blog post on that. + +Last but not least, because people are asking about that, graduation is in progress. I can share a little bit more in the tech overview. So yeah, that’s on the community updates. Let me hopefully go into the technical highlights. + +This was the text I wanted to show first, right? There are so many updates. I will not touch on everything that the Hotel project is doing right now; it’s almost impossible. But I picked the things that showed up recently. Since I saw this question coming up very early on when we started this session today: how can I stay updated? The OpenTelemetry blog is a really good place, and we try to motivate SIGs to put things there. For example, the profiling SIG will come out very soon with, "Hey, we are moving towards alpha." I think that’s a great thing that we’re seeing in the community. + +Equally, we had a blog post just recently about sampling, and they're going to adopt the W3C trace context level two. That’s probably worth another "What’s New in Hotel." I’ll try to do a two-sentence summary of that. It’s very difficult, but in a long story to make a long story short, it’s mainly about randomness and trace ID to ensure enough randomness in the trace ID, allowing you to do certain things with sampling. If we have time in the Q&A, I can try to dive into that a little bit. + +The mainframe community has been doing a survey recently, and I don’t know if people know that we have a mainframe community with OpenTelemetry. They just asked a bunch of questions to people on the mainframe: Do you know about OpenTelemetry? How do you want to see it used? They were talking about needing good Python and Java support and seeing that the OpenTelemetry Collector is able to take metrics out of a mainframe. Of course, the question about what to do in COBOL is still open. I have no more details on that, but it’s a very exciting community, even for the mainframe, which is a technology that has been around for many years. + +We are looking forward to hearing more about the declarative configuration myself. We have the Android team working towards a stable version, and they have done a ton of work over the last few months. There’s even a demo app now in the OpenTelemetry demo with Android, so you can play around with that. I think that’s a really good starting point. They’re looking for feedback, so go to the OpenTelemetry website, check out the OpenTelemetry blog. In that blog, there’s a link to see what you want the OpenTelemetry Android community to do for their next release. + +As I said, we are working towards graduation. We met with the TOC of CNCF. There were a few adopter interviews where we discussed what the next steps are. There’s an issue in the CNCF TOC repository, issue number 1739, which you can look up. There’s a summary of what the next steps are. One big thing, and this is maybe exciting because this is not new feedback for us, is that people are looking for how we can improve stability, how we can communicate stability better, and how we can make it easier for people to get started with OpenTelemetry. + +Probably this slide alone would have covered another 20 to 25 minutes if I had talked about browser phase one, the bail donation (now called OBI), the injector weaver, OpenTelemetry Arrow, and the call for contributors for Kotlin. There are a lot of good changes in SIGCOM, so I tried to start with that. I really hope we can do these kinds of sessions more regularly so people can stay on top of things. I hope this helps. + +**Bree:** This was amazing! + +**Severin:** Oh, sorry. Go ahead! + +**Bree:** No, I was going to say thank you so much. That was really helpful for me. + +**Severin:** Yes, definitely. I was going to say this proves that we need more of these. + +**Bree:** Yes! I mean, I was learning a bunch while preparing. I was like, "Hey, let me ask around and talk with people about what’s going on in your SIG!" + +**Adriana:** Our project is growing, and we are adding so many cool things to it. I’m very excited about a lot of those things. And again, I only recommend two things: go to the OpenTelemetry blog from time to time. We have those kinds of announcements there. If you’re a maintainer or contributor, come to SIGCOMs and let us know about the latest and greatest of your SIG so we can share this with the wider world. + +If you have any questions, please put them in the chat of whichever platform you're tuning in from, and we will get to them after we speak with our next guest. Thank you so much, Severin. + +**Severin:** Yeah, thank you! Speak to you in a minute. + +**Bree:** Thank you! Now we are getting ready to bring on Marilia Gutierrez to talk about... what is she talking about? Ree, remind me. + +**Adriana:** Amazing! Hi, Marilia! + +**Marilia:** Hello everyone! Continuing on the theme of where you're tuning from, I'm also tuning in from Toronto. So yeah, me and Adriana. + +**Adriana:** And we’re both Brazilians tuning in from Toronto, which is a coincidence! + +**Marilia:** Small subset! That’s awesome. So, I do a lot in the community. Would you like to share some of the work that you do before we get into declarative configuration? + +**Marilia:** Yes, of course! I was even talking with colleagues yesterday that I keep finding more things in Hotel, and I’m just trying to get involved and spread the things that I work on. For example, I am a maintainer for the contributor experience. If you want to be a contributor and you're having some challenges, that would be the group that will help you. I am an approver for the JavaScript SDK for Portuguese localization, database semantic conventions, and as of a couple of days ago, also for the communications SIG as well. + +**Adriana:** Yay! Amazing! + +**Marilia:** Thank you! So, even for this one, we were like, "Okay, we need to pick one topic to go into deep." There are so many things, like, "Okay, I was working on this. I’m working on that." Let's pick one. I feel like the declarative config is something that is really cool and is getting to a very almost stable place. We’re going to see this implementation across several SDKs now, so I think it’s a really cool one to talk about. + +**Adriana:** Cool! So, let’s start. Before I start with the config, I feel like I should give a little context: what is this? Why do we need this? What are we talking about? + +So, how would you set up your SDK? You’re using OpenTelemetry, and you want to set up some specific things for your case—things like my server's name or what is going to be my tracer exporter. How it was done up until now is basically using environment variables. It was very easy to do this because environment variables are universally available in all languages. It doesn’t matter the SDK; there was some way to do this. But then more complex things came up: "Oh, I need this." + +We just published a blog post yesterday talking about the story of the declarative config. There was an issue created five years ago on the Java SIG: "I just want to be able to filter out health checks traces. I don’t want to keep sending this." There was no easy way to just filter that specific one. I have my sampler, but how do I specify what I want to filter? It started to get very complex to do just with an environment variable, so we needed something more robust. So that is the idea behind working on the declarative config. + +At this point, we have disallowed environment variables just so we don’t keep adding more when we have to transition. Then we started with declaring config, which is pretty much a YAML file where you can put all the things that you care about. It is a little more robust compared to environment variables and is easier to expand because you can have all the objects and types you need there. + +Of course, I’m the one talking about it today, but there were several people working on this for months. We are now in the phase of really implementing it across several SDKs. The process of creating the convention has its own repo. It takes a lot of time to get feedback from people and see what’s going to work with all languages. People have concerns about not using environment variables anymore, but you can still have environment variables in your config file. + +I have an example here of how to use this. If I have this, use it; otherwise, use this other thing. To just start using it, it is available today in Java and is starting in JavaScript. I do recognize the irony that you have to set up the environment variable config file to use the config file. I do see the irony in that: "Don’t use my environment variables; use the file." But that’s how you do it. + +Here’s a basic example. In YAML format, we have some resources. Here we have an example that still points to environment variables if you care. I have an example of the tracer provider. I only care about the endpoints. This is anything I put in here. + +The example I mentioned for the issue that was created is how to filter health checks. This is an example of how you do this. On Java, it’s already working. You can see here I have a sampler, and I say, "My rule is to drop if I have these attributes with this pattern." It’s a lot easier than having to go into the code or your instrumentation to create your sampler. You just create this object, and that’s it. You don’t have to care about how the SDK would do this. We are the ones instrumenting, doing that for you. + +Of course, there are a lot of things. I’m just trying to put a few screenshots here, as this can be quite big. You can see here, "Oh, I have a processor. I have a batch with this schedule, with this timeout. My exporter is this type." We have a file in that repo called "kitchen sink," where every single spec created is added to this file, which is the one I’m showing here. If you want to see examples of everything you have here, it has comments on what it means, what the values are, and what the default values are if you don’t put them. + +We also have one for migration. If you want the basic because you’re using your environment variables and don’t want to copy, just copy that file, and it’s still going to use your environment variables for some basic stuff. + +If you want to know what is available today, we have this matrix of compliance. Java is fully compliant, PHP, and JavaScript are partially compliant. We have a lot of other languages that are also working on it. I think they need to update this for a few of them, like Go and Python, which also have some implementations. When I say it’s fully compliant, it doesn’t mean it’s complete. There are still features that need to be added, but at least the basics are working for those languages. + +Here, I can probably copy this in the chat, and we can share. I put a few links for resources: the blog post that I mentioned, which is in English and Portuguese, the file in the repo for the configuration, and the examples I mentioned. If you have questions about the config file in general, you can use the Slack channel for the hotel config file. For specifics about the SDK, you can go to the specific channels like hotel Java, JavaScript, Go, and so on. + +That’s what I have for an overview today. + +**Adriana:** This is great! I have a follow-up question for you, Marilia. These configurations sound like they are language-dependent. You do them at the language level, not at the collector level. Is that correct? + +**Marilia:** Yes, the config is supposed to be completely agnostic. You can use the same file regardless of which SDK you’re using. However, there are parts that are also for the collector. For example, the collector is already using the declarative config for a couple of things. + +**Adriana:** Right. How does the config file get loaded? Does it get loaded as part of... I'm having trouble understanding where in the flow it fits in. + +**Marilia:** Are you talking specifically about the collector or in general? + +**Adriana:** Oh, just in general—the declarative config, like for the environment variables. + +**Marilia:** For example, I can talk about the JavaScript, which is the one I’m working on. When you initialize the SDK, usually you do something like `new SDK` and sometimes you have to pass parameters. This time, you don't need to do this. Your file is going to exist in the same place that you’re adding your instrumentation. + +**Adriana:** So, does it pick it up from a specific directory in your code? Is that the idea? + +**Marilia:** Yes, as long as you have that file in a particular directory, it’ll pick it up. The environment variable points to the file, so when you initialize the SDK, it checks if you have set a config file. If you don’t, it ignores it and uses whatever you were doing before. But if it has a config file, it checks what you have in there. For some things, it’s pretty much saying, "If you have this value, I’m going to use it; otherwise, it has default values for many things." This happens at the moment you initialize the SDK. + +**Adriana:** Okay, that makes sense. This is cool because it also lets you... I guess if you’re using just the old way of environment variables, that was very flat, right? This kind of gives it a little more structure, which is cool. + +**Marilia:** Yes, we have sections for everybody, so it’s supposed to be very agnostic. Now, my application is in Python; I can use the same config file. However, there are things that are very specific to languages. For example, agents only exist in Java. But I still want to configure this. The config file has sections for languages as well. + +**Adriana:** That’s awesome! Ree, do you have anything you wanted to chime in on? + +**Ree:** I’m still processing! + +**Adriana:** It’s so cool, right? + +**Marilia:** Yes! + +**Adriana:** Well, I think we can get right into some Q&A because I have questions, and I think our audience has some questions. So, Lisa, we would love to bring you on. + +**Lisa:** Hi, everyone! I’m Lisa Jung, a staff developer advocate at Grafana Labs. I’m part of the communications and end-user SIG. Now we are gathering questions from the audience to ask our lovely featured speakers here. + +The first question is from Cecil. He said, "If you have both options—environment variables or the configuration—in a project, which one takes precedence?" + +**Marilia:** If you do have the environment variable saying, "This is my config file," that is the one that takes precedence. It’s going to use that one and not use the environment variable as a backup at all. It’s going to ignore that unless the config file itself has the environment variable. + +**Lisa:** Gotcha. Thank you! We have a question from Kieran: "Rust is still a four-letter word. Long way to go?" + +**Marilia:** For the specific declarative config, I believe this question came up during your session. + +**Severin:** Yes, it is a complicated topic. When people create the semantic conventions and stuff, they try to get people from different languages to know what’s going on. They go to their own SIG and start implementing. A few SIGs have very small groups, so it’s hard sometimes for them to handle because they have to review PRs and do releases. It’s hard to keep up with everything going on. If you really care about Rust, that is your chance to join the SIG and actually be a contributor. + +**Lisa:** Gotcha! Another question for Severin: "What does W3C trace context L2 for tail sampling bring to the table?" + +**Severin:** I had to take some notes on that myself because it’s a very complex topic, and I’m also not a sampling expert, but I’ll try my best. We have been doing sampling in OpenTelemetry for I think over four years now, and it has been evolving. We added a lot of good things. One of the things we had is this trace ID ratio-based sampler. This sampler could only sample on root spans. You had not a lot of good guarantees on probability downstream. + +What’s happening is that you use the trace ID as a random n-sized word. You use part of the trace ID and compare it to a boundary that’s 2 to the power of n times the ratio. For example, if you take 50%, it’s like 128. If the number is smaller or bigger, then it comes in or does not come in. + +The big problem was that until level two, there were no guarantees on how many of those bits are truly random or are encoding additional information. This is what has changed in that new standard, and OpenTelemetry is adopting that. You send an additional flag when using that, and with that, you get 56 bits of sufficient randomness. + +I recommend reading into that. If we talk about sampling, we all know that with the amount of data that people are consuming these days with OpenTelemetry, this is urgently needed, and they need a lot of good hands to try these things out. So, give this a look! + +**Lisa:** Gotcha! To piggyback on that, Henrik asks, "What about consistent probabilistic sampling?" + +**Severin:** I’m not exactly sure. That’s maybe a good topic for the next round. We can invite someone from the sampling SIG and have them talk specifically about that. + +**Lisa:** Great! We have a question for Marilia from Fernando: "Can I have more than one config file, like a file with generated configs or another with only specific JavaScript configs?" + +**Marilia:** The file itself is going to be one because otherwise, we don’t know which one takes precedence. For example, if you have things that are specific to language, they are still part of the same file. You have sections that are specific for Java, for example. At the moment, there’s nothing specific for JavaScript, but if you have things different for different applications, you need one file for each one. + +If you have the same thing but want to have the service name different, you can say use the environment variable for this case. Then you have your two environments that use the different names. + +**Lisa:** Gotcha! Another question from Henrik for Marilia: "Would the OTL operator provide a config CRD that could be deployed across Kubernetes objects?" + +**Marilia:** I’m not sure because I don’t see a reason to add new config. The idea is to have all possible configs that exist in that file. It’s just a matter of getting feedback from the community about what’s missing and what we can add there. + +**Lisa:** Gotcha! We have a question from Cecil: "Is there any guidance you can share around OpenTelemetry for long-running background processes?" + +**Severin:** Yes, this is actually a very complicated topic. I looked it up, and there’s an issue on the OpenTelemetry specification from 2019 on this topic. The first guidance I can give is to go to that issue in the repository and let them know you’re interested in that and maybe even interested in helping with that. + +What can you do until then? It depends on what "long" means to you. If we talk about spans and traces taking a few minutes, then you should be just fine with the tracing that you’re doing today. + +But if you say you have a background job—maybe this is doing some video encoding—and you want to do something around that, one thing you can do today is use logging and events for that and announce those kinds of things. It’s not perfect, and let’s be super honest with that. There are probably still things you need to do on the backend. + +**Lisa:** Thanks, everyone! As we wrap up our Q&A, I’m going to ask one question to both Severin and Marilia. You’ve been contributing to Hotel for a long time. How did you get your start in contributing to Hotel, and do you have any advice for new contributors? + +**Severin:** I can start. My reason to start with OpenTelemetry was very selfish. This is something I always tell people: have your own good reason to start with open source. Have something where you say, "I want sampling to be better in OpenTelemetry," for example. + +My situation back then was that I was in a consulting position, and I was talking with a lot of people about APM. People had questions about OpenTelemetry. They told me, "Hey, what is this thing? Should I use it or stick with what I have?" I recognized I had to level up on that, so I went to the community to learn what OpenTelemetry is, helping with documentation to explain it better to others. + +**Marilia:** I can go next. My journey was a mix of things that I found interesting and what made sense for my own company. I wanted to focus on observability, so I was looking for a job where I only had to think about observability. This was why I went to Grafana. The team was like, "You have to contribute to Hotel." That was exactly what I wanted. + +**Severin:** I do actually have— we just created a post that I can share here. There are ideas on how to contribute. You can have several reasons: you're working on something because your company needs it, your project needs it, or you want to learn. Pick something that interests you, whether you have knowledge in it or are just curious. Our project is huge, with so many components and languages. You will find your place there. + +**Marilia:** Yes, exactly! Join a couple of calls, listen in. You don’t have to say anything. If something seems interesting, continue, but if it’s not for you, go to the next one until you find what excites you. + +**Lisa:** There’s one more burning question for both of you: what kind of interesting usage trends in terms of implementation, observability stack, docs page views, or anything have you seen over the past year? + +**Marilia:** I think it’s people realizing that what they didn’t think was important before to have observability is important now. They’re adding more and more. It’s a lot of old applications that people didn’t think to add observability on. You’ll find a lot of everything—new things, and even things people do for fun, like monitoring the quality of the air for their plants! + +**Severin:** I can try to make it quick! I had five or ten things in my head. One thing is that I saw more databases being used for observability. People are recognizing that every software can be traced, logged, and monitored. Also, people are now realizing that with OTLP, they can send data everywhere. They have more freedom with their observability data, which is super interesting! + +**Lisa:** Thank you all for following along! If you have additional questions, please find us on the CNCF Slack instance at hotel-sig-end-user channel. There’s a QR code on the screen for you to use. Huge shout-out to Henrik Rexet, who is doing all this live-streaming stuff for us on the back end. Thank you so much to Lisa for hosting our Q&A, and to Adriana for being my co-host, and to Severin and Marilia for taking the time to come on and keep us updated. + +**Severin:** Thank you for having us! + +**Marilia:** Happy to be here! + +**Lisa:** We got a question asking when the next one is. This means we’ll have to put on another one. We definitely plan to do another. We’re hoping for a monthly cadence, but we also have KubeCon coming up in November, so we’ll see. + +Speaking of KubeCon, as Severin mentioned, the Hotel Observatory is happening. If you haven’t been to the Hotel Observatory booth, it’s loads of fun. This is where all the cool OpenTelemetry people come to hang out. We’re also doing a live stream from KubeCon. I guess it’ll be something like a "Humans of Hotel" live stream, which will be to a certain extent a "What’s New in Hotel." + +Keep an eye out for that! I believe it’s happening on Wednesday, November 12th, starting at 2:30 PM Eastern time. That’s it! Do you have anything else to add, Bree? + +**Bree:** No! Thank you all for being here! + +**Everyone:** Thank you! Bye! + +## Raw YouTube Transcript + +Heat. Heat. Hello. >> Hello everyone. Welcome. >> Welcome. Uh, if you're on Pacific time like me, it's early 800 am. Still dark. >> And thank you for joining if you're on Pacific time. >> You're welcome. >> Well, hi. Welcome to this special edition of Open Telemetry and User Sig Live. We're very excited to bring to you a brand new segment called What's New in Hotel. I am Bree Lee. I'm a senior developer relations engineer at New Relic. Um, and I get to work with the lovely Adriana Villa. Would you like to introduce yourself? >> Yes. Uh, yeah. My name is Adriana Vila. I work with Ree in the hotel and user SIG and I am a principal developer advocate at Dino Trace. And like Reese said, this is our very first what's new in hotel. And I think this is really exciting because um I don't know about you, but for me I I can never keep up with all the stuff that's going on in hotel. So I kind of need someone to tell me. So this is a little selfserving and we're we're we're basically bringing folks in from the hotel community to tell us some of the cool stuff that is going on. >> Yes. And hello, Michael. >> Hi. Yay. Thanks for joining us. I know it's it's lateish for you. >> It's way 5 5:00 pm for you. So, thank you for joining, Michael. >> Yes. And by the way, um I am coming in from Portland, Oregon. And if you would like to share where you're tuning in from uh in the chat of whichever app you're using, we would love to see where you're from because there's a pretty good >> there's a pretty good variety of time zones on this call as well. So it so for example, Adriana is coming from Toronto. >> That's right. Yeah. And you said it right. You said Toronto, not Toronto. >> Amazing. Yes, I'm I'm from Canada, so uh rep representing um yeah. Um so I guess this is uh a good time to bring in our first guest. >> Oh yeah, sorry. Sorry. Yes. >> So yes, so just to give you all um a sneak peek into what um our new segment is going to look like. Um, we are gonna speak with Severn Newman. And then we're gonna speak with Marilia Gutierrez. And finally, we are gonna bring on Lisa Young to Lisa Jung. I don't know why I said young. Um, and she's going to host our audience Q&A um, at the very end. So, if you have questions, um, put them in the chat and we will get to them um, at the end of both of the first two interviews during the audience Q&A. Hopefully that'll make sense. Um, and yes, like Adriana said, let's bring on Severign. Oh, Egypt is also watching. >> Hey, >> happy to be here. >> Thanks for joining us. >> Yeah, sure. >> Where are you? Um, where are you coming to us live from? >> Yeah, I'm based in Germany. Uh, so I'm close to Nuremberg. So, yeah, central European time. >> So, it's it's evening for you. So definitely. >> Yeah, it's 500 p.m. So it's it's getting dark again. So summer is officially over >> unfortunately. >> And and then next week you guys get the time change of 1 hour back and then North America gets the time change of 1 hour >> back this week because like it's like screwing up my whole calendar. So it's like oh everything is now at 5:00 p.m. which should be at 6 p.m. and everything. So yeah, le let's get over it with it. So >> yeah. Well, at least we have like one one more week of not have having to worry about that. So, >> yeah. Yeah. >> Um, so do you want to introduce yourself briefly? >> Yeah, sure. So, >> all things >> Sorry. >> Tell us all the things you do for hotel. >> Right. Thank you so much. >> I I try my best. Yeah. So, I'm uh Se No, uh, however you pronounce it, I I always also pronounce it differently when I speak English or German. Um I'm head of community and developer relations at Cosley. Um and I'm part of the open telemetry governance committee and I'm also one of the co-maintainers of what we call sikcoms. So what we do is the website, the blog, uh the social media channels and like everything around that. So so that's uh that's a very short version of of what I do for the open telemetry community. Yeah. and I'm super excited to be here today to give you a a first start into what's new in open telemetry. Um, so yeah, that's me. >> Amazing. So I guess uh let's let's get started then. Uh do you have some goodies that you can share with us? >> Yeah, absolutely. So I have prepared a little bit of slides not to like bore everybody out with like a lot of tax and more like to make a few of these things I wanted to share with you a little bit more visual. So I have three topics to talk about right a little bit hotel and numbers because I'm always excited about seeing how big this project has grown um and then a little bit about the community and then a little bit about uh the tech stuff. I think that's the stuff uh the things that most people are excited about. So yeah, let me share my screen and get started with a little bit on the numbers. Um, yeah. Uh, so those numbers are like for all the time, right? So since open telemetry was created what is it now 6 years ago we had almost 20,000 contributors from 120 countries and like I call it organizations but think about it's like companies and and yeah foundations whatever like almost 2,000 organizations. Uh I also looked up the numbers for the last 12 months which is still like 6,000 contributors over 70 countries over 600 organizations. So you see like there's a lot of velocity in in in our project, right? So um and and then this is like growing and growing every year and we see it also like with the PRs and and and and like the the work that's happening on GitHub. I mean we have to add here right something like this session here is not recorded by anything that you see on CNCF Devstat, right? There's a lot of work going on in our community that's not tracked on those boards unfortunately. Uh so also big shout out to everybody doing let's say the work outside of the repositories but I still think like those numbers are impressive right in since foundation of the project we had like those 60,000 APRs uh reviewed as you can see like most of them three to four times and then like over half a million comments right so that means like uh there were over half million times someone wrote something on a below our issue or pull request or something like that which I think is really great and Like what's what's really exciting is like uh in the last year I think 20% of those PRs have been just done in the last year which is like huge growth and especially like from the reviews 30 30% of them have been done in the last year meaning like we're leveling up our reviews right so so we have more and more repositories where we say like hey you're not done with one review you maybe need two reviews or for example in our open telemetry IO repository, we need a review from docs maintainers and from like the co-owners sig and and I think this this really helps with the quality of the project. Um, and who's doing that, right? So, and and this is some number I'm I'm super happy to be able to share because we build a a page for that specifically on the open telemetry website where we list all the triagers, all the approvers and all the maintainers. Um and as you can see like those are like people that stepped up and and took over a role where they said like hey I help us triaging I help us like managing the repositories I help with running the sik meetings and and if you do the math there's like almost 300 people just having those kinds of roles which is I think really really great to have. So yeah, a big shout out to everybody who's who's helping with that and also a big shout out to all the people I'm not listing here because there's so many people in our community that that are helping or that they're that just landing in our community and then wanting to to step up and and do much more things. Um yeah, and then of course who is using open telemetry, right? And um there's always like three kind of groups that we mention. Um there's first of all of course like what we call the vendors or like the backends. So this is of course companies that consume open telemetry data and do something with it. Uh but also open source projects, right? This also includes Jagger or open search. Um but then there's also what we call integrations, right? Um and that number is probably much much bigger. So this is just what we have on our website. Um those are projects that said like hey we add open telemetry to our project natively. So things like uh Kubernetes has their API server supporting open telemetry or uh there's the PHP server called roadrunner runner that has open telemetry. I always love mentioning that one because like it's written in Go and then doing PHP. So you can even do tracing from Go into the PHP in this web server. This is this is really something I always love seeing. Oh and also Docker, right? So if you I think it's Docker Buildkit and Docker another project of Docker like you can see your containers being built with open telemetry. I think that's also very fun. Um and then of course thousands of adopters right I I I mean every time I'm in a presentation or in a room and I ask people like hey who of you knows about open telemetry most of their hands go up and then if I say like hey and who of you is using open telemetry also like 80% of the people are raising their hand. Unfortunately, this is something we don't have that many listed on our website. So, here's the the first call to action. If you are an open telemetry adopter, if you have uh maybe even written a blog from your company where you say like, hey, here's how we use open telemetry. Here's how it's helping us. Um just use that link um and you can reach out and and and we can have you add it there, right? Um so, yeah, that's a little bit open telemetry numbers. Um, I think that's just a good starting point and let's talk about the community or is there anything you you'd like to deep dive on that slide? I think it's just good to get started. >> Yeah, I think uh I I think let's dive into let's dive into the next topic. >> Yeah. Awesome. I think that's that's now more exciting. Right. As I said, I always love to start with those numbers to just emphasize how how big this community has grown. And that also means like there's a lot of community updates and especially like we all know CubeCon is coming and for whatever reasons September, October, November is like all those community changes that are going on. Um, and one I'd like to highlight because like I'm one of the maintainers of this project is like if you go to the Open Telemetry.io website. There's now nine languages that we have started to translate the open tele telemetry website to. So that includes English just like the base language. But then we have Bengali, Spanish, French, Japanese, Portuguese, Romanian, Ukrainian and Chinese. I I have written it down so so that I don't forget anybody. But I think that's amazing, right? I mean we have like started with that a little bit more than a year ago with four languages and it has grown to nine languages. And this is really amazing community because I think that's a really great way to get started with open telemetry, right? So you can translate the things into your native language, learn about them and then maybe move into different projects of the open telemetry project with with this additional understanding. Um yeah, then maybe a big call out the technical committee has uh added two members. So David Ashpole and Josh McDonald have joined the TC. David is uh like subject matter expert on metrics Prometheus. He's in Kubernetes contributor as well. Um and Josh has stepped down from the TC a while ago is now coming back in and he's like in the sampling sig. He is one of the people behind the hotel arrow and so many other things. So so I'm I'm probably not doing him justice only only calling out those two things. Uh but yeah big congrat congratulations to the two for for joining the TC and we will see the the impact they will have on on the project very soon. Um then another one the governance committee election is coming up uh next week. So we just announced the the candidates if you are member of standing how we call it in the open telemetry community you should have uh a notification about you being allowed to elect. If you think that you're allowed to be uh in in that election as a as a voter and have not get let's say your your access to that, reach out. There's a link for that also. You you can find it from here um to to be added to that. Um there for for people that are new to the open telemetry project, a new slack channel that we called open telemetry new contributors. So if you want to start contributing go over there um and you can get started or can ask your questions and say like hey I picked up that issue and I need some help with that or hey I want to help with documentation I want to help as the collector can somebody guide me with that. So I think that's a really great opportunity to get started. Um yeah another call to action I think I have a few more coming. So so so take down your your paper and and and take some notes. We we have community awards, right? So, we started with this last year and we wanted to redo this once again. So, at CubeCon, uh we will call out a few members of the community where the community thinks like they deserve an award, right? So, so everybody is allowed to nominate someone and this also does not only include like people contributing to the project. This also includes people that are promoting the project in any way, right? So let's say you have a person in your company that's running around and convincing everybody to use open telemetry. Uh you can nominate them, right? If you find four or five friends in your company and say like, "Hey, let's nominate that person. Let's let's add them to that list." Perfectly fine with us, right? We we want to hear those stories. We want to hear about people that are promoting open telemetry. >> Um and speaking about CubeCon, right? I mean, the observatory is back to CubeCon. Uh so if you want to speak If you if you want to speak with us at CubeCon, I will be first time for me at CubeCon North America America this year by the way. So So if you want to catch up with you, if you want to speak with us, uh you will find us there. Um I think we also have some schedule and some program there planned. Um you can probably tell more than I can do on that, but yeah, I'm very excited about that. >> Yeah. And uh if I if I may interject really quickly on on a couple of things, um >> I wanted to uh just dig in quickly to the uh on the GC elections. Um because that's that's pretty much open to anyone, right? I know you said like it's it's it's happening next week. Um but anyone who's been involved in open telemetry can um can nominate themselves right for the GC elections. >> Sorry, can you now I lost you for a minute. >> Oh uh anyone anyone uh can nominate themselves for the GC elections that's been working in open telemetry. Is that correct? Yeah, but I think the nomination is closed since like Okay. Okay. Yeah. Yeah. Yeah. Yeah. So, so that's why I did not call that the nomination. But yeah, no, fair enough. Fair enough. >> I think I think even you have don't have to be like >> So, so yeah, everybody can be nominated. I think you need two supporters and then you're you're in. I said it's closed for this year, but next year, yeah, we will do this once again. So maybe maybe it would be good for anyone who's interested in in uh uh running next year to uh just give them a heads up heads up of what the process might be like. >> Uh yeah, sure. So um what what we what we normally do is um we will send out a blog post I think end of September, early October where we say like hey elections are coming up. Uh if you want to run as a candidate create a pull request against the community repository and have two people uh supporting you or saying like yeah I think that person is a good candidate. Um, I can recommend that like if you want to do that for the first time, wait for the first people to submit their uh their nomination or self-nomination. Let's be honest, most people self-nominate and and there's nothing wrong with that. I do that. Um and just use that as a template and then think about like if you would be elected in the governance committee of open telemetry which is like responsible for managing the whole project for the health of the project for the community as a whole what what are the things that that that you would do for the project right so so the GC just to call this out this is less about like oh I want the collector to do x y and zed or I want to to have I don't know added some specific language to the open telemetry project of course from a GC perspective let's let's say you can engage in those conversations with those communities but but the GC is much more around like how can we grow and and make our community stronger and better and how can we manage the project as a whole so that means like how can we get projects added to to what we are doing how can we make sure that they're sustainable how can we make sure um that that our project is thriving right um so yeah next year around the same time window if you want to run. Uh that that that's where where you should should take a look. >> There you go. And for anyone who's planning for next year, I guess uh keep keep an eye on the nominees this year. See what like you said, see what they've what they've written for their nominations to get a an idea of of the types of things that that gets you that gets you elected. You you're you're on the you're on the GC, right? >> Yeah. Yeah. Um so yeah I um what what gets you elected I think is like um that that you show that interest in the community right I mean at the end of the day it's also like >> very oft that's it's very hard to say that since I'm member of the the GC myself. So, so what I look for but um before I was member of the GC right and before I was like voting with this insight in perspective I always looked at like >> um what have those people done before that right sure putting on your nomination is the one thing like writing down hey here's the things I want to do for the community um is is is the one thing um but but but um yeah it's more about like what have you done in the last I don't know few months to uh to to help people with that right so so that's that's the thing yeah >> yeah and I think from a voter perspective it definitely helps if you've been visible in the community and you know and and it could be you know in a number of ways right code contributions talks um blog posts um you know being active in the channels answering questions Yeah. Yeah. And that's maybe a very important point, right? You don't have to be part of the GC to do that, right? I mean, we we we need a lot of people to do a lot of work outside of of writing all the code, outside of writing all the documentation. Um there there's so many great things you contri can contribute to this project and and I say it very often to people that that want to uh either break into the open telemetry community or in any open source community. I mean, you have skills already, right? You could be a good marketeteer person or you could be a designer or you could be a good technical writer or you could be a good developer, right? And and think about the things that you bring to the table. Um, and and that's helping you to to to be appreciated in the community, right? Don't think like, oh, I have to be a good go go go programmer to to be valuable to the open telemetry pro community. That's that's just not true. That's a great point. And and I think before we transition over to Marilia, you still had one more thing that you wanted to show us really quickly. I think you're going to show hotel unplugged. >> Yeah, I I have like the slide and another one. I'm not sure how we doing on timing. I'm talking a little bit long now. So I still have like one more. Um but let's see that we can get through that. I think people are more excited about the declarative configuration part, but people should be very excited about open telemetry unplug. This is like an an unconference we're planning for happening after FOSTM next year. So Monday after FOST day if you're still around uh register for this event and then you can like meet with the open telemetry community. There's a blog post on that. Um very excited about that one. Uh and then last but not least because people are asking about that graduation is in progress. I can share a little bit more in the in the tech overview. Um so yeah that's on the community updates. Um, let me hopefully go into the technical highlights. Um, this was the text I wanted to show first, right? There's so many updates. I will not touch into anything and everything uh that the hotel project is doing right now, right? So, so this is almost impossible. But I picked like the things that that showed up recently and since I saw this question coming up uh very early on when when we started like this um this session today is like how can I stay updated the open telemetry blog is a really good place and we try to motivate also six to to put things there for example the profiling s they will come out very soon with like hey we are moving towards alpha I think that's a that's a really great thing that that we're seeing in the community And equally, we had a blog post just recently about sampling that they're going to adopt the W3C trace context level two. Um, that's probably worth another what's new in hotel. I I try to do like a two sentence summary of that. It's very difficult but but in a long story to make a long story short it's about mainly about randomness and and trace ID to to ensure that like enough there's enough randomness and the trace ID and that allows you to do certain things and sampling. Um I can like if we have time in the Q&A I can try to to dive into that for a little bit. Um the mainframe community has been doing a a survey just recently and I don't know if if people know that like we have a mainframe community with an open telemetry and they just ask a bunch of questions to like people on the mainframe like do you know about open telemetry how how do you want to see it used and they were talking about like hey we need good Python and Java support we need to see that the open telemetry collector is is able to take uh metrics out out of a mainframe and of course the question about like what to do in cobalt. I have no more details into that but but it's a very exciting community that like hey even even the mainframe which is like a technology for how many years around um probably longer than I'm living um and they're looking into open telemetry just to close this gap for for end to- end visibility declarative configuration I just have it here for completeness looking forward to hear about this more myself um and then we have next Android road to stable. Uh so so they're working towards a version 1.0. They have done a ton of work over the last few months. So also very excited about that one. There's even an demo app now in the hotel demo with Android. So so you can even play around with that. I think that's a really good starting point into that. And they're looking for feedback, right? So So go to the open telemetry website, go to the open telemetry blog. in that blog there's a link to like hey what do you want the hotel Android community doing for for their next release um and then what I said about like we are working towards graduation we met with the TOC of CNCF there were a few adopter interviews we were like discussing what what are the things to go next so so there's a an issue on the CNCF TOC um repository it's number I I wrote it down 1739 you can look this up and there's a summary of like what are the next step right and and one big thing and and this is maybe exciting because this is not new feedback for us that like people are looking for like how can we improve stability how can we communicate stability better and how can we look into into ease of use how can we make it easier for people to get started with open telemetry right um and I said probably this slide alone would have covered like um another 20 to 25 minutes if I would have talked about browser phase one the bail donation which is now called OBI the injector weaver hotel arrow they're doing a new phase we have a call for contributors for cotlin a lot of good changes in samcom so yeah um I tried to start with that and I really hope we do those kinds of sessions more regularly so people can uh stay on top of things uh so yeah this is how you say like scratching the surface and then let's dig deeper next time. So yeah, hope it's helping. >> This was amazing. And >> Oh, sorry. Go ahead. >> I know. I was going to say thank you so much. Like that was really helpful for me. >> Yeah, definitely. I was going to say this proves that we need more of these. >> Yeah. Yeah. No, I mean I was learning a bunch of things while preparing. I was like, "Hey, let me let me ask around and let's talk with people like what should what what's going on in your sick? What what's happening?" Right? But I mean, our project is is is is growing and growing and we are adding so many cool things to it. And um yeah, I'm very excited about a lot of those things. And again, I only can recommend two things. Uh go to the open telemetry blog from time to time. We have those kinds of announcements there. And if you're a maintainer or contributor, come to uh sikcoms and and let us know about the latest and greatest of your sik so so we can share this with with the wider world. Right. >> And again, if um seven covered a lot, if you have any questions, please put them in the chat of whichever platform you're tuning in from and we will get to them um after we speak with our next guest. Thank you so much, Severin. >> Yeah, thank you. Speak to you in a minute. Thank you. And and we are getting ready now to bring on Marilia Gutieres um to talk about What is she talking about? Reese remind me. >> Amazing. Hi Marilia. >> Hello everyone. Well continue on the team of where you're tuning from. I'm also tuning from Toronto. So yeah, me and Nana. Yeah. >> And and we're both Brazilians tuning in from Toronto which is Coincidence >> small subset. That's awesome. Um, so Marilia does a lot in the community. Would you like to share some of the work that you do before you get into configur declarative configuration? >> Yes, of course. Uh, so yeah, that is one thing that I was even talking like a colleagues yesterday that I keep finding more things in hotel and I just trying to get involved and I just like keep spreading the things that I work on. So for example, I am a maintainer for the contributor experience. So if you want to be a contributor and you're having some challenges, that would be the group that will help you with. I am an approver for the JavaScript SDK for Portuguese localization uh database semantic conventions and as of a couple of days ago also for the communications s as well. >> Yay. Amazing. >> Dang, that is so productive. No, >> you're an inspiration. >> Thank you. >> Seriously. >> Yeah. So, even like for this one that we were like, okay, we need to pick one topic to go into deep like so many things like, okay, I was working on this, I'm working on that. Let's pick one. But I feel like the cratical effect is something that is really cool that is getting to a very like almost stable uh place and we're going to see this implementation several like SDKs now. Uh so I think it's a really cool one to talk about. Cool. So yeah, let's let's start. Uh so before I start with the K config, I feel like I should give a little context like what is this? Why do we need this? What are you talking about? So how you would set up like your SDK? You're like you're using hotel and you want to set up some specific things for your case. things like even like my server's name or like what is going to be my tracer exporter. So how it was done up until now is basically using environment variables and it it was very like easy to do this and because environment variables is just universally available on all languages. So it doesn't matter the SDK there was like some way to do this but then more complex things came up and like oh I need this and just we we just published a blog post yesterday talking about the story of the critical config there was an issue created five years ago on the Java CX like I just want to be able to filter out health checks traces I don't want to keep sending this there is no easy way to just like filter that specific one like I have my sampler but how do I pass what is the thing that I want to filter so that start to get very complex to do just with an environment variable so we needed something more robust so that is the idea that came for let's work on with the character config so at this point we now disallowed environment variables so just so we don't keep like adding more when we have to transition and then we started with now declaring config so this is pretty much a YAML file and you can just put all the things that you care about on that one. It is a little more robust with just compared to environment variables is easier to expand because you can have like all the objects and types that you need there. Uh and because this was a project of course I'm the one talking about it today but there were like several people that working on this for several months and now we're just on the phase of really implementing on several SDKs but the process of actually creating the convention there is this own like uh repo specific for this one it it takes a lot of time to get feedback from people and see what's going to work with all the languages And also people have concern like I'm not going to use environment variables anymore but I'm used to them. So you can still have environment variables on your config file. So I can't I put it an example here like how do you use this and like if I have this use it otherwise use this other thing and to just start using which is uh available today in Java and also starting on JavaScript is I I do recognize the irony that you have to set up the environment variable config file to use the config file. I do see the irony on that. Let's say like don't use my environment variables, use the file. I do. Yeah. But that's how you do it. And then from that means like the SDKs are will be able to just use your config file instead of having set up all the environments. So if you're curious, this is how a basic one look like. So here you see like the ammo. So we have like some resources. Here we have like an example that you still points to environment variable if you care. Uh and then I have like some example like tracer provider. I only care about the end points. So this is all anything that I put in here. And then here the the example that I mentioned for like the issue that was created like I just want to filter my health checks. This is an example how you do this uh on Java is already working. Uh and you can see here just like a sampler and I say like my rule I want to drop if I have this attributes with this pattern. So is a lot easier than having to actually you go into the code or your instrumentation trying to like create your sampler or like trying to have this environment somehow. Now you just create this object and that's it. You don't have to care about like how the SDK would do this. We are the ones that instrumenting like doing that for you. So you don't have to do it and of course has a lot of thing like I'm just trying to like put a few screenshots this can be quite big. So you can see here like oh like oh I have a processor I have a batch with this schedule with this time out my exporter is this type and is a good one that we have a file called like kitchen sync on that repo that every single like spec that is created is added to this file which is the one that I'm showing here. So if you want to see examples of everything you have here and it has the comments of what it means, what are the values, what are like the default values if you don't put them. So is a great one and we also have one for migration. If you're like I just want the basic because I'm using my environment variables and I don't want to have to like copy. So you just copy that file and it's still going to use your environment variables for some basic stuff. uh if you want to know what is available today. So we have uh this matrix of compliance which Java is fully compliant PHP uh JavaScript is partially and we do have a lot of other languages that are also working on it. uh I think they we just need to update this for a few of them like for example go uh and I believe Python are also have some implementation and when I say like it's fully compliant does not mean that it's ended is like complete that is not what it means there are still features that still needs to be added but at least the basic of what you're seeing here is is working for those languages uh and here I can probably copy this on the chat and we can share but here I put it a few links for resources. So the blog post that I mentioned which is in English and in Portuguese. So that is like see one of the cool things about localization that I already created like both of them. Uh I have the the file like the repo for the configuration and the examples that I mentioned. So the kitchen sync the migration itself and if you have question about the config file in general you can use the the select channel. So the hotel config file if you want to have specific for the SDK. So for example, how this is working on the Java one or the JavaScript one. So then you go to those specifics uh channels. So like hotel Java, JavaScript go and so on. And yeah, that's what I have for an overview for today. >> This is great. Now I I had a follow-up question for you, Manidia. Um so these configurations, it sounds like they are um language dependent. So you do them you do them at the language level not at the collector level. Is that correct? >> So the so one thing is supposed to be like completely agnostic. So you can use the same file doesn't matter which SDK you're using. Uh but there are parts that are also for the collector. So for example the collector is already using the clarative config for a couple of things >> right? >> So is is pretty much like already added there. uh I don't have a list with me easily to just show what are the things that are working but the idea is to have like all components will be able to have the craft config is just a matter of now we need to go and implement on all those places >> but does like how does how does the config file get loaded up does it get loaded like as part of yeah I I guess I I'm having trouble like understanding where in the flow it it fits in >> is you're talking specifically about the collector or in general. >> Oh no, just the just in general like the the like the declarative uh config like for the environment variables. >> So for example, I can talk about like the JavaScript which is the one that I'm working on. So when you initialize the SDK usually you do like for example new SDK and sometime you have to like pass parameter. >> This time you're not going to you don't need to do this. So your file is going to exist on the same place that you're adding your instrumentation. Uh >> so it picks it up from does it pick it up like from as long as you have that file in a particular directory in your code it'll pick it up. Is that the idea? >> Yeah. So yeah because the because the environment variable it points to the file. So as long as you put the like this is the path to the file. So when you initialize the SDK it's going to check like does this do you have set a config file? If you don't, it's gonna ignore and use whatever you were doing before. But if it has, it's gonna say, "Okay, let me check what you have on that file." And for some things, it's pretty much saying like if you have this f value, I'm going to use otherwise it have default values for a lot of things. So it's basically that is the moment when you initialize like the SDK that is going to read and do whatever it needs to do. >> Okay. Okay, that makes a lot of sense. This is cool because I guess it also lets you um I I I guess if you're using like just the old way of like just the environment variables that was very flat, right? And this kind of gives it a little bit more structure too. So it kind of lets you like lump configurations together which is very cool. >> Yeah, because we have a session for example this works for everybody. So it's supposed to be like very like agnostic. So ah now my application is in Python IP like use the same. But of course there are things that are very specific for languages like this like agents only exist in Java. So but oh but I still want to configure this. So the config file has sessions for languages as well. So if you have like this is specific for Java do this. We have those type of things there as well. >> That's awesome. Reese, do you have anything uh that you wanted to chime in on? I'm still processing. >> It's so cool, right? Like my mind is blown. I'm like where were you like five years ago? >> Yeah. Yeah. And the idea is even like for some of the things you don't have to like restart some of components and can just like read as is. So it's also that is an advantage as well. But yeah. Well, I think we are can get right into some Q&A because I do have questions and I think our audience has some questions. Um, so Lisa, we would love to bring you on. >> Hi Lisa. >> All right, thank you so much. Hello everybody. I'm Lisa Jung, a staff developer advocate at Graphana Labs. I'm part of the communications and the enduser sake. So yeah now we are um gathering questions from the audience and asking our lovely feature speakers here. So the first question is from Cecil. So he said if you have both options environment variables or the configuration in a project which one takes precedence. So if you do have the environment verb saying like this is my config file that is the one that takes precedence and it's going to use this and it's not going to use the environment variable as a backup at all. It's like it's going to ignore that unless you have your config file had the environment variable on that file. But that takes the precedent. If then if you don't have the that environment var set up then uses even if you have the file existing somewhere it doesn't know. So it's going to ignore and use environment variables. >> Gotcha. Thank you. And then we have a question from Kieran. Russ is still a four-letter word. Long way to go. Question mark >> for specific for the classical fig. I I believe this question came up during your >> Okay. So yeah it is so that is the thing like because we when we people like are creating the semantic convention and stuff they try to get like people from different like languages so you know like what's going on and then you go to your own sig and start implementing but a few of segs have very small like small group so it's hard sometimes for them to handle because they have to like reveal PRs that come in and do like releases so it's hard sometimes to do and like be able to keep up with everything's going on. The same thing like database and conventions came out a lot of SDKs still don't have implementation. So if you for example you really care about rest that is your chance to join the sik and actually be a contributor. So people are really all the materials are always happy to help and have people joining in. Uh you can say like this is something that I'm interested at and I really care about how do I start contributing and it's a way for you to be able to have things faster what you want and if your goal is to be like an approver or maintainer for small things that is going to be happen faster because your contribution is going to have a lot of weight if there are not a lot of people helping out. So that is something also to keep in mind. >> Gotcha. All right. We have a question for Severine. So what does W3C trace context L2 for tail sampling uh bring to the table? >> Yeah, I said I I I I had to to take some notes on that myself because it's like a it's a it's a very very complex topic and and I'm also not like a a sampling expert, but but I try my very best. Right. So uh we have been doing sampling and open telemetry for I think over four years now and it has been evolving and like we added a lot of good things. One of the things that we had is this trace ID ratio based sampler right um and this sampler had always like this is just me quoting straight from that blog post. always had this to-do where we said like hey all you can do safely with that is sampling on root spans right so so you had not uh a lot of good um guarantees on probability downstream right because what's happening is that like you use the trace ID as a random n-sized word like you use some part of the trace ID and then you compare it to like uh uh let's say a boundary that that's like 2 to the^ of n times the ratio. So you can think about it like um if if you take 50% it's like I don't know 128. If you have six seven six bits something like that and if the number is smaller or bigger then then it comes in or does not come in. But the big problem is that like in the trace ID until level two, there were no guarantees on how many of those bits are truly random, right? Or are maybe encoding some some some additional information, right? And and this is what uh what what has been changed in in that new standard and and open telemetry is adopting that, right? So, so you send an additional flag uh when when you use that and with that you get 56 bit of sufficient random randomness and you can use that right so and again this is me just talking from a few notes that I took from the blog post. I can highly recommend reading into that. I think if we talk about sampling, we all know that like with the amount of data that people are consuming these days with open telemetry, this this is urgently needed and and they need need a lot of good hands to to also try out those things. So so give this a look and and and yeah, try it out. >> Gotcha. And to piggyback on that, Henrik, what about consistent probabilistic sampling? Uh, I don't know. Very, very honest. Um, I I'm not exactly sure. So, so that's like um yeah, maybe um that's that's maybe a good good thing for the next round, right? Maybe invite someone from the sampling sig and have them talk about that specifically, right? It looks like there's some interest into that. >> Yeah, absolutely. So Murelia uh Fernando has a question for you. Um can I have more than one config file like a file with generate configs or another with only specific JavaScript configs? >> So the file itself is going to be one because otherwise we don't know which one takes precedence. Uh but for example if the part that I mentioned that are specifics to language is still part of the same file. So you have for example like oh like tracer exporter like resources and then there is a session that we call like you should put at the bottom that like okay now it's specific like some resources and stuff for like JavaScript for Java. So that would be the parts that you have that is specific for JavaScript. Uh at the moment there's nothing specific for JavaScript. So you wouldn't have that at the moment. But if you have things different for different applications, you need one file for each one. So this is is something like for example that you can take advantage of some things environment variable related. Imagine that you have oh is exactly the same thing but I want to have the service name different on those two. So what you do then your config file point just say like use the environment variable for this case and then you have your two environments that use the the different name. Gotcha. Another question from Henrik for Melia. Uh would the OTL operator provide a config CRD that could be deployed across Kubernetes objects? Not sure because I don't see like for example a reason to add new config because the idea is to have like all possible config that we want to exist on that file. It's just a matter of like does it exist already or something that needs to be added because it's a little more also about like getting now feedback from the community just because we have something a little more stable and say like what is missing and what we can add there. So those are the points that we need feedback from people. I can check this one specifically but from the top of my mind I don't know the answer if we have something already for this one. >> Gotcha. Um so we have a question from Cecil so please feel free to jump in Stephan or Morelia. Um is there any guidance you can share around open to labor for longunning background processes? >> Yeah I can pick that. Um so so this is actually like a a very complicated topic right and I just looked it up. There's an issue on the open telemetry specification from 2019. So since the project was founded uh by Armen around that topic and the first guidance I can give is >> um open to labor that's >> yeah no worries um um now now I lost track for a minute yeah so so um the very very first guidance I I I can give you is like um go to that uh issue to go to that repository um and call out that that you're interested in that and maybe even interested in helping with that. It's a very gnarly topic, right? Um so what can you do do until then, right? I mean at the end it depends a little bit on of what does long mean to you, right? I mean if we talk about if we talk about spans and traces talk taking a few minutes um then then then we are totally fine. you maybe should be just fine with with the tracing that you're doing today. But if you say like, hey, and this is the classic thing, right? You have a background job and maybe this is doing some video encoding and maybe you want to do something around that. One thing you definitely can do today, right, is is use is use um logging and events for that and maybe announce like those kinds of things and say like, okay, now I'm starting, now I'm ending. It it's not perfect, right? and and let's be super honest with that and there's probably a still of things you need to do on your back end of things. Um but yeah, you have to work around that and just let us know that like um we we we need to do more on that topic. So so that's the guidance here. Yeah, >> thanks everyone. All right, I need to stop just reading straight I need to process a little bit more before like reading the questions out loud. So sorry about that. Okay. So, as we uh we wrap up our Q&A, I'm gonna I'm gonna ask one question to both Sever and Amilia. So, you've been contributing to hotel for a long time. So, I'm going to take you all the way to the beginning. Um, how did you get your start in contributing to hotel? And do you have any advice for new contributors? Who goes first? You want to start or should I start? I >> You can You can start. You started first. >> Yeah. Yeah. Uh they let me get started. So my reason to start with open telemetry was a very selfish reason, right? And and this is this is something I always tell people, right? If you want to get into open source, have your own good reason to start with open source, right? have something where say like I don't know I want sampling to be better in open telemetry because like I have way too much telemetry so let me get started with sampling and then make this thing work. So, so, so my situation back then was like um I was in a consulting position, right? And I was talking with a lot of people what we back then called APM and we were still transitioning to call everything observability and people had questions about open telemetry, right? They told me like, hey, what is this thing? Should I use that or should I stick with this thing I have already? And this was like every time like something where it was like recognizing like, hey, I have to level up on that, right? So I'm I went to the community and I said like okay let me get started on that. Let me learn myself what this open telemetry thing is so that I can speak better uh to my users to our customers to people that that want to to get into our product. Um so so I started to help with documentation help us getting started. I started actually also in the JavaScript community but then I stuck with documentation because again this was the most helpful thing for myself to just explain open telemetry better to to other people. So yeah, for me it was basically I I have working on observability like a few companies before and then I really like oh I like this observability thing and I really want to focus on that. So I was like I looking for a job that that is basically my job. I only have to think about like the observability and this is why like I went to graphana that the team was like you have to contribute to hotel so that it was like oh exactly what I wanted. So this is kind of how I started contributing. So was a mix of things that I found it interest like oh I think this is important I should contribute it and a mix of what also makes sense for my own company. Uh, and I do actually have I was going to say we we just created a post that I can share here. There's like ideas on how to contribute it because if you wanted again you can have like several reason for it or like you're just working on something because your company needs or something your own personal project needs and you want to like develop is something that you just want to learn. You want to increase your network uh people that you want to meet. So pick something that or really like interested you like oh I already have knowledge and I want to like focus on that or something like oh I'm just curious about thing I don't know what is this but I want to learn can have like both ways and really like dig deep and into those ones that open tele is a huge project and like so many components so many languages you will find it's just a matter of finding what is your place there and you can try it out. There is no thing like I picked this no now I have to go through this one until the end like no join a couple of calls just listen in you don't have to say anything like oh this thing seems interesting like continue a little more other no this is not for me go to the next one until you find what is the thing that really excited you >> nice thanks Melia so Melia Severin Adriana and Ree have been amazing guides and mentors when I was first getting started by the hotel. Now we I lied there's one more burning question for both of you. So the question is what kind of interesting usage trend trends in terms of implementation observability stack docs page views or anything have you seen over the past year? I think this is more of like whatever people did not think it was necessarily important before to have observability on they're like oh yeah that would be good if I actually know what the hell was going on. So I think and then just like time people keep adding more and more. So it's a lot of old type of applications that people did not think about like oh yeah I should adding but also new things but I I guess you're going to find a little of everything like people like really now until like mobile like nobody was paying attention or like the browser one that's being really active. So it's just like what we forgot to add it okay let's work on all this thing because it's really important. So it's just like all over. So you're going to find from every single place and then even things that people do for fun like I want to monitoring the quality of the air and humidity on my plants. There are people that do this to like oh no I'm actually working at NASA and have want to know what's going on. So you're going to find so much spread. Yeah. So that is the cool thing about hotel >> 7D. >> Yeah. I can I can try to to make it quick because there's so many things I could now answer that I was just thinking about it like yeah I had like five or 10 things in my head but let me let me pick like a few like the one thing is like and and this maybe is is close to like hey let's measure like the humidity in our room. Uh I saw more more also like databases being using observability inside the database right I mean we saw like hey tracing going up down to the database but I know that like now people are thinking about like why not do tracing inside of the database right I mean that's that's like something I I found very funny similar to what I mentioned before right your docker build um so so this this is just this thing where we need to recognize like every software can be traced locked and metrics metric Is there even a verb? I don't know. Um but but I think you get the gist, right? So so you can put open telemetry into just everything and and get out out amazing data. Um and the other thing and and then I I tried to make it quick to so so that we can get to an end is like I think people start to recognize that like with OTLP they can send their data everywhere, right? I mean Adriana you wrote about that don't send your traces, logs and metrics into into different places. That's not what I'm talking about. It's more like, hey, wait a minute. I can send a bunch of my data over into that back end and I can send maybe another portion of that data into into another specific tool. And and people start to recognize like they have a lot of freedom with the data, right? They have a lot of freedom with the obser with the observability data beyond let's say what what observability vendors offer today or what we do with like trace explorers, alerting and all the capabilities that we had before. This is actually a topic I'm I'm very very curious and excited about right now that we say like wait a minute now we have all this data what else can we do with that right I think that's that's super super interesting >> thank you I yeah it has been really interesting um and thank you all so much for following along um if you have additional questions please find us at um on CNCF Slack instance at hotel-s SIG dash and dash user channel. Um there is a QR code on the screen for you to use. Um also huge shout out to Henrik Rexet um who is doing all this live streaming stuff for us on the back end. Thank you so much to Lisa for hosting our Q&A. Um Adriana for being my co-host and Sean and Maria for taking the time to come on and keep us updated. Thank you so much. Um >> yeah, thank you for having us. >> Yeah. happy to be here. >> We got a question asking when the next one is. So, um, this means that, uh, I guess we'll have to put on another one. We definitely plan on putting on another one. Um, hoping for a monthly cadence. Um, but we also have CubeCon coming up in November, so we'll see. There will be uh, speaking of CubeCon, um, as Sephron mentioned, Hotel Observatory is happening. Um, if you haven't been to the Hotel Observatory booth, it's loads of fun. This is where all the cool hotel peeps come to hang out. Um, and we're also um doing uh as part of uh uh a thing that we've been doing the last little while for CubeCon, we're doing a live stream um from CubeCon. And I guess it'll be uh we call it the humans of hotel liveream. I guess it will be to a certain extent a what's a what's new in hotel. So uh keep an eye out for that. I believe it's happening on the Wednesday. Let me just double check. Uh Wednesday, November 12th from uh starting at 2:30 p.m. Eastern time. So, keep an eye out for that. And um I think that's it. Uh do you have anything else, Ree, to add to that? No, I thank you all for being here. >> Yeah, thank you. And also if you have a cool you uh hotel story to share um whether it's like how you use hotel in your organization or you have like a presentation that you want to do about cool stuff you've done with hotel reach out to us in the hotel and user sig again it's hotel- sig-end user on Slack we would love to hear from you. >> Yes. Thank you so much everyone. We'll see you next time. >> Thank you everyone. >> Bye. + diff --git a/video-transcripts/transcripts/2025-11-13T08:33:00Z-humans-of-otel-live-from-kubecon-na-2025.md b/video-transcripts/transcripts/2025-11-13T08:33:00Z-humans-of-otel-live-from-kubecon-na-2025.md new file mode 100644 index 0000000..e8036c6 --- /dev/null +++ b/video-transcripts/transcripts/2025-11-13T08:33:00Z-humans-of-otel-live-from-kubecon-na-2025.md @@ -0,0 +1,258 @@ +# Humans of OTel: Live from KubeCon NA 2025! + +Published on 2025-11-13T08:33:00Z + +## Description + +Streaming live from Atlanta, Georgia at KubeCon NA 2025 -- We are pleased to be speaking with Jacob Aronoff and Diana Todea! + +URL: https://www.youtube.com/watch?v=NzbDui8hDdo + +## Summary + +In this video, Reese and co-host Sophia present a live discussion from KubeCon North America 2025 in Atlanta, featuring guests Jacob Marino and Diana. Jacob, a maintainer of OpenTelemetry, shares insights about his recent talk on Kubernetes and OpenTelemetry, highlighting the high attendance and diverse range of questions from beginners to experienced users. He emphasizes the importance of community feedback for improving the project and discusses ongoing initiatives like the new injector project using Zig programming language to enhance compatibility across environments. Diana, a developer experience engineer, discusses her involvement in localization efforts for OpenTelemetry documentation, aiming to make it accessible in multiple languages, including Romanian, and shares her passion for community engagement and the importance of empowering developers in open-source contributions. The conversation touches on the need for better onboarding practices, best practices for instrumentation, and the overall excitement surrounding developments in the OpenTelemetry community. + +## Chapters + +00:00:00 Introductions and welcome from Atlanta, Georgia +00:01:30 Introduction of guest Jacob Marino and his background +00:03:15 Jacob shares his experience giving a talk at CubeCon +00:05:00 Discussion on audience engagement during Jacob's talk +00:07:45 Importance of user feedback in open source projects +00:10:00 Introduction to the OpenTelemetry injector project +00:13:30 Overview of the Zig programming language being used in the project +00:16:00 Discussion on stability and maturity of the OpenTelemetry project +00:19:45 Diana introduced as the next guest and her background in observability +00:22:15 Diana discusses her involvement in localization and translation efforts +00:25:00 Conversation about community engagement and upcoming trends in OpenTelemetry + +# CubeCon North America 2025 - Live Transcript + +**Reese:** Just the light. Oh, action. + +**Sophia:** Oh, you did. You did. + +**Reese:** Hello everybody! If you've been waiting around, we are so sorry for the delay, but we are very excited to be coming to you live from Atlanta, Georgia. Pretty much everybody was very surprised by a cold front that had us looking at 40°F (about 3°C) the past couple of days. Luckily, it's warming back up, so thank goodness! We are coming to you live from CubeCon North America 2025. I'm Reese, and I am joined here by my co-host, Sophia. + +**Sophia:** Hi! + +**Reese:** And our first guest, Jacob. + +**Jacob:** Hello, my name is Jacob Marino. I'm a maintainer for OpenTelemetry operator, OpenTelemetry Helm charts, hotel injector, and I maintain a few random collection things. I contribute to OpAmp, and I'm missing one; I can't remember. I was wondering who had hands for all ten, honestly. + +**Reese:** If Josh Sar were here, he's in every single group. That's wild and awesome at the same time. + +**Jacob:** No, it was very impressive talking with him. I learned so much. + +**Reese:** So, how do you feel talking to us all? Jacob, you did a talk or was it two talks? + +**Jacob:** One talk. I gave a talk about two hours ago, three hours ago. + +**Reese:** Cool. Very fresh! + +**Jacob:** It was packed. I was worried because this is my second time giving a talk. The first talk I gave was in Chicago two years ago. We got a very low slot, like towards the end of the day on Thursday. It was a tougher day to give a talk. We had a solid audience, and for this one, I was worried that with the content we were doing, which is like introductory to hotel and Kubernetes, I always worry with these introduction ones. You know, is this going to be enough? Is it going to be too much? My fear is that nobody shows up and I'm speaking to an empty room. This was the reverse, where it was a full room, and people were standing in the room, and then people got turned away from that door. + +**Sophia:** Oh my gosh! + +**Jacob:** I was happily surprised by that. We only reserved five minutes for questions at the end because I was like, you know, I don't know what questions people will have. Usually, people aren't asking a ton of questions after; they'll come up to you later. But it's like usually a Q&A session does not last very long. So I thought five minutes for questions is fine. We do questions, and there's a line of like 15 people waiting to ask questions. It just kept growing and growing. We stayed after for another 30 minutes just answering questions. + +**Sophia:** Interesting! What level of questions? Was it more, you know, people who are newer to the topic, or who were asking more like stage two kind of questions? + +**Jacob:** Huge range! We had some people who were very introductory level, just like, "I'm getting my bearings here. How does this part of it work? What's the deal with Prometheus and hotel? What do you use for a backend?" That's like a solid starter question. Then other people came in, saying, "I've been using this chart for the past year and there's this one thing that I want to be able to do. When are you going to do the work to enable that? How can we architect this?" Very big and each one was from a different part of the area. + +**Reese:** Okay, that's so cool! + +**Jacob:** Yeah, very, very cool and very surprising. I'm always happy because, as a maintainer, we get people who come to us with issues on GitHub, and it's like you'll get someone complaining, "Hey, this thing broke. The most recent release broke me." But we don't get a lot of positive feedback. We don't hear what usage is. It's kind of the benefit and the curse of open source—we don't necessarily get that user feedback directly. It's why I love what you all do, because getting user feedback is one of the most valuable things to the community. In order for us to push forward, we need to know what isn't working right now. + +**Sophia:** Right! It's a bit of a challenge because we feel like we're talking about it at our work and in the OpenTelemetry community, and then, like you said, we have all these conscious people who don't know. + +**Jacob:** Exactly! We're trying to figure out how we can make them aware of all these cool things that are happening. One of the things we started doing is the "What's New in Hotel" segment, which we just had our first one about a month ago or so. We're hoping to have something we can point people to. + +**Sophia:** Of course! + +**Jacob:** You mentioned the injector. + +**Reese:** I did! Tell us more about it, please. + +**Jacob:** I’m a maintainer for it, but I'm mostly doing reviewing because this is actually a joint effort between Splunk and D-Zero. Both of them have written their own injectors for auto instrumentation, and they are essentially donating a merge of both of those projects to the alpha version of this for their customers. We are just testing out that everything is working the way we need to. It's a very exciting project because it introduces a new language to Hotel, which is rare for us. We're now using Zig in the project. + +**Sophia:** Zig! + +**Jacob:** Zig is a fun language that is sort of a merge between Go, Rust, C++, and C. The real value of it is it doesn't require a lot of core Linux dependencies. For example, on AWS, not all of the nodes have GCC. When you're trying to do injection, like say you're trying to run a load balancer like Envoy, it used to not work on certain types of AWS nodes. Zig is pre-built with a bunch of these C libraries and has its own dependencies. So the benefit of this injector project is that we'll be able to meet users where they're at without needing to know a ton about their environment. + +**Reese:** That's exciting! + +**Jacob:** I think we're a few months away from it being in alpha or beta. Ted Young is thinking a lot about the release timeline. + +**Sophia:** That’s so cool! What parts of the project are you excited about? + +**Jacob:** Good question! I think the biggest thing for the project right now that everybody cares about is stability. We're trying to get to a place where users can reliably get a new update without needing to do any configuration changes, without needing to worry about components being deprecated. I talked to a developer at a major bank who has developed all of their own custom components for the collector, and every time we do these reworkings, they need to refactor a bunch of their code as well. It becomes a weeks-long project to keep up to date. + +**Reese:** Wow, that sounds challenging. + +**Jacob:** Yes! Users want to see that level of maturity and stability. If we can get to that graduated status with the CCF, it would be huge. Our adoption has already been really high, but that would take us to the next level. + +**Sophia:** Can you explain a bit about the CCF stages? What does it mean to go from incubated to graduated? + +**Jacob:** Sure! Right now, the project has relatively lenient standards. There are standards, but graduation is really strict for good reason. You want to be sure that when you are considered a graduated project, you have a healthy community, which we definitely do, and stability guarantees, which is the main thing we're working towards. Lastly, adoption has to be at a certain amount of companies, and you have to have testimonials from those companies about how the project has been helpful. + +**Reese:** That makes sense! + +**Jacob:** Stability can be thought of as the main goal. You can think of Kubernetes as the example here. Kubernetes is, you know, the first graduated project. Every time Kubernetes improves their security and reliability, it raises the bar for everyone else to continue to improve. We are getting towards stability, and as we do, we will be introducing more security features and guarantees, which will be very helpful for enterprises. + +**Sophia:** That is super valuable! + +**Jacob:** Yes! Later on, we will be talking about mobilization efforts and documentation as well. + +**Reese:** If you are an OpenTelemetry user and your company is using it, we would love to hear your story! It will help us graduate the project and take us further. + +**Jacob:** Absolutely! Documentation is one of the best ways to contribute to the project. It is so valuable. We had someone for the operator group contribute a whole bit of documentation for the target allocator, and it was so well done. + +**Sophia:** That’s wonderful! + +**Jacob:** It's been a fantastic resource for me to point users who come into our Slack or our meetings to just say, "Hey, you have a question about this? We have a full document to answer everything you might need to know about it." + +**Reese:** And the local community has been super helpful! Anything you can take on, like documentation, is something that you're taking off their plate. + +**Sophia:** Yes! Join the communication SIG for that. I'm currently in it, and we're working on a lot of refactoring of the documentation. + +**Jacob:** Exactly! The beginner stuff is especially important. From the talk today, it was so useful to learn where people are at in their implementation journeys. + +**Reese:** Definitely! Getting started is always going to be the most important thing for us to nail. + +**Jacob:** Yes! I heard from the event in New York that reference architectures and showing how we can do hotel in various environments are also needed. + +**Sophia:** That would be really useful! + +**Reese:** Are we planning to host them in a repository or on sites directly? + +**Jacob:** I think that would be the best. Centralizing it and making sure that it's well organized is a lot of the work that you're doing. + +**Sophia:** Absolutely! + +**Jacob:** So, we should bring all that together. + +**Reese:** Yes! There's a ton of companies contributing, so there are definitely references that our community could use. + +**Jacob:** Exactly! People are so friendly, and the engagement with end users is so valuable. + +**Sophia:** Do you have any general trends that you are seeing during your time here at CubeCon? + +**Jacob:** Yes! The main concern seems to be around OpAmp, which has been getting a lot of attention. There are at least four talks that mentioned it, which is a lot. OpAmp is the Open Agent Management Protocol, part of the OpenTelemetry project, designed to be vendor-neutral. The goal of the protocol is to enable things like component status reporting, health reporting, and remote configuration. + +**Reese:** That sounds interesting! + +**Jacob:** Yes! What users really want is a way to dynamically update what the collector can do, what the SDKs can do in real-time, but that’s challenging. The way the collector was designed was never really with remote configuration in mind. + +**Sophia:** That does sound tough! + +**Jacob:** Yes! But we want to realize this vision that people can dynamically update their configurations. + +**Reese:** That’s exciting! + +**Jacob:** Thank you so much! + +**Reese:** We will bring on our next guest, Diana, in a moment. I want to show off this baseball jersey. It looks really good! + +**Sophia:** It does! + +**Reese:** Our friends at Honeycomb have been kind enough to sponsor us for the OpenTelemetry community. Thank you, Austin Parker, for the design! + +**Sophia:** It looks sharp! + +**Reese:** Yes! And when we get our second guest set up, I also want to show you her nails because they are amazing! + +**Sophia:** Yes! + +**Reese:** I brought my winter puppy out. It was intense, but we made it through day two of the main event. I hope everyone is doing well and staying hydrated! + +**Sophia:** Staying hydrated is so important! + +**Reese:** I think we're ready to speak to our next guest, Diana. + +**Diana:** Thank you! Congratulations on winning the Open Source Community Star 2025. + +**Reese:** Thank you! Can you please introduce yourself? + +**Diana:** Yes! I'm Diana, a Developer Experience Engineer at Parametric. I have worked all my life in engineering and observability for the last five years. I discovered that I really enjoy explaining things to the community. I had a chance years back to be a speaker at one of the first conferences I ever attended. I was super nervous, but I liked it a lot, and I saw so many women on stage giving talks that I felt I needed to do more. + +**Sophia:** That's amazing! + +**Diana:** So, I did that for two years. Last year, I heard so much about OpenTelemetry and wanted to get involved. I got lucky because the foundation was looking for observability people. We wanted to create the first OpenTelemetry Learn Fundamentals, and I got in. It was a kickstart for me to delve into the documentation and engage with the community. + +**Reese:** That’s wonderful! + +**Diana:** I wanted to explain OpenTelemetry to my entire company, getting software engineers involved in instrumenting things in SDKs and testing everything in production. It was a cool experience, and I’ve been talking about it since in my talks. + +**Sophia:** That's great! + +**Diana:** I also thought about how, if I'm all alone and still want to contribute, I could translate documents. + +**Reese:** Nice! What does localization refer to in this context? + +**Diana:** Localization not only refers to translating documentation into different languages but also includes how to automate specific pipelines and how you communicate a message in a language. + +**Sophia:** That's interesting! + +**Diana:** Back when I got involved, Spanish and French were available. Since then, three more languages have emerged: Bengali, Ukrainian, and I started translating into Romanian. + +**Reese:** That's beautiful! + +**Diana:** The hardest part about localization is translating specific terms into your own language while giving them the right flavor and essence. + +**Sophia:** Yes, I can imagine! + +**Diana:** It’s challenging to find the right words that capture the meaning without sounding dry. + +**Reese:** That’s a great point! + +**Diana:** Yes! There are many languages that lack technical terms directly in English, and translating them appropriately can be tough. + +**Sophia:** That’s wonderful that you’re addressing that! + +**Reese:** Is there a way to see the rate of adoption or involvement from the communities that have these translations available? + +**Diana:** Currently, we are kind of primitive in terms of metrics. It’s important to correlate it with ground meetings or local communities that can promote the translations. + +**Sophia:** That makes sense! + +**Diana:** For example, I started with Romania and reached out to people, encouraging them to get involved. + +**Reese:** That's fantastic! + +**Diana:** It’s important to keep motivation high, especially when starting in free time. + +**Sophia:** For sure! + +**Diana:** It's a gateway project into contributing to open source. + +**Reese:** Absolutely! What are you looking forward to in the coming year for OpenTelemetry? + +**Diana:** I’ve seen a lot of interesting talks about instrumentation lately. Users need more best practices to understand how to implement instrumentation easily. + +**Reese:** That’s a great point! + +**Diana:** Yes! They want to know the best starting points and onboarding processes. + +**Sophia:** That's super important. + +**Diana:** I’m also interested in contributing more to the engineering side, particularly around the collector and how to implement various configurations. + +**Reese:** That sounds exciting! + +**Diana:** Thank you so much for having me, and congratulations again on your award! + +**Reese:** Thank you, Diana! Thank you, Sophia, for being my co-host. + +**Sophia:** Yes! Thank you so much, Reese, for being our amazing host! + +**Reese:** We hope to see you again soon, maybe in Amsterdam or at next year's event. + +**Sophia:** Yes! Check us out, and talk to us if you like! + +**Reese:** See you, everybody! Bye! + +## Raw YouTube Transcript + +Just the light. Oh, action. >> Oh, you did. You did. >> Hello everybody. If you've been waiting around, we are so sorry for the delay, but we are very excited to be coming to you by from Atlanta, Georgia, but pretty much everybody was um very surprised by a cold front that had us looking at 40° uh which I think is a 3° C um the past couple days. Luckily, it's warming back up, so thank goodness. But yeah, we are coming to you live uh from CubeCon North America 2025. I'm Reese and I am joined here by my co-host Sophia. >> Hi. >> And our first guest, Jacob. >> Hello, my name is Jacob Marino. I'm a maintainer for let's see open telemetry operator, open telemetry helm charts, hotel injector, and I maintain a few random collection things and I contribute to opamp and and I'm missing one. I can't remember. I was wondering who had a hands for all 10 honestly. >> If Josh Sar were here, he's he's in he he's everywhere. >> That's every single group. >> That's wild and awesome at the same time. >> No, it was very impressive talking with him. Learned so much. >> So, how I feel talking to you all? Um, now Jacob, you did a talk or was it two talks? >> Uh, one talk. So I gave a talk uh about two hours ago, three hours ago. >> Cool. >> Very fresh. >> It was packed. >> It was like I was worried that uh I've given this is my second time giving a talk. First talk that I gave was in Chicago 2 years ago. >> Oh, nice. >> And the room we got a very low slot like it was like towards the end of the day on Thursday, I think. So it was a tough tougher day to give a talk. uh we had a solid audience uh and for this one I was worried that with the content we were doing which is like introductory to hotel and kubernetes I always worry with these introduction ones you know it's like is this going to be enough is it going to be too much so my fear is that nobody shows up and I'm speaking to an empty room this was the reverse where it was a full room and people were standing in the room and then people got turned away from that door >> oh my gosh >> which I was happily like surprised by. Then I also like we only reserve 5 minutes for questions at the end cuz I was I was like you know I don't know what questions people will have. Usually people aren't asking the ton of questions like after they'll come up to you later but it's like usually like a Q&A session does not last very long. So oh 5 minutes for questions like that's fine. I we do questions there's a line of like 15 people waiting to ask questions. like they just keep it kept growing and growing. We stayed after for another 30 minutes just answering questions. >> Interesting. What level of questions like was it more um you know people who are newer to the topic or who are asking like more like stage two kind of questions >> huge range so >> we had some people who were you know very introductory level just like I'm getting my bearings here how does this part of it work you know what's the deal with Prometheus and hotel and what do you use for a back end you know that that's like a solid solid starter question and then other people came in they were I've been using this chart for the past year and there's this one thing that I want to be able to do and uh when are you going to like do the work to like enable that and how can we like architect this big range >> very big and each one was like a different part of the of the area. >> Okay, that's so cool. >> Yeah, very very cool and very surprising. I'm always happy to cuz you know I think as a maintainer it's like we get people who come to us with issues on GitHub and it's like you'll get someone complaining on a GitHub issue or like hey this thing broke the most recent release broke me um you know what are we doing about that broke >> but no one's broke their spirit you know I've had releases break my spirit >> but the uh the real thing is we don't get positive feedback. We don't get a lot of questions cuz realistically most people are asking track or going on Stack Overflow, >> right? It's like they're not bringing the questions to us. So, we don't really hear or understand what usage is. I mean, it's kind of the benefit and the curse of open source is that like we don't necessarily get that user feedback directly. It's why like I love what you all do cuz getting the user feedback is one of the most valuable things to the community. like in order for us to push forward, we need to know what isn't working right now where people just don't see the effort. Um we actually I hosted a meetup in New York uh last month, 2 months ago, something like that. >> Yeah. Uh, and that was eye opening because it was like wow, people are really using these things and things that I I feel like I've talked about so many times are still not known, you know, and it's like I'll mention like how many people know what like the target allocator is in a show of hands and nobody raised their hand in a room of like 40, you know, senior SRRES. I'm like, okay, uh, would anybody like it if you could do distributed, sharded, retheus scraping? and everybody raises their hand and I'm like well we have that like it's done it's there you know you can use it it's been a thing for a few years you know so I guess all to say the community just is still there's still things for people to learn about the vast ecosystem that we have here >> yeah I think you know I think a common theme just in general is people not being aware of you know different abilities and features and yeah on the end you see too you know it's it is a bit of a challenge cuz like we feel like we're talking about it um you know at our work and like in the ozone community and then yeah like you said we have all these conscious people who don't know and it's we're trying to figure out like where are they going for like how can we >> make them aware >> of you know all these cool things that are happening. Um yeah, so you know, one of the things we we started doing is um the what's the hotel segment, which we just had our first one um a month ago or so. And so we're hoping to, you know, I don't know, have something we can point people to. >> Of course. >> But you mentioned the injector. >> I did. Tell us more about please do. So I am the uh I'm a maintainer for it but I'm mostly just doing reviewing because this is actually a joint effort between uh Splunk and D-Zero. Both of them have already written their own uh injectors for auto instrumentation. Oh, right. >> And they're essentially donating a merge of both of those projects to the tot alpha version of this for their customers. Um, so we're just testing out that everything is working the way we need to. There's a lot of really random edges here. Um, interestingly enough, this does introduce a new language to hotel, which is exciting, which is it's very rare for us to not use a language in a project that is meant to instrumental languages. Right. >> Right. >> And we are now using Zig in the project. >> Zig. >> Zig um is a fun language that is sort of a merged between like Go, Rust, C++, and C. Oh, super fun. >> But so the real value of it is it doesn't require uh a lot of like core Linux dependencies. So like GIC C is one that we really think about a lot. Uh if you're on AWS, something that I've been bitten by is not all of the nodes have g. And so when you're trying to do injection like say you're trying to run a load balancer liketo or envoy >> um is similarly injected like we inject auto instrumentation and that will just not work. It used to not work. I think they've changed this but it used to not work uh on certain types of AWS nodes. So we have that same issue because we're trying to inject to all these different environments. So, Zigg is pre-built with a bunch of these C libraries, but from scratch, part of it standard library and not based on anything existing. So, it has all of its own dependencies. So, the benefit of this injector project is that we're going to be able to really meet users where they're at without needing to know a ton about their environment. And that's the real problem today. >> Yeah. >> Um, so it's an exciting project. I think we're a few months away from it being alpha beta. Other people are on the release timeline. Ted Young is thinking a lot about the release timeline there. Um, but all TC >> TD, that's so cool. >> Yeah. What are the parts of, you know, whether it's something working on or not working on directly that you are excited about? >> Good question. I think that the biggest thing for project right now that everybody cares about is stability, right? I think that uh we're trying to get to the place where users can reliably get you know a new update without them needing to do any configuration changes without needing to worry about uh components being deprecated things like that right I talked to at that meet up in New York I talked to a developer at a major bank who was you know they've developed all of their own uh custom components for the collector and so every time that we've been doing these reworkings the collector's architecture. They need to then refactor a bunch of their code as well. And you know after a few months you lose track of all these updates. So it becomes like a weeks long project to keep up to date. When you have to do that their release cycle is every quarter, right? It gets a lot to manage and a lot to update. So for the users, I think what people really want to see is that level of maturity and stability. We can get to that graduated status with the CCF would be huge. That would be massive for the project to have. Uh I think it would really I mean our adoption already has been really high. I think that's when we would come to the next level. Great. >> Can you explain a little bit for those of us like CCS stages like what going from incubated incubating projects to graduated means? >> Sure. >> Yeah. I'm not the best. I can tell you what I know. I'm definitely not the person in charge of this. There are people who are far more confident in the stability part of this governance than I. >> Yeah. >> So I I will speak on only >> that's all we're asking for. >> So right now we're in as you said uh project is relatively lenient in terms of its standards. There are standards but graduation is is really strict I would say for good reason right like you want to be sure that when you are considered a graduated project you have a healthy community which we definitely do uh you have stability guarantees which is the main thing we're working towards and lastly adoption so adoption has to be at certain amount of companies you have to have testimonials from those companies about how the project's been helpful uh that's another thing that we have plenty of so now it really just comes down to stability and stability. You can think of Kubernetes as the example here. Kubernetes is, you know, the first graduated project. I imagine it's the first one. Maybe there maybe someone beat them to it, but you don't know. But essentially, every time that Kubernetes improves their security and reliability, uh, you know, release stability, anything like that, it raises the bar for everybody else to continue to improve. So as we are getting towards stability, Kubernetes is is chugging along, you know, really positively and improving their posture constantly. So it's up to us to not only meet where we started for our uh graduation criteria, but also to continue building towards that. So that means that you're going to see more uh security features, much more uh security guarantees, which is going to be very helpful for enterprises. M um you're going to hear more about quarterly releases rather than uh the sort of disperate release calendar that we have today. >> Uh you'll hear more about documentation which is also going to be huge. So uh later when that comes on I assume that we'll be talking about some mobilization efforts and documentation. >> Ah yes, uh that is super super valuable right um all of that is going to get us to that graduation phase. Uh it's just time it's time and effort and uh we have a dedicated group of people who want to see this happen and are spending you know all of our time at all of these large companies trying to make that happen. Very exciting. >> Awesome. >> That is very exciting. And if you are yourself an open end user and um your company is using in fashion um and you want to share your story, we would love to have your story as part of um our uh legacy our catalog. >> Yeah. like to help, you know, help graduate the project and help take us out to us. Um, we'll have it in the show notes um after this. But yeah, it's pretty easy. As long as you remember me Slack, you can find us at hotel dash dash and dash. >> I think that's all the dashes. >> I thought we were going to dash zero. Oh, so we're like dash. >> And similarly, if anybody wants to help out with documentation, documentation is >> one of the best ways to contribute to the project. It is so valuable. We had somebody for the operator group contribute a whole uh bit of documentation for the target alligator and it it was so well done. Adriana did this as well. super helpful >> and it has been such a resource for me to be able to point users that come into our Slack, come into our S meetings to just say, "Hey, you have a question about this? We have a full document to answer everything you might need to know about it." That has been such a way to offer. Um, it's lovely. So, great way to contribute and get started. Doesn't require writing code. Requires asking questions and writing a bunch of stuff down. >> Yeah. And I can confirm that everyone that I met in the local community has been super helpful. Um, anything you know you can take on and be responsible for like documentation that's something that you're taking off their plate. So I know they're having to help you. And it's also cool to see your work on an official website. >> Yeah. Like uh definitely join the communication sig for that. I'm currently in it and we're working on a lot of refactoring of the documentation like just like the early stages since I'm more of a beginner with the open telemetry and everything. I've been working on kind of refactoring those beginner level documentations. So like it's been really helpful getting people like acquainted with that and localization efforts too. Like I know Lisa is working on a lot of localization efforts. So yeah, it'll help. >> Super helpful and the the beginner stuff especially. I mean >> again from the talk today I think it's just so it was so useful to learn where people are at in their implementation journeys >> right >> uh we do have people that really need reference architectures a ton of code examples right >> these things help project adoption that also make it so that users actually are delighted by the experience right that and ultimately that that's what I think we all want to see and that's what users want like you want to be able to install hotel wherever you need it and not have to scour the internet to figure out how to how to make it work, right? >> Different companies have so many different needs and so getting started is always going to be the most important thing for us to really nail. Uh the next thing that I heard from uh when I did the event in New York is reference architectures and showing how we can do hotel in all of these various environments. so that people can have some code samples to copy from uh some architectures to you know design around. I was just talking with an engineer at and they >> are really interested in understanding how to do various trace sampling architectures, right? where we can really have like a whole as you said catalog these architectures to show people and we all of our companies like you know when I was at our flight we had all these architectures internally that we would post blog posts about to share we have a lot of companies that are using hotel as part of the ingestion path as well which involves a lot of this uh architectural work similarly a lot of companies are like that are contributors to hotel also are obviously using the tooling themselves for their own company's observability. We should reach out to the people who are doing that and try to publish what are the reference architectures that hotel maintainers use ourselves, right? >> I think that would be really useful uh for people who are trying to implement this right now. >> Absolutely. >> For sure. Um are we planning to host them like in a repository or like on sites directly? >> Oh, I think would be the best. I think you know we have documentation places the operator group and I think the idea of everything being the doc is going to be the way to go >> right >> like centralizing it and making sure that it's well organized which is a lot of the work that you're doing super valuable >> of course >> uh but I think it just it needs to be there right now I think it exists on a few different company blogs a few different companies >> head scattered >> bringing that together I think would really really valuable. >> One reason let's do it, >> right? There's so many companies like contributing. So there's definitely a bunch of references that people in our community could use. >> Yes. >> Yeah. >> Absolutely. And so many people are people are so friendly, right? like back to that like going to uh the hotel booth. We've had so many people who have never come up to us before and are just asking great questions and we have so many people who just, you know, we hang out there cuz we love to talk to people about this. It's rare that you get to have this level of engagement with end users, right? Uh that direct type of engagement is so valuable and so people coming by has been one of my favorite experiences of it always. I think the reason that I love to come to these is just getting to talk to people, hearing people's problems, digging in because then it helps us plan what I like what I need to do for the next 2 years, right? I can take that feedback on it. What um so in during your time here at coupon like from the audience questions that you had at your talk and people who come through the open tree observatory do you have are you seeing any like general trends as to like what people are leaning toward or needing more help with or >> Yeah. So I say the the main thing that is of concern to me is around amp which has been getting a lot of there are a lot of talks in opamp this coupon. I don't know if that if we've noticed that there were at least four that mentioned it which is a lot. >> Could you explain that a little bit more for us? Nothing. Sorry. >> We're slowing down a little bit. So, opamp is the uh open agent management protocol uh part of the hotel project but is also designed to be in in the same way the hotel is very vendor neutral it's neutral to agents as well. So, it's not just for the collector necessarily to be for a lot of different uh agents out there. The goal of the protocol is to enable things like uh component status reporting, health reporting, and remote configuration. And remote configuration is definitely the top of the town is what I would say. >> Yeah. Okay. Very cool. >> Um and so in that vein, what it sounds like users really want is a way to dynamically update what the collector can do, what the SDKs can do uh in real time. And that's challenging. The way that the collector was designed was never really with remote configuration in mind. There are ways to reload the process. Like we can actually go in and restart the process from the collector binary. Like you just do that, right? >> But it's different than a remote. What users really want is a way to push a rule to a collector and the collector accepts it and continues going. >> Yeah. So that's what a company like bind plane has already built. Uh and so what we want to do is take a lot of the lessons from what BL has been successful with and obviously we're working with them as well. They are big contributors to uh the idea is that we work all together to improve the provider posture here and improve the SDKs as well so that we can actually realize this vision that people can dynamically update their configurations wherever they might be. Uh but it's challenging. I mean that that's like a really tough topic. >> Yeah. >> There's a lot of a lot of companies in hotel have done this work beyond my plan before. Elastic comes to mind. They have their own >> I work here. >> Oh yeah. >> Elastic has uh their own protocol for doing road configuration already. >> Okay. Okay. >> We've talked with a few of their containers and there's definitely some like room for some good collaboration that we've talked about. >> Awesome. >> I think it's all to be seen but this is definitely the thing that we hear a lot about. The thing that I hear a lot about on the operator side is in Kubernetes, how can I do uh call it not just remote configuration but layered configuration. So what this means is how can I make it so that if I'm running in an enterprise company that I'm in a multi-tenant environment, I want like a base collector. I want to overlay. I want my teams to be able to overlay different pieces of config on top of that collector for their needs. >> Right. >> It's a really complicated topic. We >> Yeah, it's really >> sounds like it. And the thing that I sort of struggle with is how do we make this extensible and configurable but also simple to understand. The problem with this that it starts getting into a lot of moving parts and as soon as you start to introduce more moving parts is when complexity and usually reliability both complexity increases and reliability decreases right you need consistency and more moving parts uh will reduce that it's sort of that there's a great quote in reliability that I love it's u the most reliable system is the one that does that never that you right we know this quote. I'm totally botching it right now. >> No, I hear it. I hear it. >> The idea is that like the most reliable system is the one that like doesn't exist, >> right? >> Right. Cuz it's always achieving its goal of non-existing. >> And so if you want to really improve the reliability of the service, you delete it >> because then you don't have to care then it's at 100%. Cuz it's always it's always giving you the same answers which is nothing. Yeah. >> Sorry. >> No, I'm here for it. I'm here for it. Yeah. >> Someone I I'll say that to I don't know, someone at the Ozel booth and they'll be like, "You totally messed up." >> You know, we'll have the correct throw in the shout out. >> Yeah. Thank you so much, Jacob. Um, we are going to bring on our next guest, Diana, in a moment. Um, and in the meantime, I really want to show off this baseball jersey. >> Looks really good. >> But, um, our friends at Honeycomb have been kind enough to um, sponsor for the hotel community. Um, thank you, Austin Parker, for the design. Um, I would stand and show the back. It says uh the number 11 actually I don't know why it says 11 like there. But yeah, it says maintainer on the back. Um, we'll have some pictures hopefully to show you. But yeah, you've got the logo over here and it says open too across the front. Very, very snazzy, very sharp. Um, yeah. And it also comes in black, which I had the option, but I was like, the client looks sharp. Yeah. Um, and also, you can see the blue and the yellow very clearly. Yeah. Just wanted to show you a little bit of a conference fashion. Um, and then when we get um our second guest in a setup, I also want to show you her nails because they're amazing. >> Oh, yes. >> I don't know how close I can get. >> Yeah. And like I said, we have a group right here. I'm very excited that it's for today. So that's why we're able to, you know, have lunch on right now. But yeah, I brought my winter puppy out. Um it was it was intense, but we made it through and we are now day two of the main event with one more day to go and I hope everyone is doing well. >> Staying hydrated. staying hydrated is so large and important. And yeah, I think we're ready to speak to our next guest, Diana. >> Yes. Awesome. >> Diana, welcome. And thank you guys. Congratulations on uh winning the Open Salt Community Star 2025. She was one of several winners. Um, the only one I want to point out, but that's why it's so important. Um, Diana, can you please introduce yourself? And also, the earrings. >> Yeah. Oh my gosh. Yes. Oh, those are gorgeous. Oh my goodness. Okay. Sorry. Continue. >> Like, hold up your treasury. >> Yeah. Yeah. I'm Diana. Uh I'm a developer experience engineer parametric and I work all my life in engineering uh in observability the last couple of like 5 years. Uh and I discovered like I really like going and explaining things to the community. So I had a chance like years back to be a speaker on one of the first conferences I've ever been in my life. Uh I was super nervous but I liked it a lot and I saw like so many women on stage giving talks that I felt like I needed to know to do more. >> Of course. >> Uh so I kind of did that for 2 years. Uh and actually last year like I was hearing so much about open I said like oh I want to get involved. I want to do something. uh and I got lucky because in the same moment when I thought about it foundation was like oh we are looking for observability people we are looking for uh let's get involved we want to create the first open and learn fundamentals uh I got in so it was like first start with the documentation reading questions questions about and that's like simple months into certain words. I knew it was finally polished. So that was like for me the kick into like sens and slowly I kind of like I was like okay I'm sorry things how about I'm going to do my company. that was like uh for my current company actually. Um so uh it was like really interesting because I I wanted like from explaining open telemetry to the entire company, getting software engineers involved like instrumenting some things in like SDKs uh testing everything. Um so yeah production so that was like decision. Uh but that was like a really cool experience that I've been talking about since one of my talks and explaining how you know companies some particular companies that to be honest they're not super big times to build in takes a very important. So I need to be involved in the community. So for me that was like a very very experience also like understand Yeah, I thought about myself. Okay, but if right now I'm all alone and I still want to contribute, how can I do it? And I saw translating documents. >> Oh, nice. That's interesting. I can do my own time. start something translations and yeah, can you um so global organization the projects or localization just refers to translating the documentation to different languages. >> Exactly. Yeah. So the idea that okay there may be a bit more um other terms around localization. So it's not necessarily only the the technical terms or the tech writing part but there are like also like um how you're going to automate some CI going to automate specific u pipelines documentation. Right. Right. >> Can also be that and also how are you communicating a specific message uh in a language right interpretation of of concepts and terms and technology in other languages. So I think that's that's how I see things. >> Well, you mentioned Spanish. Um are you involved with other languages? Um and can you also tell us like what other languages are being are for the project right now? >> Yeah. Yeah. Yeah. Um so when um back when I got involved I didn't think of this year uh Spanish was available obviously English, French. >> Ah yes >> and that was right. So from the beginning of the year up to now I think three more languages emerged. So three more communities. Uh I know for sure Bengali. >> Oh nice. >> And that's like so they really got together and and they started to contribute more and more like their only technology. >> Yeah. >> Um Ukrainian that's locked up recently. So I know for sure they need more contributors. Uh so somebody came with the idea and they started to do that. Um, and I started uh for my own language Romanian. I'm native to Romanian. Uh, I started like a couple of months back and I said, "Look, yeah, I think it's an amazing opportunity for for my language to represent and for developers back home to understand a bit more language. It's going to really perspective like it is a really nice nice community." Yeah, that's really beautiful. So like what is the hardest thing about localization that you find when you do that kind of work? Yeah, I think um definitely translating specific terms into your own language or in that particular language, but giving it the flavor or giving it like that that really gets the essence so it doesn't have to sound like a very dry terminology and that word captures the essence of of the meaning. So for example, I don't know about If you translate really like like literally a word from English or another language, it might sound a bit weird. So you have to like al oh does it sound really like this in that language? I have to like you know modify it a bit. I have to give it like a nice ring to it. So like you can get it you know it's kind of like >> uh people like like it immediately and they want to use it right. Um so that's I think that's the difficult part and also the difficult part is that there are many many languages that without technical terms directly in English right >> and when you want to translate it in that language quite like the appropriate meaning is like okay I know you know uses that word in my language and they always something in English right oh that get immediately you know it doesn't sound strange so those are like words basically That's so wonderful. >> Yeah, it's really cool. >> Is there um like a way or um to kind of see, you know, the rate of adoption or involvement from the communities that now have these translations available. >> Um in terms of metrics or in terms of adoption, I think that we are uh kind of primitive. Um, and I think that you need to correlate it a bit with a ground meeting event or with some type of local community that you could uh speak about it on site, right? >> Or this makes some noise back home for example. Uh, and say look, we are starting this, we are doing this. We need more contributors. Uh, we need, you know, like the people from the local communities. So like that for each one of is is very much needed whether online or on site. Uh so for me for example I just started this with Romania. I started going like to people like I contributed before or new from like events and I'm also Romania. around there like can you put it can you promote it like we are like actually entitling our language to to be heard in the open community like really jumped into it and yeah for sure let's do it like a lot of people started it's really nice >> I think as you know native English speakers we get so used to everything being >> to English And honestly, you know, a lot of um non-native English speakers, you know, they usually speak multiple languages. Um and so I think it's really impressive um and awesome the work that you all are doing. >> Yeah, I appreciate it. And also I think down the line people get motivated not only because of modernization but they start thinking okay uh do I like overall open source uh should I do something else maybe in another project uh does it help me in like my work projects whatever I'm doing uh and I think this is how I practically you know promoted it to to my community I said if you guys like me uh to say like I'm doing a open source project as your contributor like if your employer needs you to have this experience or maybe you want to go to a conference and talk about it. So all of these are very very nice to have you know on your screen and you know different age groups you know there university or later on their career and I think it's really important to keep motivation because it can be sometimes really hard to start in your free time or remind yourself that you know you have an objective online. It's kind of like a gateway project into contributing to open source a little bit. >> I always keep telling them that I I didn't necessarily start with the organization and I share my new stuff here. I'll definitely going for example I being an engineer I still like doing engineering work. >> Oh yeah. And I want to get involved in the codes part as well because I've been keeping an eye on everything that's happening talking like there are many many things that for sure my company or other people will definitely find. Um, so yeah, you know, one project or one S is is not enough. >> We're going to jump into something else at some point and then the community especially is really really and they keep talking about the same issues over and over again. >> Yeah. So just recently talking about I heard about the same kind of issues in Kubernetes. So I'm kind of like went back and forth because I'm kind of like struggling with similar issues. So I think it's very good to keep an open mind open and keep going. For me at least personally this is what motivates us going forward you can diversify your contributions not stop and we always find something. >> Thank you. >> Yeah that's awesome. And then kind of pivoting a little bit. Um I would love to know what you're looking forward to in the coming year for hotel. Like what are you excited about? What kind of trends you're seeing like implementation? Anything that has so um just like the last couple of weeks been to like specific events. I saw a lot of interesting talks about yeah obviously projector and uh I think these are two things that are really interesting at the moment so instrumentation particular end users need a lot more best practices so people don't see users and say oh but we don't know how to do all this instrumentation or we don't know like which to add. I don't know like like what's the easy stuff you can implement like can we can you add some like best practices can you right >> you know kind of like tell tell us you know like what would be like best starting point on boarding let's say in this case uh and I've been hearing like uh community and for open tele Yeah. >> You know, >> like get them in. Yeah. >> I know that you're complaining, but tell us more about your use, >> right? >> Yeah, that's so cool. Yeah. Um, I just wanted to know, you said that you wanted to get involved in more of the engineering side, the coding side. Is there somewhere specifically that you're looking forward to or or some somewhere specifically that you would like to work in? Like anything exciting to you specifically? >> Uh, for instance, I'm very interested in the river and the river. >> Can you explain it a little bit to us? >> Uh, yeah. Well, I I don't know. >> No, right. Right. You're getting >> I think people just last maintainers and people that got involved and all the idea about this convention around it uh and although I I I'm not sure like how they are implementing it like practically something that I want to do is stop from going to so many events like my laptop stuff like this. So, um I think it needs a bit of time and time and you know with my team and my brother. >> That's awesome. I love it. Really cool. Awesome. Well, thank you so much, Diana. Congratulations again on your movie star award. It is so cute. And thank you so much, Sophia, for being my co-host. >> Yes. Thank you so much, Reese, for being the host host. >> You're amazing. Our co- co-host co-host. >> Um, yeah, we hope to see you at a future soon. um6 Amsterdam in EU and soon for our next year. Um and we'll have some show up. So check us check it out there and talk to us if you like please. >> See you everybody. Bye everybody. + From d697275b35dd021db421a04002305cdb2a1d45d6 Mon Sep 17 00:00:00 2001 From: Nicole van der Hoeven Date: Mon, 24 Nov 2025 11:58:57 +0100 Subject: [PATCH 10/28] Strip AI preamble from chapter generation - Add clean_ai_preamble() function to remove conversational preamble lines - Remove phrases like 'Sure! Here are the key moments...' from chapter output - Preserve only actual timestamp lines in generated transcripts --- video-transcripts/transcripts.py | 57 ++++++++++++++++++++++++++++++++ 1 file changed, 57 insertions(+) diff --git a/video-transcripts/transcripts.py b/video-transcripts/transcripts.py index 44cdee2..663c8ad 100644 --- a/video-transcripts/transcripts.py +++ b/video-transcripts/transcripts.py @@ -388,6 +388,61 @@ def openai_cleanup(transcript, video_id): return [summary, cleaned_up] +def clean_ai_preamble(text): + """ + Remove AI assistant preamble lines from the response, keeping only the actual content. + This strips lines like "Sure! Here are the key moments..." or "Here are 10 key moments..." + @param text: The raw text response from AI + @return: Cleaned text with only the actual chapter timestamps + """ + if not text: + return text + + lines = text.split('\n') + cleaned_lines = [] + + # Common preamble patterns to remove + preamble_patterns = [ + 'sure!', + 'here are', + 'here\'s', + 'below are', + 'i\'ve identified', + 'i have identified', + 'key moments', + 'from the livestream', + 'from the transcript', + 'from the youtube', + 'with timestamps', + 'along with their timestamps', + 'with their timestamps', + 'with the corresponding timestamps' + ] + + for line in lines: + line_lower = line.lower().strip() + + # Skip empty lines at the start, but keep them later for formatting + if not cleaned_lines and not line_lower: + continue + + # Check if this line is a preamble (contains preamble patterns and doesn't look like a timestamp) + is_preamble = False + if line_lower and not line_lower.startswith('0'): # Timestamps start with 0 + for pattern in preamble_patterns: + if pattern in line_lower: + is_preamble = True + break + + if not is_preamble: + cleaned_lines.append(line) + + # Remove leading empty lines from the result + while cleaned_lines and not cleaned_lines[0].strip(): + cleaned_lines.pop(0) + + return '\n'.join(cleaned_lines) + def create_chapters(transcript, video_id): """ Create timestamps and chapters for a YouTube video transcript. @@ -420,6 +475,8 @@ def create_chapters(transcript, video_id): ) chapters = response.choices[0].message.content + # Clean any AI preamble before returning + chapters = clean_ai_preamble(chapters) return chapters def write_markdown(args, video): From 5a5286e4c78dcaf452709f4a1896a6a785183118 Mon Sep 17 00:00:00 2001 From: Nicole van der Hoeven Date: Mon, 24 Nov 2025 16:42:08 +0100 Subject: [PATCH 11/28] Add timestamp insertion and validation to transcript generation MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Insert chapter timestamps inline in cleaned transcripts at matching text positions - Add video duration constraint to prevent AI from generating timestamps beyond video length - Improve transcript cleanup to preserve exact wording and speaker names - Add post-processing validation to verify chapter timestamps match content - Limit timestamp corrections to ±60 second window around original time - Use window-based matching with chapter titles for accurate timestamp placement - Lower AI temperature for more accurate and deterministic results --- video-transcripts/transcripts.py | 491 +++++++++++++++++++++++++++++-- 1 file changed, 463 insertions(+), 28 deletions(-) diff --git a/video-transcripts/transcripts.py b/video-transcripts/transcripts.py index 663c8ad..b1e6968 100644 --- a/video-transcripts/transcripts.py +++ b/video-transcripts/transcripts.py @@ -184,10 +184,11 @@ def fetch_transcripts(args, videos): print(f"Processing video {i+1}/{len(videos)}: {video_id}") # Use improved transcript fetching with retry logic - transcript_text = get_video_transcript_with_retry(video_id) + transcript_text, raw_transcript = get_video_transcript_with_retry(video_id) if transcript_text: video['transcript'] = transcript_text + video['raw_transcript'] = raw_transcript processed.append(video) write_markdown(args, video) else: @@ -222,7 +223,7 @@ def get_video_transcript_with_retry(video_id, max_retries=5): Fetch transcript with retry logic and exponential backoff. Uses youtube-transcript-api 1.2.3+ API. Handles YouTube rate limiting (429 errors) with extended wait times (60-120 seconds). - Returns cleaned transcript text or None if unavailable. + Returns a tuple: (cleaned transcript text, raw transcript data) or (None, None) if unavailable. """ for attempt in range(max_retries): try: @@ -238,7 +239,7 @@ def get_video_transcript_with_retry(video_id, max_retries=5): # Validate transcript structure if not transcript or not isinstance(transcript, list): print(f"Invalid transcript format for video {video_id}") - return None + return None, None # Safely extract and clean text text_parts = [] @@ -253,26 +254,26 @@ def get_video_transcript_with_retry(video_id, max_retries=5): if not text_parts: print(f"No valid text found in transcript for video {video_id}") - return None + return None, None full_text = ' '.join(text_parts) # Minimum length validation if len(full_text.strip()) < 50: print(f"Transcript too short for video {video_id}: {len(full_text)} characters") - return None + return None, None - return full_text + return full_text, transcript except TranscriptsDisabled: print(f"Transcripts are disabled for video {video_id}") - return None + return None, None except NoTranscriptFound: print(f"No transcript found for video {video_id}") - return None + return None, None except VideoUnavailable: print(f"Video {video_id} is unavailable") - return None + return None, None except YouTubeRequestFailed as e: # Check if this is an IP block error error_str = str(e) @@ -284,7 +285,7 @@ def get_video_transcript_with_retry(video_id, max_retries=5): print(f" 2. Use a different network/WiFi connection") print(f" 3. Set up cookie-based authentication (see README)") print(f" 4. Use a residential proxy or VPN (not cloud-based)") - return None + return None, None elif '429' in error_str or 'Too Many Requests' in error_str: if attempt < max_retries - 1: # Use much longer wait time for rate limiting (60-120 seconds) @@ -295,7 +296,7 @@ def get_video_transcript_with_retry(video_id, max_retries=5): else: print(f"❌ YouTube rate limit persists after {max_retries} attempts for video {video_id}") print(f" Consider waiting 10-15 minutes before running the script again.") - return None + return None, None else: # Other YouTube API errors if attempt < max_retries - 1: @@ -304,7 +305,7 @@ def get_video_transcript_with_retry(video_id, max_retries=5): time.sleep(wait_time) else: print(f"YouTube API error persists for video {video_id}: {str(e)}") - return None + return None, None except Exception as e: error_str = str(e) # Check if this is likely a rate limit error disguised as XML parse error @@ -318,7 +319,7 @@ def get_video_transcript_with_retry(video_id, max_retries=5): else: print(f"❌ Persistent error for video {video_id} after {max_retries} attempts") print(f" This may be due to YouTube rate limiting. Wait 10-15 minutes and try again.") - return None + return None, None elif attempt < max_retries - 1: # Standard exponential backoff for other errors wait_time = (2 ** attempt) + random.uniform(0, 1) @@ -326,9 +327,9 @@ def get_video_transcript_with_retry(video_id, max_retries=5): time.sleep(wait_time) else: print(f"All {max_retries} attempts failed for video {video_id}: {error_str}") - return None + return None, None - return None + return None, None def openai_cleanup(transcript, video_id): """ @@ -368,11 +369,28 @@ def openai_cleanup(transcript, video_id): transcript that contains few sentence breaks, indications of music in the video, and filler words. You will need to clean up the transcript into a set of logical sentences and paragraphs. - Please output your results as markdown text; you may use markdown formatting for emphasis, bold, - bulleting, and so on. - - Stay as close to the original transcript as possible but make it more readable. Do not interpret, - add, or remove any words unless they are filler words. + CRITICAL INSTRUCTIONS: + - Use the EXACT words from the original transcript - do not paraphrase or rewrite + - ONLY remove filler words like "um", "uh", "ah", "you know", "like" (when used as filler), "so" (when used as filler) + - Fix sentence structure by adding proper punctuation and paragraph breaks + - Do NOT change technical terms, names, or any substantive content + - Do NOT summarize or condense - keep all the original content + - Do NOT add words that weren't in the original + - If someone repeats themselves, keep the repetition + - Preserve the natural speaking style and word choice of each speaker + + SPEAKER FORMATTING: + - Identify when different speakers are talking + - Format speaker names as **Name:** at the start of their dialogue + - Use bold ONLY for speaker names, not for emphasis or highlighting other text + - Each time a speaker changes, start a new paragraph with their name + - Keep the same speaker name format throughout (e.g., if someone introduces themselves as "Jacob Marino", use **Jacob:** consistently) + + OUTPUT FORMAT: + - Use markdown paragraph breaks (blank lines) between speakers + - Do NOT use bold, italic, or other formatting except for speaker names + - Do NOT add bullets or numbered lists unless they were explicitly spoken + - Keep it simple: speaker names in bold, everything else as plain text """ print(f"Cleaning up transcript for video {video_id}") @@ -443,11 +461,398 @@ def clean_ai_preamble(text): return '\n'.join(cleaned_lines) -def create_chapters(transcript, video_id): +def parse_timestamp_to_seconds(timestamp_str): + """ + Convert a timestamp string like '00:05:30' to seconds (330). + @param timestamp_str: String in format HH:MM:SS or MM:SS + @return: Integer seconds + """ + parts = timestamp_str.strip().split(':') + if len(parts) == 3: + hours, minutes, seconds = parts + return int(hours) * 3600 + int(minutes) * 60 + int(seconds) + elif len(parts) == 2: + minutes, seconds = parts + return int(minutes) * 60 + int(seconds) + return 0 + +def parse_chapters(chapters_text): + """ + Parse the chapters text into a list of (timestamp_seconds, timestamp_str, title) tuples. + @param chapters_text: Text with lines like "00:05:30 Guest introduction" + @return: List of tuples (seconds, timestamp_str, title) + """ + chapters = [] + for line in chapters_text.split('\n'): + line = line.strip() + if not line: + continue + # Match timestamp at start of line (HH:MM:SS or MM:SS format) + import re + match = re.match(r'^(\d{1,2}:\d{2}:\d{2})', line) + if match: + timestamp_str = match.group(1) + seconds = parse_timestamp_to_seconds(timestamp_str) + title = line[len(timestamp_str):].strip() + chapters.append((seconds, timestamp_str, title)) + return chapters + +def insert_timestamps_in_transcript(cleaned_transcript, chapters, raw_transcript): + """ + Insert timestamp markers into the cleaned transcript at appropriate positions. + @param cleaned_transcript: The cleaned transcript text from OpenAI + @param chapters: List of (seconds, timestamp_str, title) tuples + @param raw_transcript: Raw transcript data from YouTube with timing info + @return: Transcript with timestamps inserted + """ + if not cleaned_transcript or not chapters or not raw_transcript: + return cleaned_transcript + + # Build a mapping of time (in seconds) to text at that time + # Store multiple entries per time window for better matching + time_to_texts = {} + for entry in raw_transcript: + if isinstance(entry, dict) and 'text' in entry and 'start' in entry: + text = entry['text'].strip() + if text and text not in ['[Music]', '[Applause]', '[Laughter]']: + start_time = int(entry['start']) + if start_time not in time_to_texts: + time_to_texts[start_time] = [] + time_to_texts[start_time].append(text) + + # Split transcript into lines + lines = cleaned_transcript.split('\n') + + # For each chapter, find the best line to insert it at + line_to_timestamp = {} # Maps line index to timestamp to insert + + for seconds, timestamp_str, title in chapters: + # Skip 00:00:00 as it's at the beginning + if seconds == 0: + continue + + # Get text from a window around the timestamp (±5 seconds) + window_texts = [] + for time_sec in range(max(0, seconds - 5), seconds + 6): + if time_sec in time_to_texts: + window_texts.extend(time_to_texts[time_sec]) + + if not window_texts: + # Fallback to closest time if window is empty + closest_time = min(time_to_texts.keys(), + key=lambda t: abs(t - seconds), + default=None) + if closest_time: + window_texts = time_to_texts[closest_time] + + if not window_texts: + continue + + # Combine all texts in the window + combined_text = ' '.join(window_texts) + + # Extract key words from the window text + sample_words = [w.lower() for w in combined_text.split() + if len(w) > 3 and w.lower() not in ['that', 'this', 'with', 'from', 'have', 'were', 'been', 'they', 'them']] + + if not sample_words: + sample_words = [w.lower() for w in combined_text.split() if len(w) > 2][:10] + + # Also use words from the chapter title itself + title_words = [w.lower() for w in title.split() + if len(w) > 3 and w.lower() not in ['the', 'and', 'for', 'with', 'about', 'discussion', 'introduction']] + + # Find the best matching line in cleaned transcript + best_line_idx = None + best_score = 0 + + for idx, line in enumerate(lines): + # Skip empty lines and lines that already have timestamps + if not line.strip() or line.strip().startswith('['): + continue + + line_lower = line.lower() + + # Count matching words from the window + text_matches = sum(1 for word in sample_words if word in line_lower) + + # Count matching words from the chapter title + title_matches = sum(1 for word in title_words if word in line_lower) * 2 # Weight title matches more + + # Bonus for lines that start with speaker markers + is_speaker_line = '**' in line and ':' in line + speaker_bonus = 1 if is_speaker_line else 0 + + # Prefer lines that appear after already-placed timestamps (chronological order) + position_bonus = 0 + if line_to_timestamp: + last_placed = max(line_to_timestamp.keys()) + if idx > last_placed: + position_bonus = 0.5 + + score = text_matches + title_matches + speaker_bonus + position_bonus + + if score > best_score: + best_score = score + best_line_idx = idx + + # Insert timestamp at the best matching line + if best_line_idx is not None and best_score > 1: # Require at least some confidence + # Only insert if we don't already have a timestamp for this line + if best_line_idx not in line_to_timestamp: + line_to_timestamp[best_line_idx] = timestamp_str + + # Build the result by inserting timestamps at marked lines + result_lines = [] + for idx, line in enumerate(lines): + if idx in line_to_timestamp: + # Insert timestamp at the beginning of this line + timestamp = line_to_timestamp[idx] + result_lines.append(f"[{timestamp}] {line}") + else: + result_lines.append(line) + + return '\n'.join(result_lines) + +def format_duration(seconds): + """ + Format seconds into HH:MM:SS format. + @param seconds: Integer seconds + @return: String in HH:MM:SS format + """ + hours = int(seconds // 3600) + minutes = int((seconds % 3600) // 60) + secs = int(seconds % 60) + return f"{hours:02d}:{minutes:02d}:{secs:02d}" + +def get_video_duration(raw_transcript): + """ + Calculate the video duration from the raw transcript data. + @param raw_transcript: Raw transcript data from YouTube with timing info + @return: Duration in seconds, or None if unavailable + """ + if not raw_transcript: + return None + + max_time = 0 + for entry in raw_transcript: + if isinstance(entry, dict) and 'start' in entry and 'duration' in entry: + end_time = entry['start'] + entry['duration'] + max_time = max(max_time, end_time) + + return int(max_time) if max_time > 0 else None + +def validate_and_fix_chapters(chapters_text, raw_transcript, video_id): + """ + Validate that chapter descriptions match what's actually said at those timestamps. + Fix any mismatches by finding the correct timestamp for the description. + @param chapters_text: The generated chapters text + @param raw_transcript: Raw transcript data with timing info + @param video_id: The YouTube video ID + @return: Validated and corrected chapters text + """ + if openai_key == "" or not raw_transcript or not chapters_text: + return chapters_text + + # Parse the chapters + parsed = parse_chapters(chapters_text) + if not parsed: + return chapters_text + + print(f"Validating {len(parsed)} chapters for video {video_id}") + + # Build a map of timestamp to text and also create text segments + time_to_text = {} + all_segments = [] # List of (start_time, text) for searching + + for entry in raw_transcript: + if isinstance(entry, dict) and 'text' in entry and 'start' in entry: + text = entry['text'].strip() + if text and text not in ['[Music]', '[Applause]', '[Laughter]']: + start_time = int(entry['start']) + if start_time not in time_to_text: + time_to_text[start_time] = [] + time_to_text[start_time].append(text) + all_segments.append((start_time, text)) + + client = openai.OpenAI(api_key=openai_key) + validated_chapters = [] + + for seconds, timestamp_str, title in parsed: + # Skip 00:00:00 as it's typically correct + if seconds == 0: + validated_chapters.append(f"{timestamp_str} {title}") + continue + + # Get text around the current timestamp (±10 seconds window) + context_texts = [] + for time_sec in range(max(0, seconds - 10), seconds + 11): + if time_sec in time_to_text: + context_texts.extend(time_to_text[time_sec]) + + if not context_texts: + # If no text found, keep original + validated_chapters.append(f"{timestamp_str} {title}") + continue + + context = ' '.join(context_texts[:50]) # Limit context to avoid token overflow + + # Ask AI if the timestamp matches the description + validation_prompt = f""" + You are validating a chapter marker for a video transcript. + + Chapter description: "{title}" + Current timestamp: {timestamp_str} + + Here's what is actually being said around {timestamp_str}: + "{context}" + + Does this text match the chapter description "{title}"? + + Respond with ONLY one of these: + VALID - if the description matches what's being said at this time + INVALID - if the description does NOT match what's being said at this time + """ + + try: + response = client.chat.completions.create( + model="gpt-4o-mini", + messages=[ + {"role": "user", "content": validation_prompt} + ], + temperature=0.2, + max_tokens=20 + ) + + result = response.choices[0].message.content.strip().upper() + + if "VALID" in result and "INVALID" not in result: + # Keep original timestamp + validated_chapters.append(f"{timestamp_str} {title}") + print(f" ✓ {timestamp_str} - Timestamp validated for '{title}'") + else: + # Need to find the correct timestamp for this description + print(f" 🔍 {timestamp_str} - Searching for correct timestamp for '{title}'...") + + # Find the next chapter's timestamp to ensure we don't go past it + next_chapter_seconds = None + current_idx = parsed.index((seconds, timestamp_str, title)) + if current_idx < len(parsed) - 1: + next_chapter_seconds = parsed[current_idx + 1][0] + + # Search through the transcript to find where this topic is discussed + correct_timestamp = find_correct_timestamp(title, all_segments, time_to_text, client, seconds, next_chapter_seconds) + + if correct_timestamp and correct_timestamp != timestamp_str: + validated_chapters.append(f"{correct_timestamp} {title}") + print(f" ✏ Timestamp corrected: {timestamp_str} → {correct_timestamp}") + else: + # Keep original if we couldn't find a better match + validated_chapters.append(f"{timestamp_str} {title}") + print(f" ⚠ Could not find better timestamp, keeping {timestamp_str}") + + except Exception as e: + print(f" ! {timestamp_str} - Validation error: {str(e)}, keeping original") + validated_chapters.append(f"{timestamp_str} {title}") + + return '\n'.join(validated_chapters) + +def find_correct_timestamp(description, all_segments, time_to_text, client, original_seconds, next_chapter_seconds=None): + """ + Search through the transcript to find where a topic is actually discussed. + Only searches within ±60 seconds of the original timestamp. + @param description: The chapter description to find + @param all_segments: List of (timestamp, text) tuples + @param time_to_text: Dict mapping timestamps to text + @param client: OpenAI client + @param original_seconds: The original timestamp in seconds + @param next_chapter_seconds: Timestamp of next chapter (don't search past this) + @return: Corrected timestamp string or None + """ + # Define search window: ±60 seconds from original + search_window = 60 + min_time = max(0, original_seconds - search_window) + max_time = original_seconds + search_window + + # Don't search past the next chapter + if next_chapter_seconds is not None: + max_time = min(max_time, next_chapter_seconds - 5) # Stay 5 seconds before next chapter + + # Create text windows every 10 seconds within the search range + windows = [] + current_time = min_time + + while current_time <= max_time: + window_texts = [] + for time_sec in range(current_time, min(current_time + 10, max_time + 1)): + if time_sec in time_to_text: + window_texts.extend(time_to_text[time_sec]) + + if window_texts: + windows.append((current_time, ' '.join(window_texts[:30]))) # Limit text per window + + current_time += 10 + + if not windows: + return None + + # Build a prompt with the windows + windows_text = "\n\n".join([f"[{format_duration(w[0])}]: {w[1][:200]}" for w in windows]) + + search_prompt = f""" + You are fine-tuning a chapter timestamp for a video transcript. + + Topic: "{description}" + Original timestamp: {format_duration(original_seconds)} + + Here is text from within ±60 seconds of that timestamp: + + {windows_text} + + Which timestamp is the BEST match for when "{description}" begins? + The timestamp should be close to {format_duration(original_seconds)}. + + Respond with ONLY the timestamp in HH:MM:SS format (e.g., 00:05:17). + If the original timestamp is already good, respond with: {format_duration(original_seconds)} + """ + + try: + response = client.chat.completions.create( + model="gpt-4o-mini", + messages=[ + {"role": "user", "content": search_prompt} + ], + temperature=0.2, + max_tokens=20 + ) + + result = response.choices[0].message.content.strip() + + # Check if result is a valid timestamp + if ":" in result: + # Validate it's in HH:MM:SS format + import re + if re.match(r'^\d{1,2}:\d{2}:\d{2}$', result): + # Make sure the result is within our search window + result_seconds = parse_timestamp_to_seconds(result) + if min_time <= result_seconds <= max_time: + return result + else: + print(f" Timestamp {result} outside search window, keeping original") + return None + + return None + + except Exception as e: + print(f" Error searching for timestamp: {str(e)}") + return None + +def create_chapters(transcript, video_id, raw_transcript=None): """ Create timestamps and chapters for a YouTube video transcript. @param transcript: The raw transcript of a YouTube video @param video_id: The YouTube video ID + @param raw_transcript: Raw transcript data with timing information (optional) @return: A string containing formatted timestamps and chapters """ if openai_key == "": @@ -455,13 +860,32 @@ def create_chapters(transcript, video_id): return "Chapters not available" client = openai.OpenAI(api_key=openai_key) - - chapters_prompt = """ + + # Get video duration if available + duration_seconds = get_video_duration(raw_transcript) if raw_transcript else None + duration_constraint = "" + if duration_seconds: + duration_str = format_duration(duration_seconds) + duration_constraint = f"\n\nIMPORTANT: This video is {duration_str} long. DO NOT generate any timestamps beyond {duration_str}. All timestamps must be less than or equal to {duration_str}." + + chapters_prompt = f""" This is a transcript of a YouTube livestream. Could you please identify up to 10 key moments in the stream and give me the timestamps in the format for YouTube like this?: 00:00:00 Introductions - 00:01:30 What is structured metadata? - - Always start with 00:00:00 and use simple but descriptive language that makes it easier for users to understand what is being spoken about. + 00:01:47 What is structured metadata? + 00:03:22 Discussion about metrics + + CRITICAL INSTRUCTIONS: + - Always start with 00:00:00 + - Use the ACTUAL timestamps from where topics begin in the transcript + - DO NOT round timestamps to neat intervals like :00 or :30 + - Use precise timestamps like 00:05:17, 00:12:43, 00:08:09, etc. + - Look at the actual flow of conversation to determine when topics change + - The chapter description MUST accurately describe what is being said RIGHT AT that timestamp + - Do NOT describe what happens later - only describe what is happening at the exact moment of the timestamp + - Read the text carefully around each timestamp to ensure your description matches what's actually being discussed + - If a guest is introduced at 00:05:30, don't put "Guest introduction" at 00:20:00 + - Be precise and honest about what's happening at each moment + - Use simple but descriptive language that makes it easier for users to understand what is being spoken about{duration_constraint} """ print(f"Creating chapters for video {video_id}") @@ -471,12 +895,17 @@ def create_chapters(transcript, video_id): {"role": "system", "content": chapters_prompt}, {"role": "user", "content": transcript} ], - temperature=0.7, + temperature=0.3, # Lower temperature for more accurate, less creative responses ) chapters = response.choices[0].message.content # Clean any AI preamble before returning chapters = clean_ai_preamble(chapters) + + # Validate and fix chapter descriptions + if raw_transcript: + chapters = validate_and_fix_chapters(chapters, raw_transcript, video_id) + return chapters def write_markdown(args, video): @@ -491,6 +920,7 @@ def write_markdown(args, video): title = video['snippet']['title'] description = video['snippet']['description'] transcript = video.get('transcript', '') + raw_transcript = video.get('raw_transcript', []) if transcript == '': print(f"Skipping video {video_id} / {title} because it has no transcript.") @@ -500,7 +930,12 @@ def write_markdown(args, video): filename = file_for_video(args, video) [summary, cleaned_up] = openai_cleanup(transcript, video_id) - chapters = create_chapters(transcript, video_id) + chapters = create_chapters(transcript, video_id, raw_transcript) + + # Parse chapters and insert timestamps into cleaned transcript + parsed_chapters = parse_chapters(chapters) + if cleaned_up and parsed_chapters and raw_transcript: + cleaned_up = insert_timestamps_in_transcript(cleaned_up, parsed_chapters, raw_transcript) with open(filename, "w") as file: file.write(f"# {title}\n\n") From 7dbd5e17244cbe5c9487dec1367d24a5cc4ba172 Mon Sep 17 00:00:00 2001 From: Nicole van der Hoeven Date: Mon, 24 Nov 2025 19:13:39 +0100 Subject: [PATCH 12/28] Enforce 10-chapter maximum limit in chapter generation - Move chapter limit to CRITICAL INSTRUCTIONS section - Use stronger mandatory language (MUST, NO MORE than 10) - Add reminder at end of prompt to reinforce limit - Prevents AI from generating 19-24 chapters per video --- video-transcripts/transcripts.py | 127 +++++++++++++++++++++++++------ 1 file changed, 102 insertions(+), 25 deletions(-) diff --git a/video-transcripts/transcripts.py b/video-transcripts/transcripts.py index b1e6968..699daeb 100644 --- a/video-transcripts/transcripts.py +++ b/video-transcripts/transcripts.py @@ -778,20 +778,31 @@ def find_correct_timestamp(description, all_segments, time_to_text, client, orig if next_chapter_seconds is not None: max_time = min(max_time, next_chapter_seconds - 5) # Stay 5 seconds before next chapter - # Create text windows every 10 seconds within the search range + # Use actual timestamps from the transcript within the search range + # This preserves second-level precision instead of rounding to 10-second intervals windows = [] - current_time = min_time + seen_times = set() - while current_time <= max_time: - window_texts = [] - for time_sec in range(current_time, min(current_time + 10, max_time + 1)): - if time_sec in time_to_text: - window_texts.extend(time_to_text[time_sec]) - - if window_texts: - windows.append((current_time, ' '.join(window_texts[:30]))) # Limit text per window - - current_time += 10 + for segment_time, segment_text in all_segments: + if min_time <= segment_time <= max_time: + # Group by 5-second windows to avoid too many entries + window_key = (segment_time // 5) * 5 + + if window_key not in seen_times: + seen_times.add(window_key) + + # Collect text from a small window around this timestamp + window_texts = [] + for time_sec in range(segment_time - 2, segment_time + 3): + if time_sec in time_to_text: + window_texts.extend(time_to_text[time_sec]) + + if window_texts: + # Use the actual segment time, not the rounded window_key + windows.append((segment_time, ' '.join(window_texts[:30]))) + + # Sort windows by time + windows.sort(key=lambda x: x[0]) if not windows: return None @@ -805,12 +816,13 @@ def find_correct_timestamp(description, all_segments, time_to_text, client, orig Topic: "{description}" Original timestamp: {format_duration(original_seconds)} - Here is text from within ±60 seconds of that timestamp: + Here is text from within ±60 seconds of that timestamp (with ACTUAL precise timestamps): {windows_text} Which timestamp is the BEST match for when "{description}" begins? - The timestamp should be close to {format_duration(original_seconds)}. + Use the EXACT timestamp shown in brackets where the topic starts. + Do NOT round or approximate - use the precise timestamp from the list above. Respond with ONLY the timestamp in HH:MM:SS format (e.g., 00:05:17). If the original timestamp is already good, respond with: {format_duration(original_seconds)} @@ -847,6 +859,53 @@ def find_correct_timestamp(description, all_segments, time_to_text, client, orig print(f" Error searching for timestamp: {str(e)}") return None +def build_timeline_from_transcript(raw_transcript, sample_interval=10): + """ + Build a timeline showing what's being said at regular intervals. + @param raw_transcript: Raw transcript data with timing information + @param sample_interval: Sample interval in seconds (default: 10) + @return: String representation of the timeline + """ + if not raw_transcript: + return "" + + # Build a map of timestamp to text + time_to_text = {} + max_time = 0 + + for entry in raw_transcript: + if isinstance(entry, dict) and 'text' in entry and 'start' in entry: + text = entry['text'].strip() + if text and text not in ['[Music]', '[Applause]', '[Laughter]']: + start_time = int(entry['start']) + if start_time not in time_to_text: + time_to_text[start_time] = [] + time_to_text[start_time].append(text) + if 'duration' in entry: + end_time = entry['start'] + entry['duration'] + max_time = max(max_time, end_time) + + # Sample at regular intervals + timeline_lines = [] + current_time = 0 + + while current_time <= max_time: + # Collect text from a window around this time (±5 seconds) + window_texts = [] + for time_sec in range(max(0, current_time - 5), current_time + 6): + if time_sec in time_to_text: + window_texts.extend(time_to_text[time_sec]) + + if window_texts: + # Take first 100 characters of the combined text to keep it manageable + combined = ' '.join(window_texts)[:150] + timestamp_str = format_duration(current_time) + timeline_lines.append(f"[{timestamp_str}] {combined}...") + + current_time += sample_interval + + return '\n'.join(timeline_lines) + def create_chapters(transcript, video_id, raw_transcript=None): """ Create timestamps and chapters for a YouTube video transcript. @@ -868,24 +927,42 @@ def create_chapters(transcript, video_id, raw_transcript=None): duration_str = format_duration(duration_seconds) duration_constraint = f"\n\nIMPORTANT: This video is {duration_str} long. DO NOT generate any timestamps beyond {duration_str}. All timestamps must be less than or equal to {duration_str}." + # Build timeline from raw transcript if available + timeline = "" + if raw_transcript: + print(f"Building timeline from raw transcript for video {video_id}") + timeline = build_timeline_from_transcript(raw_transcript, sample_interval=10) + if timeline: + timeline = f"\n\n=== TIMELINE WITH ACTUAL TIMESTAMPS ===\n{timeline}\n=== END TIMELINE ===\n" + chapters_prompt = f""" - This is a transcript of a YouTube livestream. Could you please identify up to 10 key moments in the stream and give me the timestamps in the format for YouTube like this?: - 00:00:00 Introductions - 00:01:47 What is structured metadata? - 00:03:22 Discussion about metrics + This is a transcript of a YouTube livestream. Identify key moments in the stream and provide timestamps in the format for YouTube like this: + 00:00:00 Welcome and intro + 00:01:47 Structured metadata + 00:03:22 Metrics discussion CRITICAL INSTRUCTIONS: + - You MUST provide a MAXIMUM of 10 chapters - NO MORE than 10 - Always start with 00:00:00 - - Use the ACTUAL timestamps from where topics begin in the transcript - - DO NOT round timestamps to neat intervals like :00 or :30 - - Use precise timestamps like 00:05:17, 00:12:43, 00:08:09, etc. - - Look at the actual flow of conversation to determine when topics change + - Use ONLY timestamps that appear in the TIMELINE below - these are the ACTUAL timestamps from the video + - Pick timestamps where topic changes or new segments begin + - DO NOT make up timestamps - only use timestamps that appear in the timeline + - Look at what's being said at each timestamp in the timeline to determine good chapter points - The chapter description MUST accurately describe what is being said RIGHT AT that timestamp - Do NOT describe what happens later - only describe what is happening at the exact moment of the timestamp - - Read the text carefully around each timestamp to ensure your description matches what's actually being discussed - - If a guest is introduced at 00:05:30, don't put "Guest introduction" at 00:20:00 + - If a guest is introduced at a specific timestamp, use that exact timestamp - Be precise and honest about what's happening at each moment - - Use simple but descriptive language that makes it easier for users to understand what is being spoken about{duration_constraint} + + CHAPTER DESCRIPTION FORMAT: + - Use SHORT PHRASES, not full sentences + - Keep descriptions 2-5 words when possible + - Avoid articles (a, an, the) at the start + - Avoid question format - use noun phrases instead + - Make descriptions easy to search for + - Examples of GOOD descriptions: "OpenTelemetry basics", "Demo walkthrough", "Q&A session", "Kubernetes integration" + - Examples of BAD descriptions: "What is OpenTelemetry?", "A discussion about metrics", "The team talks about features" + + REMEMBER: Generate NO MORE than 10 chapters total. Select the most important moments only.{duration_constraint}{timeline} """ print(f"Creating chapters for video {video_id}") From 91a521b6e11e66595797972d7deb963601829b32 Mon Sep 17 00:00:00 2001 From: Nicole van der Hoeven Date: Mon, 24 Nov 2025 23:12:28 +0100 Subject: [PATCH 13/28] Regenerated transcripts --- ...etry-q-a-feat-iris-dyrmishi-of-farfetch.md | 303 ++++++++++-- ...-whats-an-observability-engineer-anyway.md | 28 +- ...8Z-teaser-observability-is-a-team-sport.md | 28 +- ...o-you-foster-a-culture-of-observability.md | 19 +- ...-a-team-sport-with-iris-dyrmishi-teaser.md | 27 +- ...eam-sport-with-iris-dyrmishi-full-video.md | 278 ++++++++--- ...ob-aronoff-of-lightstep-from-servicenow.md | 408 ++++++++++++++-- ...10Z-opentelemetry-q-a-feat-hazel-weakly.md | 286 ++++++++++-- ...-threading-the-needle-with-hazel-weakly.md | 285 ++++++++++-- ...n-distributed-tracing-with-doug-ramirez.md | 234 +++++++--- ...in-practice-fireside-chat-december-2022.md | 334 ++++++++++++-- ...-end-user-discussions-amer-january-2023.md | 162 ++++--- ...otel-migration-story-with-jacob-aronoff.md | 284 +++++++++--- ...he-evolution-of-observability-practices.md | 278 +++++++++-- ...ng-parsing-data-with-the-otel-collector.md | 307 ++++++++++-- ...T23:34:44Z-otel-q-a-feat-jennifer-moore.md | 214 ++++++--- ...Z-otel-q-a-feat-luiz-aoqui-of-hashicorp.md | 275 +++++++++-- ...:51Z-the-humans-of-otel-kubecon-na-2023.md | 175 ++++--- ...ty-majors-amy-tobey-and-adriana-villela.md | 361 ++++++++++++--- ...ow-to-train-your-teams-on-observability.md | 277 +++++++++-- ...:24Z-otel-collector-user-feedback-panel.md | 155 ++++--- ...us-interoperability-user-feedback-panel.md | 176 ++++--- ...:59Z-the-humans-of-otel-kubecon-eu-2024.md | 204 ++++---- ...aniel-dias-and-oscar-reyes-of-tracetest.md | 173 ++++--- ...ile-apps-with-hanson-ho-and-eliab-sisay.md | 198 +++++--- ...liab-sisay-and-austin-emmons-of-embrace.md | 175 ++++--- ...4T15:54:54Z-otel-q-a-with-steven-swartz.md | 295 ++++++------ ...T21:03:07Z-otel-q-a-with-dan-ravenstone.md | 376 ++++++++++++--- ...umans-of-otel-live-from-kubecon-na-2024.md | 343 ++++++++++---- ...2:14:29Z-humans-of-otel-kubecon-na-2024.md | 148 +++--- ...d-user-conversation-with-ariel-valentin.md | 294 ++++++++---- ...31T05:58:14Z-cfp-writing-q-a-livestream.md | 260 +++++++---- ...el-for-beginners-the-javascript-journey.md | 102 ++-- ...-otel-me-with-jerome-johnson-cal-loomis.md | 136 +++--- ...on-with-austin-parker-marylia-gutierrez.md | 310 +++++++++++-- ...6:00:52Z-otel-me-with-eromosele-akhigbe.md | 257 +++++++---- ...humans-of-opentelemetry-kubecon-eu-2025.md | 142 +++--- ...025-05-spring-starter-for-opentelemetry.md | 262 ++++++++--- ...05-leveraging-ai-for-opentelemetry-data.md | 193 +++++--- ...h-oluwatomisin-taiwo-and-andrei-morozov.md | 265 ++++++++--- ...practice-alibabas-opentelemetry-journey.md | 127 +++-- ...aled-kafkalog-ingestion-for-otel-by-150.md | 173 ++++--- .../2025-10-22T04:08:27Z-whats-new-in-otel.md | 316 +++++++------ ...umans-of-otel-live-from-kubecon-na-2025.md | 436 +++++++++++++----- 44 files changed, 7321 insertions(+), 2758 deletions(-) diff --git a/video-transcripts/transcripts/2023-06-01T17:22:03Z-opentelemetry-q-a-feat-iris-dyrmishi-of-farfetch.md b/video-transcripts/transcripts/2023-06-01T17:22:03Z-opentelemetry-q-a-feat-iris-dyrmishi-of-farfetch.md index abdc363..615df39 100644 --- a/video-transcripts/transcripts/2023-06-01T17:22:03Z-opentelemetry-q-a-feat-iris-dyrmishi-of-farfetch.md +++ b/video-transcripts/transcripts/2023-06-01T17:22:03Z-opentelemetry-q-a-feat-iris-dyrmishi-of-farfetch.md @@ -10,93 +10,300 @@ URL: https://www.youtube.com/watch?v=9iaGG-YZw5I ## Summary -In this YouTube video, Iris and Edith, both engineers at Farfetch, engage in a Q&A session discussing their experiences with OpenTelemetry and observability practices within the organization. Edith, who works on the observability team, shares her journey from a backend developer to a platform engineer, highlighting the importance of OpenTelemetry in improving their monitoring architecture, which encompasses a complex setup of cloud-native systems and Kubernetes. They discuss the adoption process at Farfetch, emphasizing a supportive culture around observability, the implementation of OpenTelemetry for traces and metrics, and the gradual transition from legacy systems. Edith also shares insights about their current use of tools like Grafana, Tempo, and Prometheus, the challenges faced during implementation, and the importance of collaboration across teams to achieve effective observability. The conversation emphasizes the positive reception of OpenTelemetry and the ongoing efforts to enhance their observability practices. +In this YouTube video, a discussion is held about OpenTelemetry and its implementation at Farfetch, featuring Iris, a platform engineer at the company. Iris shares her journey from a software engineer to her current role in observability, emphasizing the importance of OpenTelemetry in managing complex architectures with around 3,000 engineers. The conversation highlights the tools used for observability, including Grafana, Prometheus, and Tempo, as well as the challenges faced during the implementation of OpenTelemetry, particularly in integrating tracing and metrics. Iris notes the positive culture around observability at Farfetch, where the team has embraced OpenTelemetry without significant resistance, and expresses pride in contributing to the OpenTelemetry community. The video also touches on the significance of collaboration among engineering teams and the continuous evolution of observability practices at Farfetch. Overall, the discussion serves as an insightful look into the practical applications and benefits of OpenTelemetry in a large organization. ## Chapters -00:00:00 Introductions and Overview -00:02:30 Iris introduces Edis and her role at Farfetch -00:05:00 Edis shares her background and journey to observability -00:09:15 Discussion on how Edis discovered OpenTelemetry -00:12:30 Overview of Farfetch's complex architecture -00:17:00 Importance of OpenTelemetry in their system -00:21:45 Edis explains their CI/CD pipeline and deployment process -00:25:30 Tools and technologies used for observability at Farfetch -00:30:00 Edis discusses the adoption of OpenTelemetry within the organization -00:35:45 Challenges faced in implementing OpenTelemetry -00:40:00 Edis shares experiences with OpenTelemetry's operator and community contributions +00:00:00 Welcome and intro +00:01:00 Iris introduction +00:02:10 Edis background and journey +00:04:30 OpenTelemetry architecture discussion +00:06:00 CI/CD pipeline overview +00:07:10 Observability tooling at Farfetch +00:08:10 OpenTelemetry journey at Farfetch +00:10:00 Team enabling observability practices +00:12:30 OpenTelemetry implementation process +00:14:40 OpenTelemetry collector usage -# OpenTelemetry Q&A Session +**Iris:** Thank you. I guess we can get started. It's a cozy group today, which is cool. I like cozy groups. It means we get to have a more intimate conversation. But like I said, there will be ways of consuming this information as well because what Edis has to say is awesome. She's very passionate about OpenTelemetry, so I'm very excited to have her join us. Edith works at Farfetch. -Thank you all for joining us today! It's a cozy group, which means we can have a more intimate conversation. I'm excited to introduce our guest, Edis, who works at Farfetch and is passionate about OpenTelemetry. +Do you want to do like a brief little intro for everybody? -## Introduction +[00:01:00] **Edis:** For sure. Hello everyone, again. My name is Edis. I work as a platform engineer. Yeah, I know this title changes every time, so in my LinkedIn it could be something different. You know how it is. I'm a platform engineer, part of the observability team currently in Farfetch. I belong to the central team that provides tools for all the engineering teams across Farfetch to monitor their services, including traces, metrics, logs, and alerting. Again, I'm very excited to be here. -Hello everyone! My name is Edis, and I work as a platform engineer as part of the observability team at Farfetch. My role involves providing tools for all engineering teams across Farfetch to monitor their services, including traces, metrics, logs, and alerting. I'm thrilled to be here and look forward to our discussion. +So we're just gonna do this like just a regular Q&A. And because we have such a small audience, also if we have time at the end, y'all are more than welcome to ask questions. You know, the purpose of this is to understand what Edis is doing at Farfetch around OpenTelemetry observability to help the rest of the community share use cases across the community so that we can all learn from each other, right? -## OpenTelemetry Journey +That is that. So first things first, how did you come about to your current role at Farfetch? -### How did you get to your current role at Farfetch? +**Edis:** So observability has been a part of a build-up for me. When I started my career, I actually started as a software engineer, a back-end developer, and because it was a type of company that offered service to different clients, I became a DevOps engineer. I was like, okay, I was put there. So I started working very small scale with monitoring, mostly in AWS and Azure with CloudWatch, a little bit with insights. And then it started becoming more of a passion for me, the more I learned about it. -I started my career as a software engineer, specifically as a back-end developer. I transitioned into a DevOps engineer role, where I began working with monitoring tools, primarily in AWS and Azure. Over time, I developed a passion for observability and started working with more equipped observability platforms. My first exposure to OpenTelemetry was when I created a proof of concept (POC) after hearing about it on LinkedIn. Now, I'm excited to be at Farfetch, where I've been continuously learning and evolving my skills in observability. +Then I changed the position that I was in, and I started working in a more, let's say, well-equipped observability platform. And that's when I was like, okay, I really, really like this. I heard about OpenTelemetry for the first time and had my chance to touch it a little bit. Prometheus, Grafana, so I saw there was a lot of potential there. -### What does the architecture look like at Farfetch? +And then where I currently am, of course, my previous experience really helped to come now. It's been one year and a little bit of learning and continuously evolving in observability. So now I think I've become pretty good at it. I started from zero. Yay, that's the best! -Currently, we have around 3,000 engineers at Farfetch, and our architecture is quite complex. We have a mix of cloud-native services, Kubernetes, and virtual machines running on Ubuntu Linux across three different cloud providers. Each team follows certain guidelines, but there is a lot of legacy processes that still exist, which makes uniformity a challenge. +But how did you hear about OpenTelemetry specifically? Is it like one of those things where someone mentioned it to you, you saw it on the interweb somewhere? What's your story? -For metrics, we previously relied heavily on Prometheus, but some applications found it cumbersome. That's where OpenTelemetry came in, allowing us to collect telemetry signals consistently and uniformly. +**Edis:** I think it was LinkedIn somewhere. I know that I was working—we were working with no traces at the time. I know, a blasphemy. But we were looking into tracing solutions and somewhere like that I saw OpenTelemetry. So I was like, okay, I'm gonna give this a try and made a small POC for my manager. It never went more than that to the POC. It was almost more than one year ago. But that's how I heard about it. I really liked it. I had it at the back of my mind. -## Deployment Process and CI/CD Pipeline +So now that there was an opportunity here in Farfetch, OpenTelemetry came up again. I was like, okay, I like that. I'm gonna go for it. -Our CI/CD pipeline primarily uses Jenkins, managed by a separate team. My role focuses strictly on observability, overseeing the tools, deployments, maintenance, and the release of new features. +So at Farfetch, what can you tell us a little bit about the architecture of the system that you're working with and why observability and OpenTelemetry are so important to that? -### What OpenTelemetry tooling are you using? +[00:04:30] **Edis:** So I guess we'll start with like, let's give us a sense of what the architecture is. We are around 3,000 engineers currently in Farfetch, only in tech. We have an extremely complex architecture because we have different types of different sides of the business. So we have cloud-native, we have Kubernetes, and we have virtual machines, Ubuntu Linux, the three types of different cloud providers. -We primarily use open-source tools, including Grafana for dashboards, Tempo for tracing, and Prometheus and Thanos for metrics. We also have Jaeger in use to some extent as we transition to OpenTelemetry. +Every team, of course, we have uniformity and we have guidelines that need to be followed, but it's a lot of years in the process and some things are still like in the past. Every team decides to do things the way they think is best. So it is a lot of information coming from everywhere, and it's not uniform. -## OpenTelemetry Implementation at Farfetch +For example, we were relying heavily on Prometheus to collect metrics, but some of our applications, some of our engineers, Prometheus was just not a good idea. So we were like, okay, what could be better? Here's OpenTelemetry. The same for the traces. And of course, this will help us not only collect these telemetry signals from places where we couldn't before and from services that were not possible, but also it's helping us put everything in a uniform way, which is amazing. -### What was your organization's OpenTelemetry journey like? +[00:06:00] Very cool. And now what about your deployment process? What do you mean by building in the organization? What's your CI/CD pipeline like? Is it something that you're involved in to any extent or not really involved? -At Farfetch, we have a strong observability culture. When OpenTelemetry was introduced, it was welcomed with enthusiasm, and we quickly made space in our yearly plans to implement it. The positive reception was heartening, and I’m proud to be part of that culture. +**Edis:** More in the sense that I use it a lot, and I'm in close communication. But yeah, we currently use Jenkins in Farfetch, and we have a separate team taking care of that, which tells again how complex everything is. We have teams and everything is segregated in my position, which is strictly observability—basically everything with observability, all the tools, deployments, maintenance, and releasing new features on top of what we have. -### What about your team's journey to enable OpenTelemetry? +Okay, cool. So what observability tooling are you using? And you know, if you're using an observability vendor that you'd rather not divulge, that's totally okay. But just give us a sense of maybe your OpenTelemetry stack, how it's set up, that kind of thing. -When I joined the team, much of the groundwork for implementing observability had already been laid. Most engineers had already embraced the importance of observability, making it easier for us to introduce OpenTelemetry. +[00:02:10] **Edis:** Yeah, currently we're mostly open source and some things that have been created in-house by us, but mostly open source. We use Grafana for dashboards, we use Tempo as a data source, which is also part of Grafana, mostly for as a tracing back-end. That was something new. Sorry for the baby cats in the background. -To skill up, we focused on a phased implementation to minimize disruption for engineering teams. I took the lead on research and development, and over time, we successfully transitioned to using OpenTelemetry in production. +We use Prometheus and Thanos for metrics, and now we have added OpenTelemetry there as well. We used to have Jaeger, and we still do to a certain degree because we still haven't completely moved up on OpenTelemetry when it comes to the frameworks that some of the teams are using. So they're working hand in hand, Jaeger, and that's pretty much it. That's all that I can think of right now, but these are the main ones. -## Challenges and Experiences +[00:08:10] Cool. And what about in terms of your organization's OpenTelemetry journey? We hear it's always a mixed bag, right? Some organizations are like, giddy up, more OpenTelemetry observability, and others are like, what was it like at Farfetch? -### What challenges did you face during implementation? +**Edis:** Well, I'm very proud to say actually that we have a very good observability culture in Farfetch. It was actually one of the ideas that was welcomed immediately. Okay, OpenTelemetry, let's do it, let's go for it. It took us a bit more time to make some space on our yearly plan for it, but the moment that it was mentioned, everyone was like, okay, let's jump to it. We only see positives there. We couldn't see any negatives other than the time spent, which is a necessity. And yeah, it was very well accepted. I was surprised. I'm very happy, obviously. -One of the biggest challenges was figuring out how to release OpenTelemetry in manageable parts to avoid negatively impacting the engineering teams. We also faced technical challenges with the OpenTelemetry operator and integration with other systems, but we found solutions and adapted accordingly. +That's amazing! Yeah, so it sounds like it came from the top then. -### How are you currently handling traces and metrics? +**Edis:** Yeah, exactly. Of course, I don't want to mention names, but some of our senior leaders are very involved in the community, and they're always seeing new technologies and saying, "Hey, what do you think about this? Do you like it?" And of course, me, I'm extremely ambitious, and I'm always on the internet on LinkedIn seeing for new things to implement. It's a great combination there. -We are currently using a combination of manual and auto instrumentation. Our goal is to have all applications using OpenTelemetry instrumentation over time. We're also working on improving our tracing capabilities, gradually increasing the sampling size to provide better insights. +Oh, awesome! Now in terms of your team starting to enable observability practices and OpenTelemetry, what did you and your team have to go through in order to make that happen? -### How do you manage the volume of data generated? +**Edis:** Well, the moment that I joined the team, at a point in time, I think the biggest struggle had already passed because observability became a very new thing three years ago in Farfetch. I think the people that struggled the most were the engineers that worked in the team back then. Of course, it was a new thing. Observability, why do we need it? Why is it so important? -We take a cautious approach to data management. While we have a high volume of data, we monitor and adjust our resources accordingly. We implement guidelines and review processes for significant changes to ensure stability and performance. +[00:10:00] But I think at the point that I actually joined, everyone had already embraced how important observability is for everyone. Of course, that's an overestimation, but most of the engineers had already embraced it. So for us saying that, hey, we are implementing this amazing new thing, OpenTelemetry, that everyone had heard about, it was very popular right now and it was embraced immediately. And of course, knowing the benefits that they were getting from it—more metrics, more traces, more uniform way of collecting everything—it was a blast immediately. -## Community and Contributions +And how did your team skill up in terms of being able to start implementing OpenTelemetry? Because it sounds like some people were kind of familiar with it. Maybe, I'm assuming it might have been newer to other folks. You said actually when the directive came that your team was gonna start implementing observability principles, OpenTelemetry, that was something that you know there was like work involved. So what was the kind of work that was involved in order to enable that for your team? -### Have you made any contributions to OpenTelemetry? +**Edis:** Well, I think the biggest challenge—what we thought at the time was a challenge—was how to release this in patches and in parts so we could... we didn't do any damage or that the engineering teams that use observability every day didn't feel the change for the bad, of course. We would like them to see the improvement. -Yes, we recently contributed to the OpenTelemetry operator regarding certificate management. It was a collaborative effort, and the community was very welcoming, which made the process smooth. +So we thought that that was going to be a challenge. Of course, we were going to move from one technology to the other. But actually, we always have someone who is a driver for the project, and in this case, in my team, it was me. So I did a thorough investigation, and every time I was reading something, I was like, wow, okay, this is cool. Wow, okay, this is amazing because everything managed to fit together very well for us. -### Any thoughts on areas for improvement in OpenTelemetry? +Of course, the fact that we use open source really helps, with OpenTelemetry having compatibility with Prometheus, with Jaeger, and with everything that we were working with, it became easier over time to just put everything out there. -While my experience with OpenTelemetry has been overwhelmingly positive, I believe some documentation could be improved, particularly for certain features and configurations. I plan to contribute to that as well. +So it sounds like you were the primary driver for getting people leveled up on OpenTelemetry. -## Audience Questions +[00:12:30] **Edis:** Yeah, I'm proud of that. Together with the ones who were the biggest pushers and supporters of OpenTelemetry, and now we're in production. So I think we were pretty successful. -If anyone has questions, feel free to ask! +That is super cool! I don't know that we hear too many stories of organizations using OpenTelemetry in production, so I think that's really awesome. And how long did it take you guys to get to the point where you've got OpenTelemetry in production? ---- +**Edis:** So let me see. We planned to start implementing OpenTelemetry around January, and we started the first investigation gathering information. I think by mid-March, we were already ready in production. But again, we are still not there, not 100%. We're still relying on a lot of things with Prometheus and Jaeger. We're just using OpenTelemetry to transport. We still need to do instrumentation with OpenTelemetry. Some parts of our engineering teams are still not using it, but currently, OpenTelemetry is our main transport of our telemetry signals, basically. -**Thank you all for joining today!** I hope this session has been insightful. If you're interested in future discussions or have stories to share, please reach out. We’ll be providing a blog post summary of today’s Q&A, so stay tuned! +Are you relying heavily on the OpenTelemetry collector then to do that for you? + +**Edis:** Yeah, especially with traces. Traces were one of the neglected parts of the observability stack, you know. It's typical because traces are a bit, let's say, more modern and it takes more time to implement and to actually understand how good they are. Now traces are becoming the eat of observability. So it really helped us. OpenTelemetry really helped us get the best of the tracing in Farfetch. + +We were actually doing some numbers today with our architect, and we were moving around 1,000 spans per second in the past, and now we have 40,000 that flawlessly without even needing to lift a finger. And we're not there yet. We still have a lot to do there. + +And so how are you collecting your traces right now? Is it like manual instrumentation, auto instrumentation, a combination of both? Where are y'all at with that? + +[00:14:40] **Edis:** It's a combination of both. So we're currently doing... We have another team working with us that helps provide the instrumentation of the frameworks because we have a huge variety of languages. So we still have the OpenTracing framework that teams are still using. We also have some teams using manual instrumentation, and we're also implementing the OpenTelemetry operator with auto instrumentation, mostly for .NET and Java currently, but looking forward for Go because we have a lot of applications running in Go. + +So our main goal is that very soon we want to have all applications using OpenTelemetry instrumentation, but it's going to be a process, as slow and as fast depending on the team's pace. + +So for the teams who are using Go, which to my understanding there is no auto instrumentation, is there any kind of support that your team provides around that in terms of helping these teams instrument? + +**Edis:** Yeah, of course. One of the main teams that this is for Go, especially because of the open source, it's my team. But yeah, of course, we provide documentation, we provide guidelines, and there is a lot of very good documentation in the OpenTelemetry as well about manual instrumentation. So basically, we have sessions with engineers—not to train them because I think they can do it better than we can because it's their code—but just to show them the best practices and to introduce them to that and to tell them that, hey, our tools are here for you, and they take it from there. It's a team sport, let's say. + +Awesome! I love that. Because yeah, I think what you said is really important that you're not instrumenting their code for them because it's their code, but that you provide the guidance on the instrumentation, which I think is super important. + +Also, I want to call out, Ubuntu just shared a link in the chat that indicates that there is auto instrumentation for Go. So yay! Something to look at. + +**Edis:** Yeah, it's a more recent thing, and it's still a work in progress, but I came across this last week only. Since you mentioned Go, I was like, okay, go take a look if there's something interesting. + +I've been following it as well, actually, because it's my stack and the things that I'm—it's very close to me that I usually want to use Go. So I'm like, every day, is there news? Is there news? + +**Iris:** That's very cool! Now I want to ask on the auto instrumentation because, you know, keeping on that thread, I think you mentioned you're leveraging auto instrumentation for Java through the OpenTelemetry operator. Can you share a little bit of your experience around using the OpenTelemetry operator? + +I came across it like maybe a couple of months ago, and for me, I was like, this thing is amazing! So yeah, like, it's cool to hear someone who's actually using it. Is that in production right now for you? + +**Edis:** Yes, it is partly in production, but it's very isolated. It's not available for everyone. And yeah, the operator is amazing. We loved it when we just discovered it. We were like amazed. + +The thing is that it is a bit getting used to. We had some challenges when we started—well, we still do—but especially in the beginning because the operator and the instrumentation object and the collector, well, it was my bad, obviously. We were having some certificate issues and trying to rotate certificates. I was trying to delete something, and I just deleted and created—it was like a big mess. I would just like leave my computer and come back because of how coupled the operator and the collector and its orientation is. + +But yeah, now we've gotten the hang of it, and it is amazing. One other challenge that I didn't think of—that's the beauty of it—we were having a discussion with another engineer, and I was complaining that Prometheus cannot target the operator, the collector, for some reason, and we cannot create alerts from it. + +He told me, "Well, have you tried using OpenTelemetry and then sending everything to Prometheus?" And I'm like, hmm. It's the beauty of it. You know, it's compatible. + +Yeah, I guess there's challenges every day. One day it's time. + +That's very cool! And speaking of like, you mentioned you're ingesting traces. I know the log signal is like a newer player in the land of OpenTelemetry. Have you and your team or anyone at Farfetch started playing around with OpenTelemetry logging as well? + +**Edis:** Very, very little. Mostly consuming from a Kafka topic and see how that goes. It's pretty good, but we know that this is not there yet, and it's not stable. So we don't expect it to go into production or to have it part of the day-to-day because currently we have a huge volume of logs going through Farfetch, more than places, so a lot more. + +So yeah, it's not worth risking, but we're definitely experimenting. We expect in one year from now OpenTelemetry will be the only receiver that we're going to use for everything. + +Oh cool! For the little experiment that you've done in using OpenTelemetry logs, have you taken advantage of the log-to-trace correlation, or is that just the logs in isolation? How's that been going? + +**Edis:** Actually, no. That's a very good suggestion because I've been focusing mostly just getting things across and mostly the processing of the obfuscation of the data because that's something that we would like to use OpenTelemetry, which is amazing for. But no, that's something that it's definitely worth for me to investigate into the next. + +Awesome! What about the metric signal? How are you ingesting the metrics? Is Prometheus passing the metrics over to you, and are you using the Prometheus receiver? I know like from personal experience when I started playing around with the Prometheus receiver, it was like barely a thing. It was like so unstable. There was a disclaimer. I'm just wondering how you use the Prometheus receiver and if so, what's been your experience around that? + +**Edis:** Yes, actually, we use the Prometheus receiver. So throughout the instrumentation, just to get this out of the way, we do get some OTLP metrics as well, but the majority of our metrics is Prometheus. So the Prometheus receiver works very good for us because we already had an observability system in place, and we use Console for target control. So it was extremely easy for us to use the receiver, and it can handle a huge amount of data. + +The scrape configs are the same as in Prometheus, so technically nothing changes. You're just using another tool to scrape all these metrics. It is very straightforward for us, and currently, we're using both OpenTelemetry for some scenarios and Prometheus. But again, in the future, the goal is to use only OpenTelemetry; it's just that it's a process. + +Do you think then the Prometheus receiver will be the way to ingest all of your metrics and so you'll be able to scrape Prometheus altogether as your end goal? + +**Edis:** I would say so, but it would be maybe the majority because we have a lot of our services running on virtual machines as well, and we have Prometheus exporters there. So it's best that those, let's say, remain in touch, and it's going to be more difficult to adapt them. But when it comes to our Kubernetes, I think we're going to go more with OpenTelemetry because everything is Prometheus, but we have the Kubernetes SD configs for the Prometheus receiver, which is amazing. + +We also have the target control target allocator through the operator, which I have been testing, and I really liked it. I think we're going to go completely, especially on Kubernetes and in the cloud. + +Cool! And going back to Kubernetes, how many clusters are you typically involved that you're observing? + +**Edis:** I would say maybe a hundred in total for different data centers. Yes, and thousands of virtual machines. We have a huge stack. + +So your team is responsible then for ensuring that if there's an issue with the operator, you and your team are managing the OpenTelemetry operator across all these Kubernetes clusters? + +**Edis:** Exactly. That's why we're also trained in Kubernetes, all of us that are part of the team, because it's a very important part of our job to actually maintain everything. + +**Iris:** Oh nice! Nice! Okay, so you wear multiple hats. So you maintain the clusters as well, is what it sounds like? + +**Edis:** Or I don't know. + +Okay, cool! And I seem to recall—I want to say like version 1.26 of Kubernetes—they enabled some OpenTelemetry capabilities as an experimental feature. Is that something that you've ever dabbled with or heard of? Just curious. + +**Edis:** Not yet, actually. No, I haven't come across it, but something that I can note and test. There's always a lot to learn. + +**Iris:** Yeah, I'll see if I can find a link around that. I feel like it was a very niche thing that was not often talked about, but I'd love to hear if anyone's played around with that. + +Now, for me, one of the things that I always love to hear is how teams are structuring their collectors. Do you have one collector, multiple collectors? If so, how are you deploying them? + +**Edis:** So different configurations. Yeah, we currently have an agent and a central collector type of organization. Well, as I mentioned, we have especially when it comes to Kubernetes clusters, we have a huge number of them. So having just one point of interest is going to be overwhelming. + +So we are deploying—well, currently we have one OpenTelemetry agent in each of the clusters, and we are starting to substitute it with OpenTelemetry operator, which has the collector and auto instrumentation. So that's the end goal. That's where we're getting it. So everything is sent to a central collector where we do the observability of data, the sampling, and everything, and all that is sent. Currently, we're using Tempo, but in the future, we might use a vendor. + +Basically, there's going to be a central collector collecting all that per data center. + +Okay, so you basically like each Kubernetes cluster has its own collector, and then they feed into your central collector. Where does your central collector reside? Does it reside on a VM? Does it reside on another Kubernetes cluster? + +**Edis:** It's a Kubernetes cluster, and it's... yeah, we call it the central collector because most of the stock on that cluster is dedicated to us. We have a lot of data, so we have a lot of requirements for memory. So most of the applications running in this cluster are for observability and very few for the platform. + +So yeah, that's where everything is. + +Okay, cool! And then how do you ensure, you know, if everything's being sent to that central collector, that becomes a single point of failure. So how do you ensure that, you know, if that goes down, what's your backup plan? + +**Edis:** Well, we have fallback clusters with fallback collectors. So if this one fails, it goes to the other one immediately. We have also implemented—well, it's not currently running, but we also have implemented the—you know that the OpenTelemetry collectors can send to as many exporters as possible? So we are currently using a fallback cluster to send, for example, if one fails, send to the other one, and we can immediately enable it without an issue. + +It's like a background. Yeah, and we have equipped our collectors with very—we well, we use auto-scaling a lot. It's based on our metrics, so we will make sure that our collectors have a lot of memory and CPU liberty so their queues will be big as well. If there is a small downtime, everything will be saved into the queue and then sent to the central collector. + +Awesome! And on that central collector—well, speaking like continuing on the collector thread—what were some of the challenges that you experienced initially when you started implementing the collector? Because I would imagine, you know, you said that you want to make sure you have enough memory allocated. I'm assuming that might have been perhaps an initial challenge. Are there any others, or can you talk more about that? + +**Edis:** Well, yeah, I think knowing the collector and how it works is a new technology introduced to us was the biggest challenge. We're very fortunate in Farfetch because we rely heavily on auto-scaling. So the first thing that we did was to enable auto-scaling. But yeah, we had to do some tests to see how it was working with a small amount of data because again, it was completely new, and it took us a while to know the memory and CPU requirements. + +So at the same time, we're not wasting a huge amount of money just to have this available, but at the same time, we need to have this available because it is so crucial. So yeah, definitely the resources, memory, and CPU are very important. Everything else, I would say the Helm charts in the community are so good. We just needed to use them, enable it, and of course modify everything, the configuration basically, the exporters and receivers, and that's it. But yeah, it was pretty straightforward when it comes to that. + +Okay, and in terms of configuration, are you using any processors to do any data masking or to add attributes, remove attributes? Do you have custom processors? What's your processor story like on the collector? + +**Edis:** Well, we're currently experimenting with processors. I think we only have the batch processor or whatever it is enabled by default on the charts. But we are playing a lot with the data masking processors because especially for the logging part in tracing and in metrics, we do not have any data that could be sensitive, but for logging, that's something that we are relying on heavily, and that's what we're testing the most. + +But currently, we haven't implemented anything special for the telemetry data that we're currently passing. + +Cool, cool! And I was curious because, you know, you mentioned like you're using traces, you're using metrics, playing around with logs. Within traces, are you aware of any teams using span events, for example? + +**Edis:** No, not yet. And that's something that we are really, really planning to introduce currently because of the limitations that we had with our previous tracing system. We had a very low sampling; it was 0.1 percent. The interest in the teams was not big. They didn't really care much about tracing. It was very rare to find an engineer that relied on tracing. + +So now that we implemented Tempo and OpenTelemetry, we are gradually increasing the sampling size and allowing more information to come through. I think we are getting better at it, but still not there. That's part of our package of making traces first-class citizens. + +That's what I love to hear! And I don't think this is the thing yet now, but my understanding in talking to a few of the OpenTelemetry folks is that the idea is that the logs are going to replace the span events because, I mean, span events are basically like logs embedded in your traces anyway. But with the idea that, obviously, you continue having that correlation, but I believe my understanding is also you get access to the fact that the logs specification is a lot richer than the span events specification, so you get to take advantage of having more information potentially at your disposal. + +So that's kind of a thing that I'm looking forward to personally. I noticed that Sebastian just posted a link in the chat regarding the traces for Kubernetes cluster in version 1.27. So for anyone, I'm not sure if that was the one that you were speaking of. I was just trying to find a reference. + +**Edis:** Yeah, yeah, I believe that is the one. I believe that is the one. + +Yeah, that's super cool! I do seem to remember that for enabling it, you have to go into deep, deep in the bowels of Kubernetes configuration to be able to enable that feature. So I think if you were using a cloud provider, do so at your own risk kind of thing. I think it was a lot easier to use if you're doing Kubernetes locally on your machine. But anyway, still a cool feature nonetheless. + +But I definitely think it's something that is worth exploring as it matures. Just going back to our discussion, I had a question also with regards to have you encountered any folks who were resistant to this whole OpenTelemetry thing? Like on development teams or even within your own team? What was the vibe around that? And if so, what did you do to help alleviate their stress? + +**Edis:** To be honest, not really. It's curious, actually. I haven't really met anyone that opposes it. Yeah, I've met plenty of people that simply do not care for it. Like they're like, okay, we have it, it's okay, we do not have it. + +In this situation, it's mostly, for example, someone asks for something related to observability and traces, and I'm like, hey, look at this cool thing that we did with OpenTelemetry. Now it is available for you, and they're like, okay. I just keep sending things that I consider that are so cool and it's good for them to use, and that's pretty much it. I know that they're going to use it because it's a but never had someone that was against it or like, no, we don't need it. + +It's interesting! + +**Iris:** Awesome! I think we need more companies like yours where people are like, yeah, if it's a lemon tree. + +**Edis:** Yeah, we have a very good observability culture. I'm actually really, really proud of that. I mean, I'm proud because I'm helping continue and build it, but the previous team where that worked for that could also... + +**Iris:** That's awesome! Yeah, and I think like that's where we really see success in OpenTelemetry is always having like a group of people who are like amazingly enthusiastic about it and are just like out there and believe in it and want to make sure that it happens. + +So, you know, and I know like Shubanchu, who's also on the call, like he's done a lot of evangelism around OpenTelemetry in his organization, which is super awesome. So, you know, hats off to y'all who do that in your organizations because I think it's gonna keep helping make OpenTelemetry awesome. + +Now, is your organization at the point now where you've started to like make contributions to OpenTelemetry, or is that still something where you're like not there yet? + +**Edis:** Yes, actually we made the contribution recently to the operator because of the certification. I think the certification was relying on search manager and not with custom certificates, and that didn't work for us. So there was a feature request and then a request to fix that, and that's why we are working so heavily now with operators so we can use our own certificates basically until search manager is available for us as well. + +That is so cool! How did it feel like making the contribution? + +**Edis:** It was great! Well, actually, it was a joint effort. Our architect was the main figure, let's say, after this. But yeah, it feels great, and the community is super welcoming, and like it was approved so fast because when he submitted the feature request, we were thinking that it was going to take like months or like, okay, yeah, we will have to wait. And then a week later he's like, hey, guess what? It's more! Let's go forward and test it. It's amazing! + +That is so cool! I'm always like a huge fan of that, you know, like don't wait around for the feature request, just do it yourself. So it's really cool that you and your team were able to achieve that. I think that's like, you know, I think it's a great accomplishment because like putting yourself out there for open source is like you have to be vulnerable, and you have to be okay with people saying, like, well, that's not really correct. + +It's scary, right? So yay, congrats! That's super amazing! I hope the team continues to make OpenTelemetry contributions. One thing I wanted to pivot to is, you know, like I am so happy that you have awesome things to say about OpenTelemetry. Is there anything that you think you and your team have encountered where OpenTelemetry could improve? Because that's, you know, that feedback is also super important so that we can continue to improve as a whole. + +**Edis:** Well, to be honest, I've had a super positive experience with it. Working with it at every step of the way has been super easy, and there's been a lot of support from the community. The only thing that I would say maybe is that it is a bit lacking in documentation in some parts. For example, when I was implementing the Prometheus receiver first, it was very confusing to use. I was using the console, the console SD config, and it was a mess. I couldn't find information anywhere on how to implement it or some good documentation. + +Of course, I'm planning to contribute to that, but that's all that I would have to say. It takes a bit longer to figure it out, or you have to dig very, very deep to find some documentation for certain features, exporters, or receivers. But other than that, it's been an amazing experience. + +Cool, cool! Awesome! Yeah, I have to agree with you; sometimes it is a bit of an archaeological dig. So anything that can be done to help bring that up to the surface is most welcome. We definitely look forward to a contribution to the docs. + +Also, for anyone on here who's played around with OpenTelemetry, you can always submit a pull request to the OpenTelemetry.io repo if you want to write a blog post about anything OpenTelemetry-related. Because I know the comms folks are always looking for contributions. + +And so also if you're looking for a first contribution on OpenTelemetry, that is a great place to start. + +Now, you know, let's turn the tables around to our lovely audience here today. Does anybody have any questions for Edis? + +**Audience Member:** No burning questions? We all good? + +**Iris:** I haven't even wanted to say thank you for doing this. I appreciate it. + +**Audience Member:** Yeah, I will second that to both of you. I think you covered a lot, Adriana and Edis. You gave pretty good insight into her company. It's amazing to see how little objections there are to change, not just observability, but change in general. So it's nice to see. Almost wish like, can I have some of it? + +I've been grinding for the last year basically talking observability and OpenTelemetry non-stop, and I finally made some inroads, I think. But yeah, it's nice to see that there are easier companies or easier paths. Not everybody has struggles the same. It's good to know. + +**Edis:** Yeah, that's so true! I think my experience in the past has also been like it's the uphill battle to OpenTelemetry. So it is so refreshing to hear like such a lovely positive story that there is light at the end of the tunnel. There are people who get it. + +**Audience Member:** There isn't it, but now it's becoming—I think it is the second most contributed project and one of the most well-known. So more people are getting a smell of it, so it's getting more and more accepted, I'd say, than it was one year ago. + +**Edis:** Yeah, so true! Like when I started dabbling around in OpenTelemetry, the traces specification wasn't even finalized. I was pushing the organization I was at the time to be like OpenTelemetry is going to be the big thing, y'all! And they're like, uh-huh. So it was an uphill battle. I feel like now at least like a lot of the specs are finalized, so it makes for a very compelling narrative, and you get more and more user stories of people actually using this in production, even if it's like, you know, kind of like where Farfetch is at—where you're not fully productionalized, but you've got some stuff running in production. + +And I think that's a compelling story to tell as well, right? Which is like get it out there, start using it! + +I think that's a really great message to share with folks. Do you have any parting thoughts? + +**Edis:** I was gonna say something as well. We're, yeah, it's not fully there, but we're passing a crazy amount of data after those collectors, and they're tough. Trust me, they're tough. It's worth it. It might be difficult to convince your peers to start implementing it, but it's worth it. They are durable collectors! You can pass thousands and millions of data per second, per minute. + +Yeah, if you do a good job with auto-scaling, Eden's gonna manage how much memory and CPU are they using while doing that kind of load. Do you know offhand? + +**Edis:** When we were tracing around 30,000 spans per second, I think we were like at around eight gigabytes of memory, and I don't know how many instances of the collector we were running, but maybe like four. But around eight gigabytes. + +Cool, thank you! + +**Iris:** Any other questions or comments for Edis? + +Well, if you like what you heard today, you just will be back for our OpenTelemetry in practice session on June the 8th, and it's gonna be, I believe, at the same time. So keep an eye out for an invite then. + +Edis, why don't you tell folks what you'll be presenting about? + +**Edis:** Well, my topic will be observability as a team sport. It's something that I'm extremely passionate about, and I see that in many companies, observability teams are the ones that are making the guidelines, instrumenting the code, creating the alerts, and responding to them. I am completely against that. I think that observability, everyone should do their part. Engineers know their code and their product better. We, as observability engineers, are there to help them and empower them and provide the tools, but they should be the ones taking charge when it comes to this part. So that's what I'm going to be talking about. + +**Iris:** Awesome! I cannot wait to hear this talk! So that'll be on June 8th at one o'clock Eastern, which is 10 o'clock Pacific and plus six in Central European Time. I think it's going to be awesome, so I hope you all can join and tell your friends. We would love to have more people here to hear what Edis has to say because I think it's awesome. + +I think you're a great champion of OpenTelemetry, so keep on keeping on! This was great! Thanks so much for joining us here today. + +And, you know, if you have a friend who—or if you yourself are interested in participating in one of these Q&As or even OpenTelemetry in practice, please reach out to either Therese or Rin or me on the OpenTelemetry end-users Slack. We are more than happy to hear your stories and help folks share them with the world. + +We will be providing a blog post summary of today's Q&A for all to see. Thank you so much, everyone! + +**Edis:** Thank you so much, Adriana! + +**Iris:** Thank you, everyone! ## Raw YouTube Transcript diff --git a/video-transcripts/transcripts/2023-06-01T17:22:03Z-whats-an-observability-engineer-anyway.md b/video-transcripts/transcripts/2023-06-01T17:22:03Z-whats-an-observability-engineer-anyway.md index 494522c..6be0ced 100644 --- a/video-transcripts/transcripts/2023-06-01T17:22:03Z-whats-an-observability-engineer-anyway.md +++ b/video-transcripts/transcripts/2023-06-01T17:22:03Z-whats-an-observability-engineer-anyway.md @@ -10,30 +10,22 @@ URL: https://www.youtube.com/watch?v=KRjYXPS3_so ## Summary -In this video, the speaker discusses the importance of documentation and guidelines provided by OpenTelemetry for manual instrumentation. They emphasize the collaborative approach taken with engineers, not to train them, but to share best practices and highlight the availability of tools to assist them. The speaker conveys that the responsibility lies with the engineers to implement these practices, framing it as a team effort. +In this video, the speaker discusses the importance of providing documentation and guidelines for engineers regarding manual instrumentation in OpenTelemetry. They emphasize that while they are not training engineers, they facilitate sessions to showcase best practices and introduce the tools available to them. The speaker highlights the collaborative nature of the process, suggesting that it is a "team sport" where engineers can effectively utilize the tools and resources to enhance their work. ## Chapters -Based on your provided excerpt, here are the key moments with timestamps for the livestream: +00:00:00 Welcome and intro +00:00:10 Documentation and guidelines +00:00:20 Manual instrumentation overview +00:00:30 Best practices introduction +00:00:40 Tools and teamwork +00:00:53 Closing remarks -00:00:00 Welcome and Overview -00:01:20 Importance of Documentation -00:02:10 Guidelines for Manual Instrumentation -00:03:30 Engaging with Engineers -00:04:00 Best Practices in Instrumentation -00:05:15 Role of Tools in Development -00:06:00 Collaborative Approach to Instrumentation -00:07:30 Conclusion and Next Steps -00:08:00 Q&A Session -00:09:30 Wrap Up and Thank You +[00:00:10] **Speaker:** Thank you. We provide documentation. We provide guidelines, and there is a lot of very good documentation in the OpenTelemetry as well about manual instrumentation. -Please adjust the timestamps according to the actual content of the livestream if needed. +[00:00:30] So basically, we have sessions with engineers, not to train them because I think they can do it better than we can because it's their code, but just to show the best practices and to introduce them to that and to tell them that, hey, our tools are here for you. -Thank you. We provide documentation and guidelines, and there is a lot of very good information in OpenTelemetry regarding manual instrumentation. - -Basically, we have sessions with engineers—not to train them, because I believe they can do it better than we can since it's their code—but to show them best practices and introduce them to our tools. We want to convey that our tools are here for them, and from there, it's a team sport, let's say. - -Thank you. +[00:00:40] They take it from there. It's a team sport, let's say. Thank you. ## Raw YouTube Transcript diff --git a/video-transcripts/transcripts/2023-06-05T15:14:28Z-teaser-observability-is-a-team-sport.md b/video-transcripts/transcripts/2023-06-05T15:14:28Z-teaser-observability-is-a-team-sport.md index db409d3..22d6b75 100644 --- a/video-transcripts/transcripts/2023-06-05T15:14:28Z-teaser-observability-is-a-team-sport.md +++ b/video-transcripts/transcripts/2023-06-05T15:14:28Z-teaser-observability-is-a-team-sport.md @@ -10,30 +10,22 @@ URL: https://www.youtube.com/watch?v=HLJe2RLPlWY ## Summary -In this video, the speaker discusses the concept of observability as a collaborative effort within teams, advocating for a shift away from the traditional model where dedicated observability teams handle all aspects of monitoring and alerting. The speaker emphasizes that engineers are most familiar with their own code and products and should take an active role in observability practices. Instead of solely relying on observability teams to create guidelines and respond to alerts, the speaker believes in empowering engineers to lead these efforts, with observability engineers providing support and tools. The video aims to encourage a more inclusive approach to observability in software development. +In this video, the speaker discusses the concept of observability as a collaborative effort within teams, emphasizing that it should not be solely the responsibility of dedicated observability teams. The speaker, who is passionate about this topic, argues that engineers should take ownership of observability in their code and products, while observability engineers should act as facilitators, providing tools and support. The main points include the importance of empowering engineers to handle observability and the need for a collective approach rather than a top-down model. ## Chapters -00:00:00 Introductions -00:01:00 Overview of the topic: Observability as a team sport -00:02:30 Importance of shared responsibility in observability -00:04:15 Critique of current observability practices in companies -00:06:00 Empowering engineers to take charge of observability -00:07:45 Role of observability engineers in supporting teams -00:09:00 Tools and resources for effective observability -00:11:30 Real-life examples of successful observability practices -00:14:00 Discussion on the cultural shift needed for observability -00:16:45 Q&A session and audience interaction +00:00:00 Welcome and intro +00:00:10 Observability as a team sport +00:00:20 Role of observability teams +00:00:30 Empowering engineers +00:00:40 Engineers taking charge +00:00:55 Conclusion and wrap-up -# Observability as a Team Sport +[00:00:10] **Speaker:** Foreign topic will be observability as a team sport. It's something that I'm extremely passionate about. -The topic I want to discuss today is **observability as a team sport**. This is something I am extremely passionate about. +[00:00:20] I see that in many companies, observability teams are the ones that are making the guidelines, instrumenting the code, creating the alerts, and responding to them. I am completely against that. I think that observability, everyone should do their part. Engineers know their code and their product better. -In many companies, observability teams are responsible for creating guidelines, instrumenting the code, setting up alerts, and responding to them. However, I am completely against this approach. I believe that **observability should be a shared responsibility**. - -Engineers are the ones who know their code and product best. As observability engineers, our role should be to **help and empower** them by providing the necessary tools, rather than taking charge of this aspect. - -That's the core message I want to convey today. +[00:00:40] Us as observability engineers are there to help them and empower them and provide the tools, but they should be the ones taking charge when it comes to this part. So that's what I'm going to be talking about. ## Raw YouTube Transcript diff --git a/video-transcripts/transcripts/2023-06-08T02:24:12Z-teaser-how-do-you-foster-a-culture-of-observability.md b/video-transcripts/transcripts/2023-06-08T02:24:12Z-teaser-how-do-you-foster-a-culture-of-observability.md index 81e990e..fa19803 100644 --- a/video-transcripts/transcripts/2023-06-08T02:24:12Z-teaser-how-do-you-foster-a-culture-of-observability.md +++ b/video-transcripts/transcripts/2023-06-08T02:24:12Z-teaser-how-do-you-foster-a-culture-of-observability.md @@ -10,24 +10,15 @@ URL: https://www.youtube.com/watch?v=RxP76ahb29s ## Summary -In this video, the speaker shares their pride in fostering a strong observability culture within their organization. They emphasize the importance of contributing to and developing this culture, highlighting how it enhances overall performance and collaboration. The discussion revolves around the key elements that make up a successful observability culture, including transparency, continuous improvement, and teamwork. The speaker expresses enthusiasm for their role in this ongoing process. +In this video, the speaker expresses pride in fostering a strong observability culture within their team or organization. They discuss the importance of maintaining and developing this culture, emphasizing its role in improving operational efficiency and transparency. The speaker highlights their contributions to this initiative and reflects on the positive impact that a robust observability culture has on the team's overall performance and collaboration. ## Chapters -Sure! Here are the key moments identified from the transcript: +00:00:00 Welcome and intro +00:00:10 Observability culture discussion +00:00:20 Building observability culture -00:00:00 Introductions -00:01:15 Discussing the importance of observability culture -00:02:45 Personal pride in contributing to observability -00:03:30 Strategies for building a strong observability culture -00:05:00 Challenges faced in promoting observability -00:06:50 Tools and technologies that support observability -00:08:15 Sharing success stories within the observability framework -00:09:30 Future goals for enhancing observability practices -00:10:45 Q&A session on observability topics -00:12:00 Closing remarks and thank yous - -Thank you! I have a very good observability culture, and I'm actually really proud of that. I mean, I'm proud because I'm helping to continue and build it. +[00:00:10] **Speaker:** Thank you. I have a very good observability culture. I'm actually really, really proud of that. I mean, I'm proud because I'm helping continue and build it. ## Raw YouTube Transcript diff --git a/video-transcripts/transcripts/2023-06-12T23:45:47Z-teaser-observability-is-a-team-sport-with-iris-dyrmishi-teaser.md b/video-transcripts/transcripts/2023-06-12T23:45:47Z-teaser-observability-is-a-team-sport-with-iris-dyrmishi-teaser.md index cf62003..e13ec4c 100644 --- a/video-transcripts/transcripts/2023-06-12T23:45:47Z-teaser-observability-is-a-team-sport-with-iris-dyrmishi-teaser.md +++ b/video-transcripts/transcripts/2023-06-12T23:45:47Z-teaser-observability-is-a-team-sport-with-iris-dyrmishi-teaser.md @@ -10,30 +10,17 @@ URL: https://www.youtube.com/watch?v=IHeqL36AlK0 ## Summary -In this video, the speaker discusses the importance of involving team members in the creation of their own dashboards and alert systems for effective observability. They emphasize that without team involvement, alerts would be overly generic and potentially ineffective, as the creator wouldn't have the necessary context about the products or code. The speaker argues that if team members handle the development of these tools, they would be more tailored and responsive, allowing for quicker incident response. Overall, the video underscores the significance of team ownership in observability processes. +In this video, the speaker discusses the importance of internal teams being involved in building their own observability dashboards and alert systems. They argue that if external teams are responsible for these tasks, the alerts and dashboards will lack specificity and relevance, leading to delayed responses to incidents. The speaker emphasizes that an internal team’s familiarity with their products and code allows them to create more tailored and effective monitoring tools. Overall, the video highlights the necessity of team involvement in incident response systems to ensure timely and accurate reactions to potential issues. ## Chapters -Sure! Here are the key moments identified from the provided transcript excerpt: +00:00:00 Introduction to dashboard building +00:00:10 Discussion on alerting systems +00:00:20 Observability data challenges +00:00:30 Generic alerting thresholds +00:00:40 Importance of team involvement -00:00:00 Introduction to the topic of observability -00:01:15 Discussion on the involvement of foreign ERS in dashboard building -00:02:30 Importance of team-specific alert creation -00:03:45 Consequences of generic alerts on incident response -00:04:10 The role of product knowledge in effective monitoring -00:05:00 Comparison between generic and tailored dashboards -00:06:30 Significance of team ownership in observability tools -00:07:15 The impact of proper alerting on incident management -00:08:00 Summary of best practices for dashboard creation -00:09:00 Closing thoughts on improving observability in teams - -Feel free to adjust the timestamps or descriptions based on the actual content! - -Foreign ERs were not involved in building their own dashboards or alerts, which means they don't know what data they're sending for their observability needs. As a result, they won't be able to respond to an incident in time. - -If I were the one creating the alerts for them, I would have to make them extremely generic with some thresholds that I thought might be appropriate, since I don't know their products or their code. Consequently, the dashboards would also be completely generic. - -However, if those dashboards and alerts were created by someone within the team, it would be much easier for them to manage and respond effectively. +[00:00:10] **Speaker:** foreign ERS were not involved in the part of building their own dashboard, building their own alert, knowing what data they're sending for their observability reasons. They will not be able to respond to an incident in time. Simple as that. If I was the one making the alerting for them, it would be something extremely generic with some thresholds that I just thought would be right because I don't know their products or their code. Obviously, the dashboards would be completely generic, but if those were done by a person inside the team itself, it would be so easy for them. ## Raw YouTube Transcript diff --git a/video-transcripts/transcripts/2023-06-12T23:45:52Z-otel-in-practice-presents-observability-is-a-team-sport-with-iris-dyrmishi-full-video.md b/video-transcripts/transcripts/2023-06-12T23:45:52Z-otel-in-practice-presents-observability-is-a-team-sport-with-iris-dyrmishi-full-video.md index c999093..8dba675 100644 --- a/video-transcripts/transcripts/2023-06-12T23:45:52Z-otel-in-practice-presents-observability-is-a-team-sport-with-iris-dyrmishi-full-video.md +++ b/video-transcripts/transcripts/2023-06-12T23:45:52Z-otel-in-practice-presents-observability-is-a-team-sport-with-iris-dyrmishi-full-video.md @@ -10,110 +10,274 @@ URL: https://www.youtube.com/watch?v=U1yLXnMONkc ## Summary -In this YouTube video, Iris, a platform engineer specializing in observability at Farfetch, leads an open discussion about the importance of observability as a collaborative effort within engineering teams. She emphasizes that observability is not the sole responsibility of a central team but rather a shared duty among all engineers, regardless of their specific roles. Iris outlines what she considers a perfect observability system, highlighting the necessity for centralized access to metrics, optimized data costs, reliable alerting, and team involvement in incident response and monitoring. Throughout the session, she encourages audience participation, discussing challenges such as excessive data, the need for structured telemetry, and the integration of observability into team processes and culture. The conversation also touches on practical tools and frameworks used in observability, such as OpenTelemetry, Prometheus, and Grafana, and the significance of creating a culture that values observability across the organization. The discussion concludes with an invitation for further community engagement and sharing of best practices in observability. +In this YouTube video, Iris, a platform engineer at Farfetch with a strong focus on observability, discusses the importance of fostering an observability culture within engineering teams. She emphasizes that observability is a collective responsibility rather than the duty of a single team, advocating for active participation from all engineers in creating and maintaining observability systems. Key points include defining a perfect observability system, the significance of team ownership in monitoring and alerting, the challenges of managing telemetry data, and the need for a collaborative approach to optimize observability practices. Iris also highlights the role of observability engineers in guiding teams without taking over responsibilities, and she shares insights into the tools and frameworks used at Farfetch, such as OpenTelemetry and Grafana. The discussion touches on various topics, including incident response, auto-scaling, and the need for structured data to avoid excessive costs associated with observability. The video concludes with an invitation for further dialogue and collaboration in the observability community. ## Chapters -00:00:00 Introductions -00:01:30 Overview of Observability Culture -00:05:00 Discussion on Responsibility of Observability -00:10:00 Defining a Perfect Observability System -00:15:30 Importance of Team Involvement in Observability -00:20:00 Role of Observability Engineers -00:25:00 Challenges in Observability and Incident Response -00:30:00 Custom Solutions and Tools for Observability -00:35:00 Addressing Anomaly Detection in Data -00:40:00 Centralizing Observability Tools and Data +00:00:00 Welcome and introduction +00:01:10 Observability as a team sport +00:03:05 Perfect observability system overview +00:05:30 Importance of observability engineers +00:08:42 Incident response and team involvement +00:10:40 Custom dashboards and alerts +00:12:30 Challenges of observability data +00:15:00 Observability culture and team ownership +00:20:00 Changing observability culture +00:24:00 Q&A session -# Observability Culture Discussion +**Iris:** Thank you for joining us again. We had her for the Q&A last month, a couple of weeks ago I guess, and she did a bang-up job, so we asked her to come back and I'll let you take it away. -**Iris**: Thank you for joining us again! We had her for a Q&A last month, and she did a fantastic job, so we asked her to come back. I’ll let her take it away. +Hello everyone, thank you so much for joining today. My name is Iris, I'm a platform engineer with a focus on observability working at Firefits currently, and I'm a super passionate person when it comes to observability. So yeah, I like to talk about it as much as I can, and I like to share the observability culture all over, not just in my company. ---- +[00:01:10] So today I wanted to have a talk, and I would like to have it as an open discussion. I would love to hear about the observability culture in your companies as well. I believe that observability is a team sport, and it is not something that is being done just by one central team in a company, no matter how big or small it is. I think everyone should put their hands and be involved in the observability. -**Iris**: Hello everyone! Thank you so much for joining today. My name is Iris, and I’m a platform engineer focused on observability at Farfetch. I’m passionate about observability and love to share its culture, not just within my company but across the board. +But of course, let's say observability has been there forever, but now it's becoming this huge thing, and now we have this amazing culture that is spreading, but we're still not there. I feel like this conversation and these discussions are so important to put out there for us to discuss and then to take them to our companies and to learn obviously from each other. -Today, I want this to be an open discussion. I would love to hear about the observability culture in your companies as well. I believe that observability is a team sport; it shouldn’t be confined to a single central team, regardless of the company size. Everyone should contribute and be involved in observability. +So yeah, this was just a small intro. Please interrupt me because I do not have a good visibility, so if you have a question, please don't hesitate to unmute yourself and just speak. If you have any opinions about something that I'm saying, I just want to have a spoiler alert: these are mostly things that I've learned from my job and things that I believe in. So of course, if you have a completely different opinion, please let me know. Let's discuss it; it would be amazing to have a conversation about this here. -Observability has existed for a long time, but it’s becoming a significant focus now, and we have this amazing culture spreading. However, we still have a long way to go. Conversations like this are essential for us to learn from each other and improve our practices. +Okay, so the first question that I ask all engineers that are being interviewed for a job in my team is: whose responsibility is observability? Because, let's be honest, observability is being implemented differently in many different companies. I've seen so many ways everywhere that I've interviewed for; it's different. So of course, observability is everyone's responsibility. It's simple. If your engineer says that answer to me, then I'm like, "Okay, okay, we're on board, we're getting somewhere." -### Introduction +[00:03:05] And why do I believe this? First of all, in order to get deeper into why I believe that everyone is responsible for the observability in their company, I want to give an overview of what I think a perfect observability system is. And that's perfect—let's take it with a grain of salt, of course. It's perfect for me; my company works in another person's company, doesn't work. -Please feel free to interrupt me if you have questions or opinions about what I'm saying. A spoiler alert: these are mostly insights I've gained from my job, and I welcome any differing opinions for discussion. +So if you have any more ideas here, this was just what came to the top of my mind. For me, a perfect observability system has a centralized view of systems observability. Data engineers can go, and they can access their traces, metrics, logs, dashboards, alerts, everywhere. They don't have to memorize a lot of URLs or have to go to many places to find the information. -The first question I ask all engineers interviewing for my team is: *Whose responsibility is observability?* Let’s be honest, observability is implemented differently across various companies. I've seen many approaches, and the consensus is that observability is everyone’s responsibility. If an engineer answers that question affirmatively, I know we are on the right path. +So that's the first thing. The second one is optimized data will optimize costs because doing observability doesn't necessarily mean that you will send everything that you can and rack up huge bills. But it needs to be optimized data and some good old data that is actually going to help other teams troubleshoot and monitor their systems. Obviously, reliable alerting, reliable and custom dashboards, rich traces, correlation between the different telemetry signals—so this is something that makes a perfect observability system for us. -### Defining a Perfect Observability System +And my job as an observability engineer, especially in perfect, is to provide that for the engineers. But I cannot do it alone because no matter how much I try, many—let's say most of these characteristics—are a teamwork and something that other people also have to contribute. -To understand why I believe everyone should be responsible for observability, let me outline what I think a perfect observability system looks like. Of course, this is subjective and may not apply universally, but here are the key characteristics I envision: +This means the engineers that are part of, I would say, our customers. I don't know if anyone has any other ideas about what a perfect observability system is for them. I would love to hear more about it or if you completely agree with my goals of having this system. But at least I'm with you all throughout me any moment. -1. **Centralized View**: Engineers should have easy access to traces, metrics, logs, dashboards, and alerts without memorizing multiple URLs. - -2. **Optimized Data**: Sending everything doesn’t mean better observability. It’s crucial to optimize data to control costs while ensuring it remains useful for troubleshooting. +**Derek:** Thank you! I definitely agree with a lot of the things that you said. Like, it's just, yeah, spot on. Sorry, I'm not able to see the comments here, but if there is something interesting, do let me know, please. -3. **Reliable Alerting**: Alerts must be dependable and customizable. +[00:05:30] **Iris:** Okay, so the question is: okay, you have this vision of a perfect observability system. How do we get to this observability system? Well, observability engineers like myself, my team with the knowledge to shape observability systems into the correct path, I'm not saying that observability is something that a simple engineer that is working can't get into. They can be participants and active participants in it, but of course, it is important that there are engineers who know observability better than everyone, who know what guidelines to implement, what tools to use, or how to optimize data. It's very important that these people exist in the company. -4. **Trace Correlation**: There should be correlation between different telemetry signals. +Because as I said before, observability is one of those fields that is moving so fast that it is impossible for a person that is working, let's say, with databases or in Java and their full-time job is called getting back-end, whatever else, to keep up with this. So it is important for people like us to be in the company and to preach, let's say, these values and to bring all these good tools and good things that are being developed and shared in the community. -My job as an observability engineer at Farfetch is to provide these systems for engineers, but this is a collaborative effort. Many characteristics of a perfect observability system require teamwork from various individuals who understand the nuances of their products. +Of course, as I mentioned, observability engineers also bring the data to collect and process telemetry data. But my fourth favorite so far, of course, is open telemetry, which I'm a huge fan of. We have fans, Grafana, Prometheus; those are some of the tools that are open source that are being used so massively right now by observability engineers. -I would love to hear if anyone has different thoughts or additional ideas about what makes a perfect observability system. +And now I'm getting to what the rest of the presentation will be about. It's not only observability engineers that are needed when it seems to actively participate in optimizing the data that they're sending and actively building their monitoring. Five people, 10 people, 20, depending on the size of the observability team, cannot do the work of two thousand, three thousand, depending on the work of so many engineers. Everyone needs to have control of their own code. -### The Importance of Team Involvement +So some of the things why observability is so important to be done as a team is because of how crucial it is in a company. I don't—in some, let's say that in some companies, it's still not understood how important it is to have observability. But that is, well, what I believe is because of the small scale. But the more that the companies grow, they understand how important it is to have highly available and reliable systems. It is crucial to have optimized performance because you could have—you could allocate a lot of resources, and you are not saving on cost. Basically, you're spending too much money or you're cutting back on resources, and you are basically not giving a good performance or a good experience to your users, depending on the type of product it is. -Observability is crucial in a company, especially as it grows. Many organizations still don’t grasp its importance until they face issues. Here’s why I believe teams should be involved: +[00:08:42] Also, it is crucial for incident response. This is something that is very important—in my company, incident response is very important, considering that we have a lot of customers that rely on our product, very expensive products at times. So having all the teams involved in the crucial incident response is extremely important. We have around 2000 engineers. I actually thought that we were 3000; we were actually 2000 that are working in completely different areas, some of them I don't even know, except for what they are. So when I hear about them, I'm like, "Wow, okay," because it's so big and so diverse in our company. -- **Incident Response**: In our company, incident response is vital. With a large user base relying on our product, it’s crucial for all teams to be involved in incident management. If engineers are not engaged in monitoring their systems, it could take hours to respond to incidents instead of minutes. +And if these engineers were not involved in the part of building their own dashboard, building their own alert, knowing what data they're sending for their observability reasons, they will not be able to respond to an incident in time, simple as that. If I was the one making the alerting for them, it would be something extremely generic with some thresholds that I just thought would be right because I don't know their products or their code, obviously. -- **Custom Alerts and Dashboards**: Custom alerts created by the engineering teams will be more effective than generic ones set by someone outside the team. This ensures that alerts are relevant and actionable. +The dashboards would be completely generic, but if those were done by a person inside the team itself, it would be so easy for them because they know it inside out, their work, what are some of the issues that they might have. So if an incident happens, you reduce the response from hours that it would be if it was for me dealing with their incident to seconds or minutes if someone from their engineering team is involved. -- **Auto-scaling**: Effective auto-scaling relies on telemetry data. Engineers who know their systems can set appropriate scaling rules based on their unique needs. +[00:10:40] Also, it is crucial for incident detection, as I said before. Us as engineers, the rebuilt engineers, we can provide a set of alerting dashboards that are very generic that each team can have as a baseline to implement their monitoring system, but I don't feel comfortable in a way to go and implement custom alerts, custom dashboards for these teams. Again, it goes to this: even if I wanted to, there are thousands and thousands of engineers, technologies, hundreds of teams. I would never be able to do that properly. -To summarize, observability isn’t just about having a system; it’s about ensuring everyone involved in building and maintaining the product takes ownership of observability. +That's why it's so important that each team is involved and does it themselves. If they need it, they change it; they don't need our permission; they are owners of that solution. It's extremely important and also critical for auto-scaling. I say this because in perfect, we rely heavily on telemetry data to auto-scale, because you never know the amount of traffic that you're going to have in your application. -### Challenges and Solutions +But for example, let's say for me or for another person in the observability, if we were to set some auto-scaling rules for you to be based on CPU or on memory, it would be something that is completely generic. But an engineer that knows their system so well, they would go and say, "Let's do a custom auto-scaling depending on the number of throughput," or whatever. -Observability can get messy, especially in large organizations. It’s common for engineers to add custom metrics and logs, leading to a cluttered environment. Here are some challenges I’ve encountered: +Basically, my bottom goal here would be to say that all this requires people from the teams that are monitoring the system to take part in. If it is done by people that are outside—I am an infrastructure engineer; I work with Kubernetes, I work with VMs, with Azure, with cloud, and I'm not very good at back-end coding, let's say. So if I was the one doing this, I wouldn't do it properly, no matter how much I tried and how much effort I put. -- **Too Many Metrics/Logs**: Engineers may flood the system with unnecessary data, making it hard to sift through and find valuable information. +[00:12:30] So let's not do observability for the sake of it. Yeah, let's have some generic alerts just for having them. No, let's do it properly. Let's have the people that are working day-to-day with the product implement them, make it their baby as well. It's very important to show it to the engineers. And I am very proud to say that in my company, we have a pretty good observability culture that started a few years back, so now it is widely accepted. -- **Lack of Structure**: Unstructured logs can be particularly frustrating. If engineers can’t easily query and analyze data, it hampers their ability to troubleshoot effectively. +But I've seen it happen that teams rely heavily on other people to set up alerting for them, and that is something that scares me. If something happens, if an incident happens, and I wouldn't be comfortable working in a team like that if I wasn't the one setting up my own monitoring for the system. Also, observability can be so messy depending on the scale of the company. -- **Cost Concerns**: Observability can be expensive, and it’s essential to strike a balance between collecting sufficient data and managing costs. +And so many people that come and go, everyone wants to add the custom metric, the custom log. It can get extremely messy. It can get too many metrics, too many logs, especially—for example, living debugging logs flowing all day. You know, because it can—yeah, you just sense, "I'm an engineer; I want to have full observability of my application." It's not affecting my performance, so I'm sending everything: debugging info, warning metrics. I create like a bunch of them on top of each other, or the same metric with different names. -### Creating a Culture of Observability +And yeah, what can I, as an observability engineer, do about it alone? I detect it; I see it, and I say, "Okay, this is completely wrong." But of course, I'm going to need the collaboration of the engineers that are working with our product to actually change it. Otherwise, it would never be optimized, and it should never be, let's say, good observability. -For a successful observability culture, we need to: +Too many traces—well, I'm a firm believer that there is no such thing as too many traces. I'm a big advocate of that. But yeah, it's possible that we are sending 100% of traces with a mess and completely no information there. And what can I do, considering that I'm not the one doing the day-to-day developing? I have to go and talk to the teams that they need to be active participants on that. Without it, for me, there is no durability, or it is something very generic just for the sake of it. That happens so much, and to be honest, it rates me. -- **Educate and Empower**: Provide engineers with the tools and knowledge to take ownership of observability. It’s crucial to avoid judgment and create a supportive environment where engineers can learn and adapt. +[00:15:00] But there are also too many dashboards, too many alerts. If someone else creates the alerts or they're just created automatically, you have them ringing back and forth, and nobody actually looks at them because they know that it is spam. So it is very important for people to go and for engineers to go there and say, "Hey, okay, I don't need this alert; I need this. I need to create this." And the same for the dashboards. -- **Be Patient**: Transitioning to a robust observability culture takes time, especially for new team members who may have different experiences from previous roles. +It can be thousands and thousands of dashboards, and when you actually have an incident, that's at midnight, 3 a.m. in the morning, you wake up, your eyes cannot open, and you have to scroll through thousands of those or try to find a person that might know what this is. It's very important to be about as simple as that. -- **Foster Collaboration**: Encourage a collaborative atmosphere where observability is a shared responsibility, and everyone feels empowered to contribute. +Also, it is not easy; it makes it easy. I think they go hand in hand. It's, let's say, mostly the same thing with too many alerts and dashboards, and you just cannot get the hang of it, and you just give up. So basically, you do not rely on this to troubleshoot them. On what do you rely? Maybe the application logs or it takes you hours to solve an incident, which is not great either. -### Conclusion +Too few or too many telemetry signals—I talked about too many, but there is also one thing: it is possible that you don't know how metrics are, how logs are generated, how traces are generated. In this case, that same place is considering right now, it’s this huge movement to make them first-class citizens. Many engineers, let's say, do not know how important tracing is and not how the great capabilities that they can achieve to that. -In summary, we all need to work together to cultivate a great observability culture and system, making our lives significantly easier. If anyone has questions or would like to share their experiences, please feel free to speak up! +And my job as an engineer would be to advise them and to help them with it, but not to do it for them because I don't know the type of information they might need. It's simple. And it is not cheap. Absolutely, it is not cheap. Depending on the scale of the information that we are having, it could rack up to millions. ---- +I was reading some articles recently about some companies that were paying a huge bill in observability, and the question in those articles was, "Who wants to blame?" And the stack of different billing is a team sport. It is also the blame is to share because the engineers of observability are clearly doing a great job at educating, or maybe they're not even there; that could be one of the cases, or the teams are not participating. -**Derek**: One comment related to what you said about observability is the ability to form rich queries against data once it's in your analytics backend. If you can’t query effectively, it leads to frustration. +Basically, you have this framework that just sends 100% of logging, tracing, metrics, and that is all sent somewhere. So yeah, observability is something that is a very delicate topic when it comes to costs because so many engineers consider it as not a very important thing until they actually need it. For example, when something happens and they don't receive an alert, "Oh my God, it is actually important!" Or "Oh my God, the information on that dashboard now would be so nice!" -**Iris**: Absolutely, I completely agree. +But at the same time, we have to know what to collect, how to, and to also advise our team to do it and to empower them to do it themselves. It's not an easy task, especially when it's so comfortable for them to have all that information. Let's say, for example, logging. I have a vision of talking in general, but nothing against it; it's used so much. They say, "Okay, I just will have the debug logs, and even if something happens, I will have it there." -**Derek**: What was the observability culture like when you joined your company, and how did you make improvements? +Yeah, it's our job to convince them to show them and to actually make a plan and follow it through together. I would say our mission as observability engineers is not just to teach people and to show them because sometimes that sounds like, I don't know if it is the right word, but like stack ups. "Oh yeah, you do it." It's not that; it's just trying to do it in a collaborative manner to consider it as a teamwork. -**Iris**: I was fortunate because my company had already laid a strong foundation for observability. We continue to improve it by holding presentations, sharing articles, and being available for questions. +That's a real team. It's not an external team that just works completely away from you, but it's, let's say, part of your team. And at the same time, it's not—it serves as an advisory. -**Derek**: How have you encouraged application teams to take ownership of observability without overwhelming them? +So how do we change to this observability culture? It is very, very difficult. Because of my passion for observability, I talk a lot about it, and it is very, very difficult to convince an engineer with a higher grade than you to follow a certain observability culture if sometimes it's not even the engineers themselves. -**Iris**: We promote ownership by making it clear that each team is responsible for their monitoring and alerting. We’re here to guide them with documentation and support, but the responsibility lies with them. +[00:20:00] It could be senior leadership; it could be higher-ups. They just do not believe it, and some companies—this is very prevalent and it's very difficult to crack in there to actually implement it. But I think that our job is to make a change. We need to be ambassadors in our companies, and I think what we're doing today and all these sessions are a great way for us to talk and to share opinions and to know how to approach our teams better, how to be better at what we're doing, and to actually show the importance of our work. -**Derek**: What has been the reaction from teams when you enforce this? +We need to show the importance of the observability of the data. In fact, you can just go and say, "Hey, observability, a rainbow is amazing, beautiful." We have to show data, and that's very important to collect and very easy as well, considering the amount of incidents that are happening. Or you just show how easy it is to get information about one aspect of the application that they never thought about. -**Iris**: Most teams respond positively. They often ask for guidance, and we have the support of management to reinforce this approach. +It's important to provide up-to-date tools. Our job is amazing because we can work with someone in modern technologies that are moving so fast. At the same time, it is very difficult because you need to provide tools that aren't mature enough. You have to provide tools that the engineers need and are looking for, and sometimes this can get overwhelming. But it is part of the challenge, and actually, this is my favorite aspect of the job: how fast it moves and how fast you have to adapt. It's part of being an observability engineer. -**Iris**: Thank you all for the insightful discussion! If you’re interested in sharing your stories or participating in future sessions, please reach out on Slack. Thank you for having me today! +Yeah, we have to provide guidelines. Not everyone knows—I mentioned this before, but I'm a firm believer of not judging. It might sound like a childish thing to say or kindergarten issue, but I know that tech people judge each other. "Oh, you don't know that much about Kubernetes parts of durability." It's very important for us to offer help and to avoid judgment of that sort because observability is not difficult, but at the same time, it's not easy, as I said before. + +It's very fast; it's moving; it's modern. You need to adapt very fast, and the person that is completely detached from that will not be able to do it. So there is no place for judgment here, and I think that if there is judgment, that doesn't make you a good observability engineer. I think we're like the saints of the engineers. + +And also, we need to give space and time to adapt. In some companies where the durability cultures exist, it's easy, obviously. If you have a majority of engineers that follow these practices, let's say, it becomes very easy for it to spread. But when you're starting from zero, it needs time. People that have been working in other companies that have completely different working mentality, it's going to be difficult for them to just land in the company and be like, "Ah, okay, now I accept observability as this amazing thing that it is." + +No, we have to give them space and time to adapt, show them, be patient, be there to help, to implement. Even an engineer that is excellent at his job but has no idea how observability works, we have to be there for them. I think the human aspect of this job is more important than the technical one because it is good; it's important to be technical and to adapt and to build, but also the human aspect and spreading the culture and talking to people and being a people's person professionally. + +You don't have to be an extrovert and have these beautiful conversations. It's a very important aspect of observability. + +[00:24:00] These are all the slides that I had prepared for this presentation, but I just want to say my summary, or let's say the message that I have through this, is that we all need to work as a team to have a great observability culture and a great observability system, and that's going to make our lives much easier. + +So that's pretty much it. I don't know if you have any questions, please let me know. + +**Derek:** It was a comment from Derek, actually. I think when you're talking about what is important in observability, they mentioned the ability to form rich queries against the data once it's in your analytics backend. It's when you're just asking what's important in the design of a perfect system. You can pop all the perfect data that you want into your backend, but if you don't have a way to quickly and effectively query that to get the answers you want, then you're just going to be frustrated. + +I think that's one of the reasons why we get frustrated with unstructured logs so much is because you end up having to craft these complicated regexes and pattern matching of unstructured strings, and it's really frustrating to try and find what you want. So if we can do our part to make sure the data going in is nicely structured, it makes it a lot easier to get the signals you want out of it. + +**Iris:** Absolutely, absolutely agree. + +**Derek:** I had a question too, but I can go to the back if someone else has one. + +**Iris:** Oh no, go ahead, Derek. + +**Derek:** I have one, but I can go after. + +So, I guess my question would be: what was the observability culture like when you joined your company? And kind of like what was ground zero? Where were you starting from? And then kind of how did you make improvements to that, or how did you kind of watch that culture improve? + +**Iris:** So actually, I'm pretty lucky when it comes to that extent where in this company, when I joined, I think the previous team and just a few members that are currently part of my team had done most of the heavy lifting. They had started the spreading of the observability culture and how to do it correctly. + +So when I joined, I think everything was put in place. But of course, we are improving that day-to-day because you can have a culture there, but you have to, let's say, you have to water it every day and you have to keep it and to improve it. + +So what do we do? We have presentation sessions with other engineers in the company showing them how to use our tools best of or what our tools can do. We are always open—the team—for just a Slack message to help them on how to create metrics dashboards. Even though the simplest questions, we're always there for them. We also share articles, share presentations for observability, and it's actually interesting because many people read them. + +And we're also making sure that we're also always on top of the new technologies. In this case, for example, it was open telemetry; it was something that was very well accepted in Firefits, and some people were actually requested, so we provided. And just saying like always providing with the newest technologies, with the newest updates, so nothing is missing. And if, I don't know if it's called "bribing," but for example, we make sure that it's one of our engineers who reads something on the Internet. It's like, "Oh, this new cool feature in Grafana that I saw! Can we have it?" We make sure that we've had it before we always make sure that we're on top of things because that makes people want to use the system more and use it well and implement all the new changes. + +So it's great for us. + +**Derek:** That's really good. If I can ask one more quick follow-up question, then I can give someone else a turn. So in your slide deck, you talk about how developing ownership by everybody of observability is super important for success. Because ultimately, like application engineers, they're the ones who have the context for how their applications run and for the things they need to look for. + +So how, in your experience, have you gotten application teams to kind of take that ownership without making it feel like it's just one more thing you're trying to chuck over the wall at them? Right? Because again, like it's a unique spot that we're at as infrastructure platform people because we're not like the best at writing code, like we're not—that's not going to be our strongest skill. So it feels weird to me, at least in the past, saying, "Hey, can y'all go learn this thing and put it into practice, and I'll coach you from the sidelines? But I can't do it as well as you." + +**Iris:** Yeah, I think what helps, as I said again, we are lucky because we have a very good observability culture already ingrained. So most of the engineers just do it. But yeah, I think what really helped was that we have given full ownership to the teams on building their own dashboards, alerting, and sending their own telemetry signals. + +So basically, when they come to us like, "Hey, can you help us set up an alert?" We're like, "Well, say you are full owners of that part of the observability," and that actually helps the teams knowing that they're owners of that. They want to make it grow and improve it rather than just leaving it there. + +And we've also made it very clear that we will not be going there and implementing anything related to monitoring. So if you need monitoring, you have to do it yourself. We are here for you; we provide guidance, of course. We have tons of documentation on how to do everything. They're not alone, but we've made it very clear that we're not going to do that for you. It's simple. + +And considering that all our incidents are coming from the alerting that is set up on our platform, let's say they're kind of forced to implement it. But I think just the fact that they're owners of that, and it's part of their team responsibility coming from the higher structures of the company, "Okay, you'll have this amazing product that you are building and maintaining, but you also have observability that is your full ownership." So if it's not good, then it is your responsibility, not somebody else's. + +So I think that has motivated the teams to really work on that. Sometimes it happens that when a junior enters the team, they're the ones that have to take care of all the monitoring without knowing well the code, but I think they soon realize how important it is for everyone to put their hands in to contribute there; otherwise, it does not work out. + +**Derek:** That's great, thank you! + +**Iris:** I have a question for you. It is, um, so you're saying like you guys hold your ground as far as like not being the ones to instrument code for folks. What kind of reactions do you get from teams when you say that? Like, are they like, "Oh, okay, I'll do it," or did they—like defensive pushback? Like, what's it like? How do you deal with it? + +**Iris:** Most of the time, they're like, "Oh, okay. Is there a framework? Can you send me to the framework so that I can find it?" And we usually also have a team that helps build this framework, so we send it to them. But most of the time, yeah, we just hold our ground. If they say that, "Oh, but I really need to help me," like, "Okay, I'm gonna help it. I don't know how to do it for yourself; I don't know your code." + +And of course, there are cases that people are not happy, and they could go to a higher structure in the company and complain. But considering that this is not just something that as the durability team have decided today, that, "Oh, we're not going to build anything for anyone because it's their responsibility." It's not something that is, of course, supported and signed by upper management. They're like, "Sorry, but you have to do it; it's documented, and you have to do it." And it's simple as that. + +**Derek:** It's great having support; that's awesome. Because I think you've nailed it: that is the key thing. Because you're right, like if you don't have the management supporting that decision to like, you know, have an observability team, we're not going to instrument your code. Yeah, you're gonna have back channeling, and I mean, I've experienced that—it's really annoying. + +And it ends up kind of ruining the thing that you're trying to build, like as far as that observability culture. + +**Iris:** Yeah, exactly. As I said, yeah, in Firefits, we are very, very happy in this aspect because, yeah, being forced to instrument somebody's code, that would be the breaking point for me as an engineer in believing that observability is lived preaching with everyone and talking about it and then being forced to go against these values that we have. That would be the breaking point for me; I have to stay. + +**Derek:** Yeah, I don't blame you. + +**Iris:** I have a question. Firefits is an e-commerce site, correct? And so I was curious—and Derek, I guess actually this question is inspired by your question in the hotel end-users channel this morning. How do you—or do you—like, what do you use for client-side instrumentation? Are you— + +**Iris:** Well, we are using a vendor currently. I’d rather not say, but yeah, we're using a vendor for that part, which is very, very small. The rest—the rest of the platform is all instrumented by our own custom framework. + +**Derek:** Okay, okay. Do you have that integrated at all, or is it kind of like its own view of the client? And then you have trouble correlating signals with your back end? + +**Iris:** Currently, it is a bit segregated, but we're working on changing it because it's such a small part of our platform. But yeah, it's not fully integrated. We're focusing more on, let's say, on our own custom solution that we have locally, and that one is a bit segregated from it, and it has only specific information, which is not great. But that's our goal for 2023 and 2024 to integrate everything together. + +**Derek:** What's your custom solution involve? Like, what makes it custom, I guess, is the question? + +**Iris:** I mean, it's using open telemetry now. We're using thousands, Prometheus, Grafana, Alert Manager—basically all open source—and we built it, let's say, we have around 100 Kubernetes clusters. We have created our agent based on this open source and just adding them on each of the classes, collecting information, centralizing it. + +**Derek:** Oh, so custom solution, like you're talking specifically around tooling and not like—I thought maybe you meant like having some abstraction layer on top of like telemetry, which I've seen some organizations do. + +**Iris:** No, no, no. + +**Derek:** Okay, cool. + +**Iris:** Any other thoughts or questions for Iris? + +**Derek:** Yeah, I guess I'm a little curious. So you said, you know, there's no such thing as like too many traces. And I know some people, you know, prefer to do some form of tail sampling. So they're just—they're still getting like a subset of, you know, randomized 200s and then like, you know, whatever traces with errors or latency or whatever attributes that they might be interested in. + +So I was just curious if you could expound on that a little bit more, especially like, you know, when talking about like increased costs with excessive amounts of data. + +**Iris:** Yeah, so tracing is something that we are working a lot because it's not one of the telemetry signals that are being used a lot in the company. So we are trying to advocate it and push it a lot. And for your—when I say too few traces, I would say, uh, 0.01 sampling, like normal sampling, let's say, in younger. So the teams were considering it extremely low, and they weren't even caring about it, even though some of them you could find very good information there. + +For example, I use it a lot to troubleshoot our Thanos queries; it's amazing. So where we are right now, we increased because we were able to change our back-end solution from Cassandra to Grafana Tempo, which is also open source. So of course, before we were spending a crazy amount of money for 0.01 tracing, we were spending so much money. I don't want to say numbers, so there was no place for us to grow more; it was completely out of budget. + +So now we implemented Tempo, and we are able to send more traces, and we increased to five percent and increasing that just a few applications. And actually, uh, implemented this change because, of course, it's a custom framework, and they can do it however they want. And we already went for 1,000 spans per second to 40,000 from just one application, and this application is not even using most of what they're sending. + +And imagine we have hundreds of applications, all of this sending—all that is going to be millions of spans per second. It's going to require a lot of computation power, and it's going to require a lot of storage. It's going to be a mess, and the cost will be out of this world, in conclusion. So I think that we really need to go there, like we're in the process of doing, and try to see—to implement better tracing because just normal sampling is not working. + +Yeah, we're looking into a tail base; that's why open telemetry was brought up in the first place. So the perfect—to look at the tail base—and yeah, we were sending too little or basically there was not enough information. Now we're sending too much, so we are trying to find something that is in between. Tail sampling is one of them, and also we're also making changes to the framework so not everything goes through. Some of the information is just not usable, and the engineers are not needed. + +Yeah, it's a work in progress that is taking a lot of our time. I'd rather spend more money and send more information, obviously, but at the same time, we could not spend millions of dollars a year for information that's not going to be used. + +**Derek:** I see a raised hand. + +**John:** I did. I wanted to ask if you're doing anything around anomaly detection, especially as you increase the volume of information coming in, to avoid having to look at everything to look at those anomalies that are out of the time frame. + +**Iris:** You mean anomaly detection in the—? + +**John:** Yeah, well, so the scenario of, you know, here's a high volume event, but it's normal, and here's a high volume at what's normally a low time of day, right? Just to look for anomalies and everything that you're receiving without having to have a human look at it. + +**Iris:** Yeah, we actually have some alerting implemented there based on standard deviation, and it's like a simple PromQL-based alerting for that. And they are accurate, but we haven't gone about that because it's a custom solution, and we actually need to apply some machine learning there, which we haven't really been able to get to. + +So it's do some alerts with the standard deviation that are okay; they can detect some anomalies and some very big changes, but also are highly inaccurate. So it's not my favorite, but they are there. We need to work better on it, but yeah, for example, if we have a huge anomaly or a huge flow of sudden that is not predicted, it is okay, let's say, it's not perfect. + +I know you can get into some really interesting solutions with that that can come with a high cost, but there's some really interesting things you can do with that too. So I would love to know more if you have any links, any suggestions for me, please let me know, because the more solutions we can implement, hahaha! Like you mentioned, cost is always a worry, obviously, but if it's actually a quality solution that we are introducing, of course, it is a cost that is needed. + +And that's it, you know. It's really enjoyable when you start to use tools like that to solve the problems because then you see things you didn't expect to see. We were experimenting with something a few years ago, and we intended to point at the top 10 issuers. And, you know, the engineer mistakenly didn't put the top 10 in, so he was actually looking for anomalies across the entire user set. + +So, you know, thousands of customers, and they were in entities. So, you know, you can start to see, "Oh look, they're not sending at this time of day when they usually are! That's a low volume, low center, but they're sending nothing, and they usually send 70 per hour!" And we could see that before it finally blew up from running out of memory. But you know, you really get to say, "Now what can I do with that for my business?" + +So that was interesting. We presented to them at the team who wanted to own that and didn't want to develop the expertise in that tooling, even though we could show them what it could do. So a 500 gigabyte EC2 instance was running that before it blew up. + +**Iris:** Yeah, a challenge that we have currently is with our metrics as well, and detecting which applications are sending higher technology, more number of times series. And we have dashboards and alerts for that, but nothing is based on machine learning. It's just pure statistical data, and it is being evaluated for standard deviation or simple queries. + +So some more insight there would be amazing. + +**John:** Yeah, that too much data problem is very interesting to have, especially as the open telemetry standards evolve and add more fields. So now you have more fields to consider. + +**Iris:** Yeah, and they're good. It's good that those are coming, absolutely. + +**John:** One thing that I wanted to ask you is: is your team involved at all in the creation of SLOs? + +**Iris:** Foreign, yes, but also no. Well, let's say we are the ones that are providing the tool for creating SLOs. + +So basically, the teams can implement the structure or basically create alerting dashboards based on it, but we are not the ones that are providing the guidelines and the instructions on how to create them. We just provide the tooling. + +**John:** Is it like an open-source tool that you guys are using, or is it some like commercial? + +**Iris:** We're actually using—we have our own alerting solution, so we are doing the same for SLOs as well, and we are currently using Thanos ruler to send the alerts and to evaluate them, Alert Manager to send the alerts, and we also have a custom tool that reads the SLOs that are written in Terraform and makes them compatible with a Thanos ruler, so Thanos ruler can read them. + +**John:** Okay, and do you—so the reason why I'm asking specifically about SLOs is because one of the, I'd say, one of the practices a team should aim for is like using observability data to help generate their SLOs or not generate, to, you know, create their SLOs based on observability data. + +So I'm just wondering if that's something that your team is planning on providing guidance on? Like, what's—are there any plans around that? + +**Iris:** So basically, we are providing the data and the tooling to do it. Most of the guidance, like how the SLO should be built, most of them are our business, and we're not really involved in that part. + +So we cannot give guidelines, so that's completely different. But yeah, for example, for infrastructure, we provide some basic guidelines on SLOs. But I would say that, yeah, we just want to provide the data; they feed off of our metrics and the tooling that they can do it and how they can actually build them, but not really how we can go and or like how to calculate the SLOs or what is best and what is worse. + +And we haven't gotten there. I don't think we have the plans currently to move into that as we have other teams that give advice or recommendations on that matter. There is so much to do in observability in our company. I think we've done a pretty good job so far, but there is so much more that we can do. + +**John:** Is there something that your team is working on implementing, like that you're excited about in the near future? + +**Iris:** Well, we decided that because of the huge amount of data and we do not have capability to process it as good as we would like, we are currently trying to see if we can—well, let me put some precedent. We are using different tools for different things. We're using Prometheus styles for metrics, we're using open telemetry for traces and open telemetry for metrics to a certain degree as well, Grafana for dashboards, and we're using a different tool for logging. + +And currently, it's all a mess. So what we're trying to do right now is we want to centralize them in one platform. What we don't know yet if it's going to be an outside platform that we are feeding the data to or like a vendor or if we're going to provide it ourselves. So that's something that we're working on right now: centralizing everything in one place and sending all telemetry signals through one transporter, that is open telemetry, so we could have better correlation, which we are also lacking currently. + +**John:** Awesome, so exciting times! + +**Iris:** It is amazing! I love the work that you're doing right now, and it's going to be the next two years at least. I know after that, it's going to be very exciting for us. + +**John:** Well, it's very, very awesome. We're coming up on time, but I did want to give folks an opportunity to ask any other final burning questions. + +Nope, everyone's questioned out. Thank you, everyone, so much for joining. And if anyone who's in the audience is interested in giving a presentation for hotel and practice or would like to participate in one of our hotel Q&As, please reach out on Slack. We are more than happy to hear your stories, share your stories so that we can continue to build this amazing open telemetry community. + +Thank you so much for having me today! ## Raw YouTube Transcript diff --git a/video-transcripts/transcripts/2023-07-02T17:46:37Z-otel-q-a-feat-jacob-aronoff-of-lightstep-from-servicenow.md b/video-transcripts/transcripts/2023-07-02T17:46:37Z-otel-q-a-feat-jacob-aronoff-of-lightstep-from-servicenow.md index fc17e4f..60c2ed1 100644 --- a/video-transcripts/transcripts/2023-07-02T17:46:37Z-otel-q-a-feat-jacob-aronoff-of-lightstep-from-servicenow.md +++ b/video-transcripts/transcripts/2023-07-02T17:46:37Z-otel-q-a-feat-jacob-aronoff-of-lightstep-from-servicenow.md @@ -10,77 +10,405 @@ URL: https://www.youtube.com/watch?v=dpXhgZL9tzU ## Summary -In this YouTube Q&A session, Jacob Arnoff, a staff engineer at Lightstep, discusses his experience with migrating to OpenTelemetry (otel) from OpenTracing and OpenCensus while working at Lightstep. The conversation, led by the host, covers the challenges and strategies involved in this migration, emphasizing the importance of a safe and efficient transition for observability in software applications. Jacob details the initial use of proxies for metrics collection, the performance issues encountered, and the iterative approach taken to migrate tracing and metrics to otel. He explains the various deployment options for collectors in Kubernetes, including sidecar, daemonset, and stateful set, and highlights the benefits and considerations for each. The discussion also touches on the significance of keeping telemetry systems up to date, the advantages of using tools like Dependabot, and the ongoing developments in the open-source observability community. Overall, the session provides valuable insights for organizations looking to adopt OpenTelemetry and improve their observability practices. +In this YouTube video, Jacob Arnoff, a staff engineer at Lightstep, discusses his experience migrating to OpenTelemetry (otel) from OpenTracing and other telemetry solutions within the company. The conversation, facilitated by the host, explores the challenges and strategies involved in the migration process, emphasizing safety and performance improvements. Jacob shares insights into the initial migration from OpenTracing to OpenTelemetry, the issues encountered while migrating metrics, and the advantages of using features like views for managing metrics effectively. The discussion also touches on deployment strategies for collectors in a Kubernetes environment, comparing sidecar, daemon set, and stateful set approaches. Throughout the conversation, Jacob emphasizes the importance of keeping dependencies up to date and how this impacts the overall performance and reliability of observability systems. The session concludes with an invitation for Jacob to share more of his insights in future talks, highlighting the ongoing evolution of telemetry practices. ## Chapters -Here are the key moments from the livestream along with their timestamps: +00:00:00 Welcome and intro +00:00:20 Guest introduction: Jacob +00:02:30 Jacob's background and role +00:03:50 Migration to OpenTelemetry +00:05:41 Migration challenges and strategies +00:09:00 Q&A session begins +00:12:01 Metrics migration details +00:16:02 Performance improvements discussion +00:18:00 Metrics views and aggregation +00:22:02 Logging and span events +00:24:50 Collector setup and architecture -00:00:00 Introductions and welcome message -00:01:30 Introduction of Jacob Arnoff from Lightstep -00:03:00 Discussion about Jacob's experience with OpenTelemetry -00:05:30 Overview of Lightstep's migration from OpenTracing to OpenTelemetry -00:10:00 Challenges faced during the migration process -00:15:00 Explanation of the metrics migration strategy -00:18:30 Discussion on the performance issues encountered -00:25:00 Insights on using attributes and metrics in OpenTelemetry -00:30:00 Overview of the differences between deployments, stateful sets, and daemon sets -00:40:00 Explanation of the Target Allocator and its benefits -00:45:00 Discussion on keeping dependencies up to date and the importance of regular updates +[00:00:20] **Host:** Thank you. Welcome everyone to Hotel Q&A. Thanks for joining. Today we are super lucky to have Jacob Aronoff from Lightstep from ServiceNow join us. It's a cool treat because I tapped Jacob because he was telling me that he had submitted a CFP to one of the, I think it was KubeCon, right Jacob? -These moments encapsulate the essential discussions and insights shared during the livestream, making it easier for viewers to navigate the content. +**Jacob:** Yeah, it was a CFP on migrating to Hotel from our organization—migrating to Hotel, right? If I understand correctly, which I thought that's a pretty freaking cool story to tell. So I asked him to join and have this Q&A and hear the story because I think it makes for a compelling narrative when an observability vendor talks about using OTEL themselves on their own products. So here we are. So welcome Jacob. Do you want to do like a quick little intro? -# Hotel Q&A with Jacob Aronoff +**Jacob:** Sure. Yeah, hi, my name is Jacob Aronoff. I'm a staff engineer at Lightstep on the Telemetry pipeline team. I've been in lead stuff for almost two years, and the first year of that journey was solely focused on Hotel migrations—doing them internally and making them easier for customers, sort of that whole process. So yeah, I could get into this anyway that is interesting. How do you think we want to do this? Very Q&A style or should I just go right into the story? -Thank you for joining us for today's Hotel Q&A session. We are fortunate to have Jacob Aronoff from Lightstep, part of ServiceNow, with us today. Jacob recently submitted a CFP for KubeCon, focusing on migrating to OpenTelemetry (otel) within his organization, which is an exciting topic to discuss. +**Host:** We can let it evolve organically. -## Introduction +**Jacob:** Cool. Yeah, I like that. Why don't we start like— I mean I think you're rearing to go, so maybe why don't you describe, like, set the scene for us? -**Jacob Aronoff:** Hi, my name is Jacob Aronoff. I’m a staff engineer at Lightstep on the Telemetry Pipeline team. I have been working on this for almost two years, with the first year dedicated to internal migrations to OpenTelemetry and making the process easier for our customers. +[00:02:30] **Jacob:** When I joined Lightstep, we were still on OpenTracing for tracing and then a mix of OpenCensus and some hand-rolled StatsD stuff for metrics. So this meant that we had to run a proxy on every single pod that we ran in Kubernetes. A proxy, since it's a sidecar on every pod, means that every single time you run one of your applications, you have to run another little application that's going to read from StatsD and then forward those metrics off. And, you know, for us, as we're building out a metric solution, that destination was Lightstep. -## Migration Story +[00:03:50] I came in, and this was sort of at the beginning of metrics Alpha, I think. I was like, hey, it would be great for us to—we know that OpenTelemetry for metrics is going to be a huge effort for us in the next year or two. We wanted to reach stability. The Hotel team had been—we have an Hotel team internal to Lightstep. They've been working on it a lot and really wanted some immediate feedback on how to improve it. So I took on that migration for us. -When I joined Lightstep, we were still using OpenTracing for tracing, alongside a mix of OpenCensus and some hand-rolled StatsD metrics. This setup required running a proxy on every pod in Kubernetes, which was costly and complex. As we were moving towards OpenTelemetry for metrics, I took on the migration effort. Our internal OpenTelemetry team had been working hard on it and needed feedback for improvement. +We also had the theory that doing so would save us a good chunk of money because we would no longer need to run these relatively expensive StatsD sidecars. So I planned it initially to be sort of as safe as possible. I'd done some migrations like this in the past, and there are a few different ways that you can do migrations like this. You can do the all-in-one go, which for us would have been possible— we're in a mono-repo—but it's much more dangerous because you worry about, you know, am I going to push a bug that's going to take everything down, right? -During this migration, I aimed to make it as safe as possible. Since we were in a monorepo, we could have done an all-in-one migration, but that approach is risky. Instead, I opted for a feature-flag-based migration where we could disable the StatsD sidecar and enable the OpenTelemetry code. +Obviously, this is application data. It's data about your applications, which we use for alerting. We use to understand how our workloads are functioning in all of our environments, and so it's important that we don't take that down because that would be disastrous for us. But obviously, for an end user, it's going to be the same story. They want the comfort that if they migrate to Hotel, they're not going to lose all of their alerting capabilities immediately. They want a safe and easy migration. -### Performance Issues and Discoveries +So that was the only one with our initial approach for doing a sort of feature flag-based—just like part of that configuration that you run in Kubernetes. It would disable this sidecar, enable some code that would then swap to OTEL for metrics and then forward it off to where it's supposed to go. So that was sort of the path there. -Midway through the migration, I tested everything in our staging environment and found performance issues. I collaborated with the OpenTelemetry team to identify the cause, which was related to the attributes used in our metrics. This led me to theorize that by migrating from OpenTracing to OpenTelemetry, we could also see performance improvements. +[00:05:41] Midway through this journey of doing these migrations, I had tested it all out, and staging looked pretty good. I tested the container in our meta environment. So we use to monitor our public environment. I noticed some pretty large performance issues in those 2021 and I had reached out to the Hotel team, and we had worked together to sort of alleviate some of those concerns. One of the ones that we found that was a big blocker was we heavily use attributes on metrics right now, and it was incredibly tedious to go in and figure out which metrics are using all these attributes and getting rid of them. -I paused the metrics work and began the tracing migration. This time, I decided to try an all-or-nothing approach, as the OpenTelemetry path was more straightforward and backwards compatible. I managed to complete the migration with minimal issues, and it turned out to be a significant improvement in performance. +Well, I had a theory that really this one code path was the problem where we're doing the conversion from our internal tagging implementation through Hotel tags, which had a lot of other logic with it that was pretty expensive to do on pretty much every call. So I was like, you know what? There's no better time than now to begin another migration from OpenTracing to OTEL basically. While we wait on the Hotel folks on the metric side to push out more performant code implementations for us, we can also test out this theory that if we migrate the Hotel entirely, we're actually going to see even more performance benefits. -## Insights from Migration +So at that point, I said, okay, I'm going to put a pause in the metrics work while we wait for Hotel, and I'm going to begin on this tracing migration. However, I decided to try a different approach, which is the all-or-nothing approach. Basically, the OpenTracing to OpenTelemetry path was a bit more known. There were a few really small docs and examples, and they are backwards compatible. You’re able to use them in conjunction with each other, so one thing emitting Hotel and one thing emitting OpenTracing is not the end of the world. -This migration story is valuable for organizations at different stages of their observability journey. Those just starting with instrumentation and those who have dabbled in OpenTracing can learn from our experience. +So you can mix those as long as you have the propagators set up correctly. So first step, propagators. Second step, make sure that all of our plugins worked, which at the time they weren't open sourced. Now they are open source, so people can just use them. I just did it all in one swoop. Maybe I had to revert like three times from our staging environment—nothing really major. And then there was one bug that I missed where we were previously doing a lot of in-app sampling because we had a really noisy function call. So I had to implement the custom sampler, which is actually like ten times easier with OTEL than it was with OpenTracing. I was able to get rid of a good like 1000 lines of code or so and some really dangerous hacks, so that was a really good thing. -### Starting Points +And yeah, so then that went out very happy. Also stopping at any time if we want to get back to like more Q&A stuff. -When migrating, I suggest starting with smaller services that have low traffic but enough consistency to gather meaningful data. By comparing metrics before and after migration, you can identify any discrepancies. +[00:09:00] **Host:** I have so many questions from this already, but do you mind taking like a quick pause? -## Challenges and Solutions +**Jacob:** Sure. -One challenge was ensuring that all attributes remained consistent after the migration. This required close monitoring and sometimes rewriting processors to ensure compatibility. +**Host:** A couple of comments. One thing that came to my mind as you're saying this, I'm like, this is actually really freaking cool story because I think we tend to see like two different types of organizations. We see the ones where there's like zero code instrumented—like this is their first foray into instrumenting their code—and then we see the organizations that have either dabbled in OpenTracing. I think it's a really cool story because this is like a real-life migration story where you can actually provide advice on this is what you can do if you find yourself in this situation, which is really cool. -### Future Directions +I want to call that out because I think that's a really important thing, especially if you're starting to get into OpenTelemetry. The other thing that I wanted to ask you about because you said that it’s a monorepo, so in that case, did you find it—and especially since you did the all-or-nothing approach—did you find having a monorepo more challenging than if you'd been dealing with microservices instead? -With logs maturing in OpenTelemetry, there are potential plans to replace span events with logs, which is an exciting development. However, my focus has recently been on improving infrastructure metrics in Kubernetes. +**Jacob:** Yeah, well, so we do use microservices. It's just that we have a repo of microservices—sorry, my bad about you guys. In that case, yeah, then how do you know—because I mean yes, you're going for like a big bang approach, but you got to start somewhere. So then like where do you start? -## Collector Setup +**Jacob:** I started with—and it was the same with how I started the metrics migration—I started with really small services that my team owned that were really low traffic, but enough for it to be constant. The reason that you want to pick a service like that is if it's too low traffic, if you're only getting like one request every minute, one request every like ten minutes, you have to worry about sample rates. You might not have a lot of data to compare against. Really, that's like the big thing that you need to have is some data to compare against. -We run various types of collectors at Lightstep, including metrics and tracing collectors. Currently, we leverage a mix of deployments and daemon sets depending on the use case. +[00:12:01] I wrote a script early on for the metrics migration that just queried different build tags that are on all of our metrics. So you would say, you know, query all of the metrics for service X, grouped by release tag, and if you see that the standard deviation for the newer build tag is, you know, greater than one—right? So if it's one or more standard deviations away from the previous release, then there's probably something going wrong in your instrumentation library. -Daemon sets are particularly useful for node-level scraping, while deployments are more efficient for stateless collectors. Stateful sets are less common but beneficial for certain scenarios where consistent IDs are necessary. +If you assume that your metrics are relatively stable, then if they're not, it's important to know. The other thing I had to check for was that all of the attributes were still present before and after migration, which is another thing that matters. Sometimes they weren't because something might be something that StatsD just adds automatically that we don't really care about, and so those were acceptable. I just like hand waved and said those are fine; we don’t care. -## Keeping Up to Date +For tracing, it's sort of the same deal where I picked a service that had both internal-only traces—traces that stayed within a single service—and then traces that spanned multiple services with different types of instrumentation, so coming from like Envoy to Hotel to OpenTracing. What you want to see is that the trace before has the same structure as the trace after. So I made another script that checked that those structures were relatively similar and that all of them had the same attributes as well. -To ensure everyone stays updated with the latest versions of OpenTelemetry, we use tools like Dependabot. It’s crucial to keep dependencies current to avoid security vulnerabilities and compatibility issues. +**Host:** Right, right. Because tracing attributes, again, I was doing an attribute migration. That was really the point of doing the tracing one, so what matters is that all the attributes stayed the same, right? -## Conclusion +**Jacob:** Yeah, it's interesting too because you're starting from the point where you were migrating from an existing thing. You have that frame of reference, which I guess is kind of a double-edged sword, right? Because on the one hand, it's like you pretty much know that you've instrumented the things—hopefully. Maybe you’ll discover as you go along that there's like more stuff to instrument, but at least you have a baseline to start from. But then I guess on the other hand, if something's missing, you're like, oh damn, why is that missing? -Thank you all for joining us for this informative session! I hope you found the discussion on the migration process and collector setups helpful. If you have more questions or topics you'd like to explore, feel free to reach out to me or engage in future discussions. +**Jacob:** Yeah, and those, you know, "why is this missing" stories are the really complicated ones because, of course, sometimes it's easy to just like, you know, oh I forgot to add this thing in this place, and that's usually pretty simple. But sometimes it's like, oh, there's an upstream library that doesn't emit the thing or Hotel. And now I need to—again, this is like early stuff. Most of these have all been upgraded and are fine now. -Thank you again, Jacob, for sharing your insights and experiences today! +But there is an example with like our gRPC. Actually, this is like an interesting one. I had done a gRPC migration on a gRPC util package, which I think is now in like token trip—the Hotel gRPC trip. There was an issue with propagation. I was trying to understand, you know, what's going wrong here? And when I looked at the code, it just tells you how early in this story I was doing this migration, where there was supposed to be a propagator. There was just a TODO. + +**Host:** Oh no. + +**Jacob:** So, and the TODO was from someone on it. It was from Alex Book, which I'll call him out. So I sent it to Alex, and I was like, hey, this is a funny TODO because I just, you know, took down an entire service's traces in staging. So I spent some time to fix that TODO. It wasn't that difficult; it was just that they were waiting on another thing. I mean, that's how it goes. You're waiting on someone else, which, you know, another person—it's just endless cycles of that type of thing. + +**Host:** Yeah. + +**Jacob:** But then I got it working, so that was like one of the main blockers for us. + +**Host:** Nice, nice. + +[00:16:02] **Jacob:** And was able to upstream it as well. It wasn't just a fix for ourselves; it was a fix for the community. There were a few things like that. A lot of the metrics work actually resulted in big performance boosts for Hotel metrics, like Hotel Go metrics, and it also has given the specs folks some ideas about how descriptive the API should be or various features. + +So things like views and the use of views is something that we did heavily in that early migration because we were worried about—can you just— + +**Host:** Yeah, definitely. Just tell folks what you mean by that. + +**Jacob:** Yeah, so a metrics view is something that's run inside of your metrics provider in OTEL, your media provider in Hotel. What it's doing is it's saying you can configure it to do kind of a lot. It could just be drop this attribute whenever you see it. If you're a centralized SRE and you don't want anybody to instrument code with any user ID attributes because that's a super high cardinality thing, it's going to explode your metrics cost, right? So you could just make a view that gets added to your instrumentation that says just don't let this attribute from being recorded—just deny it. + +So that's like probably the most common use case. There are other ones though—more advanced use cases for dynamically changing things like the temporality or the aggregation of your metrics. So temporality being cumulative or delta for like a counter. You know, am I recording zero, one, three? I'm trying to—two, three, zero, one, three—or am I recording one and two, right? + +And then your aggregation is going to be about how do you—oh man, being on the spot is so much harder because it's like I want to look it up, but feel free to look it up. That's totally cool. + +**Host:** Well, aggregation is like how you send off these metrics basically. We had an aggregation that instead of doing—well, for histograms, this is like most useful when you record a histogram. There are a few different types of histograms. Datadog's histograms, StatsD's histogram is not a true histogram because what they're recording is like aggregation samples. + +So they give you a min, max, sum, count, average. And so I actually don't even think they give you sum. I looked at this last night. They don't give you sum. They give you like min, max, count, average, and like P95 or something. And so the problem with that is in distributed computing, if you had multiple applications that are reporting a P95, there's no way that you could get a true P95 from that observation with that aggregation. + +The reason for that is that in order to get P95, you can't—if you have five P95 observations, there's not an aggregation to say give me the overall P95 from that, right? You need to have something about the original data to actually recalculate it. You could get the average of the P95s, but that's not a great metric. That's not really like—it doesn't really tell you much. It's not really accurate. And if you're going to alert on something, if you're going to page someone at night, you should be paging on accurate measurements. + +[00:18:00] Initially though, we did have a few people who relied on this min, max, sum, counter instrument, so we used Hotel views in the metrics SDK to configure custom aggregation for our histograms to emit what some would call a distribution or what OTEL calls an exponential histogram, or the min, max, and the min, max count. So we were dual emitting. This works because they're different metric names that we were emitting, so there was no overlap between them. So what we did was we migrated. After we did the metrics migration, we were able to then go back and say any dashboard, any alert, anything that was using a min, max, sum, count metric just changed it to be a distribution instead. + +Because we had enough data in the past, like, you know, a few weeks, months of running Hotel metrics in our public environment, that was possible to do. So that was like one of the key features because we had it, it was ten times easier, and we were able to do it from the application. We didn't have to introduce any other components, which is pretty neat. + +[00:22:02] **Host:** Right, right. Cool. Another question that I had for you. So like, when you were doing this migration, traces and metrics were in existence; logs I believe would not have been like the specification would not have been ready—possibly not even in the works? + +**Jacob:** No, it was still really early for that. + +**Host:** I guess in lieu of logs, there's span events that you could use, so it's not something like that was leveraged as well? + +**Jacob:** Definitely. We've heard a long time have you used span events and logs or a lot of things internally. I'm a big fan of them. I am not a huge fan of logging. I find it to be really cumbersome and really expensive. IOPS for tracing and trace logs, whenever possible, I find it easier for myself to reason about. There are other people who are like logging first, and that's great, but that's just not who I am. I like logging for local development and tracing for distributed elements; that makes sense. + +But we use this heavily. That was one of the first things that I checked worked. It's actually an interesting bug where we had some custom code or OpenTracing that allowed us to serialize JSON blobs in the span events, and that stopped working because we didn't emit them in the same way. It's a little hazy, but I had to rewrite a processor to make that work and then update some downstream code like in Lightstep as platform to Facebook. + +**Host:** Cool. So now how about keeping that in mind—now that logs are more mature, are there any plans to do any conversions? And please correct me if I'm wrong, but my understanding too is that with the log specification maturing more and more, the span events are going to be replaced by logs in some form—like it's going to be the log specification for span events. Have you heard anything around that? + +[00:24:50] **Jacob:** No, this is a bit outside of where my recent focus has been, so I'm not positive. I think right now the way that we do—I think the thing that we would change is how we collect those logs potentially. Right now we use—how do we do this right now? It changed recently. I don't want to say something incorrect, but we previously did it by just using like Google's logging agent, where they basically are running Fluent Bit on every node in the GKE cluster, and then they send it off to GCP and they just like tail it there. I think this changed though, and I'm not sure what we do now. + +**Host:** Okay, cool. Speaking of GKE, I have many questions on GKE specifically. Do you know if there's like a feature now in newer versions of Kubernetes where there's like some— I think there's some telemetry collection. Do you know if that's been enabled in any of the clusters? + +**Jacob:** Yeah, so I think that Kubernetes now has the ability to emit like the Hotel traces natively. + +**Host:** Yeah, yeah. + +**Jacob:** I'm not sure if we're collecting those yet. I don't know what version that's more of a question for the SREs. I don't—Kubernetes came out like I think even last year, starting whatever, like last fall kind of thing. + +**Host:** Yeah, that's a really good question that I want to look into because I want to see if really what I would like to do is see if we can collect the traces that we get from those to use the span metrics processor to generate like better Kubernetes metrics from those traces. I'm very focused on infrastructure metrics—like Kubernetes infrastructure metrics—and I find them to be very painful in their current form. + +It would be really cool; right now, I prefer to use the Prometheus APIs for them currently. It's just a bit more ubiquitous in the observability community to use Prometheus to do that because that's what Kubernetes natively emits, right? + +**Jacob:** Right, go ahead. + +**Host:** Oh, no, go ahead. I'll let you complete the thought. Maybe it answers my questions. + +**Jacob:** And so that's what we do right now. I use the target allocator, which is, you know, a nutshell component that I work on to distribute those targets, which is, you know, a pretty efficient way of getting all that data. We also use daemon sets as well that we run in our clusters to get that data. In addition to that, so that works pretty effectively. The thing that's frustrating is just Prometheus. Prometheus script failures can be a super common problem, and it gets really annoying when you have to worry about metrics cardinality as well because it can explode. + +I actually found a bug in GKE maybe six to eight months ago—six, seven months ago—but they've since fixed where they weren't deleting—they weren't reconciling certificate signing requests in their Kubernetes cluster, which meant that for kube-state metrics, which reports on cluster state, it was omitting because there were so many certificate signing requests left from these abandoned nodes. + +Something on the magnitude of like six hundred thousand for like a single metric, which is huge. And so then Prometheus—the Prometheus that I was running—fell over because of that. And that's like a thing that happens constantly in this Prometheus realm, which is just like someone emits a high cardinality metric, Prometheus goes to scrape it, and then it just crashes. + +**Jacob:** Oh wow. + +**Host:** That does not sound fun. + +**Host:** I want to just take a step back because you mentioned the target allocator. I was wondering if you could expand a little bit on that because I know one of our previous Q&A folks also mentioned the target allocator. That was the first time I had heard of it, so I think it'd be like super helpful to just get a little overview. + +**Jacob:** Sure, yeah. So the target allocator is a component, part of the Kubernetes operator in Hotel, that does something that Prometheus can't do, which is dynamically shard targets amongst a pool of scrapers. Prometheus has some experimental functionality for sharding, but you still have a problem for querying because Prometheus is a database, not just a scraper. + +If you shard your targets, you don't necessarily—you have to do some amount of coordination within those Prometheus instances, which gets expensive. It's like a very experimental feature. Or you could scale Prometheus with something like Thanos or Cortex, which is Grafana's Prometheus scaling solution, I think, right? + +Which works, but you just then have to run like six more components that you then need to monitor, and then if those go down, how do you monitor all these other problems, right? In Hotel, we just basically tack on this Prometheus receiver to get all this data. But because we want to be more efficient than Prometheus, because we don't need to store the data, we tell—we have this component, the target allocator, which goes to do the service discovery from Prometheus. So it says give me all of the targets that I need to scrape, and then the target allocator says with those targets distribute them evenly amongst the set of collectors that are running. + +**Host:** Oh, okay. + +**Jacob:** So that's the main thing. It does some more stuff around job discovery now, or if you're using Prometheus service monitors, which is part of the Prometheus operator, which is a very popular way of running Prometheus in your cluster. It's what a lot of vendors use as well. So if you're on GAE or OpenShift, I think both of those natively use service monitors and pod monitors. + +So the target allocator can also pull those service monitors and pod monitors and update the collectors' scrape configs to do that. + +**Host:** Oh cool, it's awesome. And so related to the Prometheus thread, are you running like Prometheus itself, or are you just scraping the Prometheus metrics and pumping them through to the collector? + +**Jacob:** Exactly right. Just no Prometheus instances, just the collector running Prometheus receiver and then sending them off to Lightstep. + +**Host:** Oh, living the dream! That was always my dream. + +**Jacob:** That's awesome. + +**Host:** Do you use—because I remember Lightstep has like a Prometheus operator that helps facilitate that, so we used to have this thing called the Prometheus sidecar, which you might run. + +You would run it as part of your Prometheus installation, which would then sit on the same pod as your Prometheus instance and read the write-ahead log that Prometheus has for persistence and batching and all these other things. So we would read the write-ahead log and then forward those metrics. + +But if your Prometheus is very noisy—as many customers have very noisy Prometheus statistics—it's not really efficient. It can get really noisy, and not that—what's the word? It's not the best thing to run the collector as like the best way to run. + +**Jacob:** Okay, so—and it sounds like this thing still requires to have Prometheus installed. + +**Host:** Yeah, you would still need to be running a whole computer system. + +**Jacob:** Oh, okay. I thought it was—I was under the impression it was like a replacement for Prometheus and that it was—maybe I'm thinking of something else. There was a thing that I knew was like a replacement for needing Prometheus, and it was like vendor-neutral, so it wasn't like, oh, you have to use Lightstep to use this thing. + +**Host:** I think I might just be a Hotel operator collector target allocator trio. + +**Jacob:** Oh, okay, okay. + +**Host:** But maybe there's another thing out there. + +**Jacob:** Oh, unless maybe that got integrated into the target allocator as part of—anyway, it is a mystery. + +**Host:** Yeah, okay, cool, cool. + +**Host:** So then, okay, since we're talking collectors now, for me, like the two million dollar question—the one that I'm always curious about—is collector setup. So what is the collector setup that y'all have chosen? What works for you now? + +**Jacob:** Yeah, it's hard to say because we run a lot of different types of collectors. At Lightstep, we run like metrics things, tracing things, internal ones, external ones. There are a lot of different collectors that are running at all times. You have like a separate one that just collects metrics and one that just collects traces. + +Right now we don't—it’s all varying flux. Right now we're changing this a lot to run experiments and stuff. Basically, like the best way for us to be able to make features for customers and end-users is by running them ourselves and then using them internally, making sure that they work, and then sending them for the open source realm. + +So that's what we're trying to do even more of—like we're kind of reaching a point where we dogfood everything, which gets really confusing because you have to like— + +**Host:** Yeah, I can imagine. + +**Jacob:** Yeah, we're running like in a single path there could be like, I think, two different collectors in two environments that could be running two different images in two different versions. It gets really meta and very confusing to talk about. + +**Host:** Yeah, I can imagine. + +**Jacob:** And, you know, if you're sending from collector A across an environment to collector B, collector B also emits telemetry about itself, which is then collected by collector C. And it just chains—like you basically ensure that you have to like make sure that the collectors are actually working. + +**Host:** Yeah, you have to be sure that everything along this path—yeah, you just have to know which thing has the data. + +**Jacob:** Right. Well, you shouldn't have to—we do that for you, but like— + +**Host:** Right. + +**Jacob:** Yeah, and we make like dashboards to help with that. But that's like the problem when it's like we're debugging this stuff is when there's a problem you have to think about like where's the problem actually? Is it in how we collect the data? Is it in how we emit the data? Is it in, you know, the source of how the data was generated? It's one of like a bunch of things. + +**Host:** Yeah, yeah. + +**Host:** Now, like you need to work on the Hotel operator. So, and I've been reading up on the operator recently, and there's like, I think, four different deployment modes, right? There's sidecar deployment, daemon set, and—what's the other one? + +**Jacob:** Yeah, nice, yeah. + +**Host:** So my question is, which mode—or is it like all of the above, depending on the thing that you need to do? + +**Jacob:** Yeah, it's all the above depending on what you need to do and your general needs and like how you like to run applications for reliability and stuff. + +**Jacob:** So sidecar is the one that we use the least and is probably used the least if I were to just make a bet. Sidecars are really useful across the industry, you would think. + +**Host:** Yeah, across the industry, I'd be willing to bet that those are the least popular. + +**Jacob:** That's the least popular method. Sidecars are just expensive, and if you're not using them—if you don't really need them, then you shouldn't use them. You’ll really only need them like something that's run as a sidecar, like Istio, which makes a lot of sense to run as a sidecar because it's doing like traffic proxy hooks into your container network to change how that all does its thing. + +And you get a performance hit if you sidecar your collectors for all your services. You just get like a cost. It would just cost you a lot more. And you also wouldn't be able to do as much with like if you're making like Kubernetes API calls for attribute enrichment—that's like the thing that would get exponentially more expensive if you're running it as a sidecar, right? But as like a stateful set of like, you know, five pods, that's not that expensive. + +But if you have a sidecar on like 10,000 pods, then that's 10,000 API calls made to the Kubernetes API. + +**Host:** Right, right. + +**Jacob:** What would be the advantage of running your collector as a stateful set versus a deployment? I guess what's the state that you would want to persist? + +**Jacob:** Yes, this is, I don't know what the right word is, but stateful sets aren't only used for their ability to mount volumes. There are a few other things that are inherent to how stateful sets run that are really valuable in distributed computing. This is an important thing to know for not just like how the collector runs as an application, but how your applications can run, right? + +Stateful sets have consistent IDs, so if you have a stateful set with 10 replicas, they're all going to be the stateful set name dash counter number, so it goes from like zero to n. So that's a really valuable thing when you want consistent IDs, right? + +As opposed to like with deployments, like when you, like your pods are all like random crap, right? So the pod IDs are done where it’s deployment name dash replica set ID dash pod ID, right? And so with stateful sets, because we have this consistent ID range, we can actually do some extra work with—for the target allocator, which is why we require that. + +And so the other thing that stateful sets guarantee is what's called an in-place deployment, which is what daemon sets do as well, where you take the replica, you take the pod down before you create a new one. + +So the reason that this is important is that in a deployment, you normally do a one-up, one-down, right? Or what's called a rolling deployment, a rolling update. And so if we were to do this for—if we were to do this for the— with the target allocator, we would probably get much more unreliable scrapes because you would—and someone actually just asked this question in the operator channel—I mean, I'm going to give them this exact response. + +When a new replica comes up, you have to redistribute all the targets because your hash ring that you place these on has changed. So if you're doing a rolling deployment, if you're doing one up, one down, that's a really expensive operation, because then you have to recalculate all these hashes that you assign. + +So if you were to do a one down, one up, you would still have to redo this whole thing because you would lose a pod, which means it's taken out of the ring, redistribute. You would gain a new ID, and then you'd have to redistribute again, right? + +Whereas stateful sets, because it’s a consistent ID range, you don't have to do that at all. And so this means that when we do a one down, one up, it keeps the same targets each time. + +**Host:** Right, right. So it's almost like a placeholder for it. You don't have to recalculate the ring, basically. + +**Jacob:** Yeah, it's just sort of like a little—what's it called? + +**Host:** Yeah, yeah, yeah. + +**Jacob:** I can't think of the word. Cool, that's really neat. I didn't know that. + +**Host:** Yeah, it was funny because I was reading about like pods being deployed in stateful sets. I'm like, straight or sorry, not collectors. I'm like, I straight up do not understand what the use case would be, but this makes a lot of sense. + +**Jacob:** So that's really cool. And so it's not as useful—or this is really only useful for like a tracing use case I would say, or sorry, metrics use case where you're like doing complete test scores. + +We would probably run it as a deployment for anything else because a deployment gives you everything that you need pretty much, because the collectors are stateless—they don't need to hold on to anything. Deployments are much more lean as a result. + +**Host:** Yeah, yeah. + +**Jacob:** They can just run and roll out, and everybody's happy, and that's how we run most of our collectors—deployment. + +**Host:** And then at what point would a daemon set be useful? + +**Jacob:** Yeah, so daemon sets are really good for things like node scraping, which we do a lot of. So this allows you to scrape like the kubelet that's run on every node. It allows you to scrape the node exporter that's also run on every node, which is another Prometheus daemon set that most people run. + +**Host:** Right, right. + +**Jacob:** Yes, the daemon sets guarantee that you’ve got pods running on every node. + +**Host:** Right, exactly. Every node that matches its selector, right? + +**Jacob:** Right, right. + +**Host:** And so that's really useful for like scaling out. So if you have like a cluster of like 800 plus nodes, it's more reliable to run like a bunch of little collectors that get those tiny metrics rather than a few bigger stateful set pods, right? + +Because your blast radius is much lower, so if one pod goes down, you lose like just a tiny bit of data. But remember like with all this cardinality stuff, that's a lot of memory. + +**Jacob:** So if you're doing like a stateful set scraping all these nodes, that's a lot of targets, that's a lot of memory. It can go down much more easily, and you lose more data. Luckily, the collector isn't like Prometheus, where we don't care about that state. So if a collector goes down, it comes back up super fast. So usually, the blip is low, but it does mean that the blip is more flappy, right? + +Where like it could go up and down pretty quickly if you're past the point of saturation. That's why it's good to have like a HPA, horizontal auto-scaler. + +**Host:** Right, right. + +**Jacob:** But still, daemon set is a bit more reliable. + +**Host:** And it sounds like it would be useful again, like from a metric standpoint. + +**Jacob:** Yeah, yeah. + +**Jacob:** Tracing, you could do it for tracing and just send it on like a node port, but tracing workloads, again, because it's all push-based, they are much easier to scale on. + +And you can distribute targets; you can load balance. There are all these other benefits that we get from push-based workloads. Pull-based is like—the reason that Prometheus is so ubiquitous in my opinion is just because it makes local development really easy, where you can just scrape your local endpoint. + +That's what most back-end development is anyway, so you could like hit endpoint A and then hit your metric set point and then hit endpoint A again—metric standpoint. You can just like check that. It's like a very easy developer loop. + +No, it also means that you don't have to reach out side of the network. So if you're a really strict like proxy requirements to send data, local dev is much easier for that, which is why like Hotel now has like a really good Prometheus exporter so you could do both, right? + +**Host:** Right, right. + +**Jacob:** If you have that hankering for running Prometheus. + +**Jacob:** And then I'm assuming there's a centralized gateway somewhere or— + +**Jacob:** This is part of the collector chain that I was talking about. Again, we're running a lot of experiments. Cool. I can like half talk about it. + +Okay, I can be vague. A big effort within Hotel right now is around Arrow, which you might have been hearing about some. There’s been some work done by Lightstep and F5 to improve the processing speed and egress and ingress costs of OTEL data by using Apache Arrow, which is a project for columnar-based data representations. + +And so we're just like doing some proof of concepts or like proof of implementation work to see what the actual performance of this stuff looks like, right? And also, like, you know, check that everything works as expected. + +**Host:** Yeah, yeah, yeah. + +**Jacob:** As well, which it is, but that's—you always have to check. + +**Host:** Yes, absolutely. + +**Host:** Well, I'd say the main takeaway from this whole story on collectors is like it sounds like it's always going to be an evolving game, which is not a terrible thing to do. + +**Jacob:** No, it's important that you keep your telemetry up to date. I think that like library authors and maintainers are like constantly working on new performance features and new ease of use, like quality of life stuff as well. + +**Host:** Yeah, yeah. + +**Jacob:** Especially with OTEL, like we talk about quality of life a lot. + +**Host:** Yeah. + +**Jacob:** And so that is definitely a focus, and that's why it's important to keep up to date. It makes migrations easier as well. Trying to migrate from like an ancient version of something to the latest version, you're probably missing a lot of breaking changes potentially, and you have to be careful of that. + +**Host:** And on that vein then, how do you ensure that everyone's keeping up to date with the latest versions of OTEL across the org? + +**Jacob:** I think like stuff like Dependabot is pretty good. We use it internally—or not internally, we use it in Hotel. We're keeping up to date with Hotel stuff, so I find it to be really helpful. It's very frustrating sometimes because it's like Hotel packages all update in like lockstep pretty much. + +That means you have to update a fair amount of packages at once, but it does do it for you, which is pretty nice. + +**Host:** That's nice. That's nice. + +**Jacob:** But you should be doing this not just for Hotel but like any dependency, right? Like CVEs happen in the industry constantly, and if you're not staying up to date with vulnerability fixes, then you're opening yourself up to security attacks, which you don't want. + +**Host:** So, yeah, do something about it is my recommendation. + +**Jacob:** That’s fair, that’s fair. + +**Host:** I know we've got like four minutes left. Do you want to give us any like parting thoughts as we wrap up? I'm interested to hear if there are any questions from the group that we've missed in our discussion here. Any takers with burning questions? Now's the time. If not, I'm going to call on Rhys, but this was great. + +I feel like I was like rapidly trying to take notes. I probably will have more as I try to like go back and tidy up some of my notes. But yeah, honestly, I was just trying to like keep up with people with so much information, I feel like because I've been reading up on the operator like the last week and so like more questions than answers, and I feel like I have some answers now. + +I've definitely learned a lot. I hope folks on this call have learned a lot as well. I think this is like really great information. I think it gives folks like an idea of like what's involved in a migration, things to consider like when you're setting up your collectors, things to avoid doing, keeping up with those latest versions of OTEL—always a good thing. + +**Jacob:** Yeah, that's the big one is that like new fixes really often. + +**Host:** Yeah, yeah. + +**Jacob:** Like new versions every two weeks for the collector, I'd say it's like a pretty frequent cadence. There are some Prometheus libraries that like don't update like ever. Like Thanos hasn't released since March, right? + +**Host:** Which is absurd to me because there's like a bug in compatibility between Prometheus and Thanos right now. + +**Jacob:** Oh really? + +**Host:** How cheap! + +**Jacob:** Yeah, so you can use them together if you're like coding, so not fun. + +**Host:** Damn! + +**Host:** All right, last chance for burning questions, y'all. Erica said thanks for the info and your time, Jacob. + +**Jacob:** Thank you, Erica, for hopping on. + +**Host:** Yeah, this was really awesome. Thank you so much because like for real, this is a dope, dope topic. So yeah, we'll, you know, reach out to your friendly neighborhood CFP approvers. + +**Jacob:** No, thank you so much. I feel like there was so much more we could have chatted about as well, so we might— + +**Host:** Yeah, yeah. Maybe I can—if I get accepted for the talk, then I can give a sneak peek at least. I can get some feedback on it. + +**Jacob:** Actually, you know what, like if you're interested because we have Hotel in practice, which is kind of like giving like a little talk. So if you're interested in doing that even like before you find out whether or not you get accepted, we are happy to have you. + +**Jacob:** Yeah, it sounds great. Cool, I'm in. + +**Host:** Cool, cool. We'll figure out the behind the scenes on like when to schedule you. + +**Jacob:** Sounds good. + +**Host:** Yay! Cool. All right, and Daniel said thanks, Jacob, as well. And yeah, we're at the top of the hour. Thanks for joining us. + +**Jacob:** Thank you all. + +**Host:** Thank you! + +**Jacob:** Yeah, thank you everyone. + +**Host:** Bye! ## Raw YouTube Transcript diff --git a/video-transcripts/transcripts/2023-09-13T19:26:10Z-opentelemetry-q-a-feat-hazel-weakly.md b/video-transcripts/transcripts/2023-09-13T19:26:10Z-opentelemetry-q-a-feat-hazel-weakly.md index bf898a4..f93e33b 100644 --- a/video-transcripts/transcripts/2023-09-13T19:26:10Z-opentelemetry-q-a-feat-hazel-weakly.md +++ b/video-transcripts/transcripts/2023-09-13T19:26:10Z-opentelemetry-q-a-feat-hazel-weakly.md @@ -10,91 +10,283 @@ URL: https://www.youtube.com/watch?v=wMJEgrUnX7M ## Summary -In this YouTube video, Hazel Weekly shares her experiences and insights on observability and the implementation of OpenTelemetry in organizations. She discusses her journey as a novice user approaching OpenTelemetry, the challenges of over-instrumentation, and the importance of asking meaningful questions to utilize telemetry effectively. Hazel explains how she navigated obstacles in organizations, such as managing excessive data volume and ensuring meaningful instrumentation, often finding herself as a one-person team tackling complex issues. She highlights the cultural and technical hurdles in convincing teams and executives to adopt observability practices while emphasizing the need for thoughtful implementation of telemetry tools. The conversation covers the intricacies of sampling, the challenges of using various programming languages, and the significance of aligning technical capabilities with business needs. Hazel also touches on the complexities of regulatory compliance, particularly in relation to government standards like FedRAMP. Overall, the discussion provides valuable insights into the practical application of OpenTelemetry and the importance of a strategic approach to observability. +In this YouTube video, Hazel Weekly discusses her experiences with observability and OpenTelemetry, providing insights for novice users. The conversation touches on the initial challenges companies face when adopting observability, including understanding the need for meaningful telemetry and the tendency to over-instrument systems without clear intent. Hazel shares her journey of helping a company reduce unnecessary data collection and implement effective instrumentation, highlighting how cultural and organizational factors can complicate observability efforts. She emphasizes the importance of aligning observability practices with business goals and the need for effective communication between developers and stakeholders. The video also covers specific technical challenges encountered while working with OpenTelemetry, including issues with sampling, baggage, and the ergonomic difficulties of manual instrumentation in various programming languages. Overall, Hazel's insights underscore the complexity of implementing observability in real-world scenarios and the importance of thoughtful design in telemetry practices. ## Chapters -Sure! Here are the key moments from the livestream along with their timestamps: +00:00:00 Welcome and intro +00:00:20 Introduction of Hazel +00:01:21 Observability definition +00:03:00 Instrumentation challenges +00:05:00 Cost-driven changes +00:06:25 Sampling methods +00:09:30 Over-instrumentation issues +00:12:01 Context propagation challenges +00:15:00 Convincing the team +00:18:00 Observability improvements +00:19:01 Anti-pattern discussion +00:23:00 OpenTelemetry support in languages +00:27:00 Baggage challenges +00:30:00 Ergonomics of instrumentation +00:34:30 Libraries for OpenTelemetry +00:38:35 Infrastructure management experience +00:45:00 Making the case to executives +00:50:00 Regulatory compliance challenges +00:56:00 Closing remarks and future events -00:00:00 Introductions and welcome to Hazel -00:02:30 Hazel shares her initial experiences with OpenTelemetry -00:05:00 Definition of observability from different perspectives -00:08:15 Discussion about the challenges of over-instrumentation -00:12:45 How to encourage teams to focus on meaningful telemetry -00:15:30 The importance of asking the right questions in observability -00:20:00 Sampling methods and their impact on data collection -00:27:00 Challenges with asynchronous programming in OpenTelemetry -00:35:00 Insights into convincing executives about observability needs -00:42:00 The complexities of running OpenTelemetry in production environments -00:46:30 Wrap-up and next steps for Hazel's future talks +[00:00:20] **Host:** Thank you. Welcome everyone to the OpenTelemetry Q&A. We have the pleasure of having our good friend Hazel Weekly come talk to us about her experiences with observability. Welcome, Hazel. -Feel free to reach out if you need more information or assistance! +**Hazel:** Glad to be here! Very excited for this. Yay! -# Observability Q&A with Hazel +**Host:** I guess let's start with first things first because I know we still have a bunch of OpenTelemetry novice users just getting into OpenTelemetry, folks who join our end-user working group. From the perspective of somebody who's new to OpenTelemetry, can you share what that experience was like? What kind of landscape were you coming into? I guess for starters, drawing back on that experience, was the company at that point ready for observability? What kind of kick-started the conversation into OpenTelemetry in the first place? -Thank you for joining us for our observability Q&A session. We have the pleasure of having our good friend Hazel Weekly here to discuss her experiences with OpenTelemetry. Welcome, Hazel! +[00:01:21] **Hazel:** I would say, so I'm going to start off with my definition of observability because it's slightly different. The definition from control theory would be like, can you understand this system from the input and the outputs? Fred Herbert might talk about the definition from cognitive safety systems engineering, and that one is much more about the work required and the process required for a group of people to be able to understand everything and actually understand the system. That's how they discovered and think about that. Mine is the process through which you develop the capability of asking meaningful questions and getting useful answers. It's a process because it's evolutionary, and the questions have to be meaningful to you—whatever that means—and the answers don't have to be correct, but they have to be useful. -**Hazel:** I'm glad to be here! I'm very excited for this. +[00:03:00] So when I look back at the company when we were starting to think about observability, or rather like distribution, literally, what are the questions that the company is starting to ask from the perspective of the engineers? One of the questions the company was asking from the perspective of the managers and what was it asking from the perspective of the executives? From the engineers at one of the companies that were early adopters of OpenTelemetry, they had it and they had instrumented things with it, but more from the perspective of “we know we need this,” but they didn't necessarily have that motivation yet. They weren't asking the types of sophisticated questions that motivate people to add their own telemetry. They had a bunch of telemetry, and they had a huge amount of instrumentation, and they had just the volume turned up to 11, and no one was actually querying that information. -Let's start with the basics, since we have a lot of novice users just getting into OpenTelemetry. From your perspective as someone who was new to OpenTelemetry, can you share what that experience was like? +What I was able to do was turn down that volume significantly. I essentially said, “You're not using this, and you're not asking any questions from your system.” So when you do, don't turn the volume back up; add that instrumentation in there with thought and intent. That was an interesting hurdle to happen because in a way, people would ask questions to understand the system. They were asking them to solve a problem, and consequently, they just wanted every bit of information ever so that whenever they needed it, they could just go through a giant stack of noise and find a needle, right? That's not really what OpenTelemetry is for. Online distribution is for, can you understand the state of your system to the point that you only collect what you actually need? That is, in and of itself, a second layer of understanding the system, and they didn't have that second layer. -### The Initial Experience with OpenTelemetry +**Host:** Right, so it sounded like they were instrumenting for the sake of instrumenting and just sort of throwing everything at the wall and hoping something would stick. -**Hazel:** Sure! When I first joined the company, I would say that the definition of observability I hold is slightly different from the traditional one. From control theory, it’s about understanding a system from its inputs and outputs. In cognitive safety systems engineering, it’s about the processes required for a group of people to understand everything about the system. My definition focuses on the capability of asking meaningful questions and getting useful answers. This process is evolutionary, and the questions must be meaningful to the people involved. +**Hazel:** Yeah, that also sounds like a very “I came over from Vlogs and I'm used to being able to search everything ever” approach. They also still had logs, which is a surprise. Basically, nobody of course. -Looking back, the company was at a stage where they were starting to think about observability, but they weren't quite ready for it. They had a lot of instrumentation but lacked the motivation to ask sophisticated questions. They had a high volume of telemetry data, but no one was actually querying that information. I managed to turn down the volume of telemetry significantly and encouraged them to only add instrumentation thoughtfully and intentionally. +**Host:** In terms of like, how did you convince them to kind of tamp it down and direct it, make it more directed so that it could actually work for them? -### The Hurdles of Over-Instrumentation +[00:05:00] **Hazel:** Convincing them to champion it down and turn the volume down was actually pretty easy. That came down to cost. They were about 300% over budget from the vendor, and so I said, “I need to get you under budget.” I actually have a very funny image from that time period where you see the ingestion at about like two to three hundred million events per day, and you know I'm trying to tweak it down to about like 200, 250, or like, you know, 180 to 250 million. That was with very, very aggressive sampling. Finally, I figured out a bunch of misconfigurations in the sampling information and dropped it to about like three to five million events per day. -It seemed they were instrumenting just for the sake of it, throwing everything at the wall and hoping something would stick. How did you convince them to dial it back and make it more directed? +Once I had done that, they had concerns about, “Oh, what if I can't find something?” I said, “Well, do you need to ask that question?” Then we were able to actually start the useful dialogue of, “Now that you have a question you need to ask, go and build what you need in order to get that answer.” Naturally, that sort of feedback loop that you kind of need with observability, whether or not you need that feedback loop end-to-end is a different question, but that is currently what OpenTelemetry requires. -**Hazel:** Convincing them to reduce the volume was easier than expected, largely due to cost concerns. The company was over budget with their vendor, and I emphasized the need to get under budget. I managed to reduce the ingestion from hundreds of millions of events per day down to about three to five million. Once we established that they needed to ask specific questions, we could start building what was needed to get those answers. +[00:06:25] **Host:** I think that's really great because being able to know what questions to ask makes it a much more meaningful experience. In your words, not searching for that needle in the haystack. I want to go back to sampling for a second. So sampling to a sampling was, we did the sampling through two methods. We had the OpenTelemetry collector, and I'll show you now, but at this company, we only had the Honeycomb Refinery set up until we had that set up. At other companies, we've had the OpenTelemetry collector setup, and I've used that. My preference is actually to use both, regardless of whether or not using Honeycomb. -### The Role of Sampling +The reason for that is because the OpenTelemetry collector doesn't have the best steel sampling configuration, and the OpenTelemetry collector has like the most open-source line economic sort of configuration setup to consent things to multiple sources and multiple sinks so that is actually really useful. I like to use that one, and then aggregate everything and send it to Refinery and then do useful tail sampling. But running two pieces of information is kind of complicated for a lot of people. It's not going to be a huge hurdle of adoption to like my current company. They use DataDog, and one of the challenges there is they're not at the point where they can ask sophisticated questions of their infrastructure. -You mentioned sampling earlier. Can you elaborate on that? +I'm really still more at the point of “Is it on?” The question there is, well, even if I set things up for success by switching us to the OpenTelemetry line instrumentation tooling and then still send it to the same place, we now need to run like the collector in multiple places or run like a collector and Refinery just in order to do the same thing that they already do with their built-in tooling. That can be a bit of a lift because rather than saying “install library,” it's “install a library and have like five things also.” -**Hazel:** We used two methods for sampling: the OpenTelemetry collector and Honeycomb's Refinery. My preference is to use both, but running two pieces of information can be complicated. At my current company, we use Datadog, and they’re still not at the point of asking sophisticated questions about their infrastructure. Transitioning them to OpenTelemetry would require running collectors in multiple places, which complicates the setup. +**Host:** Going back to the first organization where they were sending like way too much data, after you convinced them to chill and do more directed instrumentation, what was the next sort of hurdle that you experienced? -### Challenges with Instrumentation +**Hazel:** There was a notch hurdle there. The way that company structured work was very much the over-platformed thing, this little towards unusual, but they had like developer productivity teams, and they didn't really have a platform team. What they had were engineers that didn't do work. The engineers didn't do infrastructure stuff; they wrote additional code, and they would have a bunch of people working on build tooling and build other stuff like that. -After convincing the first organization to streamline their instrumentation, what hurdles did you face next? +[00:09:30] The question was, in the way that they do work in order to adopt OpenTelemetry successfully, you would need to essentially build libraries for things or write stuff into the code in a way that engineers weren't necessarily writing their own instrumentation, and that requires working at a level of abstraction that OpenTelemetry is really, really difficult. You can't propagate certain types of information down to child spans very easily or really at all. Setting up baggage is still very difficult. -**Hazel:** One major hurdle was the way the company structured its work. They had developer productivity teams but no dedicated platform team. Engineers weren't responsible for infrastructure, which made adopting OpenTelemetry challenging. Many engineers added instrumentation without understanding the full context of where it was implemented, which led to a lot of dropped information. I spent a lot of time manually tracing errors back to the source code and fixing small issues. +I'm working with context can be really tricky. If somebody uses an async function, then you might drop that and have to reconnect it somehow, and if the lifetimes aren't like already set up nicely, people can even add their own instrumentation, and then it'll get dropped from something because they don't actually understand the full context of where it is. They're just random code, and they add something, but in OpenTelemetry, you have to understand the call stack, not just the functionality, and those aren't always the same. -### Navigating Team Dynamics +**Host:** Interesting. So how do you get over that hurdle? -Were you working alone during this process? +**Hazel:** At that company, I was never able to fully get over that hurdle. I just made it better. One of the ways that I made it better was essentially following through all of the spans manually and looking for errors. When I found errors, I would trace it down to the source code, and I would do that. I ran into a bunch of very small things of like propagating information wasn't correct. So it was probably the information of like the helper functions rather than the actual call sites to activate that. Then I had to fix like 500 small things. -**Hazel:** For a period, yes. I was the only one tackling these issues while my colleague was on parental leave. It was tricky, but I had a good intuition for what the code was doing at runtime. I had to engage in a lot of dialogue with the team to ensure they understood the issues and could trust my judgment. +Even when I did that, I found that there were a bunch of places where these looking as contexts weren't being complicated, so functions were just being launched or, for example, calling tracing, and that was being disconnected from the actual whatever. Sometimes things had to end before they started or start before they ended, and that got confusing because the language that they were using, it's very easy to write code in that logic, and it actually is more correct to do it that way. -### Building Trust with Predictions +But OpenTelemetry, in its design, is very much a chicken-and-egg call stack, a tree-shaped, which is really, really ergonomic for a language like Java. It's not necessarily ergonomic for languages that aren't like that. So like React is a really common example because it's a runtime that's asynchronous, and it's really difficult to write code in React to the deadline nicely. -How did you build trust with your team regarding the issues you identified? +[00:12:01] I suppose a good one closer is a good example of one that doesn't work very well. Watch does work, but for a very interesting reason, and that is because the Rust language cares about lifetimes. So you think about the lifetime of your functions, and so you always know what those are, and those end up being the natural tracing point protocol stack-based tradition library. -**Hazel:** I started making predictions about system behavior, and when they turned out to be correct, I gained their trust. For example, I predicted that a spike in database activity would cause performance issues, and when I provided a solution, it worked. However, I struggled to convince them of more fundamental architectural issues before I left the company. +In terms of what we were able to do and how I was fixing it was really mostly in guacamole and doing some weird clever things to pack into the language runtime in order to make things hit the semantic model differences between OpenTelemetry and other language things about calling code. -### Observability Improvements +**Host:** So were you like a one-woman show doing all of this? -By the time you left, did you see any positive results from your efforts in observability? +**Hazel:** So the other person on this was off on parental leave, so for the duration, I was actually one person. -**Hazel:** Yes, there was a better understanding of observability and how to use it. I fixed a number of issues that led to more accurate error tracking, which made people more willing to utilize the tools we had. Although they had over-instrumented through auto-instrumentation, the manual instrumentation was often useless. I helped them understand how to structure their data better to facilitate useful querying. +**Host:** That is mad impressive. Did you want to like pull your hair out some days? -### Observability Challenges in Organizations +**Hazel:** It was tricky. A lot of it was that one of the benefits of using OpenTelemetry is that you get like this intuitive sense of what the code is doing at runtime if you play with it enough. They had to sit there and kind of doodle with it and look things up and ask questions and get answers. You have to have that dialogue running, and I knew how to do that, and no one else at the company was doing that except like one or two people. -You mentioned that many organizations struggle with OpenTelemetry. What do you think is the main issue? +**Host:** Wow, and so you had like a very small select power users, favorite people, and then you had everyone else who may not have even like used OpenTelemetry at all or even knew like a vendor in any way whatsoever. -**Hazel:** A lot of times, organizations face cultural challenges. Engineers can be very self-directed, leading to incomplete implementations. They might finish 80% of the work and then move on, leaving things half-done. This leads to a lot of noise in the telemetry data without meaningful insights. +**Hazel:** Yes, and I would get an intuitive sense if one was broken and what was wrong or something, and I would say, “Hey, like this is an issue,” and then people wouldn't necessarily believe me. -### Making the Case for OpenTelemetry +**Host:** Interesting. So how do you end up convincing them that it is an issue, or could you convince them? -Can you share your experience in making the case for OpenTelemetry to executives? +**Hazel:** I was able to take advantage of certain people of the things, and one of the ways that I did that was essentially my predicting things, and then my predictions would turn out to be correct, and then I would give a solution to the position, and then the solution would work. One was the database was very, very spiky, and I mentioned that this was going to cause some sort of issues. One of the reasons that the database was spiky was because we had like basically the disk would slow on a database, and so we sped up the disk on the database quite a bit. -**Hazel:** Making the case involves two migrations: a technical migration and a social migration. You need to show that the time and effort saved by migrating to OpenTelemetry will pay for itself in a reasonable timeframe, typically around four months. If you can demonstrate tangible benefits, executives are more likely to invest in it. +[00:15:00] When that happened, despite this level down, these sets were still very spiky in general, but despite going high enough to lock things down, there were small things like that added up over time, and people started to believe me a lot more when it came to certain issues. But other ones that were more fundamental in the architecture, I never actually got around to being able to convince people of that, and I actually ended up leaving that company mainly due to that. -### Conclusion +**Host:** Interesting. By the time you left, did you find there was at least like they were getting more out of their observability at that point? Because of your efforts, did you at least see some positive results because of that? -Thank you again, Hazel, for sharing your insights! This has been a fantastic discussion, and I look forward to having you back for the next session on September 14th. We appreciate your expertise and the valuable lessons you've provided on navigating the complexities of OpenTelemetry! +**Hazel:** Yeah, I did. There was a lot better understanding of where they were with the observability. People had more of an understanding of how to use it. I had like a couple of things so that it was more useful. They had a lot of it, but some things were wrong. If the error stack was there, the error stack died at the wrong spot, so it wouldn't actually tell you where the error happened. It would tell you where you defined the OpenTelemetry like the tradition helper, which is pointless. + +So I fixed that, and then all of a sudden the errors were correct again, and people were happy about that. Then they would actually use it. I had fixed a bunch of things, and I had reduced the usage from like two to three hundred events per day or even maybe 50 million events per day to like right. When that happened, then people could continue to add OpenTelemetry and add instrumentation everywhere without running into the physical—they were going to blow their budget even more than their own work. + +That allows people to continue to add more instrumentation. + +**Host:** So then I guess there it seems like there wasn't an issue per se as far as like getting people to add the instrumentation at that point because they were obviously—they had already over-instrumented the system previously. + +**Hazel:** Yes, it was interesting. They had over-instrumented it, but only via auto-instrumentation, so there was very little manual stuff done, and the manual stuff that was done was often done in a way that made the questions you could ask relatively useless. So like one example was, there's like at the top of the watched halfway down when in the life cycle of the call stack is when they would get the user ID associated with it because of how the database calls worked. + +But the user ID was never propagated to the top, so it was actually impossible to figure out how many pages a user was visiting, like procession. These types of questions whether or not I usually like it's on the same thing over and over because the question we wanted to ask was, “How effective would database account should be?” But would it be effective to put radish there somewhere? Probably don't have a 60-second something TTL. We had no idea because we couldn't actually ask that question. All the information was there, but not actually in a way that made anything possible to look for. + +[00:18:00] Because you couldn't look halfway down the span in order to get this span of correlated with this one, so that tree—you can only ask for things on the same level and down and filter that way. They needed to fix that, but no one necessarily knew how to, and no one necessarily knew that they wanted to because they weren't asking that type of question. + +So I brought that up, and then people started to get more of a sense of, “Oh, here's kind of where we put the information in a way that's visible.” To this day, I still don't have like a really good way to explain to people how to think about that other than you do kind of need to display the system. I couldn't understand where the data needs to be in order to be useful for asking questions. + +[00:19:01] **Host:** Yeah, that makes a lot of sense. So yeah, that's so interesting. I find this such an interesting sort of, I don't want to use the word “use case,” but scenario, if you will. Because oftentimes we hear the stories of people just struggling with bringing OpenTelemetry into the organization and then starting to instrument, and this feels like it was like one giant anti-pattern—one giant OpenTelemetry anti-pattern—which I think is a very important thing to be able to discuss because, you know, like all tools, OpenTelemetry can be grossly misused, and then you end up with scenarios like this where you're instrumenting but you're not getting anything out of it. + +**Hazel:** It was really interesting to see how this happened because a lot of what happened is due to how the company works and how it thought about working. It was very anarchic in the sense that engineers were very, very self-directed, and they would do things. The company had a huge cultural issue of intricate 80 percent done with instrumentation, maybe like 90 percent done, and then they would just drop off or someone outside traffic had to do something else, and no one else would pick it up. + +No one would sit there and make sure the things got fully finished, and so that essentially happened where we had one employee set up the main amount of instrumentation, and then one or two people added like some things to it, and one or two people added some things to it, but no one did like that last 20 percent of actually tuning things down. They turned everything on, they turned it all up, they added a bunch of things, and the only instrumentation, and then no one actually necessarily used it. + +For example, the OpenTelemetry instrumentation library was something they had to write because they used Haskell as a language. What that meant was they had each database query; they had four different spans. It would make a span from the start of the query instead of a transaction, an end of a transaction, and an end of a query. So you had, and then you have the command of a query itself. Every single database call cost at least four to five spans to appear. + +**Host:** Oh wow. + +**Hazel:** And there's no way to configure that or turn that down or cool unless things or anything like that. What happened was that word created, and then anytime you had like an N+1 pattern just appear instantly in the database, you would have given millions of events, millions of spans all over the place. They were like, “Oh, well, what do we use? Span events instead?” Most vendors either charge per ingestion, or they charge for like the telemetry that actually doesn't shift the cost anywhere. + +It sometimes meets the user interface slightly better, but it doesn't shift the cost, which is one of the reasons why people say, “Have one wide event when you put things in there, then like, you know, have timestamps and don't be afraid to use that.” But they were using the SDK essentially as the timestamp functionality to do a query event anytime, at any time, mark that something happened with a crazy fan anytime something happened, and it formed all the way down to the auto-instrumentation. + +That was tricky. That was probably like 80 percent of like, “Now we have instrumentation in the database,” but no one actually said, “Well, it turns out literally every other database SDK lets you turn 90 percent of the auto-instrumentation off,” and for a good reason. Same for like instrumenting the HTTP server, like the web server. You highlight the web server and then you have the web framework, and both of them had almost identical levels of instrumentation. + +Depending on which endpoint handler was in place, you either got one top-level or the other top-level or both, and so you had like a massive amount of certifications of the amount of spans that you might not need. It turns out it's largely because there wasn't really an ergonomic way to say, “Add this to a span if it exists; otherwise, create your own span.” That pattern is really necessary for libraries, and it's not actually really relevant in the SDK; it's not really mentioned anywhere, and it's not really possible for a lot of languages to do. + +[00:23:00] **Host:** Right. Yeah, that is an interesting problem to have. So now when you were helping this organization with OpenTelemetry, how well-versed were you in OpenTelemetry at the time? + +**Hazel:** I was knowledgeable of it, and I thought about it. I had done some things with it, and there had been like a previous company where I had really played with OpenTelemetry and React. I had set up like some very interesting and overly clever TypeScript helpers that made it really easy for people. I had familiarity there, but this company is where I had to really dig into the Refinery, into configuration, into operationalizing it, and to controlling the cost and understanding it from like that to the perspective of the operator rather than the end user. + +So that was an interesting change in perspective, and I ended up reading pretty much the entire RFC for everything, most of the SDK code, and a whole bunch of other things. I dug really, really deeply into it, and to this day, I will still not necessarily understand baggage. Like, I understand it, but it's kind of not well implemented. + +**Host:** Yeah, you were mentioning earlier that it's funny because I was reading up on baggage today, and you were mentioning earlier that baggage is still kind of wonky to work with. Specifically, what were some of the challenges in working with baggage that you experienced? Because I think this would definitely be good feedback for the OpenTelemetry folks, the maintainers. + +**Hazel:** It comes down to baggage is a really generic tool that ends up being used for like five different things, and they mostly come up when you want to stitch together multiple services or when you want to write a library or do something like that or write like a platform for people so that they can instrument things very easily and not have to worry about certain underlying implementation details. + +You can also use baggage to paper over the API of a lot of things and do what you want to even if you shouldn't necessarily do that. For example, package or just context application in general is more or less the only way you can have a directed acyclical graph. Otherwise, you only really have like a linear constant, and you have like some ability to form links, but even that is pretty under economic. But you need to know where you're linking to and where or what you're linking from. + +Baggage will let you like put things in a header so that you have cost service cost registration. The actual distributive part, the markets, is also the thing that you need if you want to navigate information down to public information upper chain, and there's also what you need if you want to write some sort of demand processor as a way to work around issues. So if you want to have like something added to multiple spans down, then the only way to do that, really, as far as I can tell, is to write a span processor and use this package to pull that information out. + +You would add something to the package and then ask the span processor if it's really thrown the spans, and it'll reconstruct that whole tree and figure everything out and then add that in there. You've more or less rewritten the Refinery in your code base, or rewritten the OpenTelemetry collector in your groupies in order to use packets in order to add certain features that aren't in the SDK yet. + +**Host:** Interesting. Did you find that with other aspects of OpenTelemetry as well that were kind of limiting for you when you were working with it, when you were digging deep? + +[00:27:00] **Hazel:** I think the main thing that was limiting outside of that aspect was that it's really hard to make adding instrumentation that's manual ergonomic. It's just difficult, and a lot of that comes down to OpenTelemetry really wants you to like have written the code right there, and you need to kind of know how to do that, and it's very hard to wrap the libraries themselves without adding like that extra layer of interaction that makes things hard. + +So unless your language has a very massive macro system and you can use that ergonomically, it's difficult to do that. Languages like TypeScript don't. In TypeScript, what you really want is something like a decorator; you can't get that. The other issue that I really ran into ergonomically with OpenTelemetry would have been based around async/await. The async/await one was probably one of the larger beams of my existence, and the reason for that is there's multiple different ways you can write, and especially as you're going to smoke, there is one way that OpenTelemetry works, and that would be if the parent function follows a child function. + +You're going to sleep, pass the context into it, and then waits and outlives the child function and then end-to-end span. So you have like a circuit historic, but actually, this only inside the lifetime of the parent's span. Anything other than that use case gets really weird. + +**Host:** What about span links? Aren't they supposed to kind of alleviate that though? + +**Hazel:** Span links do work, but you have to know how to use them. The parent still has to pull the child, and then the child has to have the parent ID in order to link to that, or I think vice versa. I don't know if you can do that, but essentially I couldn't write a function generically so I was unaware that it was being called from the parent and then had to link up to the parent. + +That was tricky because a lot of languages, you want to write like a library or something, and the library can be very agnostic. So I don't necessarily want to say this function is called pharmaceutical context all the time; I want to say this function is called basically in this context that's relevant to the application domain, and sometimes that's interesting context, and in which case it's been called asynchronously. But sometimes it's more synchronously, and sometimes it's called a different context altogether, and I can't express that with OpenTelemetry very easily, if really at all. + +**Host:** Got it. And speaking of languages, what languages were part of that landscape in that organization? + +**Hazel:** This organization had TypeScript and Haskell as its main languages, but I've also done OpenTelemetry with React. I don't know what TypeScript done with, and I've done it with a little bit of your name. I thought which one was the least annoying to implement? The least annoying to instrument would be probably Python because so many other people use Python, and Python is used in more of a service-sound context, and so OpenTelemetry is often used there. + +That was actually more ergonomic. Python also has like more decorators and metaprograms, how many things like that to make it easier to work with. It's interesting because it's both a front end and a back end, and so a lot of things that you can do with it may have to run both in a voucher context and server context. OpenTelemetry works really well on the service side and very not great on the client side. + +Dealing with all of those issues can be really tricky, and it may survive an API that works while I'm bullet is really hard to like on the client side. You want to send as little data as possible; on the service side, you want to send as much data as possible to the collectors so the collector can filter it. And so you have a completely need to there. You also have the issue of like how do I actually send data to the cluster on the client side, which is kind of solved, but not really. You end up like writing an API endpoint that supports things to the collector so that you can transparently authenticate or not authenticate in a way that actually works, and you can start to try and filter out noise signal. + +**Host:** I guess someone calling the endpoint or are they just like—because of where did they answer something? + +**Hazel:** I was reacting. The most annoying one to instrument or was there another language that you've worked with that was even more annoying? Haskell was the most annoying one to instrument. + +**Host:** How big is like the OpenTelemetry support around Haskell? + +**Hazel:** The OpenTelemetry support for Haskell is actually pretty decent, which is really funny because Haskell has a bigger usage community-wise than Master does for OpenTelemetry. Yeah, and so Haskell is an ergonomic language; it's very expressive, it's very interesting, but the one time of Haskell is absolutely absurd and makes OpenTelemetry extremely difficult. + +So Haskell is a funny lazy language, which means that if you write the code, you don't have any control over the lifetime of when the code is executed or actually evaluated or how deeply it's evaluated, and OpenTelemetry really, really wants you to explicitly call things, have it start, then have that happen and turn like point. + +In Haskell, laziness makes it interesting because you end up peppering the laziness with the ones of strict functions, which are the tracing stuff, and that can mess with your memory usage, can mention your performance, your interest, and it can match a couple of other things. It's very, very difficult in that language to have something include transparent; you want the tracing to be transparent. + +Otherwise, the tracing is really tricky and brings a bunch of other things, and so making it transparent actually requires the use of unstable primitives exposed by the internal implementation details of the compiler and the runtime. + +[00:34:30] **Host:** Interesting. I feel like I learned something new today. I want to go back to another point that you had made earlier on, and hopefully, I understood it correctly. I think you made a point saying that generally organizations do tend to like create libraries around OpenTelemetry. Did I understand you correctly? + +**Hazel:** Yeah, and it's not necessarily that organizations tend to do that; it's that that is one of the best ways to multiply efforts when you have like a platform team when you're thinking of things in a platform engineering manner. So in general, if you have like something people need to care about life testing or instrumentation or runtime performance or some aspect of the code that isn't necessarily functionality, getting everyone to care about that equally is really hard. + +It'll live on to have the same level of knowledge; it's really hard. In fact, it's kind of not even so much as impossible, but I think it's, I think, oh, people try too hard to hit, and consequently, it ends up being very, very useful from an organization perspective to accelerate your developers by having like libraries built around things or my platforms putting on things. + +So you have like maybe a standard template and then your instrumentations just don't work and your CI/CD is just dealt with, and a whole bunch of other things you're done with it, and you can use it, but these structures are already set up to give people capabilities for handling things, taking care of like distribution, so that cross-service instrumentation to support the box. All those things would be things that I would want to deal with from a platform team perspective of making telemetry easier to do. + +**Host:** Yeah, that makes a lot easier at home really hard. So was there, at this previous organization, or even other orgs, even subsequent organizations, is that something that—it's one thing to, I think, have the aspiration to abstract that stuff from folks to, you know, make sure that they're at the same level when it comes to instrumenting. It's another to like be able to achieve that goal. Do you feel that in any of the organizations where you worked that that has been actually achieved with these kinds of libraries? + +**Hazel:** I've been able to get things to a point where it was achievable; whether or not I actually achieved it is a different question. But so what I mean by that is I was able to figure out ways to like wrap and abstract away the vast majority of the setup for OpenTelemetry when I did TypeScript or Haskell, and I was able to write wrappers around functions in a way that the wrappers are transparent. + +So we could have like a bunch of environment variables properly in and a bunch of like sort of stand up for things dealt with in a way that people just needed to set up their application in a certain way, and then you got like the very base skeleton of the tracer and all those things were just ready to go. They can like get the, you know, without the span sort of function and have like a more limited understandable API where you didn't need to pass in certain types of contexts manually. + +I've been able to do that for both, and I've even been able to do that in a way that works isometrically in JavaScript or TypeScript. + +**Host:** What does that mean? + +**Hazel:** It means you can have the same code running in server and the browser, and it works. That involved a mild amount of crimes; it involved a lot of crimes. It involves a lot of crimes. I abused how Node modules cached a bunch of things, and then I imported certain globals and then the runtime certain stable code here and there. If you import everything in the right order in the right place and then you rely on trade shaking, then your bundle on the server can be different than your bundle on the client, and you can end up splitting that out in a way that doesn't break the global singleton pattern of observability of the OpenTelemetry stuff and also not explained anywhere. + +[00:38:35] **Host:** Switching gears a bit to more managing the infrastructure side of OpenTelemetry, how was that experience for you in contrast to having to, I guess, previous roles of being more, I guess, holistically focused or having to like clean up the OpenTelemetry mess? How was that in contrast? + +**Hazel:** Running OpenTelemetry is interesting because like you have to open the OpenTelemetry collector to run, and the collector, like, then you have to set it up. If you go outside of essentially any of the very small documentation examples, you end up having to read the RFCs for things or you end up having to read the technical design document, and they are not obtuse, but they're understandable by like a select amount of people, and you have to really dig in to understand that. + +To like even for the tail sampling configuration of the OpenTelemetry collector, like an entire document of how to do it is the most difficult thing I've ever seen. It has like different layers of things; each one has its own entire language and specification setup, and it's wild. The Honeycomb Refinery is much better in that regard, but running Refinery in production is a janky mouse in a lot of ways, and that largely comes down to Refinery is really a tool that they built for themselves, and then they made it available to other people. + +So here's Refinery; if you know how it works, great, you just run it. But otherwise, you can run Refinery successfully if you're an organization with high-performance CI/CD, the ability to implement things and look at them, and you have that ability to like dial things down very rapidly. Otherwise, Refinery is not going to work super well for you, and to the organization that I ran Refinery in, split the difference; they had some amount of rapid CI/CD and some men of that, and I was able to look into logs and stuff like that and get some things down. + +But they used its pattern and Refinery due to how broken the OpenTelemetry stuff was. We ran into essentially every error case and edge case of it. For example, one of the edge cases that we had was many spans were not like, you know, a minute or two long. Finished spans were hours long or even days, and we had some spans that were multiple weeks in length, and we also had some spans that had over three to five million events in them. + +What ended up being was you would have something that would go, you know, I started to ask, and then that starts to span, and so this would be like an asynchronous job. Into the asynchronous job would, for every single user in the database, do a whole bunch of validation stuff, and so that would be about like 1,000 demands per user times 500,000 users or 20,000 spans per user times 40,000 users. + +It's ridiculous! Or like for every single item in the database, check its consistency in one span, so that would take like 20 hours, and that would break every possible configuration of Refinery. You just can't have this span that's not broken and also have Refinery store 50 million things in memory. + +I got things improved, and ultimately I ended up saying this entire way we do OpenTelemetry in like this with an asynchronous job is actually now close enough to streaming services that it doesn't work that way. The wheel dealer streaming services is to do it the way Honeycomb does, which is you just send a snapshot of this span every minute, so any rule of things in the code, and that requires you to have written your code in a way that you can roll things up and then send it every minute. + +You need to completely rewrite how you do most of your task handling to have some sort of task handling manager that sits there and can collect a bunch of things in every minute, do that, and start a new span. That is not really possible the way most people write asynchronous jobs, and fixing that is really tricky because there's not really a wheel if it's now; you're not actually just rewriting the code. + +It requires you to really just start a million different tasks, and each task is linked together casually by like span information. Then the collector sort of like does some things in there, or maybe your span processor can roll that up, but otherwise, you can't actually do that. + +Now, you should do your tasks in that manner anyway because then it allows you to have a right-hand lock, and then you can ensure your task consistency is a longer think. It's not interested in the middle; you should do that anyway because that's how you do it if you understand distributed systems. But you run into this thing in OpenTelemetry; it was written by systems-minded people, and so a lot of design decisions that they make make sense if you know how to write a standard stability system. + +Of course, you use a write-ahead log; of course, you want to just spawn an event from like respond one task somewhere for like 20 hours. Who does that? Everyone does that until they know better. + +[00:45:00] **Host:** That's a really good point. We've got about six minutes left, but before we wrap up, I did want to talk briefly about some of your experience in trying to make the case for OpenTelemetry to executives because that can be challenging. + +**Hazel:** Yeah, so there's two sides of it: there's making the case for any sort of monitoring or observability in general, and I was making the case for more OpenTelemetry specifically. The second case usually happens when you have some pre-existing stuff, and you're arguing, “We should not use the existing stuff; we should use this other thing instead.” + +What you're arguing for there is a migration, two migrations: a technical migration and a social migration, and those are very tricky. You want those both migrations to happen, and so you need to be able to essentially show that the time and effort saved by via this migration will pay for itself in about four months. If you can't show that it will pay for itself in about four months, then you probably shouldn't actually do it, and you should figure out how to make it pay for itself in four months and then do it. + +Otherwise, you'll end up with a migration that takes like a year or something, and it has no perceivable difference in benefit, and then people are not going to be sold on it. People need to actually tangibly feel the benefit, or the migration is going to feel on the social aspect technically. + +For conventional organizations that you want to get like any sort of monitoring in general, that relies on being able to convince people that the life cycle leads to encompass more at the life cycle, and the easiest way to do that is, in my opinion, you tie the code lifecycle to the business value delivery lifecycle. + +So what can happen in organizations is you have like the developer lifecycle, which is over here, and it's, “I write code, I commit code, I merge code,” and you have the business lifecycle, which is we could do some market research, or we get like some seamless research, and then we design a product, and we get the final features, and then that final feature is implemented, and then we see what the feedback is. + +At no point do those two overlap or cross. So that leads to people to my product owners handing features over to developers and developers writing tune and just handing lots of production. Instead of playing frisbee, you need to merge those. When those get merged, then essentially the developers have to care about things all the way to the point where it delivers value to the customer, which is absolutely what every executive already wants to have happen. + +The state they have within separated is what they think is like has to be that way, and the second is it's possible and even desirable to get developers to care about like the outcomes from the customers, get them talking to salespeople, and get them talking to product people. + +That will not only improve things for the product, improve the internet, but it'll actually improve the productivity of the developers and their experience and their ability to do this. That usually sounds executives, and you need this. + +The next checking after that is once they have, “You need this,” if they just look for like observability, they're going to get one of like three or four different vendors and probably going to end up an expensive vendor. You need to pick the vendor based on what the business needs and not necessarily what you like the best. Like for example, my current company, we have advanced monument compliance requirements. + +We have it for only one environment that provides more than one codebase, and so because of that, no one in the company can use anything that isn't FedRAMP compliant, which significantly limits the amount of advantage we can have to basically one, and I don't necessarily want to use that vendor necessarily, but I have no plans to migrate off of it. If I were to migrate off of it, it would be in a very interesting manner. + +What I would do is I would split the environment into two environments and have them be identical, one FedRAMP certified, one not FedRAMP certified, and then essentially would have a FedRAMP program, but that would be like a separate sales funnel, and anyone who actually needs it can have that because the FedRAMP is valuable in two aspects for the business. One is that it gets them certain customers, and one is that it gets some certain conversations, and for the conversations, those people may not be on federal, but they may want to eventually when it means that I can run the same codebase in both locations and use a different vendor and the one that isn't FedRAMP and that one but that whole migration for the current company would be allowed two years of work. + +Are we going to do that? Who knows? Maybe. But there's a lot of other questions to ask first and a lot more that we need to do before we can even get to that point. + +[00:50:00] **Host:** Yeah, speaking as a vendor, we've had a lot of conversations about FedRAMP certification, and being that not 100% of our employees are in the U.S., we'd have to do exactly what you described. We tried to do that before for HIPAA, and it created kind of a weak experience for some customers, so yeah, it's a tricky problem. + +**Hazel:** Interesting. I think one of the most interesting things about HIPAA is that any sort of regulatory environment—there's like one regulatory environment, and then there's like FedRAMP, and the overlap between people that need any regulatory environment and people that need or really want FedRAMP is basically a circle. + +So if you're going to go like, “Oh, we're going to have like a HIPAA environment, we're going to have a FIPS environment, we could have like, you know, SOC 2, like it used to be SOC 2,” whatever. But if you have one of those, you probably would actually get as much benefit, if not more, if you went straight to FedRAMP, even though it's like three times as expected to do that at least. + +And the ongoing maintenance burden of that is ridiculous, but it gets you on the other customers you need, like HIPAA plus FedRAMP or FIPS plus FedRAMP, and that is a really interesting business thing that's non-obvious. Any type of regulatory environment completely changes the game for OpenTelemetry, what you can do with it, how it works, your vendors in general, and even like your hiring practices, and it's really weird that you can't really have an asset, but you can't just get HIPAA; you really should probably go all the way to a FedRAMP just because of who the market is that needs one or both. + +**Host:** I totally agree with you based on like our experience with that HIPAA server because it turned out there were companies camped on it that just had a desire for like more privacy, stricter controls because of how their company culture and their data worked without actually having the need for the federal regulation. + +I think you're right that it makes it easier to sell. I think probably we wouldn't end up going in that direction again unless we managed to partner with somebody who was actively like bringing in FedRAMP customers, like some agency that worked with the government or whatever, like we would have to be. But I wish we had known that at the start, and we didn't. + +One thing that would be interesting there would be to have OpenTelemetry have some sort of agency that—and that agency gets FedRAMP compliance. Then what you can do with that is you can have that agency essentially facilitate the onboarding of OpenTelemetry vendors or just other observability vendors into the space, and you can use that to sort of start bridging the gap and giving people more capabilities and sharing that my FedRAMP specific architecture concerns in the way that spreads their compliance load among multiple companies. + +**Host:** That would be a really interesting idea. + +**Hazel:** Yes, it could maybe make it work through the project, and I would say for us as a small vendor, that really is a problem that if we are not allowed to have our engineers who are in other countries touch the system, we may only have one engineer working in a particular area, and so we have to hire a second person in the U.S., which is undoable. + +There are certain ways, and you would need somebody to talk to someone who's experienced in FedRAMP to negotiate that, which would have to be—100% U.S. citizenship is a non-seer own requirement. It gets nuanced, and it gets tricky. + +[00:56:00] **Host:** Oh, interesting. Well, I think we have to leave it at that before we got into a funny side. Thank you again, Hazel, for sharing your insights. This has been really cool. I don't think we get to talk to too many people who have had like such advanced use cases for OpenTelemetry, and being able to share that with the community is really good because I think a lot of us get, you know, we do like those intro tutorials. We’re like, “Hey, that's pretty easy!” and then you get into like the guts of it, and then that's when you start getting into some of those gnarly use cases that can be really tricky in life. + +**Hazel:** Yeah, definitely. + +**Host:** So definitely appreciate that feedback and the viewpoint, and you will be back for OpenTelemetry in practice, I believe, on September 14th. Looking forward to that! + +**Hazel:** No idea. I'm going to figure it out, but it's going to be a lot of fun! + +**Host:** Awesome! Really looking forward to that. Thank you again, as always, super, super awesome insights. Yeah, and we will hopefully see everyone for OpenTelemetry in practice with Hazel on September 14th. Thank you! ## Raw YouTube Transcript diff --git a/video-transcripts/transcripts/2023-09-16T04:37:50Z-otel-in-practice-presents-context-abstraction-threading-the-needle-with-hazel-weakly.md b/video-transcripts/transcripts/2023-09-16T04:37:50Z-otel-in-practice-presents-context-abstraction-threading-the-needle-with-hazel-weakly.md index c4f65dc..13186ed 100644 --- a/video-transcripts/transcripts/2023-09-16T04:37:50Z-otel-in-practice-presents-context-abstraction-threading-the-needle-with-hazel-weakly.md +++ b/video-transcripts/transcripts/2023-09-16T04:37:50Z-otel-in-practice-presents-context-abstraction-threading-the-needle-with-hazel-weakly.md @@ -10,71 +10,272 @@ URL: https://www.youtube.com/watch?v=1a7GyarlGAQ ## Summary -In this video, the speaker, Hazel, discusses the complexities of observability in distributed systems, focusing on concepts such as context abstraction and the challenges associated with implementing observability in real-world applications. Hazel provides a primer on observability, exploring its definitions from consumer theory and cognitive systems engineering, and emphasizes the importance of asking meaningful questions to derive useful insights from systems. The discussion covers the role of context in open telemetry, the significance of tracing in understanding system interactions, and the difficulties encountered when trying to instrument code without intrusive methods. Key points include the challenges of managing context propagation, the lifecycle of spans in tracing, and the implications of abstraction on observability practices. The session also includes audience engagement, with questions and reflections on the current state and future of observability practices, highlighting the need for effective collaboration among engineers to enhance system understanding and problem-solving. +In this YouTube video, Hazel discusses the complexities of observability and distributed systems, focusing on concepts like context, abstraction, and the challenges of building observability frameworks. The presentation begins with a primer on observability, exploring its definitions and the importance of asking meaningful questions to derive useful answers about system performance and behavior. Hazel highlights the role of context in linking disparate components of a system and the difficulties faced when attempting to instrument code for observability without intrusive methods. The talk also touches on issues related to graph structures in tracing, life cycle problems, and the limitations of current frameworks like OpenTelemetry. The session concludes with a Q&A, where Hazel addresses audience inquiries related to post-processing of trace data and the evolution of observability practices. The video emphasizes collaboration among engineers to enhance system understanding and improve observability. ## Chapters -00:00:00 Introductions -00:02:30 Overview of the topic: Context Abstraction and Observability -00:04:00 Definition of Observability from Consumer Theory -00:06:15 Cognitive Systems Engineering definition of Observability -00:08:45 Personal definition of Observability -00:12:00 The importance of asking meaningful questions in observability -00:17:00 Introduction to Context Propagation in OpenTelemetry -00:22:30 Explanation of different types of context: In-Process and Cross-Process -00:30:00 Discussion on the Life Cycle Problem in tracing -00:36:00 Graph Rewriting Problem in instrumentation -00:45:00 Conclusion and final thoughts on Observability challenges +00:00:00 Welcome and intro +00:01:40 Definitions of observability +00:04:20 Personal definition of observability +00:06:50 Context in observability +00:09:28 Types of context in OpenTelemetry +00:12:30 In-process context explanation +00:15:00 Cross-cutting concerns in instrumentation +00:19:00 Life cycle problem in tracing +00:24:01 Graph rewriting problem +00:35:56 Conclusion and Q&A -# Transcript Cleanup +**Speaker:** Foreign -**Introduction** -Hello everyone! I’m having a lot of fun hoping to run innovation and developer experience in my current company. I also serve on the board of directors of the hospital foundation, and I have a lot of opinions online. You can find me pretty much everywhere, and I’m really excited to be here today talking about context abstraction and typing the needle. +I am having a lot of fun hoping to run Innovation and developer experience in my current company. I also serve on the board of directors of the hospital foundation and I have a lot of opinions online. You can find me pretty much everywhere, and I am really excited to be here today talking about contact abstraction and typing the needle. -**Observability Primer** -I’m going to cover a primer on observability and discuss how I think about it, what the context is, and the historical reasons we need distribution. Then, we’ll tie that into observability, along with some components of social filtration, mainly context. We’ll also explore abstraction and some difficulties you may encounter when trying to build libraries or platform teams, rather than just manually instrumenting all your code and writing everything from scratch. +I'm gonna go over kind of like a primer of observability and talk to you a little bit about how I think about it and sort of what that context is, and the historical why do we need distribution and how does that tie into observability. Then we're going to talk about some components of the social filtration, mainly context, and then we're going to talk about abstraction and one of the difficulties that you can run into when you're trying to actually in real life build down libraries or build on platform team or build out something other than just manually instrumenting all of your code and writing everything from scratch. -When you start with observability, you have your system, and you want to figure out what’s going on. Over time, people have encountered this problem, and different groups have come up with various definitions of observability. +Don't you started with observability? You have essentially your system, you have everything, and you want to figure out what's going on. Over time, people have run into this problem, and different groups of people have come up with different definitions for observability. -**Definitions of Observability** -One definition you might hear from Journey Majors and others in the open telemetry and tracing backgrounds is from consumer theory. It defines observability as the ability to measure the internal state of a system based on its external outputs. This is a great definition, but it doesn’t tell the whole story. +[00:01:40] One definition that you've heard journey majors and a lot of other people open telemetry and tracing backgrounds and distribution system background to talk about is the one from consumer theory, and that is the ability to measure the internal state of a system based on its external objects. This is a definition that you see repeated in a lot of places. It's a great definition, I like it a lot, but it isn't necessarily the entire story. -Another definition comes from cognitive systems engineering, which defines it as feedback that provides insight into the process and the work required to derive meaning from available data. This definition highlights that observability is a back-and-forth dialogue. It’s not just about observing everything; it’s about the process of getting there and understanding what it means. +One other definition comes from a different theory of practice which is cognitive systems engineering, and so for them their definition is feedback that provides insight into process and the work required to start to mean from available data. We like this definition for a lot of reasons, mainly that it does a really good job at showing that observability is really this back and forth dialogue. You need to not just have, you know, oh, you can observe everything. Well, that's cool once everything is built out, but like how do you get there? What does it mean? And the whole process of people are involved too is something that this definition kind of highlights, and I love any technical definition that includes people. It's my favorite. -My personal definition, which blends elements from both definitions, is that observability is the process through which one develops the ability to ask meaningful questions and receive useful answers. This means it’s an evolving process; what observability means to you can change over time. +Speaking of which, my personal definition is I'm not biased, but I like this one the most. I took the other two definitions from cognitive systems and from contour theory. They're kind of blending them together. For me, my definition of observability is the process through which one develops the ability to ask meaningful questions and get useful answers. -**Meaningful Questions** -A meaningful question is one that maximizes how much you learn about the system you’re participating in. For example, if a CEO wants to know their market position compared to where they want to be, an observable system should enable them to answer that. For engineers, observability questions often relate to system performance and code behavior, which we’ll delve into more today. +The way I think about that is it's a process, which means it has to be evolved and you start from somewhere, but you're not going to stay there. You're going to continually figure out what observability means to you, and it may mean something different today than it does tomorrow, than it will not show you than it did last week. Additionally, meaningful questions is really the thing that observability to me hinges around. -**Early Observability Practices** -In the early days of observability, when you had a web server but no data, you might have resorted to stopping production, launching a debugger, and stepping through the entire system. This method could lead to meaningful questions and useful answers, but it’s not the most efficient approach. +Can you ask a meaningful question of your system, which may or may not involve computers, may or may not around people? It may involve like a whole bunch of different things, and are those questions meaningful to business, meaningful to you, meaningful to what you need out of that learning? And in asking those questions, can you get a useful answer? -As we moved forward, the desire to observe without interruption became essential. The introduction of logging led to a more structured approach, but it still required careful consideration to connect the front end and back end of systems for a complete understanding. +[00:04:20] I like this definition for me because it includes every aspect of observability in my opinion. If a CEO wants to say, "Hey, I have this meaningful question, which is I want to know where we are in the market compared to where we want to be in the market and how feasible am I going to get there?" Can they answer that to build like what we need in order to approach our problems for the next five years? I only said in Los Angeles, so I'm keeping them happy and healthy. That's a meaningful question. An observable system will let you answer that. -**Context in OpenTelemetry** -Context in OpenTelemetry is crucial for managing cross-cutting concerns. It acts as a propagation mechanism, carrying execution context values across API boundaries and between logically associated execution units. Context allows us to unify disparate pieces of information and understand the system as a whole. +For engineers, a lot of that observability questions, it's going to come down to things about the system, things about code, and we're going to dig a lot more into that today than the other questions. We should all be working together, and we should all be able to collaborate in order to actually solve all these problems into these answers. -For each span, there is a variety of context elements, including span IDs, trace IDs, and trace states, which help identify relationships between spans and facilitate tracing across services. However, there are challenges in managing these contexts, especially when dealing with baggage and how it propagates through systems. +For me, a meaningful question is one that maximizes how much you learn about the system that you're participating in. If you have that system and you want to know more about it, what does that mean? How do you get there? What are you doing right? -**Process of Context Management** -When dealing with context, you have in-process context (the normal context created in your code) and cross-process context (how that context is serialized and transmitted over the network). It’s essential to ensure that the context is correctly integrated at various points in your system. +For programming, we have all my camera disappeared. Brilliant. I love silent problems. Alright, so I'm going to figure out what happened there super briefly and then fix that, and if I can't fix it, then I'll just tell everyone what's on the slide. Oh well. Alright, let me share my screen again and I'll just tell you what was on the slide. Sorry about that. I debugged everything and we're learning information. Awesome. -When you receive context from another service, you typically deserialize it and make it your current context. However, if you don’t own your systems end-to-end, it can be challenging to ensure that context propagation remains consistent. +[00:06:50] So imagine on the slide that you have a typical system in which you have someone and they're running a web server, and they are asking themselves, "What is this web server doing?" and should they look at the ones and the lots? And once you have no data whatsoever, absolutely nothing, nothing's there, and they're like, "Okay, now what? What do I do?" -**Challenges of Manual Instrumentation** -The biggest issue with manual instrumentation is that it assumes you’re writing all your code and instrumentation in an intrusive manner. This can lead to problems, especially when you have to manage multiple spans and their relationships. +Back in the good old days, well, what you probably would have done was you would have just killed production, taken the whole thing down, launched the debugger, and just stepped through the entire system step by step, solving everything. You can ask a lot of meaningful questions that way, and it can get a lot of useful answers. Everything's solved, and all you had to do is completely change production in order to do it, which is, you know, maybe not the best approach, but it is, you know, it's in vogue; that's what we did for a long time, and in many ways it still works. -For instance, once a span is created, you can’t add information to it after it has ended. This limitation can lead to confusion and silent failures in your system. Moreover, the life cycle of spans complicates the ability to trace and sample effectively. +But one of you have to constraint of, "I want to observe without interruption." I want to, you know, keep everything running and ask me to focus into the system. Now I'm considering the system running as part of that question. -**The Graph Rewriting Problem** -The graph rewriting problem arises when you want to manipulate spans in a way that isn’t straightforward. You may want to collapse spans or add attributes without creating additional spans unnecessarily. However, existing instrumentation frameworks often don’t provide a clear way to do this. +Let me try, you know, adding some French statements, which again, we're seeing a little bit of lack of observability here in the slides. That's fine. So if we have a French statement, and we've added them everywhere, now we have a real login. This is awesome. So you can imagine us taking the web server and getting those alarms and looking through, and you can see, "Oh hey, this one usually locked in this one." -**Conclusion** -In conclusion, observability is about developing the ability to ask meaningful questions and derive useful answers from the system you’re working with. The more you can observe, the more correlations you can infer. However, be cautious about the additional complexity and noise that can come from adding more data. +I use a middle transaction; they're able to do something, and then they locked out. That's awesome. We have everything there. We have it done, and we're ready to go, right? Is everything there? Well, you have the web server and you have the back end, and unless you have very carefully, you know, done everything with your logging, you'll notice that the front end and the back end, nothing's tied together. -Context plays a vital role in turning disparate data points into a unified trace. Understanding how to effectively manage context and its implications can significantly enhance your observability practices. +The pen might say, "Oh, you should check down," and the back end might say, "This database middle transaction, this little visual transaction, this individual transaction," and you have nothing to tie those two disparate pieces of information together. You're like, "Okay, I still can't really understand anything about the system. I know what each individual piece is doing, but I have nothing to tie it all together." -Thank you all for your attention! If anyone has questions or thoughts, I’m here to discuss further. +[00:09:28] Context really ties the system together. Context, in OpenTelemetry, is for cross-cutting concerns. The documentation for OpenTelemetry says that context is a propagation mechanism which carries execution script values across API boundaries and between logically associated execution units. That is awesome. + +Let's break that down just a tiny bit. So context, let's skip on the middle bit, carries values. Context is the blue, the terms of metadata into unified trees. What does that mean for you? What does that mean when you're trying to understand it and instrument it? + +So let's break down and decent context. They're going to have for each span, they're going to have a whole bunch of context in there. You'll have like this span ID, which identifies that, the trace ID, some trace files, and a trace state. They're going to have essentially everything that identifies to spam, of which there are many spans in one trace and many traces in one system. + +Scan links or another type of context, and they say, "Hey, this advantage is related to market." So you can think of traces as a graph, spans are nodes, and then links to the edges. If you have a graph that's a tree and you're like sort of building it down, and all of a sudden you need to take this one span over here and this one span over here and say, "But they're kind of linked together," you build that link, and that's an edge in that graph. + +Links can be really confusing. They can be kind of weird. They're not always integrated very well in the whole setup, but they are needed for especially in some types and patterns, and they are what turn OpenTelemetry traces from a tree into a graph. They are the most expensive thing you have in there and also the hardest thing to work with at the same time. + +Currently, you can only add them at span creation time, but people want to be able to add them anytime, and that may happen in the future. You'll be able to have a non-directed graph and a little drag, which will be fascinating. + +Another type of context is baggage. Packets want to keep valued parents downstream. It is widely regarded as a questionable thing to use. I know many people who might ban it in their code base. It's a bit of a on, but also very, very powerful and flexible. It is one of the ways to send information downstream, but you still can't use it to send information upstream, and I'll get more into what I mean by that later. + +[00:12:30] The package is there; you may eventually need it. I kind of hope you don't, but it's there. And when you have all these different types of context and you have these different services and you have the different processes, how do you take this context and actually use it to glue together everything and make it usable? + +You have the in-process context, and then you have the cross-process, and how to complicate things from one thing to another. Then you have fun process, which is in everything, and actually taking it in the next service and going down that way. So I'm going to go over all of them in order. + +So there's some code here. Sorry about that. I've never even probably known as context in process. It's the normal context; it's very typical, very standard, and this is what you've seen in the documentation. This is the in-process context. You create an octaves fan, and then in that disadvantage, and work, you create another active span, and then that context and links the two actually happens pretty much automatically, and you never really need to think about it. + +The library and the SDK will handle all of that for you. Sometimes, however, you do need to add it explicitly. This is an invisible example showing you that you need to do it, especially when it comes to making links and doing things that way. + +I'm going to one second. Oh, amazing. I'm going to watch my screen shoot, stop sharing, and then share something different. It turns out I accidentally made my syntax highlight and only work in Firefox. That's what that was. Cool, yay. + +There you go. You can see here that we have the current span, we have these span contents, and we get that context and pull it out manually. So then we can create a span with a link. We need to do that in order to propagate things manually for this type of context information. But typically, you don't really need to do that, and things through it just naturally work with the links you need to do normally. + +[00:15:00] That can be a little tricky. Cross-process, there's two steps involved. You serialize the context. Foreign, it can actually secretly, don't tell anyone I told you this, but there's no reason context needs to be an adder. You can do anything with any transform mechanism that you want to. It's just really serializing the context needs to happen somewhere, and then that needs to be attached to the payload of your call. + +I know some people who have actually serialized the context and embedded it into an RPC account as a way of actually instrumenting and tracing embedded systems. Some people have done that with their own bootloaders, which is actually really cool. It doesn't need to be a matter; a little wild if you want to use the library should not write everything yourself down. All the OpenTelemetry stuff will stick it in each TV header for you. + +So what that looks like is you have from the sending service, the sending service is going to inject what's the publication object, the current context into some output, and so that will do the serialization process. Then you'll have a transparent and a trace state, and then output, and you can use that over the internet. The transparent is actually the only thing that you need. The twisted has a bunch of other stuff, but you don't actually kneel in and you may not want it. + +So the transparent is the trace ID and everything required to actually link the span from the net system over the trace state has on the other fun goodies that you may want but may not want. Some people I know have been using twisted and can't touch specification because of issues it has. + +Then I'll get to later the context one process. This is the second process. We've yielded everything up on the internet, and now I'm ready to receive that and actually integrate it. You'll have, you get the fabricator, you ingest everything, and you deserialize the context and you extract it, and then you did that. It's not a context, and you make it your current context. You don't need to make it the current context, but typically that's what people do. + +Sometimes, people use the trace state in order to determine whether or not they should make the contents the current context, and you can maybe want to do that sometimes if you don't always own your systems end to end. Someone may be putting something in the middle somewhere, and that can be really tricky. + +I know that some OpenTelemetry vendors, for example, have run into this problem of you need to really make sure that you're contacted your contacts before actually making it your current context. So that's a lot of fun. Once you have that context, then you've started it, you set your span and just done working from there. + +So that is moving the context, wrapping everything together, and you have all of that, and that is great, but that is all really about manual instrumentation. That assumes that at every single point in every service you are writing all of your code, you're writing all of your instrumentation, you're writing everything from scratch, and you're writing it right there in line with all the code in an intrusive manner. + +If you don't want that intrusive style and you want to be able to build something for other people to use, this is where you run into a lot of problems, and we'll get into this now. This is, in my opinion, one of the weakest aspects of fishing in general, and it's a sign of the image journey in the ecosystem. I'm gonna have a lot of hope that can be improved, and I'm looking forward to seeing what happens there. + +[00:19:00] Do you remember earlier when I said that context is about to cause any concerns when they have a cross-Canadian concern and you tie everything together? What you also have is cross-cutting breakage because one of my favorite, um, laws out there is Hiram's law, which tells them with a sufficient number of users of an API, it does not matter what you promise in the contact. All observable behaviors of your system will be dependent on by someone, which means that the more instrumentation that you add into your system, the more things that you're adding that can be observed by yourself but also the rest of the system. + +So paradoxically, when you come on service boundaries, in theory, you can only depend on the public API and service. It's a very standard service-oriented architecture and things like that. The tracing actually doesn't do that; it's out of hand intentionally invisible, not apparently API. + +If you've been with anything anywhere, it'll propagate to the rest of the system, and you'll end up having a lot of essentially convention and hand-holding and little manual pieces put together that will break every time you change something. Those out-of-band information become in-band semantics, and this problem is something that you see in a lot of different ways. + +More OpenTelemetry, it's actually specifically about this context modification, but in terms of verification, you see this in terms of the internal implementation details of a function changing the proof required to show the scratch verified for property. And in distributed systems, you can see this in the end-to-end property of networks and in lots of other places. + +You can see it probably one of the easiest ones is different agility of snapshot testing for fun and design systems. If you've ever worked on those, you know that you can meet the tiniest little change. Everything's the worst, and somehow you just broke your entire test suite because some lines moved here and there in the code itself or something changed here and there in the code itself or in like the HTML, but the actual appearance and everything's fine, and you end up dealing with this frustration and description of the out-of-band information becoming in-band semantics, and that mismatch causing wreckage. + +In OpenTelemetry, the package is not simple enough to cause a lot of honey. It's abused. One context is key, and when you have this cross-process propagation, you lose the ability to have your out-of-band and inbound semantics match up and be enforced across boundaries. + +Additionally, in Winter designing options, or you're wrapping them, that same Alabama information will be very painful because you can't necessarily build anything that actually lets you do this in a reusable way unless you build a shared library of semantic conventions. + +Speaking of which, good options are both transparent and opaque. What I mean by that is they're transparent in that you don't have to know that they're there, any condition about them at that level of abstraction. A good way to think about that is, for example, HTTP requests; they just happen and they're just there, and you don't really have to know that they're implemented on top of a bunch of wire or glue, terrible crimes, and weird packet header loss information, blah blah blah. + +There's a lot of trauma behind that, but you don't have to know that. With that said, you can't know that, and the transparency there is that you can reason about everything at that level as if the abstraction was in there, and you can write things that are aware of the fact that they actually work through it in order to know and understand the underlying system. + +Having that absorption means that you don't need to know the system underneath it, but that the system underneath it doesn't make it less understandable of inflammatory, and out-of-band information in general meets both of those goals. + +Hard to achieve, the Louisiana ban information is not actually semantically embedded in the system and becomes relevant to the system, and so you can't build an abstraction that lets you operate without knowing the internal implementation details, and you don't actually have a way to observe the internal implementation details and know what they are without disapproved forms memorizing those conventions. + +That gives you this quandary: what do you do with that? This can manifest in a couple interesting ways. But a different problem of abstraction is I'll show you this one, and this is a problem that I run into the most, which is what I call the life cycle problem. + +[00:24:01] That is you have tracing which is a graph, but you don't actually have a way to really manipulate the graph very well, and that shows up in two ways. The livestock of how much the first flame, which is that sending information up a graph or back in time, so to speak, isn't super possible or convenient or really ergonomic in any way. + +Then adding, and then adding things after the fact is not possible either. So you can't like pack to things as a graph and like incrementally build up or out of beyond build up a view of the graph that's more complete. + +So here's an example. We have a web server, a service, and a client. The web server has, you know, one event, a one span, and then a second span. Two fish has one span, and then a plan has one span, and they have overlapping timelines. There are problems that I ran into trying to deal with building this. + +So the first span in the web server can pass information to the second span when creating it. That's totally fine; that works. That's amazing. That's awesome. However, it has no real way to pass information to the second span after creating it. + +So after you've created that challenge span, the root doesn't really have any way to continue to add information. It doesn't have any way to modify that, and it doesn't have any way to like dig down into the tree and continue to add things. If you have intuitively instruments and everything, you do have the reference to this span, and in theory, you can work with it. + +One of your building apps into new building libraries, you don't have that; so you don't have a way to send information down. They have baggage, but it almost feels like a workaround. It doesn't feel like it's embedded in this idea of how do I add things into the system and build this graph. + +Conversely, the challenge span doesn't have any good way to send information to the parent span conveniently at all. It's possible if you know how to do it; it's possible if you know how to build a workaround for that. You can even build conventions to make it easier, but it's not absolutely convenient in any way. + +The lump server is able to pass information to the service and then the client via contact configuration, which I talked about earlier, and that's possible not to create networks. However, the service and the client essentially have no way to pass any information back up to the web server. Forget about it. + +Another thing that's interesting here is if you notice the life cycle of the web server watched is actually shorter than the life cycle of the whole trees, which means that Taylor sampling is not going to work as you might expect. Taylor sampling really wants you to have a full stitched treat with a scooped corn snack life cycle to the web server or the root span has to be the longest span and everything else has to be inside of it and organized nicely so that you can make a decision once you've collected everything. + +When you have these Instagram timelines and you don't actually know how long everything is going to be, if you don't know how long to wait to make your final decision, you end up discussing until it becomes heuristic faced rather than, "Oh, and let's actually do things." + +But you can really bring you, if those heuristics are used as part of filtering your traces and actually trying to sample things. + +So Taylor sampling distributing choices is actually really, really hard. Another problem is that once each fan has ended, you have no way to add information to it. So even though it is technically possible to try and add information to expand from the parent, you can add a permission to a challenge fan in some ways; that only works if the child span hasn't ended yet. + +You can have a really weird and rich conditions essentially. Speaking of that, also happens. Yeah, you can add information to the service, and also the child and parents have the same problem too. Even in context and even in process versus under severe process, you still run into the same issue. + +This means that you have a ton of internal implementation details of the instrumentation, the shape of the code, how the code works, the life cycle of their code, the column staff of everything to keep track of, which if you are the engineer writing everything, interviewing with the whole system and working end to end with it, that may not be a huge deal for you, and that may actually be there. + +That's one of the reasons why OpenTelemetry is so deeply in hand with people owning their systems end to end and having that feedback. + +There's actually, in some ways, available compensation because you can't build an abstraction unless a lot of people work on top of that. So you have to own your system end to end intrusively rather than being able to build these abstractions that let you instrument things more effectively. + +The second problem that you run into a lot is what I call the graph rewriting problem. So we dig back into this livestock problem. You have a bunch of information based on life cycle. + +So when you make lesbians, lens fans, and how that interfaces with the constant of the stimulate system in general, the graph rewriting problem is this one. The rule of thumb when doing instrumentation is that you have like this final instrumentation that can build this scaffolding. You would take all of your manual instrumentation and hang it on that. + +So in theory, you're not really creating one span. You usually understand that auto instrumentation already set up, and you're just adding attributes in my spots, which sometimes that doesn't work, especially when you're trying to write abstract code. You end up wanting to create this fan only abstract; couldn't do that. + +In general, the more you can just add attributes to an existing span created by Honor instrumentation but not sad, where do libraries go in this, and how do they work, and how do you actually do this? + +So let's look at this as an example. We have a tree. It's fast client, and then it smash client has some action. Someone called make request on the client side, and then the auto instrumentation has that git endpoint. The next step on the client side to half the function call, and then you have the honor instrumentation of the actual that response. Awesome. + +That's great. That's cool. Then we go over into the server, and the server has the server endpoint that has three manaway functions that it has a post endpoint for the sending for an internal call. Then that has the same chain; you post on one thing, and then you go to the next one, then you keep going and keep going, and you can run into the same middleware, and you're going to do all that. + +If you're going to run into a couple problems here, the first problem is how do you collapse spans? You don't actually have a good one to do that. What if I don't actually want all of the middleware to be their own spans, and I want to have just one span for everything? Just add information to the development to it with attributes as I go down to me. If you want some spam events, maybe I just want the attributes, maybe I want things like that, and I don't actually need this fan for like the middleware around, or maybe I don't need a fan for some other things. + +I don't have a way to like collapse that tree down, and that can be activated by the Fall. That's that if you want to search through your system, typically the root span is where you search, or the same level of a span is when you search when you're trying to like index through your data. + +So if you have dealer at different span levels, you essentially have no way to correlate those two pieces of datum and do a query in which you aggregate things across those spans. That's one reason why a lot of good practice is to have very, very wide events and a very wide spans and less of them so that way you have all the data at the same level and you can do all the correlation and grouping and aggregation and everything else there that you need to in order to effectively look through your data and find something. + +But if you have all of your instrumentation and your honor instrumentation building a nested tree of like 10 or 20-inch spans before you get to run any of your code, where do you stick in so that it's at the level that it needs to be? + +On top of that, every library, auto instrumentation, and things like that reinvent their weird set of configuration. I want you to try and make it to that they can help solve some of these problems. So you'll have, if you look through all the SDKs or all the contributors contributing libraries for OpenTelemetry, you'll notice this problem over and over. + +You're like, "Oh, how do I use, for example, the express server auto instrumentation?" It'll say, "You can just use it, and also here's like 15 different options. Do you want to silence some endpoints? Do you want to filter some endpoints? Do you want to silence submit awareness? Do you want to whatever?" Here's a whole bunch of functions you can use. Here's some callbacks, and we're going to call this compact before and after this; we're going to call these to these endpoints. + +Essentially, it gives you every possible hook point to you can think of to enter into the system and stick your own code there as if you haven't written it yourself. But when you don't have this way to write an abstraction and you end up having to sit there and write the code yourself in all these intuition endpoints and you need to have people having foresight in their libraries to put these insertion points for you, you have this very, very weird platform of really people wanting to like more or less patch the source to go to the libraries and the other instrumentation, and they want to have written all of it themselves graph effectively. + +Why can't we deal with that, right? What's going on? I think I cut out for a brief second there. Did everyone catch the last things that I said? Awesome. + +[00:35:56] So in conclusion, there are some problems here. There are a lot of things going on. There's some advice on what to do, and I'm gonna go through all of that now. + +Observability is the process through which one develops the ability to ask meaningful questions and get useful answers. Meaningful questions for you are going to be about learning about the system: how do you collectively get together and advance your understanding of the system, its behavior, how it interacts with you, and everything, and how do you get that knowledge as quickly as possible? + +Those are meaningful questions. Continue can use for answers from them. If you can, your system is becoming observable. However, observability is interesting because the more you can observe, the more correlations you're going to infer with your data, and the harder it's going to be to discover contributing factors. + +So just because you can find a needle, it doesn't mean you need to build a haystack. When you have more and more data and you keep shoving more things in there, then you have more and more out-of-band data, and you just add more and more layers of things that can give you observing money into the system. + +You should be really careful to look at everything and go, "Is this actually an intervention or understanding of the system, or is this adding noise?" Context plus configuration is traces; it is the glue that turns and back of data points into a unified trace. + +When working with context, it interfaces with everything that has to do with OpenTelemetry and in many ways remains invisible the entire time using until you're wondering where the entire or how to use it, or maybe you're confused. + +But if you know how to use context and know what to think about when using context, it can become your friend. One way to map contacts and map different aspects of OpenTelemetry together is thinking about the shape of those contacts of those concepts. + +So traces are a graph; spans are your nodes and links are your edges. Despite choices being a graph, they're often actually mostly thought of and best most ergonomically designed as a card stock, which is a training that is like perhaps multi-branched. + +That tree is very scooped in. The top node is always the whitest and the longest, and everything is only encapsulated all the way down, and that's the most natural and ergonomic way to do OpenTelemetry and tracing, and everything's going to push you towards that. + +But traces are a graph, and you can actually access all of that graph power when you need it, and sometimes you wish you didn't. Observability and abstraction are natural enemies. They are the chief people in high school that love to take in on each other the entire time they're doing everything. + +So even though we love OpenTelemetry and we love observability and we love my other sister meditation, the problem of needing a large amount of out-of-band context but also wanting the ability to add and annotate is a very complicated problem of your entire system, its business logic, the flow of information throughout the system. + +Those are all problems that we don't actually have good answers for. In general, we don't have good leverage for that, and it stands beyond OpenTelemetry and beyond observability. This is just one of those ways that it becomes a very, very obvious problem. + +Two ways of having an average problem are the life cycle problem, which we talked about, which is where you have this graph, you have the whole power of a graph where you don't have a graph that is like convergent. You have a graph that you kind of have to satanically know the shape of an Azure building out, and you can't really patch it up and picture them later. + +Speaking of that graph, reminding it is really difficult, if not impossible. It's very much convention and ad hoc based. + +Does anyone have any questions, thoughts, or anything else? + +**Speaker:** Hazel + +I just want to say, Hazel, that the way that you describe these concepts in distributed tracing and OpenTelemetry are like I think the clearest I've heard. So thank you for that. I definitely really appreciate that. I think it's like everything now like just fully makes sense. + +So yeah, I just want to echo that, but that's definitely the vibe; it's like, "Oh yeah, cool, now I know what I'm writing about for the next six months." The stuff that Hazel has put so clearly is very good. + +I’m really happy to hear that. The question that I had is I'm curious how you feel like post-processing fits into that picture. Like with the collector, a lot of your times revolved around like developers, DevOps people who are working with the code, like how they're able to propagate data. But what about that? + +Like not so much like tail-based sampling, where you're deciding right at the moment, but like maybe a little further down the line being able to connect data points, you know, once you're holding five minutes of trace data? + +**Speaker:** Foreign + +Yeah, so one thing that's interesting there is that OpenTelemetry feels in many ways like an event-based emitting style of instruments in your code, but it's actually not that. The collector and other ways of post-processing data is where you start to feel that sort of impedance mismatch in particular. + +And so because of that, you can run into certain problems. A great way, a great example of that is that links can't be added after expanded there. And so even, as far as I know, like once you create this fan, you can't add a link to it, even in push processing. + +If you're trying to do that, things are going to be really tricky. Maybe post-processing can fit that, but I don't know that that's now I'm gonna go like research that. + +**Speaker:** Hazel + +That's wow, okay, cool. Yeah, I mean, I realize I have not done it. I've been like digging through it; I haven't done it. So wow, okay. + +So there is interesting historical contents there in that there used to be a method in the OpenTelemetry specification of you can add a link to this fan as long as this man is active. Then they took that method out several years ago, and it's been an open discussion in the GitHub tracker for about three or four years now of this is absolutely useful. We do end up needing this a lot, and can we do that? + +Apparently, as far as I know, the only way to do that is that's fan creation in the options of the, for each band, you can add links in there, and that's the only time. + +Which means certain types of Instagram patterns or certain types of like that type of thing are really, really difficult to do, and also means, you know, when you can approach process and then after that pretty much. + +**Speaker:** Foreign + +And then another way to think about the OpenTelemetry commentary that's interesting is I actually view the counter and how to set up libraries and baggage and other things like that all as different types of convention-based out-of-band information and handling. + +So when you build up conventions in your company or in your system or in anything like that, you can start to say, "Oh yeah, if you set this information here, then it'll be there," or "You can actually just assume that we'll have this information because the clutter will add it here." + +So as an example, when it's metal and a blog post in which they talked about, they said of their collectors to actually add to every span what data center that span came from, what machine that span came from, what, and then where everything was, what cluster it was in, and all this actual information. So everyone can always clear all that information. + +You don't know that, like, it's there because you read the documentation in the internal tooling, and you can build internal tooling that works on top of that natural convention rather than something you can actually express in your telemetry itself. + +Anything else? + +**Speaker:** Hazel + +Well, I guess if there are no further questions, thank you so much for coming on, for coming on twice, and for putting this together. I know presentations are definitely a lot more work to put together compared to just, you know, sitting there answering questions. + +So definitely appreciate the time that you took to put this together and to share your knowledge with folks. I think there's, you know, so much out there on like the novice hotel stuff, but when it starts to get into the gnarly bits, we're definitely lacking in that information. + +So we definitely appreciate you sharing that, and call out to anybody if you or you know anybody who is interested in sharing some hotel goodies in hotel in practice, we would absolutely love to have you. + +Also, if you want to share your hotel story, we would love to have you for Open A. And I think we've got a session coming up later this month, right, Rhys? + +**Speaker:** Rhys + +We do. We have a special discussion actually coming up on the 28th. It's going to be about the evolution of observability practices, and actually Nika is going to be part of that panel. We'll do a full announcement. + +So thank you, Hazel, for everything, and I guess we'll see y'all on the internet. Thank you. ## Raw YouTube Transcript diff --git a/video-transcripts/transcripts/2023-09-20T19:30:01Z-otel-in-practice-context-propapagtion-in-distributed-tracing-with-doug-ramirez.md b/video-transcripts/transcripts/2023-09-20T19:30:01Z-otel-in-practice-context-propapagtion-in-distributed-tracing-with-doug-ramirez.md index fda5e3b..9172544 100644 --- a/video-transcripts/transcripts/2023-09-20T19:30:01Z-otel-in-practice-context-propapagtion-in-distributed-tracing-with-doug-ramirez.md +++ b/video-transcripts/transcripts/2023-09-20T19:30:01Z-otel-in-practice-context-propapagtion-in-distributed-tracing-with-doug-ramirez.md @@ -10,113 +10,229 @@ URL: https://www.youtube.com/watch?v=Of_ygNU_Cbw ## Summary -In this YouTube video, Doug Ramirez, a principal architect at Uplight, presents on the concepts of distributed tracing and context propagation within the realm of observability, particularly in microservices architecture. He emphasizes the importance of distributed tracing as a means to understand interactions among various services, as traditional monolithic structures provided easier debugging and observability. Doug explains key concepts like spans and traces and provides a minimal code example demonstrating how to implement distributed tracing using OpenTelemetry and adhere to W3C recommendations. He showcases a scenario involving a bands API and a reviews service, illustrating how to send and propagate trace context through HTTP headers to monitor performance issues effectively. Throughout the session, he engages with the audience, answering questions and providing resources for further exploration. The video concludes with a mention of future meetups and the importance of integrating logging with tracing for optimal observability in production environments. +In this YouTube video, Doug Ramirez from Uplight presents a session on distributed tracing and context propagation in microservices architecture, focusing on the OpenTelemetry framework. Doug explains the importance of distributed tracing for observability, especially as microservices become more prevalent, and discusses key concepts like spans, traces, and trace context. He provides a minimal coding example to illustrate how to implement these concepts using OpenTelemetry, emphasizing the use of standardized HTTP headers for trace context propagation. Doug also addresses privacy considerations when implementing tracing and encourages developers to familiarize themselves with relevant documentation. The session concludes with a Q&A, where Doug shares further insights and resources for those looking to enhance their observability practices. ## Chapters -Sure! Here are the key moments from the livestream, along with their timestamps: +00:00:00 Welcome and intro +00:00:50 Doug Ramirez introduction +00:01:50 Distributed tracing overview +00:03:00 Context propagation explanation +00:05:00 Trace and span definitions +00:06:30 W3C trace context specification +00:08:20 HTTP headers for tracing +00:12:35 Live coding demonstration +00:15:00 Initial performance issues +00:23:00 Observability with Jaeger +00:32:55 Q&A and closing remarks -00:00:00 Introductions and guest introduction -00:02:30 Overview of OpenTelemetry and distributed tracing -00:05:00 Explanation of distributed tracing concepts: spans and traces -00:10:15 Importance of context propagation in distributed tracing -00:13:30 Overview of W3C recommendations for trace context -00:18:00 Live coding demonstration of a minimal distributed tracing example -00:25:00 Explanation of the importance of observability in microservices -00:30:45 Debugging and identifying performance issues in service calls -00:35:00 Integration of tracing with logging for better observability -00:40:00 Wrap-up and audience Q&A session +**Speaker 1:** Thank you. Hi everyone, thanks for joining us for Hotel in Practice and for being patient and sticking around. We've got Doug Ramirez from Uplight joining us for a second time. This time in the Hotel in Practice capacity. He has joined us previously for the Hotel Q&A session, and in case you missed that, we do have a blog post summarizing that session in the Hotel blogs in OpenTelemetry.io under the blog section. -Feel free to ask if you need any more information! +I guess without further ado, here's Doug. -# Hotel in Practice Webinar Transcript +[00:00:50] **Doug:** Thank you. Hi everyone, I am Doug Ramirez. I'm a principal architect at Uplight. We do cool things to try to save the planet. Just a shameless plug, uplight.com/careers, we're hiring. We're a B Corp, we do cool things. But let's move on to OpenTelemetry and distributed tracing. -**Thank you!** Hi everyone, thanks for joining us for Hotel in Practice and for being patient and sticking around. We've got Doug Ramirez from Uplight joining us for a second time, this time in the Hotel in Practice capacity. He has joined us previously for the Hotel Q&A session, and in case you missed that, we do have a blog post summarizing that session in the Hotel blogs at OpenTelemetry.io under the blog section. +[00:01:50] I have a few things that I want to try to do today, and those are to introduce this concept of distributed tracing context propagation. I want to share a very, very minimal example of distributed tracing in action, trying to peel away a lot of the extraneous stuff and noise around this so that you can just see in as few lines of code how this is implemented. Then also share some resources around just more resources for people to go and explore distributed tracing. -So, I guess without further ado, here’s Doug. +So what is distributed tracing? It's an observability concept that gives us a big picture of how our myriad of services interact with each other. If you think about the three main pillars of observability—logs, metrics, tracing—those have all been around for a while. Distributed tracing has been around for a while, but as more and more of microservices architecture becomes a thing and becomes popular, it really elevates the need for the ability to trace things that span process and service boundaries. ---- +There's a lot of things that monoliths give us, and with microservices, it's really—I really think that if you're going to move towards microservices, you really need to make sure that you're treating distributed tracing as a first-class citizen as part of that implementation. -**Doug:** Hi everyone, I am Doug Ramirez, a principal architect at Uplight. We do cool things to try to save the planet. Just a shameless plug: uplight.com/careers, we're hiring, we're a B Corp, we do cool things. But let's move on to OpenTelemetry and distributed tracing. +[00:03:00] Context propagation is the mechanism that gives us the ability to observe things that happen in a service as a transactional request spans another service and another service, and kind of your call chain of microservices. Context propagation is the thing that gives us the ability to watch and see what's happening. -I have a few things that I want to cover today. Those are to introduce the concept of distributed tracing and context propagation, share a very minimal example of distributed tracing in action, and provide some resources for people to explore distributed tracing further. +A couple of concepts that are important to know about when you're thinking about distributed tracing and context propagation are spans and traces. In my experience, oftentimes I think people flip the two. But to be clear, a span is basically a unit of operation, a unit of work. You can think a database query is a great example of a span. That's a unit of work that a service does. -### What is Distributed Tracing? +There's a collection of these units of work that make up the transaction that may happen in the service. A span is nothing more than that unit of work, and a span ID is the thing that uniquely identifies that unit of work. A trace is just a collection of those spans, and it tells the story of how a request traverses all of these different services that essentially enable a use case or a feature or a capability. Each trace has a unique identifier, which is globally unique and random. -Distributed tracing is an observability concept that gives us a big picture of how our myriad of services interact with each other. If you think about the three main pillars of observability—logs, metrics, and tracing—those have all been around for a while. Distributed tracing has also been around for a while, but as microservices architecture becomes more popular, it really elevates the need for the ability to trace things that span process and service boundaries. +[00:05:00] I put some links in the readme here so that you can read more about the reasoning behind that, but I think it's important for people to know. I know this has come up in conversations that I've had with people around what a trace ID is, and it is important that it is globally unique and random for reasons that are explained better in the documentation from the W3C, which I'll get into in a moment. -Monoliths offer certain advantages, but with microservices, it’s crucial to treat distributed tracing as a first-class citizen during implementation. Context propagation is the mechanism that allows us to observe things happening in a service as a transactional request spans multiple services in a call chain of microservices. +A trace context is essentially kind of like the envelope or baggage that it's passed between services so that they can understand what uniquely identifies the request that's coming into them, have a uniquely identifiable transaction request that's going out of that service to another service. Essentially, in order for all of this to work, there has to be an agreement on what that format and what that specification is for a trace context. -### Key Concepts in Distributed Tracing +One of the things that we get as developers, which is awesome, is a recommendation from the W3C around what a trace context should look like. We also get the benefit of a lot of the work that the OpenTelemetry community has done to codify those recommendations and specifications into SDKs and into other things like the OpenTelemetry collector that give us as developers a really easy way to implement distributed tracing context propagation. -When thinking about distributed tracing and context propagation, two important concepts are **spans** and **traces**. A span is essentially a unit of operation, like a database query. It represents a unit of work that a service performs. A trace is a collection of spans that tells the story of how a request traverses through different services to enable a feature or capability. Each trace has a globally unique identifier. +[00:06:30] There's an agreed-upon format that is recommended by the W3C on how to do this. There are software libraries and specifications that the OpenTelemetry community gives us to make all of this really easy, and I'll show that in a moment. -A **trace context** is like the envelope or baggage passed between services to uniquely identify a request. For this to work, there needs to be an agreement on the format and specification for a trace context. The W3C provides recommendations on what a trace context should look like, and the OpenTelemetry community has codified these into SDKs and other resources to simplify implementation. +With that specification that the W3C gives us, part of that recommendation is to use HTTP headers to allow a service to send information to another service so that that trace context can be propagated. There are two headers that the W3C recommends for doing this: a trace parent and trace state. -### HTTP Headers for Context Propagation +Trace parent is where we package up the unique identifiers that describe a unit of work that's happening in one service, the trace that's associated with it, to send to another service so that they can unpack that tag, that unit of work in the other service with information so that as that information is emitted to an observability platform, we can tie all these things together and we can see how this call chain of things that happen that span services all work together. -Part of the W3C recommendation is to use HTTP headers to send information between services. There are two headers recommended for this: **traceparent** and **tracestate**. +[00:08:20] Back in the day with monoliths, it was pretty easy to observe and debug things that spanned different subcomponents of your monolithic architecture, and we kind of lose that ability with microservices. But we get it back by leveraging this trace context concept and the headers that are specified from that W3C recommendation to pass that information between services. -- **Traceparent**: This header packages the unique identifiers that describe a unit of work happening in one service, along with the trace associated with it. -- **Tracestate**: This header is a simple list of key-value pairs that can carry additional contextual information. +Okay, service A calls another service. It simply uses a header to package up some information. It's an agreed-upon format for other services to unpack and use to tag the spans and units of work that those other services are using. -Using these headers allows services to communicate and understand what uniquely identifies a request. +There's some details in here about what the format of that is. Let me reduce my font here so you can see what an example of a trace parent would look like. There's a version—I'm not going to get into the version today, nor will I get into the trace flags—but the really important part of this is to understand that this ID here, this unique identifier for the trace ID, is the thing that will allow us to tie all of this work together so that we can observe how microservices are all working together to enable some feature, satisfy some use case. -### Privacy Considerations +In addition to the trace parent header, the W3C recommendation also offers another additional header called trace state, which is really nothing more than a simple list of key-value pairs that you can add along with the context that's being propagated between services so that you can attribute that to things that may be unique to whatever it is you're doing. -It’s essential to familiarize yourself with privacy considerations regarding PII and GDPR when implementing distributed tracing. While following the W3C recommendations is generally safe, it’s crucial to be aware of potential data leakage. +Vendors may use trace state to provide observability into what they're doing, but you can just think of it almost as a little bit of baggage that rides along with the trace context that moves between the services. -### Implementation Example +I have some links in here to the W3C trace context specification and recommendation. One of the things that I think is worth calling out, and I would encourage anybody who's going to go down this path, is to make sure that you're familiar with the privacy considerations. In this day and age with PII and GDPR and a lot of the other things that we really need to pay attention to as it relates to the potential leakage of personal information, there are some considerations that are worth noting. -Now, I want to quickly show you a minimal example of a couple of microservices working together. Imagine we're building an Internet Band Database (similar to IMDb) that provides information about bands and their reviews. +I will say that in general, following the W3C recommendation using the specifications in the libraries that we get from OpenTelemetry, this is a very safe thing to do. But just to be clear, spend some time and just make sure you familiarize yourself with those privacy considerations. -In this scenario, we expose an endpoint to call and retrieve a band by its unique ID and its associated reviews. +I know that I ran through that quickly because I know we're kind of pressed for time, but essentially, this thing called distributed tracing, again, it's a way to observe how a transaction or a request traverses multiple microservices. Again, the good news here is that we have a specification that's widely adopted and it's widely known. We have tools that we can use from OpenTelemetry to employ this. -When I run the service, it takes a surprisingly long time. This is common; when someone reports performance issues, it often requires tedious debugging and log gathering. Adding distributed tracing can help. +For anyone who's thinking about, like, I'm building a bunch of microservices, I need to observe how they're operating, how do I do it? The good news is that that problem's been solved for you. There's a de facto standard, there's tools that you can use, and as you'll see in a moment, it's actually quite easy to do. -### Adding Tracing +Let me pause there because I know that was probably a fire hose of stuff, but I want to see if anybody who's on the call right now has any questions about these concepts before I jump into some code. -I will add some code to my service to enable distributed tracing. Using the OpenTelemetry SDK, I can extract the traceparent header and inject it into my service calls. This allows us to trace requests and see how our services perform. +[00:12:35] Okay, we're going to run with scissors now and we're going to do some live coding. What I've done here is, again, what I tried to do was to just create the most minimal example of a couple of microservices all working together to demonstrate how you can use the W3C recommendation and use the OpenTelemetry libraries to do this stuff. -For instance, when calling the reviews service from the band service, I start a span and inject the trace context. This lets us observe what the band service is doing and how long it takes to get reviews. +In this scenario, we're building the—it's kind of like IMDb. Instead of the Internet Movie Database, it's the Internet Band Database. I want to create this source of truth when it comes to band names, and also in this scenario, we have—we're leveraging a platform that just handles and reviews just a generic platform for reviews. -### Observing with Jaeger +We've created a band's API that our front-end developers are going to use to build web and mobile applications on top of, and rather than building a reviews platform ourselves, we're just going to leverage this fictitious reviews platform. -Now when we call the APIs, we can start digging into the performance. Using Jaeger, we can visualize the traces and spans. +In this case, we've exposed an endpoint that allows something to call and say, "Hey, give me this band that has a unique ID and give me the reviews associated with it." So I'm going to call the endpoint to get a band and its reviews, and wow, it took six seconds. That's a long time. Let me run it again just to see what's happening here. Twelve seconds is a long time. -When we examine the data, we see a majority of time is spent in the get reviews function, which helps us identify where the bottleneck is occurring. +You can imagine we're all familiar with this, right? Like you're building the service, you've got people that are writing front ends to it, and they're saying, "Hey Doug, I'm calling the endpoint to get a band and it's taking a long time," and that's not a lot of information to go on and debug. -### Conclusion +[00:15:00] I think we're all familiar with those scenarios where somebody contacts you and says, "Hey, you know, we've been noticing that this doesn't seem to be performing very well." We have to do that guessing game of, "Okay, well when did you call this service? Do I start grabbing a bunch of logs?" It becomes very tedious. -In conclusion, distributed tracing allows us to observe how a transaction traverses multiple microservices. The good news is that we have widely adopted specifications and tools from OpenTelemetry to make implementation straightforward. +So one of the things that helps is to employ some tracing and some distributed tracing. What I'm going to do now is say, "All right, hey, from a developer, I'm going to add some stuff to my service and all I need you to do is to make sure you pass me some information when you call me so I know that you're calling me." -Let me pause here and see if anyone has questions before I jump into some code. +Then we can kind of debug this and see what's happening. The first thing I want to do is I want to go into my bands API, and this is just a very, very simple FastAPI application, just a very simple model for a band. I have an endpoint here that lets a caller get a band by its ID. I have a little helper function here that does some work to go actually access a data repository of reviews, which happens to be this other review service that I mentioned before. ---- +So let me bring up my cheat sheet here. Let's start by bringing in some code that will provide some visibility with tracing. What I'm doing here is a couple of things. One is I'm going to bring in the library that's going to help me essentially tease out information from the header that I talked about a moment ago, this trace parent header, and inject it into essentially a hook that will call out to another service and emit that tracing signal. -**Audience Questions** +In this case, I'm going to use Jaeger, all populated about the OpenTelemetry collector. But in this case, I'm just going to bring in some code with the SDK, give the band service, the band's API, the ability to unpack that header and inject that information to emit a trace so that we can see what's happening. -1. **Audience Member:** How did you find the documentation around context propagation? - - **Doug:** It was pretty good! The OpenTelemetry documentation has detailed information on things I glossed over today. Once you understand the concepts, building a reference implementation is quick. +Oh yeah, so the get reviews. So let's go to our get reviews. Excuse me, this is the function that does the call to the reviews service. What I want to do here is say, "Okay, for my when I call the get review service, I want to start some kind of trace so I can see what's happening." -2. **Audience Member:** How do you integrate logging and metrics with observability? +In this case, I'm starting a span in my service, and what I want to do is I want to actually—I don't want to do this just yet. Sorry, let's just do this, and then back over here I'll explain what I'm doing here. - **Doug:** I recommend starting with logging because it’s familiar to most developers. By incorporating logging with OpenTelemetry, you can get Trace log correlation for free. Once you establish that connection, adding tracing becomes much easier. +What I'm doing now is I'm adding a couple things. I am creating the ability, or what I'm doing is I'm bringing in a very, very small library. This is just a helper library that is essentially provisioning the tracer that we get as part of the OpenTelemetry SDK. -3. **Audience Member:** Could you do a follow-up presentation on logging? +What this tracer will do is this is the thing that will build up the traces and send it to an observability platform. In this case, it's going to be Jaeger. In case you're wondering, there's some default stuff that happens, like the OpenTelemetry SDK knows to send to a certain port that a service is listening to—in this case, it's 4317—and Jaeger's running locally, so it will pick it up. - **Doug:** I would love to! I think there’s a lot of value in simple implementations that highlight these concepts. +That's a lot of hand waving, but in the interest of time, just know that the SDK is handling a lot of this stuff for you. Back in my band service, what I want to do is say, "Okay, anytime somebody calls the service, what I want to do is I want to allow the caller to send that trace context in the header." ---- +In my service, what I want to do is I want to tease out that trace parent header because I know that there's a standard and a specification for that. I'm going to build up a carrier, and I'm going to inject that trace context from my caller into my context so that when I build up a span, the OpenTelemetry SDK is going to do a bunch of magic and work for me and essentially emit that trace to the observability platform. -**Closing Remarks** +So I have this now built up in my code. Now the band's API can receive a request, build up a trace loop in that trace context from the caller, and when it calls its own internal function, we're going to spin up another span, which is a unit of work to make a query to the reviews platform. -Thank you, everyone, for joining today, especially given our late start. Thanks, Doug, for adapting on the fly and for your insightful presentation. We will post updates about our next Hotel in Practice session on the various CNCF Slack channels. +This will start to give us some visibility into what the caller is doing and what the band service is doing. So now let's go. What I want to say to the caller, then, the person who's building this application, is to say, "Hey, I'm adding some stuff to our service because I know you said it was slow and things don't seem right. Send this bit of information so that we can start to really observe what's happening here." -If you're attending KubeCon EU, several of us will be there, and we’d love to connect. A blog post summarizing today's session will be available on the OpenTelemetry blog on Monday. +If I go back to my client, what I want to do is I want to essentially say, "Hey, help me help you by sending this header." When you send this header, manufacture a trace and a span on your end, add that header to the call that you make to me, and then we can kind of start to dig into what the problem may or may not be. -Thank you again, everyone! +So now at this point, we're kind of wired up now to get a bit more observability into what these different services are doing. If I run my—if the client now calls me, I can see it took five seconds. Oh, I forgot one thing. Sorry. + +I also just kind of want to print out to the console a little helper thing just to link me right to Jaeger, where these traces are being sent to, so I can start to see what's happening. Now the client calls me again; we should be able to start digging into what's going on here. Wow, 15 seconds is a long time. + +[00:23:00] Now I can kind of dig into this and I can see—and I don't—I'm not going to spend too much time on Jaeger, but Jaeger is an open-source platform that helps visualize traces and spans. Now that I have the caller following that W3C recommendation, passing that trace parent with some unique identifiers of the units of work it's doing, sending it to the band service, the band service is taking that information, its context has been propagated to it, it's leveraging that to emit information about what it's doing, and now we can observe what's happening. + +I can see that the get bands endpoint was called, and I can see that the get reviews function unit of work was run. If I look at this, the spans table here, I can see that the majority of time that was spent here was getting the reviews. So that's good helpful information to know. I can see, like, okay, get reviews is taking a long time; let's dig into that. + +So let's go back to our code and I can say, "Hey guys, building folks building the client, I see something going on in my service. Let me dig into it. The get reviews is taking a long time." + +"Hey, what's this debugging thing? Somebody did something while they were debugging. I just have a random timer here. Let me just get rid of this. Sorry about that, folks." + +Right. Okay, hey, I just deployed a new version of my service. Call me again. Still taking a long time, something doesn't seem right. So now in this fictitious world, I'm going to turn around to the folks that work on the reviews platform and say, "Hey, you know, we're looking into an issue. There was some random code that was sleeping in our end. Can you take a look and see what's going on in your platform? Because when we're calling you, like, it's taking a really long time to get reviews; like, our service seems to be working pretty fast." + +So now let's go and do the same thing on the reviews end. Very similar bit of refactoring. Let's go to the reviews, especially. Let me bring—all right, let's bring all this in. + +So now the same service here is going to bring in this helper function to start up the tracer again, leveraging the OpenTelemetry SDK to do that. We'll bring in this trace context propagator so that we can have this service do interesting things with the trace context. + +Let's see. Here we want to do something that's a little bit different than the band service does, right? Yeah, I think that's right. So now the review service can also receive that trace parent, that trace context, unpack it, and associate it with its span of work. In addition to that, we want to start a span at the internal—imagine this would be a piece of code that would more than likely read from a data repository, a data store associated with the review service. + +So now I can say, "Okay folks that are building the front end of the mobile app, we've released a new band's API, we've released a new reviews API that we think is going to help give us some visibility into what's going on." + +So they're going to keep doing their development. They're going to start making calls to us. Four seconds isn't too bad. Let's try again. It's still taking too long, so now we got to figure out, okay, well what's going on? + +So let's go back to our observability platform, Jaeger, and what is happening? Not seeing the trace from the reviews out of this. This is what you get when you try and run with scissors and do live coding. What am I missing here? + +Trace parents. So this is getting—oh, I know what we did. All right folks, there it is. + +The reason that the review service wasn't getting the context propagated is because I didn't add that in the—so in the band service that's calling the review service, we need to propagate that context. Be quiet, Siri. + +Here we go. All right. I know I'm jumping around quite a bit. In the band service, it's getting that trace context from the caller, it's attaching it to a span, it's making this call to get reviews. We want to see what's happening in that unit of work, so we're starting up a new span. + +But we also want to take the context from the caller and inject it into a header so that when the band service calls the review service, that context gets propagated to it as well. So now when the caller calls the band service, it's spinning up this trace ID, this unique identifier, sending it to the band service. The band service is like, "Thank you very much. I can attach this to my span, my units of work. I'm going to send it downstream to the thing that I rely on." + +It's going to attach this information to its units of work, and we should see all of that come together in Jaeger. Boom! All right, service is still running slow, but at least now we have some visibility into what's happening. + +Oops, and if I go to my trace graph—sorry, my spans table—actually, where is it? I can see here that the majority of the time seems to be spent getting the reviews from the reviews database. So when it's getting a review, by getting the reviews for a band by the band ID, it's taking a long time. + +So let's go back and start digging into that code there to see what might be happening. So in the review service, if I look at this function that is grabbing data from some kind of data repository, again, somebody put in some kind of sleep. Maybe they were debugging, testing some timeouts; who knows what, but let's pull that out. + +Now we can say, "Hey folks that are building the mobile and web app, we think we fixed it for real this time. Can you try again?" Wow, that's better. + +Let's go back and see what happened in Jaeger. Okay, things are taking milliseconds. That feels better. I like this. Now I can observe what's happening across all of these different things. Yay team! + +Just to kind of walk through the code one last time: the client is packaging up this header based on the W3C recommendation. It's passing that header along to our bands API. Our bands API is leveraging the OpenTelemetry SDK and saying, "Hey, let me pull this stuff out and inject it into a carrier so that I can send it downstream." + +[00:32:55] By the way, when I'm creating my spans, I'm going to leverage that information. When the review service gets a call into it, it can tease out that trace parent information and inject it into its spans by attaching that context. And now we have distributed tracing with context propagation because we've got this awesome stuff from the OpenTelemetry community and we've got this awesome recommendation from the W3C. + +Let me stop there and ask if anybody has any questions about the basic minimal implementation of these concepts in these handful of FastAPI services, and maybe this is a good time just to kind of open up to general questions as well because I know we're coming up on time. + +**Audience Member:** Doug, I had a question for you. How did you find in terms of finding the information, the documentation around context propagation? How was that experience for you in terms of finding that information in the Hotel docs? + +**Doug:** It was pretty good actually. So in the readme for this repo, which I hope that other people can help me test the documentation, I would love for some folks to go and clone this repo and try to spin all these services up and make some calls and see if they can get it running locally. It was good. + +The OpenTelemetry documentation has some very good details around things that I glossed over today that I would really like to go into further detail around. I'm sure that anybody who watches this is probably going to start asking questions like, "Well, how does that actually work?" There's a lot of magic under the hood, but the OpenTelemetry documentation goes into a lot of detail around the things that I just waved my hands around, like the tracers, the trace exporters, the context propagation, the shape and format of spans, how these things all link together. + +I found it was pretty easy to find that, and I also think that the W3C documentation is also very good as well. So I was able to kind of bring everything together pretty quickly. It didn't take me very long to build this reference implementation of these few applications. + +**Audience Member:** Yeah, OpenTelemetry also publishes a bunch of integration packages which lets you hook up to like Node middleware or HTTP clients automatically, so you can just have everything instrumented and doing the correct header insertion and things like that without really having to worry about doing it manually like in this demonstration. + +**Doug:** Yes, that's a really good point. One of the things that I did as part of this reference implementation was to do a very explicit implementation of these concepts. We don't have time, and I wanted to do that thing that you always see in a demo where somebody says, "Do all this stuff," and at the end of the demo, they say, "Oh, by the way, you don't have to do anything. You just do this other thing and it all gets handled for you." + +There is a FastAPI instrumentor which will remove the need to do a lot of this explicit unpack, this explicit extraction and injection of the context. I didn't have time today to show that, and I kind of wanted to make it very clear to a developer who's learning these concepts to walk through the code and see something very procedural that shows this. + +For example, there's a header, and that header is very important, and it is called a trace parent, and it has a specification. Here's an example of teasing out that header in code. The OpenTelemetry library has this thing where it can extract this context for you. Here's the line of code that does this. When I make a call to another service, I want to pass that along very explicitly. + +There was a lot of intention behind trying to make this application a bit more verbose, even though it's very minimal, to try to show somebody who's maybe new to this what does this actually look like. If you use the FastAPI instrumentor, the good news is a lot of this gets handled for you. + +My recommendation would be for anyone who's getting started with microservices is to make sure that you include distributed tracing from the very, very beginning. I would almost suggest that you spend a bit of time and implement this explicitly so that you understand what's happening, so that other developers understand what's happening. + +Once you solve the problem of observability, then level up and start to leverage some of these other things that OpenTelemetry gives us, like the FastAPI instrumentor and these other things that we get from the community. + +I feel like that just kind of helps bake the concepts in for people. I know that for me, I'm a very tactile learner, so it helps me to see what's happening before I start to let some libraries handle that magic for me, if that makes sense. Your mileage may vary, but my recommendation would be to go ahead and do it explicitly. + +It's a handful of lines of code, you can share it with people when you're doing code reviews, you can explain to them what's happening. It's not mysterious, it's very explicit. + +**Audience Member:** Do we have any questions for Doug? + +**Audience Member:** Thank you, Doug, for sharing that. One question from me is that how does it look when you, like, so the observability problem in production, I suppose, apart from this OpenTelemetry tracing part, I suppose we also need to serve logging and metrics, and do you find any difficulty integrating the different parts of observability? + +**Doug:** The last time I spoke here, I mentioned that one of the paths that we took and have been taking at Uplight has been to actually begin with the log signal. The reason is that in terms of introducing these concepts that these problems that OpenTelemetry solves, these problems that the W3C helps us with solving is to begin with logging so that you—because logging, I think, is very, very familiar with most developers. + +I think junior and intermediate developers are familiar with this concept of logging, like a print statement. What I've recommended and what we did at Uplight when we started down this path was to start using the logging SDK, even though it's newer than the work for the signals of metrics and traces, but to start with logging so that you can at least begin to practice including the packages and libraries into your code using OpenTelemetry to emit that logging signal. + +Get that logging signal landing in your APM or whatever your observability platform is, and once you make that connection of like, "I'm writing code, I'm writing a statement that says logger.debug, my message is landing in an APM," it's structured, it's honoring the specification that the OpenTelemetry community gives us. + +Then it's really easy to then, you know, level up to add tracing, and you get trace-log correlation for free. That's one of the things I would like to actually do with this little reference implementation is to add in some trace-log correlation so that people can see, like, how do I do the thing that I'm so familiar with doing? + +I like to send out a log; I like to do a print statement. That's so we always do that all day, every day. If you start with logging and you get the SDKs into your repo and you have that information flow into your APM, then adding the layer of maturity around traces and the trace-log correlation is almost free because you've already got everything packaged up into your application already. + +That might not work for everybody. To me, that feels like a very natural way to evolve into using the specification, using the SDKs, and really kind of getting into observability nirvana, where you can walk up to your APM and say, "I have questions about the health and behavior of my services, and I have a lot of them. What's happening?" + +You can get those answers if you do this. + +**Audience Member:** Doug, it sounds like you need to do a follow-up presentation on logging. + +**Doug:** I would love to, and I think that I would love some help from other people to take this repo and clone it or fork it or just play with it. I definitely encourage others to poke around as well. I think it would be cool to contribute this back to OpenTelemetry. + +Although, to be honest, I don't know if other OpenTelemetry folks have a better idea on the process around that because I think I'm newer to the process. But anyway, I think that, yeah. + +I know that there are, like, there's a demo app and there's other reference implementations, but one of the problems I was trying to solve here was how do I strip away all the noise and give a developer with a modern operating system and Docker the ability to get clone, fire up these services, and watch what's happening so they can really dig into these concepts and understand how powerful they are in their simplicity. + +It's pretty amazing; all we're doing is just sending a header, and these SDKs are doing these wonderful things for us. It's pretty simple, but it's unbelievably powerful. + +The idea for this reference implementation was to try to highlight that as best we could with as few lines of code as possible so that people can understand this and so that people can go and be successful with building this level of maturity in their observability in their code. + +**Audience Member:** I totally agree, and I think there's a lot of value in that. Like you mentioned, the OpenTelemetry demo app is awesome because it shows all of the things, but there is definitely value in simpler implementations like this one. + +So yeah, well, on that note, I know we're a little bit over time. Thank you everyone who was able to join us today, especially given that we started late. Thanks Doug for working on the fly with our reduced time and for lunch—that's never an easy task. + +So definitely appreciate the time that you put into making this happen today. Thanks everyone for joining us once again, and keep your eyes open for our next Hotel in Practice. I don't think we have one scheduled yet, but we'll post on the various CNCF Slack channels for that. + +Rin, do you have any additional upcoming announcements that folks should be aware of? + +**Rin:** Sure, yeah. So first, there's an OpenTelemetry in Practice meetup. If you came here through CNCF, a lightweight way to get notified whenever we have something and not have to look for the Slack. If you're not in the CNCF Slack and you want to talk further to any of the OpenTelemetry in Practice folks, Doug, the audience, we're happy to add you. That information is in the chat. + +If you'll be at KubeCon EU, several of us will be, and we'd love to connect with you there. There will be a blog post on the OpenTelemetry blog on Monday with all of the activities. Yes, I am happy to do a follow-up with Doug because of KubeCon EU; that will probably be around the end of May. We can figure out what format we want to do based on hopefully talking with you all in the Slack. + +Thank you, everybody. ## Raw YouTube Transcript diff --git a/video-transcripts/transcripts/2023-09-25T04:00:25Z-otel-in-practice-fireside-chat-december-2022.md b/video-transcripts/transcripts/2023-09-25T04:00:25Z-otel-in-practice-fireside-chat-december-2022.md index df9ae02..b4af504 100644 --- a/video-transcripts/transcripts/2023-09-25T04:00:25Z-otel-in-practice-fireside-chat-december-2022.md +++ b/video-transcripts/transcripts/2023-09-25T04:00:25Z-otel-in-practice-fireside-chat-december-2022.md @@ -10,86 +10,332 @@ URL: https://www.youtube.com/watch?v=TYrIP_LkwgU ## Summary -In this OpenTelemetry and Practice series video, the hosts, including Adriana, Rhys, and Jess, discuss various topics related to implementing OpenTelemetry across different platforms. Adriana shares her experiences with observability in HashiCorp products and demonstrates running a hotel demo app on Nomad, explaining how to translate Kubernetes manifests into Nomad job specifications. Rhys introduces the End User Working Group's efforts to engage users in the OpenTelemetry community and improve feedback loops between users and maintainers. Jess presents a fun project that visualizes spans in a heat map format using OpenTelemetry, showcasing how to create engaging visuals for educational purposes. The session emphasizes community involvement and encourages users to participate in discussions and feedback sessions. +The video features a discussion on OpenTelemetry, specifically focusing on its implementation and usage within various tech environments. Key speakers include Adriana, who shares her experience with HashiCorp products and demonstrates how to run a hotel demo app using Nomad and OpenTelemetry; Jess, who presents a fun project that visualizes data in heat maps using OpenTelemetry; and Rhys, who talks about how end users can engage with the OpenTelemetry community. The main points include the importance of observability in production systems, practical implementations of OpenTelemetry, and ways for users to contribute to the community and provide feedback. The session encourages interaction and showcases tools and techniques that enhance understanding and application of OpenTelemetry in real-world scenarios. ## Chapters -Sure! Here are the key moments from the livestream along with their timestamps: +00:00:00 Welcome and intro +00:01:00 Adriana's background +00:03:30 HashiCube demo +00:05:00 Nomad console overview +00:08:20 Job specs in Nomad +00:12:01 Environment variables in Nomad +00:15:30 Health checks explanation +00:21:10 Transition to Jess's demo +00:21:10 Jess's heat map demo +00:29:00 Discussion on observability tools +00:36:00 End user working group overview +00:41:01 Q&A session -00:00:00 Introductions to the OpenTelemetry and Practice Series -00:01:30 Overview of the format change for this session -00:03:00 Adriana discusses her background with HashiCorp products -00:05:30 Introduction to the Hotel Demo App and its features -00:07:00 Adriana shares her experience running the demo app on Nomad -00:10:00 Demonstration of setting up the observability tools locally -00:12:30 Explanation of Nomad job specs compared to Kubernetes deployments -00:15:00 Discussion on health checks and service dependencies in Nomad -00:20:00 Jess introduces her fun project for creating heat maps with OpenTelemetry -00:25:00 Rhys talks about the End User Working Group and how to get involved +**Speaker 1:** Thank you. The OpenTelemetry and Practice series is traditionally a pretty serious talk series where we, every month, go deep into implementing OpenTelemetry in a particular language. We talk to an end user who is actually implementing it in production, and we talk to usually a maintainer of the project who can talk about the best ways to implement it. This is a little bit different of a format because we decided to do three or four short talks that didn't fit well together. -Feel free to ask if you need more details on any specific section! +[00:01:00] **Speaker 1:** Adriana is going to lead off today with talking about observability for HashiCorp products. She currently works at Lightstep and used to work fully in a Hashi shop. Yes. And then a little bit later we'll have Rhys talking about how end users can get involved with the project, which I expect to be highly interactive. Jess "Jessatron" is going to give us a talk about using OpenTelemetry to make ASCII art in heat maps, which is cool. You are trying to get your team to learn about OpenTelemetry and pay attention. -# OpenTelemetry and Practice Series +**Speaker 2:** Adriana, you ready to...? -Thank you for joining today's session of the OpenTelemetry and Practice Series. This series is typically a serious talk where we dive deep into implementing OpenTelemetry in a specific programming language each month. We engage with an end user currently implementing it in production and usually include a maintainer of the project who can share the best practices for implementation. +**Speaker 2:** Yeah, I'll start. So I'm ad hoc-ing this. As Rin mentioned, I used to work at a Hashi shop, so I have a Kubernetes background and was thrust into this HashiCorp world where my previous employer ran their containerized workloads on Nomad. I found myself in a situation where I had to quickly understand this Nomad thing because I was managing a platform team that supported Nomad, Console, and Vault for the entire enterprise. -Today, however, we have a different format. We will have three or four short talks that don't necessarily fit together. **Adriana** will kick things off with a discussion on observability for HashiCorp products. She currently works at Lightstep and previously worked at a Hashi shop. Later, **Rhys** will discuss how end users can get involved with the project, and **Jess** (Jessatron) will present on using OpenTelemetry to create ASCII art in heat maps. This is all about getting your team to learn about OpenTelemetry and engage with it effectively. +**Speaker 2:** That's how I began my dabblings in the Hashi world. From that, I ended up becoming a HashiCorp Ambassador because I dug in, blogged about it, and continued to do so. At the same time, at this former company, I was also managing an observability team. I came to learn about observability around the same time that I came to understand Nomad. These were two things I was doing at the same time, which led me into OpenTelemetry. -### Adriana's Talk on Observability +[00:03:30] **Speaker 2:** Now that I work at Lightstep, I've had a chance to contribute more in the OpenTelemetry community. As I mentioned before, I've worked in the comms area, so I've contributed some content to the doc site. I've made some contributions to the demo app, and I was actually pleasantly surprised to learn about the demo app, which I think launched relatively recently, right? The demo app came out, and I like it because it showcases what you can do with OpenTelemetry. -Adriana began by sharing her background. +**Speaker 2:** I think the original version of the demo app that came out, you could run everything in Docker Compose locally. More recently, the team created Helm charts for this one, and it got me thinking, "Hey, if this thing runs in Kubernetes, wouldn't it be super cool to get this thing to run on Nomad?" It's been part of my wishlist ever since I learned about the OpenTelemetry demo app Helm charts. I finally carved out some time to do this, and I got it up and running in a local Hashi environment on my machine. -> "As Rin mentioned, I used to work at a Hashi shop, which gave me a Kubernetes background. I was thrust into the HashiCorp world because my previous employer ran containerized workloads on Nomad. I had to quickly understand Nomad since I was managing a platform team that supported Nomad, Console, and Vault for the entire enterprise. This led me to become a HashiCorp Ambassador as I dug in, blogged about it, and continued to engage with the community." +**Speaker 2:** There's this really cool tool called HashiCube, which is similar to running Minikube or K3s or whatever, like one of these local Kubernetes dev environments—a similar sort of thing for the Hashi stack. This was created by a third-party company called Serbain. I love using this tool because it gives me a full-fledged Hashi environment, similar to what you would get, I mean obviously smaller scale, to what you would get in a data center. -Adriana then transitioned to her work at Lightstep, where she has contributed to the OpenTelemetry community. +[00:05:00] **Speaker 2:** But it replicates the type of setup because in real life, Nomad by itself is kind of useless. You need the power of Console for service discovery, Vault for secrets management, along with Nomad to get all this stuff to really have a robust system to manage your containerized workloads. I have this HashiCube running on my machine locally, which I can show you. I'll do a little screen share. -> "I've contributed content to the documentation site, created a demo app, and was pleasantly surprised to see the demo app, which showcases what you can do with OpenTelemetry. I had the idea to get it running on Nomad, which has been on my wishlist for a while. I managed to get it up and running in a local Hashi environment on my machine." +**Speaker 2:** Well, first I'll show you my HashiCube. Oh wait, am I sharing the right one? I'm hearing your HashiCube. I have so many windows. The correct HashiCube... I don't know. Over time... Yeah, this—well, okay, so it says this is my HashiCube startup sequence. -She demonstrated using a tool called HashiCube, a local HashiCorp stack environment, to run the demo app. +**Speaker 2:** What it is, so it's basically using Vagrant to provision either a VM, a VirtualBox VM, or I'm on an M1 Mac, and VirtualBox and M1 Macs don't play nice. I think there's possibly a version of VirtualBox that plays nice with M1 Macs now, but at the time, it didn't. The maintainers of HashiCube basically stand up all these Hashi products in a single Docker image. -> "HashiCube essentially uses Vagrant to provision a virtual machine running all Hashi products in a single Docker image. I have it running on my local machine, which is exciting because Nomad alone is pretty useless without Console for service discovery and Vault for secrets management." +**Speaker 2:** This version of HashiCube that I'm running, where you can see the tail end of the startup sequence here, it's running Vault, Console, and Nomad all in one Docker image. If I share, I'll share my Nomad console here so you can see. The problem is when I lose sight of all my windows. -Adriana shared her screen to show the demo app running locally. +**Speaker 2:** Oh, here we go. Okay, so this is Nomad, and these are all of the services that make up the hotel demo app running on Nomad locally on my machine, which is pretty freaking exciting. I basically had to go through the Helm chart. For the hotel demo Helm chart, there's a folder in the Helm chart repo that shows the rendered YAML. -> "This is Nomad, and you can see the services that make up the hotel demo app running on Nomad. By going through the Helm chart, I translated Kubernetes manifests for each service into the equivalent Nomad job specification. Everything is up and running, and we can check the Jaeger UI for our traces." +**Speaker 2:** I basically started with the rendered YAMLs and went through the process of translating these Kubernetes manifests for each service into the equivalent Nomad job spec, which is what you see here. For example, we're looking at the feedback service. This is running on Nomad. If we go over here, we can see there's a lot of stuff here. -She also explained the Nomad job specifications and how they compare to Kubernetes deployments. +[00:08:20] **Speaker 2:** We can see the logs of the true flag service. Nothing terrible is happening. Here we go. It's running. We can see from our dashboard here that all of the services are up and running. It's made up of all these different services written in different languages. Plus, we also have— I had to convert, I had to Nomadify Redis, Postgres, we had the hotel collector. What else? There was one more. There was Prometheus and Grafana, and I used Trafic for load balancing so I can expose my services to the outside world. -> "In Nomad, everything is self-contained in one job spec file written in HCL (HashiCorp Configuration Language). For example, we define our network settings and service definitions here, which is quite different from the multiple YAML files required in Kubernetes." +**Speaker 2:** What that looks like then, to be able to access the hotel demo app's endpoints—and hopefully my computer won't crash here—I have an endpoint called hotel-dash-demo.localhost, which is accessible. This is just something accessible off my localhost. -Adriana concluded her demo, showcasing how Nomad handles dependencies and restarts services if they fail, along with how health checks work in Nomad. +**Speaker 2:** The Jaeger UI is exposed as part of my demo app configuration, so we see Jaeger is up and running here. We've got the demo app's UI running here, so we can buy stuff, we can check stuff out. Let's buy this telephone. Super. -### Jess's Presentation on Heat Maps +**Speaker 2:** If we check Jaeger, we can look to see—sorry, the screen sharing thing is getting in my face. We can see our traces. I think the hamsters are running really, really hard here. We can see it's produced some traces here that we can see. -Jess then presented an engaging project she created for Christmas. +**Speaker 2:** In addition, when I Nomadified this, there were also some Grafana dashboards—some cool dashboards that come pre-loaded, pre-configured. You can see there's some stuff going on here with the hotel collector. Then there's also this demo dashboard which gets some stats from the recommendation service. -> "I made a fun app called 'Happy Holidays' that uses OpenTelemetry to generate heat maps. You can clone this repo and run it with your Honeycomb API key. It creates spans and traces for different spans and sends them to Honeycomb." +**Speaker 2:** That's basically it. I can show you also what a job spec looks like in Nomad. I want to see more traces. -Jess explained how the application processes images to create heat maps based on the PNG input. +**Speaker 1:** Oh, you want to see more traces? -> "It uses the blue channel in the PNG to convert it into a heat map, and you can interact with the data in Honeycomb to see how it visualizes the data." +**Speaker 2:** Yeah, right. I picked the saddest trace. That's what happens when you're ad hoc-ing it. Okay, let me—let's pull up a more interesting trace. I think the recommendation service has interesting traces that we can see. Checkout's usually good. -She demonstrated how to adjust settings in Honeycomb to view the generated heat maps and explore different attributes. +**Speaker 2:** Cool. Let's find some traces. Oh yeah, look at all those! Look at all those services! Yeah, there's some colors! Look at it go to all the different services! -> "You can see how different reindeer names are represented in the heat map, and you can customize it further if you want to use your images." +**Speaker 2:** This is actually pretty cool. You're right, I picked the dinkiest to showcase all this, right? And all this is happening on my machine in this Docker image that's running all the Hashi things, which is super cool. -Jess encouraged attendees to check out her project and see how they can customize it for their needs. +**Speaker 2:** I did want to show also what a Nomad job spec looks like. Let me just share my screen for that. So Nomad is like—it's an alternative to Kubernetes, right? -### Rhys on End User Involvement +**Speaker 1:** Exactly, exactly. -Rhys discussed the importance of community involvement and how end users can engage with OpenTelemetry. +**Speaker 2:** Yeah, and it's interesting too because it doesn't just run containerized workloads. It can run VM workloads; it can even run a JVM or IIS. So it's kind of cool that way. -> "We've found that many people are unaware of how to navigate the community or that they can get involved. Our goals in the end user working group are to foster a vendor-agnostic community and create a feedback loop between end users and project maintainers." +**Speaker 2:** Here we go. Sorry, I was looking for the one. Okay, so let me pick—let's hear it. A job spec is like a Kubernetes deployment? -He highlighted various activities and opportunities for involvement, including monthly discussion groups, end-user interviews, and community surveys. +**Speaker 1:** Yeah, exactly. -> "If you're an end user interested in sharing feedback, please reach out to myself, Rin, or Adriana. We would love to get you on the schedule." +**Speaker 2:** So here, I've got a Kubernetes deployment. I'll put it side by side here. Alright, cool. Okay, so this is the feature flag service on the right. We've got the feature flag service definition. This is our Nomad job spec. -Rhys emphasized the importance of community contributions and the different ways individuals can engage with OpenTelemetry, whether through coding, documentation, or sharing experiences. +**Speaker 2:** Unlike Kubernetes, where you have all these different YAML files that make up the various objects that make up your manifest, everything is self-contained in one job spec file, which is written in HCL—HashiCorp Configuration Language. -### Closing Remarks +**Speaker 2:** I liken it to, I mean if you've used Terraform, this looks familiar. I find HCL is like if JSON was slightly improved—that's what it would look like. I do find it's easier to read than JSON. But this is basically what it looks like here. -As we wrap up, thank you all for joining today. We appreciate your participation and encourage you to reach out with any questions or if you want to follow up on any of the topics discussed. Happy holidays! +**Speaker 2:** For example, over here in our network definition, you see that we define two ports: 8081 and 50053, which lo and behold, we defined those over here in our service YAML for the feature flag service. + +**Speaker 2:** If we scroll over to the task definition, we're saying that we're using the Docker driver. This is saying, "Okay, we're using a containerized workload." Let's find the equivalent in the manifest. + +**Speaker 2:** Alright, here we go to our deployment, and we can see where we define our image. I've defined an image pull timeout because sometimes my network doesn't like to behave, so I don't want Nomad to crap out after five minutes and say, "Sorry, I can't pull your image, sucker," and then fail my deployment. + +**Speaker 2:** Over here, we've got the ports configuration, basically saying—well, just like here, we're saying that this container requires these two ports. Well, on the left side here, in our HCL, we see that our HTTP and gRPC ports that we defined up here, that's what they're pointing to. + +[00:12:01] **Speaker 2:** The other thing I wanted to point out was over here in this end definition—these are our environment variables, which for the most part correspond to the ones that we define in our deployment. I did exclude some, like these Kubernetes ones, because obviously we're not running this in Kubernetes; we're running them in Nomad. + +[00:15:30] **Speaker 2:** But the non-Kubernetes ones, I translated over to Nomad land. I wanted to point out also that over here in what's called the template stanza, these groupings and the curly braces are called the stanzas. I'm defining two additional environment variables, but they're defined in a slightly different way because they're referencing services from other Nomad job specs. + +**Speaker 2:** To be able to reference, for example, our database service, to get the information, the IP address and the port of the database service, we can't hard code that information, right? Because that stuff can potentially change, especially the IP. + +**Speaker 2:** In order to get that information to define this environment variable, we basically have to look up this service in Console, which, as I mentioned, is for service discovery. To find out what the information of that service is, you refer to it by service name. + +**Speaker 2:** If we open over here, I'm going to open this FF Postgres. This is the job spec for Postgres. I have a service named FF Postgres service which, if we look over here—there we go—basically says, "Hey Console, I want to pull up a service called FF Postgres service, and pretty please tell me the address and port number of the service so I can plug it into this environment variable." + +**Speaker 2:** We use the template stanza to do this kind of dynamic definition, and we have to tell it that the destination is going to be an environment variable. One of the things you can do with the template stanza is use it for configuration files. + +**Speaker 2:** So similarly, very similar functionality to a config map in Kubernetes. Then here is where we define our resources. This is in megahertz for CPU, and our memory is in megabytes. + +**Speaker 2:** The other thing I wanted to point out is here I've got some rules on restarting the service. There are no dependencies that you can define in your services in Nomad, like you know how you would do in Docker Compose, saying, "Hey, this service is dependent on that service." You don't have that kind of dependency definition in Nomad. + +**Speaker 2:** Basically, the jobs start when they start, which means that, for example, this feature flag service is actually relying on the database and the hotel collector to be up and running. Well, what if the feature flag service starts before the hotel collector and the database startup? + +**Speaker 2:** If we don't put some restart rules in place, then when it starts up and these two services aren't up, or one of these two services isn't up, it'll crap out and then that's it—it's dead. So what we want to do is put some restart rules in place, basically saying within the span of two minutes, we're going to try to restart this thing 10 times with a 15-second interval between restarts. + +**Speaker 2:** Either after 10 attempts or two minutes, whichever comes first, if this fails to restart, we basically say, "Okay, we're just gonna wait a little bit and reattempt that again." The default mode for this normally is fail. If it doesn't successfully restart within 10 attempts in two minutes, it would completely fail. That means your whole application deployment fails, right, because of all these dependent services. + +**Speaker 2:** This gives you some protection saying, "Okay, we'll keep trying. We'll keep trying until things are up and running," which ends up being very convenient. + +**Speaker 2:** The final thing I wanted to point out was that we also have these checks here. These are basically health checks. They're similar to the types of health checks that you would see in Kubernetes, like liveness probes and readiness probes. + +**Speaker 2:** I wanted to show you what that looks like in Console really briefly. Let me just pull that up. + +**Speaker 1:** Yeah, and we're going to move on to Jess Citron after the Console showing, so if you have more, Rihanna, think them up, please. + +**Speaker 2:** Yes! I'll show this very briefly, but this basically shows all of our services. If we look at the feature flag service, we have two services, one for our gRPC and one for our REST API. + +**Speaker 2:** This shows us that, hey, we're able to hit our endpoints, so it sends a signal back to Nomad like, "All systems go! Hurray!" That's basically it in a nutshell. + +**Speaker 1:** Okay, follow-up questions? Last parting thoughts about— + +**Speaker 2:** Nice job! + +**Speaker 1:** Yeah, I agree. Nice job! That was a super good demo and a good example of how to use the demo app. + +[00:21:10] **Speaker 2:** So Jess "Jessatron" is actually going to show us something totally useless but super fun for training people on how to make ASCII art into a heat map using OpenTelemetry, right? + +**Speaker 2:** Right! So I made this thing for Christmas, I guess for Christmas gimmicks. It's in a repo, honeycombio/happy-ollie-days, which is cute. You can clone this repo or run it and get pod, and if you give it your Honeycomb API key—this is Honeycomb. The UI is Honeycomb-specific because it had to be specific to the display, but it does use OpenTelemetry. + +**Speaker 2:** I'm going to give it a Honeycomb API key. Oh, it's still in my paste buffer. Great! And then I'm going to run this little app, and this app is in Node, TypeScript. So it has created a bunch of spans in a trace using—where's my—here we go—start active. + +**Speaker 2:** So it does a start active span in Node, and for each, it's figured out what spans it wants to send, and then it does a start span for each of them and supplies a bunch of attributes, and then ends each one. They all go into one trace. + +**Speaker 2:** It's given me a link to the datasets. Oh, it's a good thing I'm following that link because apparently, my dataset is called Fufu Free. But then in the readme, it describes how to do this. If you do a heat map on the height field and you get the last 10 minutes, you're starting to see something. + +**Speaker 2:** But then I need to pick graph—no, no, no, up here—granularity five seconds. Oh, that's so cute! Isn't it cute? And the cool part is that you can use this program. I mean, I don't know what it's going to look like in any other system because I've customized the heights to the bucket sizes that Honeycomb uses in heat maps, so you'd have to tweak the code to make it do something in somebody else's heat maps. + +**Speaker 2:** Oh, but it actually pulls this out of—where is it? Images? Now there should be an images directory in here. Oh, input—don't peak.png. So if you give it a PNG that's between 25 and 50 pixels tall, and you probably want it between 1 and 200 pixels wide, then it can convert that into a heat map. + +**Speaker 2:** Oh, it's converting the heat map based on the blue channel in the PNG. It uses the other one to get some attributes, which does some fun things in Honeycomb because you can pick bubble up and be like, "What is different about...?" + +**Speaker 2:** Oh, check this out! Okay, my new favorite feature of Honeycomb—wow! That stupid little do-me-anything is gone! Okay, which is very useful sometimes, I'm sure, but not in here. + +**Speaker 2:** Now that I've gotten rid of that, what is different about the spans in the second reindeer? Honeycomb does its little "What is different?" analysis, and it says the reindeer name that one has a reindeer name of Dancer. + +**Speaker 2:** Then if you group by the reindeer name, you can go back and like, "Oh, oh, wait, hold on. I've got to do the trick of—wait, wait, give me the right 10 minutes." + +**Speaker 2:** It wants to like—the Santa and his reindeer are like moving on. I need to switch to absolute time. Okay, now let's group by reindeer name. Okay, now it's lost the granularity, so let's fix that granularity—five seconds. + +**Speaker 2:** Okay, but now—results tab, and we have the different reindeer names. So there's no reindeer name. This one's Dancer, and this one is Rudolph, and this one is Prancer. + +**Speaker 2:** Yeah, so it's cute that way. It's got different fields, and those are based—in case you want to use this yourself and supply your own image—those are based on the red channel in the PNG, and then there's a little key for how much red between 0 and 255 is in that color and what the fields are. + +**Speaker 2:** So that's also encoded in a PNG, and you can do it yourself. This gets officially released on Monday, and it'll be advertised and stuff, but I'm not—none of that stuff am I telling you how to fix it yourself, so y'all are getting that. + +**Speaker 2:** There's one other trick in this data, which is if you do a max of stack heights and group by stack group, then you get some nonsense, but if you change the graph settings to use a stack graph, then you start to get something. + +**Speaker 2:** If you get the order by right, stack group descending—come on, do it! A Christmas tree! Oh my God, that's awesome! And you can also generate yourself because that is based on house.png, obviously. + +**Speaker 2:** It used to be a house; now it's a Christmas tree! And of course, there are limitations. You have to have all the colors on top of one another, and you can't have a color that's both under and over, and you don't control what the colors actually show up as. + +**Speaker 2:** But if you're clever, you can make your own PNGs. Oh yeah, I've got some groups on that you can group by. It is great by stack group. That wasn't necessary, but here's the names of them—star and background and tree and stuff like that. + +**Speaker 1:** Yes, where do people go to find this app? + +**Speaker 2:** It is at honeycomb.io/happy-ollie-days. You can download this, and if you have a free Honeycomb account, or if you get it to work in another tool, that would be awesome. I would love to hear about it. + +**Speaker 2:** Oh, thanks, Johnny! Yeah, that's that. We'll find stop share. Maybe not. Daniel's like, "I'm running, not walking, to try this out!" It's fun! It's true! + +**Speaker 1:** I have a question, which is how do you think this can be useful to people as a tool to teach about heat maps and observability? + +[00:29:00] **Speaker 2:** Yeah, because you can—I mean, you can ask, "How do I draw it?" And the answer is that I send a different number of spans, which, of course, is it this one? Which, of course, I also stick all the intermediate data on the spans because that's what I debug it. + +**Speaker 2:** No, that's clearly the wrong field. Show me the trace. I haven't made the trace cute yet. Someday, I'll make this have a drawing in it too. + +**Speaker 2:** Span, you know, count amount—how many? Oh well, I send multiple spans per pixel if I want a dark color because Honeycomb says darker is more and lighter is fewer, so there's just one span in these. + +**Speaker 2:** Oh, you know what? I can find that field if I do "Bubble Up—what's different about these that only have one span at a time?" There it is! Okay, so this spans at once; these only have one span at a time. + +**Speaker 2:** Some of the darkest ones have 10 spans at a time. If you think, if you can use it to be like—how does a heat map? Mostly, it's just entertaining, and it does really illustrate bubble up. + +**Speaker 2:** Letter—all of these, most of these are letters, and some of them have different letters. + +**Speaker 2:** Yeah, so Bubble Up is pretty well illustrated by this, I think. + +**Speaker 1:** I want to say what Bubble Up is for folks who aren't Honeycomb users. Bubble Up is Honeycomb's "What is different?" + +**Speaker 2:** Exactly! + +**Speaker 1:** So you can draw the box, and it does the statistical analysis on what fields are different. It's just cute! + +**Speaker 2:** That was super good! There's not question three. Do you want to go ahead and talk about end-user working group work? + +**Speaker 1:** I mean, yes, but also, how do I follow up? We're doing systematic and important work and slowly marching through things. + +**Speaker 2:** Jessica, to do the fun creative stuff! + +**Speaker 1:** Yes! This is—I mean, I have some like one or two fun graphics, but it's not going to be anything as cool as Jess "Jessatron." + +**Speaker 2:** So you'll have to make two! + +**Speaker 1:** Yeah, I just wanted to talk real quick on something that we've surfaced just from talking to a wide variety of end users. A lot of people are not really aware of how to navigate the community on how they can get involved, or that they even can get involved. + +**Speaker 1:** So I'm not sure—I know there are a couple contributors on the call. I'm not sure everyone else is an end user, but hopefully this will be of some utility for you. + +**Speaker 1:** So a couple of things we'll go over: this time of the community at large, some of the different parts of it, and then we'll talk about how you can get involved if you're interested in some of the different ways that you can get involved. + +**Speaker 1:** We've mentioned a few times the end user working group that a lot of us are a part of. What is the end-user working group? We have two primary goals: one is to foster a sense of vendor-agnostic community for end users, and two is to create a feedback loop between the end users and project maintainers with the overarching goal of improving the project software. + +**Speaker 1:** What are some of the working group activities that we have implemented? So first of all, this is one of them. This is Rin's child, but we are very excited—OpenTelemetry in Practice is what it is. So every month, keep an eye out for more fun talks, presentations, and casual conversations. + +**Speaker 1:** We have the monthly discussion groups now with a maintainer per session, and in all regions—so America, EMEA, which I just found out stands for Europe, Middle East, and Africa, and APAC, which I'm actually not sure where those things were, but it's Asia and the Pacific region. Imagine where AC stands for! + +**Speaker 1:** Those are all on the OpenTelemetry public calendars. We also have end-user interview and feedback sessions where we will talk to an end user about their adoption, implementation, and challenges that they face, and get the feedback shared back to the OpenTelemetry maintainers. + +**Speaker 1:** If you're interested, if you're an end user and you're interested in sharing feedback in one of these sessions, please reach out to myself, Rin, or Adriana. We would be happy to get you on the schedule. + +**Speaker 1:** We are also new—for end-user interviews moving forward, we're going to turn them into profiles for the OpenTelemetry blog to make the implementation and adoption that other end users have done in their organizations more discoverable. + +**Speaker 1:** We also have a community survey, so that's another way that you can contribute. If you don't really want to get too involved but you still want to have a way to share your thoughts and opinions on how using OpenTelemetry is for you. + +**Speaker 1:** Some of the stuff that's upcoming for the working group: we want to extend our user study function, and also the governance committee is going to work to implement a project management function for the specifications SIG to streamline the feedback loop from all the feedback that we're gathering at these sessions and activities and drive prioritization for work based on user feedback. + +[00:36:00] **Speaker 1:** In-person meetups might be a thing coming to a town near you. Maybe we'll see. I think this would be a really fun thing to do, especially as a lot of us were just at KubeCon, and it seems like people are pretty into meeting in person again, so we’ll see. + +**Speaker 1:** Six—probably most of you are maybe fully familiar with these, but I'll just go over them. Special interest groups: the goal is to improve the workflow and manage the project more efficiently. Each SIG meets regularly. You can access the meeting notes and recordings through the public calendar, and that's pretty much a thing for pretty much all components, languages of OpenTelemetry. + +**Speaker 1:** There's also a governance committee and a technical committee, which I won't get too deep into. You can kind of see the roles of each committee here. + +**Speaker 1:** Show me for a second, and I can also share the slide deck too because there are some links to the resources that I mentioned in here. + +**Speaker 1:** Documentation—yes, we have a lot of questions about documentation. If you do have questions, you can always talk to the COMSIG at hotel-dash.coms channel. + +**Speaker 1:** Some languages we are aware are a little bit more comprehensive than others. There is standardization and improvement work in progress at the moment. If you would like to help contribute, feel free to jump into the channel, attend one of their meetings, or open an issue directly in the repo. + +**Speaker 1:** I personally find it kind of interesting to see what OTEPs have been proposed. They are open selling to enhancement proposals. It's a process for proposing changes to the spec, and they have to be cross-cutting changes that introduce new behavior or otherwise modify requirements. + +**Speaker 1:** It's kind of fun, personally, I think, to go and look at what people are proposing or wanting to do. There’s some interesting stuff in there. + +**Speaker 1:** Also, getting involved, the first one I want to cover is how do I get help using OpenTelemetry. There are multiple ways: CNCF Slack—you have to sign up for an account with the CNCF Slack instance. + +**Speaker 1:** But once you get in there, there’s pretty much an ozone channel for whatever it is you’re looking for—languages, components, collector, etc. There are also vendor-specific channels, or you can go to the general OpenTelemetry vendor channel if you're having problems with a specific vendor that you're using. + +**Speaker 1:** And of course, GitHub—you can open issues, jump in with comments on anything that's open. And we also have the end-user discussion groups, which I mentioned earlier. If you want to join and ask questions or share how you're using OpenTelemetry or help other people who are using OpenTelemetry in their organizations, that's another great way to get involved. + +**Speaker 1:** As far as contributions, we welcome any and all code and incoming contributions. If there's a specific language you're interested in or a specific component, go check out the SIG notes, see what they're talking about, or you can hop into any of the meetings and just kind of check it out. + +**Speaker 1:** Blog posts could be something that you know—it could be something like fun and so-called useless, but really, you know, if it brings so much joy, is it really useless? + +**Speaker 1:** So something fun that Jess did could be anything OpenTelemetry-related. Basically, we would love to see it. + +**Speaker 1:** As I mentioned, documentation—you are welcome to join the end user working group and give suggestions on things you’d like us to help with or do for you as an end user. + +**Speaker 1:** Of course, you can also just share feedback, and there are different ways to do that. If you don't really want to get engaged with an interview, you are welcome to take part in our survey, which I thought I linked somewhere. + +**Speaker 1:** I have linked the survey in here. If you would like to share feedback that way, that is great. If you would like to participate in the end-user working group by way of the discussion group or end-user interview, we would be very happy to have you. + +**Speaker 1:** And that's it! That's all I have. Thank you so much. I should have added more Jingle Bells and Christmas stuff. I feel not very festive, but I promise I am! + +**Speaker 2:** Fair! + +[00:41:01] **Speaker 1:** Justice! Thank you, Rhys! And saying that, for those not following the chat, Johnny's saying they participate in some conferences related to DevOps, and they like to share about the power of observability and how OpenTelemetry can help us make our lives easier. + +**Speaker 1:** Do folks have questions? Since we've got 10 minutes, we're happy to entertain if you have random questions about OpenTelemetry. We may or may not be able to answer them, but there’s lots of knowledge in this room. + +**Speaker 2:** Yeah! And if anyone is having specifically a question about Rhys's presentation— + +**Speaker 1:** Absolutely. + +**Speaker 2:** Well, actually, I was going to say about OpenTelemetry.net. We have a maintainer on the line. Is that Alan? + +**Speaker 1:** Yeah! + +**Speaker 2:** Yes! There are two—Mike Blanchard! + +**Speaker 1:** Hello, Mike! Very important spot! + +**Speaker 2:** That's why I'm like, "It's okay if I call him out!" I will say that I tried to pull down the Honeycomb. + +**Speaker 1:** The Happy Holidays happy hour! + +**Speaker 2:** Oh, it is! It's going to take a little bit of work, but I was going to try to see how easy it would be to get that reporting into a different tool than New Relic and see what it would look like. + +**Speaker 1:** I'm sure it would look probably amazing! + +**Speaker 2:** It will look mangled initially! + +**Speaker 1:** If you want to work at that, I'm happy to work with you on—it’s I know where in the code it's choosing the height that works, and I also know how I use the browser tools in Honeycomb to figure out what height it was needing. + +**Speaker 2:** Some of those tricks will also work in New Relic, so happy to work with you on that if you want because I would love to see what it looks like somewhere else. + +**Speaker 1:** I don't know if they're still doing them, but last year Grace and I were doing a lot of Legos for New Relic. I've got some of them here, and I wonder if they’re like amazing art! + +**Speaker 2:** Oh, that's true! Grace is the New Relic social media person and a huge Lego fan. + +**Speaker 1:** But yeah, I think Legos tie in very well with ASCII art, and you know, lots of tech people are Lego fans! + +**Speaker 2:** True, true! That works! + +**Speaker 1:** Other questions that folks have? We can also go ahead and end the call early. + +**Speaker 2:** Yeah! + +**Speaker 1:** Ellen, I'm Jessatron at Honeycomb if you want to get in touch. + +**Speaker 2:** Yes! And I'll post the link to Jessatron's calendar. + +**Speaker 1:** Sounds good! + +**Speaker 2:** Yeah, nice to meet y’all! + +**Speaker 1:** You too! Well, thanks everyone! It was great to see you all! + +**Speaker 2:** Yeah, great to see you! + +**Speaker 1:** Great to have a chatty group! Thank you all for joining, and a couple of you for putting yourself on video—that's always great for us! + +**Speaker 2:** Yeah! + +**Speaker 1:** I'll Zoom to see faces. Cool! Thank you! + +**Speaker 2:** Yeah, feel free to reach out to any of us with questions or if there's anything you want to follow up on. Happy holidays! ## Raw YouTube Transcript diff --git a/video-transcripts/transcripts/2023-09-28T04:00:36Z-otel-end-user-discussions-amer-january-2023.md b/video-transcripts/transcripts/2023-09-28T04:00:36Z-otel-end-user-discussions-amer-january-2023.md index 10adf42..b90692d 100644 --- a/video-transcripts/transcripts/2023-09-28T04:00:36Z-otel-end-user-discussions-amer-january-2023.md +++ b/video-transcripts/transcripts/2023-09-28T04:00:36Z-otel-end-user-discussions-amer-january-2023.md @@ -10,138 +10,160 @@ URL: https://www.youtube.com/watch?v=qn4x0DgG5SI ## Summary -In this YouTube video, Reese, a member of the OpenTelemetry working group and an employee at New Relic, leads a discussion about enabling observability across various programming languages within organizations. Participants, including Derek, Dan, and others, share their experiences with OpenTelemetry, discussing challenges like dealing with clock drift, defining the role of an enabler, and strategies for championing observability in teams that primarily work in languages where they lack expertise. The conversation emphasizes the importance of creating community practices, the need for documentation on successful implementations, and the idea of establishing a service catalog to improve observability standards. They also explore practical solutions like implementing connectors and routing processors to optimize data flow in observability pipelines. The session concludes with a light-hearted moment featuring Reese's kitten, Taco. +In this YouTube video, Reese, who works with New Relic and the OpenTelemetry user working group, hosts a discussion focused on challenges and strategies related to enabling observability across different programming languages within organizations. Participants, including Derek, Dan, and others, explore how to become effective enablers of OpenTelemetry despite varying levels of expertise in different programming languages. Key topics include defining the role of an enabler, strategies for championing OpenTelemetry adoption, addressing clock drift in telemetry data, and optimizing data pipelines. The group discusses practical solutions such as creating communities of practice, leveraging auto-instrumentation, and implementing routing processors to efficiently handle telemetry data. The session emphasizes the importance of clear communication, sharing success stories, and understanding the unique needs of different teams to foster a culture of observability. ## Chapters -Here are 10 key moments from the livestream along with their timestamps: +00:00:00 Welcome and introduction +00:01:00 Role of enabler in OpenTelemetry +00:03:40 Strategies for enabling observability +00:06:03 Challenges with team adoption +00:08:00 Importance of auto instrumentation +00:10:20 Community involvement and support +00:12:50 Observability group and knowledge sharing +00:15:00 Documentation and real-world cases +00:18:00 Clock and time drift issues +00:20:05 Best practices for data bifurcation -00:00:00 Introductions and overview of the session -00:02:30 Discussion on the role of an enabler in OpenTelemetry -00:05:00 Defining the concept of "champions" in different programming languages -00:12:45 Sharing strategies for helping teams adopt OpenTelemetry -00:20:00 The importance of auto instrumentation in OpenTelemetry -00:28:15 Challenges of dealing with clock drift in telemetry data -00:35:00 Best practices for bifurcating data in a telemetry pipeline -00:40:30 Introduction of the concept of connectors in OpenTelemetry -00:45:00 Discussion on scaling collector deployments and configuration -00:50:00 Closing thoughts and kitten reveal at the end of the session +**Reese:** Of course, I was looking for near to the sessions. My name is Reese. My day job is with New Relic, and I also do a lot of work with the OpenTelemetry and User Working Group, including hosting these sessions. Alrighty, let's see. Okay, let's start with this one. Does your role in Compass being an enabler, how do you deal with helping in languages you're not an expert in? -# YouTube Transcript Cleanup +**Derek:** Um, usually I run these as whoever put the topic or question in. If you can unmute and maybe give us a little bit more detail. It looks like Derek has a question on how do you define enabler. -**Reese:** Of course, I was looking for near to the sessions. My name is Reese. My day job is with New Relic, and I also do a lot of work with the OpenTelemetry User Working Group, including hosting these sessions. +[00:01:00] **Derek:** Yeah, this one's mine. Happy to expand on this. Um, so like basically the role I have in my company is kind of twofold. The first is like, you know, maintain OpenTelemetry collector deployments and kind of be responsible for the data pipeline receiving data and sending that to the various backends. But it's also like what I call the enabler, which is like being a champion for OpenTelemetry, driving adoption within the company, you know, troubleshooting, helping teams troubleshoot issues that they have in their respective services, you know, things like that. Um, so like in my company, for instance, we're probably like 70% .NET, 10% Java, 10% Go, 5% Python. You know, like we're primarily focused in one area. So like we build examples in .NET. I personally am more of like a .NET developer. My team is as well. Uh, you know, a great thing about OpenTelemetry, in my opinion, is that like the concepts are extremely similar across languages. But like, you know, I don't really know how to write Java, you know, it's not too far away from .NET. But for instance, um, so sometimes it's like troubling for me to provide specific feedback or like look at a PR and like really understand the scope of everything that's happening. And I'm curious, like if teams have this problem at the companies that they're working in. Do you try to like, you know, I don't know, create champions in those languages you're not familiar in or, you know, communities of practice around that? Um, just, you know, if anyone's experiencing this and maybe how you're solving for it. I hope that makes sense. -Let’s start with this question: Does your role in Compass as an enabler help you deal with languages you're not an expert in? Usually, I run these sessions based on whoever put the topic or question in. If you can unmute, maybe give us a bit more detail. It looks like Derek has a question about how to define "enabler." +[00:03:40] **Another Speaker:** There totally, yeah. As a former enabler, um, I can tell you that my strategy was, um, really to kind of first lead the way by showing how to enable observability within my domain of expertise, which at the time was Go. And, you know, as I was demonstrating the benefits of using OpenTelemetry, or I guess at the time it was OpenTracing, if that gives you an idea of how long ago this was. Um, but you know, one of the benefits, as I was demoing the benefits, people would come in, ask me questions, and a lot of the times these people work in different teams and different languages. Um, and you know, they would be interested in starting out, but they wouldn't be familiar with the concepts of OpenTracing, OpenTelemetry. And so, you know, my role, as I thought, was to kind of pair with those people who were experts in whatever languages that they were trying to become enablers in and, you know, really work alongside them until they felt comfortable with the concepts of the observability platform I was using. So I think that's kind of my strategy. It worked out pretty well. I think we managed to deploy things to, you know, like three or four different teams within the organization, and it worked pretty well. -**Derek:** Yeah, this one's mine. I'm happy to expand on this. Basically, the role I have in my company is twofold. The first is to maintain OpenTelemetry collector deployments and be responsible for the data pipeline—receiving data and sending it to various backends. The second part is what I call the enabler. This involves being a champion for OpenTelemetry, driving adoption within the company, troubleshooting, and helping teams with issues in their respective services. +**Derek:** Yeah, okay, cool. That's kind of I guess what I mean by like champions. So like basically leveling up, you know, maybe individuals or groups within those smaller subsets and then letting them kind of run with it. Um, do you find when you did that that those people, you know, as like I wasn't familiar with OpenTracing, but like, you know, as the spec evolves and as new features get added, do they come back to you? Do they seek information on their own? Do they jump in the OpenTelemetry Slack channels and ask questions or, uh, you know, were they routing things through you and your experience? -In my company, for instance, we're probably like 70% .NET, 10% Java, 10% Go, and 5% Python. We primarily focus on one area, so we build examples in .NET. I personally am more of a .NET developer, and my team is as well. A great thing about OpenTelemetry, in my opinion, is that the concepts are extremely similar across languages. However, I don’t really know how to write Java, which makes it difficult for me to provide specific feedback or look at a PR and fully understand everything happening. +**Another Speaker:** Um, it's kind of a mixed bag, I think. I think some people feel more comfortable with just talking to the person they know. And so, you know, to a certain extent, I became a bit of a router for questions. Um, but you know, I did see a couple of people who were really into, um, you know, talking to the community directly. And so they just went into the Slack channels or reached out to people directly in GitHub or whatever. So it’s a bit of a mixed bag. Um, you know, I think at some point as you direct people to specific issues in GitHub or specific spec changes, people tend to get more comfortable with the idea that they can go there themselves. But um, yeah. -I'm curious if teams have this problem in the companies they work in. Do you try to create champions in those languages you're not familiar with or establish communities of practice around that? If anyone’s experiencing this, I’d love to hear how you’re solving for it. +**Derek:** Cool. And then maybe one more follow-up if you don't mind. Like how long would you say like it took for those individuals you worked with to like, I don't know, get up to speed, whatever that means in your opinion? You know, is that like a matter of weeks? Was that like a longer period? You know, how would you engage like how easy it was for them to sort of like understand the domain when it is maybe not their primary focus? -**Another Participant:** I can tell you that my strategy, as a former enabler, was to lead the way by showing how to enable observability within my domain of expertise, which at the time was Go. As I demonstrated the benefits of using OpenTelemetry, people from different teams and languages would come in and ask me questions. They were interested in starting out but weren't familiar with the concepts. +[00:06:03] **Another Speaker:** Yeah, um, again, sorry to give you maybe a mixed answer. But again, it depended on individuals. I think some people who are really keen on trying to learn new ways of doing things were really excited about the prospect of trying out, um, you know, something new. And then, you know, those people kind of got up to speed a lot faster. The people that were, you know, um, you know, maybe more used to doing things a certain way took a little bit longer. Um, and then, you know, if I'm honest, I think some people, you know, I was trying to get distributed tracing adopted. Some people just never really understood what the benefits were. And I think that, you know, it really is a mixed bag. So it really depends on where people are in your organizations and how resistant to change they are, I think. -My role was to pair with those who were experts in whatever languages they were using, working alongside them until they felt comfortable with the observability platform I was using. This strategy worked out pretty well; we managed to deploy things to three or four different teams within the organization. +**Derek:** Yeah, cool. No, thank you. I appreciate that because I feel like we struggle with sometimes exactly that, like they don't really understand the purpose or they don't understand how it maybe has value outside of their individual team. You know, like when we start connecting services or something. Um, so just trying to, you know, come up with any insight I can into like improving, like the culture and that sort of thing. So appreciate the response. -**Derek:** That’s kind of what I mean by champions—leveling up individuals or groups within those smaller subsets and letting them run with it. Did you find that those people, once they got up to speed, sought information on their own or routed questions through you? +[00:08:00] **Another Speaker:** Yeah, I think, you know, I think it's really hard to take someone from their day-to-day job, which, you know, most people are already kind of strained on doing whatever it is that they're doing for the business, um, and asking them to do more by, you know, heading off into this completely, what kind of appears unnecessary aspect of their jobs, right, to implement OpenTelemetry. Um, and it's not really until people have those aha moments of, you know, understanding how much better their lives will be in the future, um, to that they can really wrap their head around the benefits. I think one of the things that I've really helped try to help people understand is, you know, how do you get started as quickly as possible? And I think that's one of the main areas of OpenTelemetry that I'm really, really excited about is all of the auto instrumentation work. Um, because I think that allows people to get some amount of benefit without a tremendous amount of investment up front. And a lot of the time I find even, you know, even if it only gets you 60 or 70% of the way there, um, it tends to give people enough of visibility into what their systems are doing through auto instrumentation that they can then like get excited about the prospect of investing more time in doing this work. -**Another Participant:** It’s a mixed bag. Some people feel more comfortable talking to the person they know, so I became a router for questions. However, I did see a few people who were really into talking to the community directly, so they would go into Slack channels or reach out on GitHub. Over time, as I directed them to specific issues or spec changes, they became more comfortable going there themselves. +**Derek:** Yeah, makes sense. Awesome. Dan, I don't really have an answer. All I can say is I sympathize with you. Uh, as someone who like codes even probably a lot less than, you know, what you described in your role, it feels weird to kind of want to lead the way as an enabler for these things in your work without necessarily being the person who's able to kind of just, "Alright, let me do a couple examples for you. Let me kind of show you from start to finish how this looks." So I don't know, I'm trying to figure it out too. Probably further behind you actually. -**Derek:** How long would you say it took for those individuals to get up to speed? +**Another Speaker:** So in my previous organization, um, like when we were trying to bring OpenTelemetry into the development teams, we faced some interesting challenges where the teams recognized the importance of OpenTelemetry, but they were constantly fighting fires. So they didn't have time to instrument their code, which was kind of ironic because instrumenting their code would probably help them fight those fires more effectively. And so like my team was, uh, their job was to be that enabler. So we, you know, defined the best practices and taught teams how to instrument their code. They kept trying to use my team as like, "Oh, well you guys know this stuff, so why don't you instrument our code for us?" Which we had to push back really hard on because like we don't know your code. So you have to instrument your code because you know it best, right? -**Another Participant:** It really depended on the individual. Some people who were eager to learn got up to speed much faster. Others, who were used to doing things a certain way, took longer. Honestly, some people never really understood the benefits of distributed tracing. It varies based on where people are in your organization and how resistant they are to change. +[00:10:20] **Derek:** I feel like everything, like we're all talking about, like we all have, like somehow are in the same boat. And I've just, it's like super interesting because I feel like I've experienced all of what you guys have said. -**Derek:** That resonates because we sometimes struggle with that; they don’t really understand the purpose or how it benefits them outside of their team. +**Another Speaker:** Yeah, what a general mentioned is something that I'm experiencing as well. Um, and even if it was wider than just one company, it goes even throughout the community. So sometimes I, um, I try to help a few open source projects to instrument, to add these orientations to their code base because I wanted to use to have good references of what other people can do, you know, so I could then point other people to that code base and say, "You know, this is a very well-instrumented application you can use it as reference for instrumenting your own application." Um, the problem with that was some projects were, they wanted to have good instrumentation, but they didn't actually, um, they weren't ready for it, right? And this is something that I'm experiencing within the company, uh, within, you know, other companies as well is that we can try to get a network and instrument things for them, but if they are not ready to have their code instrumented, it's just not gonna work. I don't know. -**Another Participant:** It's tough to take someone from their day-to-day job and ask them to invest time in something that might seem unnecessary. It’s only when they have those “aha” moments that they start to see the benefits. I've helped people understand how to get started quickly, and the work on auto-instrumentation has been great for that. It allows people to see benefits without a tremendous upfront investment, which can spark excitement about investing more time later. +**Another Speaker:** Um, I could, by the way, you sound a little distant. I was able to hear you, um, but just that way. Um, I think that's true. Like I feel like first somehow we need to like sell the, like if teams maybe recognize the benefit earlier, like it helps in all these avenues. Like it helps with them understanding the purpose and therefore like maybe wanting to contribute more how to solve the problem. So maybe, I don't know, I'm just thinking out loud here and maybe like pushing to show more like here are some examples where doing this solves some problems. Like if you're experiencing similar problems and perhaps like this would be a really good idea for you. Um, maybe like, you know, I don't know, honing in on that kind of an attack might be beneficial. -**Derek:** That makes sense. Sometimes, I feel weird leading the way as an enabler without being able to just show examples from start to finish. +**Another Speaker:** Yeah, so is my microphone better now? Am I closer? -**Another Participant:** In my previous organization, we faced challenges where teams recognized the importance of OpenTelemetry but were constantly fighting fires and didn’t have time to instrument their code. Ironically, instrumenting their code would help them fight those fires more effectively. My team defined best practices and taught teams how to instrument their code, but they often wanted us to instrument their code for them, which we had to push back on because we didn’t know their code. +**Another Speaker:** Yeah, that's much better. -**Derek:** I feel like we're all in the same boat. It’s super interesting because I’ve experienced all of what you guys have said. +[00:12:50] **Another Speaker:** Okay, that's good. Uh, yeah, so, um, I just came with a performing conversation with another person that, um, that person was mentioning something exactly like that, you know? Um, and one thing that they're doing and I found very interesting is, um, they have like an observability group, and that group is, uh, the reference group in terms of observability. And they can watch what other people, what the whole company is doing in terms of observability. And what he mentioned was, um, there was one team using this specific programming language, and they found that one specific metric helped them recover very fast from an outage. So what the observability group did then was to spread that information to the whole company and saying, "So teams that are using that programming language, you should now have to include this metric here." Um, otherwise, you know, because that metric helped that team to recover very fast from an outage. So, um, I see the role of the enabler here, like the observability enabler, as also to spread information, to get information from one title and break this item and bring to the whole company what should the whole community for that matter. -**Another Participant:** Yes, I’ve tried to help open-source projects by adding instrumentation to their codebase as references for others. The problem is that some projects wanted good instrumentation but weren’t ready for it. +**Another Speaker:** Um, would it be, you know, as far as something like the community could do with this, um, would maybe some documentation around like real-world cases like the one Gerasi mentioned, um, do you think that might be helpful if you could share like similar stories with your team? -**Derek:** It sounds like we need to sell the idea of observability more effectively. If teams recognize the benefits early on, it helps them understand the purpose and maybe contributes more to solving the problem. +[00:15:00] **Another Speaker:** It might. I mean, we try to do that already. I think it's like a combination of everything that's mentioned here. Maybe like, I don't want to say like what we're doing isn't working. I think it is working. It's just like progress is slower than I maybe would prefer, you know? I mean, yeah, I think like, you know, having a collection of like good news stories that, you know, or whatever you want to call it, like, uh, you know, like, "Hey, I work at company X and we were able to solve this problem and, you know, reduce mean time resolution blah blah blah blah." Like, yeah, I do think like that, you know, you know, if people are looking to get into the space, I think those are great. And I think like having those examples of like specifics, um, you know, like as my role here as like in the observability group for my company, like I go and I read blogs and stuff, but I don't think like everybody at my company would do that. Like maybe they are reading blogs specific to like the .NET runtime or something, you know, I don't know. Um, so yeah, I think it would help. I just, I don't know what the best, I guess format would be. I'm just wondering, would it be beneficial to have like periodic, like some sort of forum, um, where like every month or whatever, or every quarter where folks from organizations that are looking to adopt OpenTelemetry practices, OpenTelemetry in general, um, have a chance to like interact with folks in the community for like a pointed Q&A, um, to like just to sort of, you know, ease their concerns? Like, or would you all feel like this is too much of an overlap of what this group already does? I'm just wondering. -**Another Participant:** One interesting thing I heard recently is about an observability group that acts as a reference group. They watch what others in the company are doing in terms of observability. They found that one specific metric helped a team recover quickly from an outage and spread that information to the whole company. +**Another Speaker:** Um, my first thought is like it does have some overlap with this group. Um, I mean, I think, I don't know, maybe I would need to think about that a bit more. Well, one thing like Reese mentioned, like she's recording this meeting now, which I think is good. Like people can go back and like get insight into the past, which before I think was like a, there's some good topics here that are brought up that like aren't necessarily easy to get answers because there aren't really answers, like they're more gray, if you will. Um, I don't know, I would have to think about that. -**Derek:** Would documentation around real-world cases be helpful? +**Another Speaker:** Yeah, um, no, I think it's a very valid, um, observability concern. Um, and yeah, I would like to noodle on this more as well. Um, and that said, I just realized, well, I realized a few minutes ago that I haven't been time boxing this, so I apologize. Um, but it sounds like, um, we might be good on this topic. Dan, was there anything else you wanted to add before we move to the next one? -**Another Participant:** I think it would help. We already try to do that, but progress is often slower than I would prefer. Having good news stories can motivate people to get into the space, especially if they see how others solved similar problems. +**Dan:** No, I'm good. -**Derek:** Would it be beneficial to have periodic forums for organizations looking to adopt OpenTelemetry practices? +**Another Speaker:** Okay, can I hop in real quick, just share one last thought? -**Another Participant:** It may overlap with what this group already does, but putting folks who are unsure in touch with others who have experience could ease their concerns. +**Another Speaker:** Of course. -**Another Participant:** Recording these meetings is good because people can go back and gain insights. There are good topics discussed that aren’t necessarily easy to answer. +[00:18:00] **Another Speaker:** Um, this is kind of just a more general approach for trying out our organization, but we're kind of trying to drive change across various laterals through the idea of like a service catalog and scorecard for services where various like subject matter experts can define like a set of standards in a particular domain that should apply for services and kind of provide a rubric for those things. So what does like, you know, sufficient or like give a service a grading in a particular set of criteria, right? C a A plus, right? And you don't have to be responsible for implementing what is like an A look like in your service, but you're allowed to, you know, you can define those standards. And then the actual service owners that are responsible for that service can consult that rubric, that scorecard, kind of grade their own services according and then kind of refer to whatever standardized documentation you provide as a subject matter expert for figuring out, okay, how do I take my service that is maybe a great C in observability to like a grade A, right? And so in that kind of a situation, you're setting forth the standards, you're providing a clear rubric that allows service owners to kind of grade themselves, and you're providing more information if they're looking to kind of level up their service. So this is kind of something that is very new at our organization. We're going to try to drive change in a bunch of different places: things like SRE practices, how you tune your services in Kubernetes, code quality, test coverage, observability stuff. So we're just gonna try that out and see if that makes it kind of easier to gamify making strides and just improving services and things like that. So I don't know if that's helpful at all, but it kind of separates implementation from like leading the way and improving something. -**Reese:** I haven’t been time-boxing this, so I apologize. It sounds like we might be good on this topic. Dan, is there anything else you wanted to add before we move to the next one? +**Derek:** Alright, Derek, I don't know if you can see this, but Dan has put a thumbs up. -**Dan:** No, I’m good. +**Dan:** Oh, cool. Yep. -**Reese:** Can I share one last thought? +**Derek:** Alrighty, so since the next two have equal how more votes, we'll just go with the, uh, in order. How are you dealing with clock/time drift? -**Dan:** Of course! +[00:20:05] **Another Speaker:** Yeah, these are actually all mine, but today, um, yeah. Um, yeah, so like I, I don't know. I noticed, well, I noticed because our backend doesn't accept data that's like in the future. I think it's 10 minutes in the future. Um, obviously people are creating data points from the future because there are times on their servers are not correct. Um, one, I guess are people experiencing this problem? Like the naive side of me wants to just say, well, like they should just fix their servers, which I do agree with. Um, if you are experiencing this, do you notify services? Do you implement something in the collector to understand that it's happening? -**Reese:** We're trying to drive change across various areas through a service catalog and scorecard for services. Subject matter experts define standards and provide a rubric that allows service owners to grade their services. This separates the implementation from leading the way in improving something. +**Another Speaker:** Uh, yeah, that so, um, I can probably talk a little bit about how Jaeger deals with that. Um, and the way that Jaeger does, or used to do, is, um, it tries to detect, well, so I guess the first realization is that, uh, clock skew is going to happen. Um, and you just have to account for it. There is no way for clocks to be synchronized, especially on a microservices architecture. Um, and one way of dealing with that is Jaeger tries to, um, it first assumes that, um, you don't have asynchronous processing. So one trace is very synchronous. So the parent span or the very first span, um, and, uh, at most when the last span finishes, you know, so, and if that's not the case, then Jaeger by default will try to adjust the parent spans of that span to be as big or as long as the longest span that it has. Um, it did generate a lot of confusion among users, especially for users who do have asynchronous processing, so much that we ended up doing a flag to disable this behavior on the Jaeger side, on the Jaeger collector side. And I think we at some point thought about not having that behavior by default so only users who would know what they were doing would then enable the clock skew. I can't remember what is the feature name, but they wouldn't have this clock fixing feature enabled. Um, on the collector side, we're not doing anything that I know of. Perhaps Alex can share if he knows whether we are doing something like that there. But, uh, and I guess the short answer is, um, the owner of the system that generates the telemetry data is in a way better position to understand the clock drift than any general-purpose tool like the collector. So the collector cannot, in a, it's not in a very good position to detect and fix this issue for you. -**Derek:** Sounds like a good approach! +**Derek:** Um, so yeah, great. Thank you. Um, so like I don't know for, to me at least, and maybe this is not true, but, um, like there's sort of like two use cases here. There's like, like a lower mild kind of clock skew where it's like, I don't know, a couple seconds or something. And yeah, sure, maybe it makes the tracing vision like look a little awkward or something. And then there's like, like the really bad, you know, ones that are like in this case, like I, I know they were like 10 minutes off. Um, so I was thinking of like creating a processor that detects some, you know, incoming data point that is like some, some like distance, I don't know what the right word is, some, you know, if it's more than five minutes or some threshold, uh, like just, I don't know, creating a data point that like captures, you know, the service name or something and, and like, you know, just at least like identifies it like, "Hey, this is a, this is something that's like way out of sync with what we consider real-time." I don't know if like something like that would be useful. -**Reese:** Let's move on to the next topic: How are you dealing with clock/time drift? +**Another Speaker:** Um, right now our backend, there's no good way to like correlate like the error that our backend throws with like the data that's coming through. So I don't even know like what data is missing unless a service owner were to reach out to me. Um, it could be specific to my backend maybe my use case. Um, I know, so I was just like maybe just like having more visibility into the issue would like be a first step. Does that sound like a weird use case? Like I don't know, creating a processor that would kind of detect that? -**Dan:** These are all mine! I've noticed that our backend doesn’t accept data that is in the future—specifically, 10 minutes in the future. People are creating data points from the future because their server times are incorrect. Are others experiencing this problem? +**Another Speaker:** I think it's definitely an interesting use case. If you know you could have a processor that just detects things that are out in the future. Um, I would guess that the hard part is, you know, you still wouldn't catch all of your clock drifts. I guess you would catch a very obvious ones that you kind of know about, um, that you've seen in the past. But yeah, so I guess then, yeah, that's like, I guess because in my case I'm like dropping data because the backend's rejecting it. It would be good to know that, but if there was like say it was a 10-minute threshold and it was, I don't know, three minutes delayed and I didn't want to detect that or I couldn't detect that, now it still cause issues with the interpretation of the data. -**Another Participant:** Clock skew is going to happen, especially in microservices architecture. One way of dealing with that is to assume that you don’t have asynchronous processing. +**Another Speaker:** Uh, but it would be like a secondary concern. Like maybe I should just shore up like the data loss if you will and then, I don't know, just be better at, you know, having that like, like Gerasi, is that how you pronounce your name? Having like the service owners sort of interpret the drift or the skew because they're the experts, like like you said. -**Dan:** I was thinking of creating a processor that detects incoming data points that are beyond a certain threshold—like five minutes out of sync—just to identify it. +**Another Speaker:** Um, are you talking specifically about metrics or are you talking about any telemetry data type? -**Another Participant:** That’s an interesting use case. It would capture obvious issues, but it may not catch all clock drifts. +**Derek:** Um, I would have to double-check. I want to say the data that I was looking at in this case was spans. Um, but in my opinion, it would be for any type. -**Dan:** I just want more visibility into the issue, especially since our backend is rejecting data. +**Another Speaker:** Yeah, so for metrics, there's some metrics, um, stores like, you know, every single Prometheus-based storage will block new metrics that are from the past, so not from the future I suppose, but from the past. Um, so you would have a warning on some logs already because of that before using Prometheus. Um, for traces, uh, it might be relatively easy to, to find it out if the clock drift can be detected by looking at a trace, right? So by looking at the spans of a trace. And I guess it would not be too hard to make a processor that detects that. -**Another Participant:** For metrics, some metric stores block new metrics from the future but not from the past. For traces, it might be easier to find clock drift by looking at the spans of a trace. +**Another Speaker:** Now for logs, I don't know. I wouldn't even know if we would want to have something like that for logs, um, because you generate such a huge amount of logs on one specific server and supposedly all of the logs on that server are having the same timestamps for things are a lot at the same time. Um, and, uh, so perhaps just, yeah, I don't know. Perhaps this data, it's hard to think about logs in this case because it would be having a bunch of logs at once, uh, and then you have to compare those logs with a batch of logs from another server, right? Coming from another server on the collector. So your comparison base would be huge. -**Dan:** That makes sense. Thanks for the input! +**Derek:** Yeah, I guess my take there would be just do the simplest thing that would work, which in this case, if the log timestamps are still in the future, you could flag it. You know, you could create a metric that would at least capture that information. But again, this would only capture clock drift that's in the future. Nothing clear would happen if it was like drifting in the past. -**Reese:** Let’s move to the next topic: Best practices for bifurcating data in a pipeline. +**Another Speaker:** Yeah, makes sense. Okay, cool. Thanks for the input, guys. I'm good with this one. Reese and other people don't have comments. -**Dan:** I have data flowing into two different pipelines. I want one to have histograms go to backend A and the other to have non-histograms go to backend B. Is configuring two pipelines the right thing to do? +**Reese:** Excellence. Alrighty, best practices for bifurcating data in a pipeline. For example, have a filter processor with positive and negative condition clarity around fan-in. -**Another Participant:** There’s a new type of component coming up called connectors, which will allow you to connect pipelines efficiently. Currently, the routing processor routes data based on their characteristics but does that by making another network connection. +**Another Speaker:** Yeah, so this is also mine, and I don't really know how to phrase this in a simple way. Um, maybe I'll tell you a really weird case use case that I have. And yeah, anyway, so like I have data flowing. Um, let's say I have an OTLP, I have a collector, I have an OTLP receiver, I have some processors configured, and then I have a backend, let's call it backend A, and then I have another pipeline, also OTLP receiver. Well, let's just say we're using metrics here. Um, some processors in backend B. So I want to like have, um, so the same set of data is flowing into both of these pipelines, and then for like pipeline one, I want say just to have histograms go to backend A, and for pipeline two, I wanted to have non-histograms go to backend B. Is configuring two pipelines, um, like that the right thing to do? -**Dan:** I feel like I’m doubling the memory by having two receivers receiving the same data. Should I be profiling that? +**Another Speaker:** Um, I was trying to, like, it works. Like I have this configured, it works. I was thinking about like, is this the most CPU or memory efficient in terms of like, uh, I'm not, I'm not, I guess like the fan and fan out of how like the receivers and stuff work in the exporters work. I was trying to like, you know, understand if I'm doing it in like the optimal way. Does that question make sense at all? Is that a use case anyone else has? -**Another Participant:** Yes, and you might want to look into connectors as they would only keep one copy of the data point in memory. +**Another Speaker:** Um, there's a new type of component for the collector that is coming up, and that's called the connector. So it's, it's, uh, so the collector right now has four types of components, right? Extensions and then pipeline-specific components like receivers, processors, and exporters. And there's going to be a fourth one, or a fifth, uh, which is connectors. So connectors they can act as receivers and exporters. So you have one pipeline that receives OTLP for instance. Then there is some processing and, um, you end up with an exporter. An exporter for that pipeline is going to be a connector. Now the connector then connects with another pipeline, um, and as a receiver, right? So it's an exporter in one pipeline and a receiver on the next pipeline. Um, so what you can do is you can translate signals. So you can translate spans to metrics for instance. But in your case here, it would work in a way that you have like one receiver and then, uh, two exporters or two connectors as part of the same pipeline. One that is going to connect this data or this pipeline with one that is histogram specific and one that is going to connect with a non-histogram specific with the other one. And then each one of those connectors would then be able to filter what is interesting to be passed through this disconnect and block everything else. So it ended up with, uh, three pipelines in there. And so one that receives the raw data, one that deals with the histogram data, and one that deals with other data. So this is what we have planned for the future. So the basic building blocks for that are merged already. So, um, from what I remember, so Alex can correct me if I'm wrong, um, but, uh, the basic building blocks are there, and it's, we're all looking forward to seeing connectors being ready to be used because we have so many use cases in mind to implement. -**Dan:** Sounds good! I’ll definitely follow that. +**Another Speaker:** Now what we have today for that specific case is the routing processor. So we have a routing processor, and it doesn't really work the way that you want here, but it works in a very similar way. So what we do is we route the data points based on their characteristics, and we rush them to specific exporters. But we do that by making another network connection, so it's very inefficient, but it works. So those are the two possible solutions for that. With the router, I'll obviously look into this. Thank you. I didn't know what it is. Maybe this is a specific use case, but what would be the advantage of like the router solution versus like having a filter processor that like, you know, having the two pipelines and having the filter processor just like get rid of a, you know, part of like the part A of the set or part B of the set? Would there be an advantage to you either one of those? I guess that would be a third solution. -**Reese:** Does anyone else have anything to add or ask before we wrap up? +**Another Speaker:** Yeah, I guess, um, that's what I'm doing right now. Um, and it's working. Uh, there were some bugs in the filter processor that have been resolved, so now it seems to be working perfectly as far as I know. It just, I just felt like I was, because I had like two receivers receiving the same data, like I was doubling up on the memory that, you know, my collector was using. Uh, I haven't done any profiling, but maybe that's something I should do. Um, but I'll definitely look at the routing processor. Um, actually take a look at the connectors because I've made a very similar question to the author of the connectors on the PR that they introduced, and I think, I think indeed the answer is that with the connector, you wouldn't only have one receiver and only one copy of the data point in memory. It only keeps one copy of the data point in memory. Let me try to find a PR, and you can get more information from there. -**Dan:** I have a question regarding scaling my collector deployment. +**Another Speaker:** Okay, awesome. That would be great. Thank you. That sounds like a great feature. I'll definitely follow that. Thank you. -**Reese:** Please go ahead! +**Another Speaker:** Yeah, that's neat. I haven't heard about the connectors, so I'm interested to learn more as well. Well, Gerasi is getting us that PR. Does anyone else have anything they'd like to add or ask while we still have 10 minutes on the clock? -**Dan:** I’m trying to determine when to horizontally scale my pods versus modifying the configuration of an individual collector. What criteria should I use to decide between the two? +**Another Speaker:** I do if no one else does that, if someone else obviously go first since I've asked enough. -**Another Participant:** If you only have stateless components in your collector, scaling up by adding more replicas would be the solution. +**Another Speaker:** I think you're in the clarity and go ahead. -**Dan:** I have an HPA configured, but I still need to manage spikes in traffic, especially with sticky sessions. +**Another Speaker:** Okay, um, I asked this a couple of these sessions ago. Um, I was trying to, I'm trying to find, I had opened a GitHub issue. Um, it was very large in scope, so I don't think I have any responses on it. Um, it was, it was, it was about like understanding more about, um, um, I'll just pop this in the Zoom chat, uh, if you're curious. Um, it was more about just like figuring out like how to scale my collector deployment correctly. And Gerasi, I saw that you had like a post yesterday that you posted on one of the channels that talks a little bit about that. Um, there were some other things I had put in here, like, um, understanding like when the num consumers setting, uh, for instance, should be modified. Um, I'm curious if you guys have any like insight into like when I should be, I guess, horizontally scaling my pod versus when I should be modifying the configuration of, uh, an individual pod if that, or an individual collector if that makes sense. Like how do I, what wonder what criteria do I use to determine one versus the other? Does that make any sense? It's probably a loaded question as well. -**Another Participant:** It’s important to have your deployment done correctly. If one collector fails, having a headless service allows clients to connect to a list of known backends. +**Another Speaker:** So are you asking like at what point should the configuration be relegated to the pod versus to the collector? -**Dan:** That makes sense. Thanks for clarifying! +**Another Speaker:** Uh, so like I have it like an HPA configured, so like as my traffic increases, you know, presumably I spin up new pods. Um, but why are there settings like the number of consumers? Like should I be, is that purely for non, you know, deployments that can't automatically scale? Um, more so like when do I, when do I create more collectors or when do I change collector-specific configuration? -**Reese:** Thank you all for your insights. If anyone has questions about these sessions or anything else, feel free to reach out. +**Another Speaker:** Um, I'm sorry, I missed the very beginning of the question because I was looking for the issue. I found the issue and I linked here, and I linked directly to the comment that I've made that is close to your question. Um, you can read then a dance and search that. Um, but coming back to this question here, I'm, and forgive me if I'm, if I'm missing some context from the very beginning of the question, but, um, the way that I would, I would say this, the way that I would tackle is, um, if you only have stateless connect or components in your collector, you can then just dot use a specific metric and scale up or scale adding more replicas would be the solution. Um, the other answer to that is, you know, your workload, um, and you would probably have to think about that not only as in the scaling to attend to the demands, uh, like the load that I have, but also scaling to improve my high availability. So if a node goes down, what am I going to, what are the effects across my observability pipeline? So how would I want to isolate failures on my observability pipeline? So is it better to have one per namespace? Is that, and is that good enough? Or, um, or should I have a one per tenant? Or perhaps a, even within tenants in my cluster, perhaps I should have different layers of collectors, um, because, you know, failing one specific branch of your collection of collectors isn't not going to be so critical as something that is going to fail only, you know, if you have everything going through one collector, then it's really going to be an epic failure at one point. Um, I guess there's no easy answer to that. -**Dan:** Thank you all! +**Another Speaker:** Yeah, I know there's no like one size fits all. I guess I was just trying to like, okay, like so obviously like I have some like bait, like some base throughput, like, you know, that I want to kind of, I want to serve like, you know, I get a certain amount of data points per second or whatever, uh, base, and then like I get spikes, right? So I have to like have a minimum deployment that can handle those spikes. Yeah, but then there are like some connections that are, I think like, like have like sticky sessions. So I need to be careful about like if one of those things like burst real high then like a given, you know, a given pod or given collector if you will will will like, I don't know, have to be able to handle that all right. -**Reese:** I just realized I haven’t introduced my kitten. This is Taco! +**Another Speaker:** Um, so in most of the cases, you're gonna use gRPC for the connection between collectors or between your workload and a collector. Um, and the good thing about gRPC is when a connection fails, uh, it will attempt to connect to another, um, to another backend automatically, right? So if you have your deployment done correctly, and correctly here means on Kubernetes, you wouldn't be having a headless service, and your clients would be then connecting to the headless service so that the client has a list of known backends up front. So whenever one of them fails, it just fails over to the next one. And, uh, so that’s one way of dealing with the sticky problems, the sticky session problems. So most other connections, they are long-lived when we talk about the observability pipeline. So there are gRPC connections, and gRPC connections, they are long-lived by design. Which also means that if you have like only three collectors at the very beginning of your life cycle of your cluster, for instance, and then you keep adding more collectors to that, but you don't increase the number of clients, then you're not going to see any effects until the clients reconnect, uh, due to some failure, right? So you're not going to see any advantage. The moments that you scale up, you're only going to see advantages the moment things start to fail and then you start seeing the other collectors receive some traffic or when you have new clients. So new clients are going to be load balanced through the new nodes, so then you start seeing that. -**Dan:** Hi Taco! +**Another Speaker:** Yeah, so I guess that's, there's that. Uh, one other thing you may consider is, um, charting your pipeline based on the type of processing that it's doing. Alright, so one pattern that we've seen before, and, uh, that I've recorded at some point is having one pipeline per data point. So you have one metrics pipeline, you have one logs pipeline, and you have one traces pipeline because the types of workloads or, you know, the workload for metrics is way different than the workload for traces. Um, the way that they work is really different, so you might want to split by that. And you also might want to split by the type of processing that you do on the telemetry data that has been generated by your workload. So if you have more PI information being generated by nodes or by pods on one specific namespace, then it makes sense to have one collector on that namespace with a specific processor to remove this PI and then goes to the next layer of collectors that deals with more generic data. -**Reese:** Thank you all so much! +**Derek:** Yeah, so you're only affecting like, you're only slowing down for like that subset, right? + +**Another Speaker:** Exactly, yeah. + +**Derek:** Okay, yeah, so yeah, okay, cool. Awesome. And then I know we're like almost at time, so just like harping on one specific setting, that num consumers thing, like is there when when would I ever want to change that, do you know? + +**Another Speaker:** I'm sorry, um, consumers as in? + +**Derek:** Yeah, there's on the OTLP exporter, there's a, maybe this is too technical, maybe I should just open an issue just for this. Uh, there's a property called num consumers. I think it's on the OTLP exporter. Um, it seemed like I got better at the root book when I increased it. Um, and I was just wondering what like why wouldn't I increase that? Like what trade-off am I sacrificing? + +**Another Speaker:** Okay, um, sorry. I see now. So this is about it's about the signing queue. Uh, this is a blocking queue, and, um, the way that it works is, um, whenever things are being sent from one place to another from an exporter and into a backend, um, it's placed on a queue and then things are picked from the queue like from like workers. And, uh, this is, you know, the number of consumers is basically the number of workers that are picking things up from the queue. Now, um, I'm not the author of this component here of this helper, but, um, I, we had a similar component on Jaeger, and I can tell you by experience that we don't actually know what is the optimal value for that. You have to find that out by yourself. Um, a, and it's complicated because, um, in Java for instance, uh, the number of workers would be typically closely related to the number of processors in a specific, like CPU processors in a machine, a CPU using a machine. Now with Go, it's not like that. So one worker thread is not a thread, it's a Go routine, which is not related to an OS thread, a Linux thread. So there's no relation between or no very explicit relation between a number of consumers with the number of processors on a specific laptop or a machine server bare metal. Um, so you have to play with that number. I, I think I kind of played with this information on an article recently. Um, and the idea is that the more, um, if you have backends that are are taking very long to answer to you, having more consumers here means that you have more HTTP connections with the backend that you're sending data to. And it might be a problem with the backend that you're, you're, I mean if you're the backend is having trouble with a specific amount of connections and you're adding more connections, you're adding more load to a server that is already overloaded. In that case, you want to decrease the number of consumers, but what it means is it is processing less data, um, simultaneously, concurrently. So what it effectively means is it increases the concurrency of data being sent to the remote server. So in theory it is related to the CPU number of CPUs that you have, uh, but at the same time, I think it is more important to the backend that we are talking to. + +**Derek:** Okay, yeah, so awesome. That's like super helpful. I think the main takeaway for me is like I should probably be doing some more specific testing around like my expected, you know, like how much data I'm receiving and expecting to send to my back ends and just, you know, trying to optimize for my specific use case rather than like having a generic, uh, this type of situation, you know, you should set it to X, Y, or Z. + +**Another Speaker:** Absolutely, okay, awesome, really, really appreciate the insight. Thank you. + +**Derek:** Thank you. + +**Reese:** Thank you all. All or a few minutes, a few minutes or so. I want to be respectful of everyone's time. Um, Dan, thank you so much for all your questions. Um, Alex, Gerasi, it was great to have you on. And, um, if anyone has any questions about these sessions or anything else they are curious about, myself, Adriana, as well as Rin are all on the End User Working Group, so feel free to reach out to any of us on CNCF Slack. Um, and for, I just have to drop real quick, but, um, Alex and C wanted to see the kitten. I'm just gonna do a quick kitten show. This is Taco. + +**Another Speaker:** Thank you. Hi, Taco! + +**Reese:** And alright, thank you all so much. ## Raw YouTube Transcript diff --git a/video-transcripts/transcripts/2023-10-02T15:08:33Z-otel-in-practice-presents-a-hands-on-otel-migration-story-with-jacob-aronoff.md b/video-transcripts/transcripts/2023-10-02T15:08:33Z-otel-in-practice-presents-a-hands-on-otel-migration-story-with-jacob-aronoff.md index 52adf9b..280d7a7 100644 --- a/video-transcripts/transcripts/2023-10-02T15:08:33Z-otel-in-practice-presents-a-hands-on-otel-migration-story-with-jacob-aronoff.md +++ b/video-transcripts/transcripts/2023-10-02T15:08:33Z-otel-in-practice-presents-a-hands-on-otel-migration-story-with-jacob-aronoff.md @@ -10,114 +10,266 @@ URL: https://www.youtube.com/watch?v=pHHINe9D94w ## Summary -In this YouTube video, Jacob Arof, a staff software engineer at Lightstep, discusses the migration from StatsD and OpenTracing to OpenTelemetry, focusing on the challenges and strategies involved in the process. Jacob outlines various migration paths, such as the "All or Nothing" approach, the "Slow Tail" method, and using a bridge for gradual migration. He emphasizes the importance of gaining organizational buy-in, the need for thorough testing, and the benefits of OpenTelemetry's API support across multiple programming languages. Jacob also shares personal insights from their migration journey, including performance issues encountered and solutions implemented. The session encourages audience participation, with attendees asking questions about practical challenges and seeking advice on transitioning to OpenTelemetry effectively. The video concludes by inviting viewers to engage with the OpenTelemetry community and participate in future discussions. +In this YouTube video from the "otel in practice" series, Jacob Arof, a staff software engineer at Lightstep, discusses the migration process from StatsD and OpenTracing to OpenTelemetry (otel). Jacob shares insights from his experience leading this migration, highlighting various strategies such as the "All or Nothing," "Slow Tail," and "Bridge" approaches to transitioning to OpenTelemetry while addressing challenges like organizational buy-in and compatibility issues. He emphasizes the importance of getting leadership support, the need for thorough testing, and managing the migration's complexity across multiple services. The session encourages audience participation, allowing viewers to ask questions and share their experiences. Jacob concludes by discussing the future steps of the migration journey, including potential log migrations and adapting to new infrastructure metrics. The video serves as a valuable resource for those navigating similar transitions to OpenTelemetry. ## Chapters -Here's a summary of the key moments from the livestream: +00:00:00 Welcome and intro +00:01:30 Migration overview +00:03:40 Challenges in migration +00:04:50 Importance of organizational buy-in +00:08:00 Migration paths discussion +00:10:15 All or Nothing approach +00:12:15 Slow Tail method +00:15:16 Bridge approach +00:17:34 Q&A session +00:32:50 Future migration plans -00:00:00 Welcome and Introductions -00:01:30 Jacob's Background and Role -00:02:45 Overview of the Migration from StatsD to OpenTelemetry -00:05:00 Challenges in Migration and Gaining Organizational Buy-in -00:08:15 Different Migration Paths Explained -00:12:30 Discussion on the All or Nothing Migration Approach -00:16:45 Exploring the Slow Tail Migration Method -00:20:00 Advantages and Disadvantages of the Bridge Approach -00:25:00 Audience Questions on Migration Experiences -00:30:00 Jacob Shares His Team's Migration Process and Insights -00:35:00 Discussion on OpenTelemetry Protocols and Collectors -00:40:00 Managing Access Tokens and Security in OpenTelemetry -00:45:00 Future Plans for Logging Migration and Infrastructure Updates -00:50:00 Closing Remarks and Encouragement for Further Discussion +**Speaker 1:** Welcome everyone to Otel in Practice. Really stoked for such a good turnout. I think this is one of our best turnouts. Yay! And for anyone who could make it today, there will be a recording. So if you have friends who are like, "Damn it, I missed this," you can let them know that we will be posting the recording and prettying it up once it's available. -Feel free to ask if you need more details or have other questions! +**Speaker 1:** Jacob presented, I want to say last month for Otel Q&A. We worked together at Lightstep from ServiceNow. And I'll let Jacob take it away. -# OpenTelemetry in Practice +**Jacob:** Thank you! My name is Jacob Arof. I am a staff software engineer. I work on our Telemetry pipeline team. Our team is sort of in charge of our internal and external open-source Telemetry efforts, whether it's like OTL SDKs and Go, or The Collector, or the operator. Sort of anything that you would be using to collect and send data to your vendor is what we do. -Welcome everyone to OpenTelemetry in Practice! I'm really excited about such a great turnout; I think this is one of our best turnouts yet. For anyone who couldn't make it today, there will be a recording available. If you have friends who missed this, please let them know we will be posting the recording soon and polishing it up once it's ready. +[00:01:30] **Jacob:** So today I'm going to be talking about a migration that we led last year and a little bit a year and a half ago, for when we migrated from StatsD to OpenTelemetry and also from OpenTracing to OpenTelemetry. I'm going to talk about the various migration paths that are out there, some of the things that went well and went wrong in our migration. But I also would like this to be, you know, feel free to ask as many questions as you have. My slides are pretty light, so I'm really interested to hear, you know, where people struggle currently, what type of challenges you're running into, what features you're interested in. -Jacob presented last month for the OpenTelemetry Q&A. We worked together at Lightstep, which is now part of ServiceNow. Without further ado, I'll let Jacob take it away. +**Jacob:** So please, you know, raise a hand or comment in the chat, and I'd be happy to answer any questions and go down any interesting avenues that you might have. So with that, I will share my screen. Can we all see my slides? Yes? Great, thank you very much. ---- +**Jacob:** So yeah, today I'll be talking about that Otel migration story. So first, let's talk about a problem. I've been in the industry for many years, and I've had to lead a few of these migrations where your hand-rolled metrics or tracing or logging library just isn't cutting it anymore. Something where it's not performant enough, maybe the maintenance is really expensive, maybe there's a feature that everybody wants that you just can't implement. You know, whatever it might be, you want to do a migration. -## Presentation by Jacob Arof +[00:03:40] **Jacob:** It can be very challenging and daunting depending on the amount of services that your company runs, depending on what the organization's willingness to undergo this migration might be. So all of these are problems that I've faced in various jobs, not just where I am now, but you know, a few past roles as well. It can be really hard to make this happen, especially when you, as maybe an SRE or a DevOps person, or maybe just an engineer, just want to make this work. When you feel very passionate about making it happen and you just can't seem to get the buy-in that you need, so that is sort of the theme for today. -Thank you! My name is Jacob Arof, and I am a Staff Software Engineer working on our Telemetry Pipeline team. Our team is responsible for our internal and external open-source telemetry efforts, including OpenTelemetry SDKs in Go, the Collector, and the Operator. Essentially, we handle everything you would use to collect and send data to your vendor. +**Jacob:** And there is a solution, right? It's like migrating to this new thing. Otel's metric API—I'm going to be talking about metrics, traces—but you know, this is really true about a lot of stuff. For Otel, the metric API, the traces API, and even the logging API is accepted by most major vendors out of the box now. The API supports all these instrument types in all of your favorite languages, probably very few that don't. The performance and compatibility is always top of mind for us throughout the stack of Otel components. Everybody is always thinking about performance. -Today, I’m going to talk about a migration we led last year, transitioning from StatsD to OpenTelemetry and from OpenTracing to OpenTelemetry. I’ll cover various migration paths, what went well, what went wrong during our migration, and I'm open to questions throughout, so please feel free to ask. My slides are pretty light, and I'm interested to hear about the challenges you're facing and the features you're interested in. +[00:04:50] **Jacob:** So, you know, with all this in mind, you're like, "Yeah, this sounds great. I want to start using this." But there's a problem, right? Using these new tools, migrating these new tools is hard. How do you break up the work? How do you know all of your Telemetry is working the same as before? What are the risks of these different migration models? What even are the different migration models? -Let's dive into the migration story. +**Jacob:** And probably the most important one here is how do you convince leadership that this is worth doing? You can try and brute force your way into doing a migration, but that to me is a recipe for burnout. Doing a migration in general, where you know there's a new architecture that you think is really going to improve quality of life for your team, for your company, without getting organizational buy-in is how you spend months in a project that may never see the light of day. And that is really demoralizing and very difficult to work around. -### The Problem +**Jacob:** So the first thing that you have to do when you want to migrate to a new tool is make the case. It's not even about showing a proof of concept, it's just can you convince people that this is worth doing? Sometimes a proof of concept helps, but that's not the key to success. The key to success is really getting a group of people who agree with you, that you're able to sell one-on-one, who can help you really make it clear to the stakeholders, you know, in your leadership that this is worth doing. -I’ve been in the industry for many years and have led several migrations where hand-rolled metrics, tracing, or logging libraries just aren’t cutting it anymore. Some common issues include performance not being sufficient, high maintenance costs, or features that everyone wants but can't be implemented. Depending on the number of services your company runs and the organization's willingness to undergo migration, these can be daunting challenges. +**Jacob:** For me, it was pretty straightforward. We're a main OpenTelemetry contributor. The Otel metrics API and SDK wanted to go into their 1.0 stable, and they really wanted some quick feedback from someone who has a lot of traffic to test it out and see where the edges are. So in that case, it was very easy to sell the migration. In past jobs, the selling point is usually, you know, you look at maintenance cost and performance overall. I worked with a hand-rolled solution in a few jobs ago, and when it came time to do a migration, the thing that really sold people was just counting the amount of tickets and hours that we had to spend on, you know, new features and maintenance on our internal library, as well as the amount of times where, you know, we've had an incident because something related to our instrumentation is just incorrect, which you know does happen a lot with your own hand-rolled stuff. -This theme resonates with many engineers, especially SREs or DevOps folks who are passionate about making things work but struggle to get the necessary buy-in. The solution, of course, is to migrate to something new, like OpenTelemetry's metric API, trace API, or logging API, which are accepted by most major vendors out of the box. +**Jacob:** So that is maybe a good basis to go off of. Does anyone have any questions sort of before I go into more specifics about this section? I'm going to do like a five count, something I learned from a teacher of mine. I'm just going to—we're going to do five seconds of silence until someone raises their hand, or I'll just keep going. -### Migration Challenges +**Speaker 1:** No cool. Oh, is there something in the chat? -When considering migration, you might face several questions: +**Jacob:** Oh, that's just me moving it a little. Cool, no worries. Thank you. So let's talk about the migration paths. -- How do you break up the work? -- How do you ensure all your telemetry works the same as before? -- What are the risks of different migration models? -- How do you convince leadership that this is worth doing? +[00:08:00] **Jacob:** The first one is what I call the All or Nothing. This method has you entirely rip out your existing instrumentation in favor of Otel. This would be, you know, a lot of people talk about, you know, replacing the engine of a running plane or running car. A lot of people will say that this would be like, you know, just getting a new plane and have everyone hop to the new one while it's flying. -Trying to brute-force your way into a migration can lead to burnout. Hence, the first step is making a convincing case for migration. Sometimes, a proof of concept helps, but the key is getting a group of people who agree with you and can help communicate the value to stakeholders. +**Jacob:** So it's difficult, but you know, maybe it has its benefits. One of the pros is that, you know, once you've pushed your code and you've confirmed things are working and you have a good enough CI/CD system, your work is really done, right? It just rolls out and everybody's happy. This reduces the time for split brain. Split brain is what happens when you're on two systems that may not be compatible together. You could imagine, you know, if you're on StatsD or if you're on Prometheus even. -In my case, we were a main OpenTelemetry contributor. The OpenTelemetry metrics API and SDK were looking for quick feedback from a team with high traffic, making it easy to sell the migration. In past roles, the selling point often revolved around maintenance costs and performance improvements, especially when we counted the number of tickets and hours spent on features and maintenance. +**Jacob:** If most of your metrics come from Prometheus, they don't have periods in them. They have, you know, pretty strict requirements about their shape overall. You can't do things like up-down counters; that's not a type that they have. They used to not have a proper histogram; they now do. They now have an exponential histogram support, which is experimental, but you know, there are things that Otel has that Prometheus or StatsD just does not have and doesn't have the ability to do. -### Migration Paths +**Jacob:** So being in a split brain where some services have it and some services don't and they're emitting different metrics in different shapes, different variables, that can get pretty unwieldy pretty fast. And so if you're not very deliberate about planning how you're going to migrate your dashboards and alerts, then you're going to be stuck with both of them for some time, which if you're on call, and if you've been on call during a migration for this type of stuff, that gets really painful. If you get paged at 3:00 a.m., you have to wake up and go, "Oh, you know, which dashboard should I check? Is this the service on Otel or is the service on StatsD?" That's really frustrating. That's the type of thing that you don't want to have an on-call engineer think about. -Let's discuss the migration paths we considered: +[00:10:15] **Jacob:** So the other benefit of doing this All or Nothing approach is that the issues are incredibly visible. If a dashboard breaks, if an alert pages or a service crashes, it's pretty obvious, right? Hopefully, you're looking at a dashboard or, you know, the person you take the pager when you're doing this migration, so that you experience that pain. And hopefully, you also page on no data. It's very important. -1. **All or Nothing**: - - This method involves completely ripping out existing instrumentation in favor of OpenTelemetry. - - Pros: You reduce the time for split-brain scenarios, where some services are on one system, and others on another. It makes issues very visible since any problems will be apparent immediately. - - Cons: This requires thorough testing in a robust development environment. If staging and production environments differ significantly, you may face production bugs that you couldn't catch in staging. +**Jacob:** And then service crashes—you should have an alert on that ideally. So all of these things, though, make it really clear that as soon as you do this All or Nothing thing, you know, if all of your services are rolling out within an hour with this new change and everything is looking good, that's a very good sign and that gives a lot of confidence. -2. **Slow Tail**: - - This method allows application developers to migrate themselves by toggling an environment variable. - - Pros: Less upfront work and the ability to confirm functionality for one service at a time. It also gives you more time to develop dashboards and alerts. - - Cons: This can be slow, especially in larger environments. Bugs may not reveal themselves uniformly if services are not instrumented the same way. +**Jacob:** This does, though, you know, the clauses here. This requires a lot more thorough testing in a really good development environment. If you have a lot of environment drift where your staging environment is entirely different than your production environment, this might be really challenging because this means you might have production bugs that you're just unable to catch in staging. -3. **Bridge**: - - OpenTelemetry provides a bridge for some migrations, allowing you to transition from one instrumentation to another without significant code changes. - - Pros: Minimal code changes required. - - Cons: You may experience performance drawbacks compared to using the new method directly. +**Jacob:** And if you do, if the blast radius is all of your services, that can be really dangerous. Also, you know, you do have those compatibility problems I mentioned. So you have to be very deliberate about observing which dashboards and alerts break and fixing them proactively or even in advance if you know what the metrics are going to be. -### Transition to OpenTelemetry +**Jacob:** The example I have here is definitely a common one where you could imagine a metric type changes but a metric name doesn't, which most vendors will just reject. Or the dashboard that you're looking at will look very strange. This also does mean that there's more upfront effort to migrate your services. It's going to take, you know, probably a group of people to help you monitor this, depending on how many services you have to migrate. If you're, you know, a company with maybe five to ten services, that's not too bad to do with one person. But if you're a company with hundreds of services, you're going to want a team of people to monitor this with you. -In our case, we began migrating from StatsD to OpenTelemetry metrics using the Slow Tail approach. We had several motivating factors, including wanting to utilize new metrics features that StatsD didn’t support, like asynchronous instruments and exponential histograms. +[00:12:15] **Jacob:** So on to the next one, we have the slow tail. This method gives your application developers the ability to migrate themselves by usually flipping an environment variable on and off for whichever method they want. The benefit here is that there's less upfront work. You can confirm that it works for one service and push it out for that service only, and you can do that in all of your environments. So if you have that, you know, if you don't have that confidence that I mentioned earlier about your staging environment versus your production environment, this would be really helpful because you could actually just push a single service without the fear of, you know, all these other services rolling out to verify that your change worked as expected. -While we started our tracing migration, we faced some performance issues due to the wrapper library we were using, which required conversion to OpenTelemetry tags. This led us to begin our tracing migration using the All or Nothing approach, as tracing had been stable for some time. +**Jacob:** And finally, this also allows you to have more time to develop dashboards and alerts to handle those compatibility issues. The problem is that this can be very slow. If you have more than 50 services in at least two environments and it takes an hour to migrate a single service because you're trying to be extra careful, then you're looking at a multiple weeks or months-long process. -We wrote the code for the migration and pushed an image to our staging and production environments, confirming that each version looked the same before and after the migration. This was a crucial step, as it allowed us to catch integration issues early on. +**Jacob:** If you leave a migration to app developers without a strong why, also they'll never do it, so it's usually going to fall on your team to make that happen. Bugs also may not reveal themselves if your services are not uniform. Imagine you have something like a queue worker that works off of Kafka, whose topology looks entirely different than something like, you know, a classic API server. If you're only, you know, instrumenting your API servers and then you go to instrument one of those Kafka servers, if there's something that's significantly different in how you did your instrumentation, that might take some time to figure out what's going wrong. -### Next Steps +**Jacob:** Ideally, Otel has figured out a lot of this, but doing any of these migrations, you always need to check. You know, we just cannot and honestly should not know how you instrument your services. You don't want to have to explain your whole observability backend to us; that doesn't really make much sense. So ultimately, it's important that you do some work to check that the migration is working as you expected. -Currently, we are focusing on migrating our infrastructure metrics to use OpenTelemetry's first receivers and plan to migrate to the new logging cases soon. +**Jacob:** One thing that I did just think of is that you could do a combination of this slow tail and All-in-One, where you do a migration for, you know, say some good sample of your services. And then once you're pretty confident with those, you could move to just enabling for everyone at once. That's another good option. I'm trying to think about the drawbacks of that. Not sure there are any. I think that's probably the way to go unless you're really nervous about some of these compatibility problems or some of these like unknown unknowns; that might be the only time that that would be a little scary. ---- +[00:15:16] **Jacob:** So the last one is the bridge. OpenTelemetry for some migrations actually provides a bridge where you can go from, you know, instrumentation A to instrumentation B without having to do any real code changes other than implementing the bridge. There are a few issues here. Well, already you can see the pros. I mean, it's pretty obvious, right? You don't have to make many code changes. -### Questions and Discussion +**Jacob:** The problem is that you're going to have some worse performance in comparison to writing using the new method. It's going to be, you know, a fair—there's a conversion cost to anything that you're doing in your application. If another option is you could just send from, you know, instrumentation A and then convert it to instrumentation B with the Otel collector. So if you wanted to go from StatsD, the Otel collector has a StatsD receiver and an Otel exporter, so that's also fine. But both of these have the same drawback, which is that if you wanted to take advantage of some of the capabilities of the Otel SDKs, you know, then this migration—you're not really doing a migration. -Do we have any questions? +**Jacob:** It means that you can, you know, try and migrate stuff piecemeal and like for dashboards and alerts, which is great, but it does have this drawback of, like, you're not actually doing the migration. You're still, you know, just putting off the hard work later. And so this can also be confusing to your app developers where someone says, "Well, my code says open tracing, but this trace says Otel. Why can't I, you know, use X feature?" And that can be a little confusing. I think that's like not the end of the world, though. It's, you know, as long as you're communicating well, that should be all right. -**Question from Rahul**: I'm currently experiencing challenges in shifting mindsets from our previous vendor to OpenTelemetry. Any tips on winning over various developer teams? +**Jacob:** But yeah, so this is really, I think, using the collector is another really good path forward. Overall, though, doing the thing where you just send some traffic to the collector from one of your services, migrate your apps and dashboards, and then you have everything sent to the collector, and then you can migrate your apps to Otel with that All-in-One approach. And then your dashboards and alerts should just, you know, already be changed, and you should be all set. -**Jacob's Answer**: It's crucial to sell on features. For instance, highlight cost savings—if you’re moving from a service like DataDog to Prometheus, the cost difference can be significant. Additionally, emphasize the local development experience for metrics and the ease of using Grafana for dashboards. Building local tooling around metrics can also help developers see the benefits firsthand. +**Jacob:** Before I move to the next portion, do I have any questions? Any thoughts? I'm going to do a five-count again. -**Question from Jay**: What is the best protocol to use within OpenTelemetry for performance and security? +[00:17:34] **Speaker 2:** Jacob, I'm actually living through this right now. One thing that I'm running into quite a bit is getting the various developer teams to shift their mindset from sort of our previous vendor that we were with into our newer one. And it's, you know, there's underlying things we're doing, like the StatsD into like Prometheus style type thing with it. And yeah, that's led to a couple of people who are like, "Oh, these things aren't apples to apples here anymore," and that's causing a bunch of stuff. Any tips and suggestions on winning the hearts and minds here? -**Jacob's Answer**: We typically use gRPC for our setups, but it can depend on your company's security requirements. I recommend using the OpenTelemetry Collector where possible, as it offers numerous options for exporting data to your desired backend. +**Jacob:** Yeah, so really, like the thing to do is sell on features when you can. So if you could say, you know, this improves our—well, the easiest one is cost, right? If you're using StatsD, you're probably coming from Datadog. You're going to say, "You know, our previous spend with Datadog was X thousands, maybe millions of dollars, and with Prometheus, it's like, you know, $100 a month or something," right? That's the easiest one. ---- +**Jacob:** But the more valuable one is, you know, you sell in the ecosystem where, for me, the real benefit of Prometheus is that the local development experience for metrics is actually much simpler. You don't have to run, you know, a Datadog agent; you don't have to do anything else. You can just hit your metrics endpoint and verify that your metrics are doing what you expect them to. And that's a really good developer flow for testing that stuff. -As we wrap up, I want to thank everyone for their participation today. If you have any further questions or would like to share your OpenTelemetry use cases, feel free to reach out! +**Jacob:** The other thing that is—I mean, people from Datadog—I used to work for Datadog—and their dashboard product is great. But Grafana, if you're using Grafana, also has a really strong dashboards product. Showing something like, you know, maybe you use Redis in your Kubernetes cluster, you install a Redis service monitor, and then you install the off-the-shelf Grafana dashboard for it, right? Like that's a pretty great experience, and that's all done at, you know, zero cost, which is pretty incredible. -Thank you, Jacob, for sharing your insights and experiences with us. We appreciate your time and expertise. Have a great day, everyone! +**Jacob:** You know, Prometheus metric cost is very small, in cents on the dollar. So the last thing you can do is trainings. I have a storied history with trainings. I think that they never really achieve the thing that you want, which is for more people to be excited. Really, what they do is can cause more confusion if you're not careful with like your language. I think maybe a strategy that I've always wanted to do is build some local tooling around the stuff for developers, whether it's, you know, writing maybe an end-to-end test or a little UI around their application metrics so they can do something with them locally to be like, "Oh yes, like this thing is working as expected." + +**Jacob:** I mean, I think that the local dev for Prometheus is just pretty fantastic, and that's probably what I would sell on. But again, you know, it's very company-specific in many ways, right? If people are really bought into the Datadog model of things, which is, you know, high cost, very low thinking, Prometheus and Grafana is not really that. It's low cost but much more thinking, and ultimately, like you don't want your developers to have to think too much about their instrumentation. + +**Jacob:** The thing that I tend to do is have a wrapper library before doing a migration so that people are used to the same signatures; it's just doing a different function under the hood. It also makes your migration easier. I didn't mention it here because most companies already have some type of wrapper because they have some needs that are specific. It's usually like a thin wrapper, but doing as much as you can to not change their workflow and make it very simple is always going to be good. + +**Jacob:** The thing with Prometheus you can do is check what metrics are available and then you can auto-generate dashboards from those metrics. And you can add into your Helm charts, you know, these are the metrics I care about, and then you could just generate alerts automatically from that as well. So there's a lot of that quality of life stuff that, again, like it's easy with the Datadog UI but is automatic with, you know, infrastructure as code. + +**Jacob:** So that's like another trade-off I would say because not a lot of people I don't think use Terraform at Datadog. + +**Speaker 2:** Thanks for that, Jacob. + +**Jacob:** Yeah, no problem! Any more questions on this part? Cool. + +**Jacob:** Okay, so now I'll talk about what I did, what the team did for our migration from StatsD to Otel for metrics. So we began migrating using that slow tail path. The reasons were, I'd say, good motivating factors. As I mentioned earlier, the metrics API and SDK weren't declared stable at the time, which meant we would have had to deal with some signature changes. + +**Jacob:** And that would have been potentially a lot of signatures. We did have a wrapper library, but still doing those types of changes can be pretty frustrating. We also wanted to use some new metrics features that StatsD didn't support. This is like asynchronous instruments. The biggest one was exponential histograms, which Otel sort of pioneered. + +**Jacob:** And then the last one is that we would, you know, because we are the group that helps write these libraries, we wanted to understand the performance and quality of life features to make it as easy as possible and as, you know, as quick a decision. It's just, you know, you don't have to worry about performance; you don't have to worry about all this other stuff. How do we make this simple? + +**Jacob:** So in migrating to Otel for metrics, we did find a few performance issues. The first one was, you know, because we were using that wrapper library I mentioned, our implementation was working off of OpenTracing tags, which at the time was our tracing instrumentation, which we then would need to convert into Otel tags anytime you wanted to make a metric, which if you think about the amount of times that you call, you know, metric.record, that's a lot of time. So that gets pretty expensive. + +**Jacob:** And so while we waited for improved performance in the metric libraries, we just began our tracing migration because the Otel team needed time to investigate some of the stuff that we brought to them. And it also meant that we could fix, you know, the very company-specific problem of this conversion. + +**Jacob:** And so then we began our tracing migration, going from OpenTracing and OpenCensus to Otel. And so we went for this one with an All or Nothing approach because Otel for tracing had been stable for some time. We weren't really concerned about compatibility. + +**Jacob:** Ultimately, at the time, there weren't a lot of guides written, but the process was relatively straightforward, and the compiler is really doing most of the work for you. The structure for it, we already supported Otel traces in our product, so there's actually, like, no—in theory, there were no real fun changes either for our own dashboards and alerts that we were going off of. + +**Jacob:** So what I did for this was write some—write the code for the migration and then push up, build, and push an image to our staging and production environments for a service that doesn't get much traffic but does get some. It's important that it gets some. + +**Jacob:** And then it confirmed that each version in our product looked the same before and after. We have a view of sort of, you know, these red metrics per version, and it shows you the difference in those versions. And if any of those looked different, if the rate, for example, dropped off, then that's a sign that we did something totally incorrect. + +**Jacob:** Doing that for one service is good, but then the reason that the, you know, the All or Nothing approach is really useful is that you get that integration test. So when I migrated all of our services to this new method, it was really clear that service-to-service trace propagation wasn't working. You know, you just did a spot check of a few different traces, and it's really—that's like such an obvious thing. + +**Jacob:** So that was a pretty easy fix. I actually had to just go into the Otel community and implement something that had a TODO around it. + +**Jacob:** Another issue that we had right before, you know, sort of in the last bit of this migration was that our sampler wasn't configured correctly. Otel provides a lot of new features for sampling, and I was under-sampling in one environment and then over-sampling in another environment because I misconfigured it. So I had to roll that back really quickly, fix it, and then push it out a day or two later. + +**Jacob:** But overall, this whole process took maybe a month of work to migrate, I think it's like a hundred or so services in all of our environments, which is, if you've ever led a migration, that's a pretty good time. I was pretty happy with that one. + +**Jacob:** It also meant that we could get back to our metrics migration because we had sort of achieved that goal with OpenTracing tags. We were able to use those—use the fact that we didn't need to do this conversion to continue our metrics migration. + +**Jacob:** It also—we came back to the Otel team having shipped some real performance and quality of life improvements that really let us continue with this without the fear of performance problems. + +**Jacob:** These changes also let us use this feature called metrics views, which let you create a new metric series to provide seamless compatibility with StatsD. StatsD emits their histogram—what's really what's called a Prometheus summary—but we wanted to emit exponential histograms, but that's really the only change that was happening here. + +**Jacob:** All of our other metrics were able to stay about the same, so we just needed to dual write the old summary, which Otel didn't really support at the time and I think doesn't and shouldn't; it's a bad metric type. And we also wanted to dual write the exponential histogram so that we could migrate all of our dashboards and alerts from this old approach to this new approach. + +**Jacob:** And then finally, you know, as I said, we put this library behind a feature flag, and then we could just flip that on and off whenever we wanted. And then we would roll it out pretty slowly over a two or three month period to all of our environments. This also let us like test the change really effectively as well. + +**Jacob:** I think what's next? Let me go back. + +**Speaker 3:** Do you have any questions about this migration process? Do a five count again. + +**Rahul:** Jacob, this is Rahul. I have one question around the Otel protocol. So I'm guessing you must have used the Otel receivers and exporters vastly during your metrics and trace migration. So what is the go-to protocol within Otel? It supports STD and gRPC, but from security and performance point of view, which is the go-to protocol for traces and metrics? + +**Jacob:** I'm not sure I follow. We actually just emitted Otel directly from our SDKs to our SaaS, so we didn't go through a collector for this one. We could have gone through a collector, but we didn't want the operational overhead at the time. It was enough migrations to handle it once. But that didn't really answer your question. Can you restate your question? + +**Rahul:** Yeah, I mean, I wanted to know what is the best protocol to use under Otel? Is it STD or gRPC in terms of metrics and traces? + +**Jacob:** Yeah, in terms of performance and security. I'm not the best with security recommendations. I would say we use gRPC for everything, just because it's, I don't know, our sort of de facto internal standard. HTTP is more accepted for some, like depending on your company security requirements. I'm not sure of the real differences—like to compare and contrast them. I don't think I'd be the right person to speak to those. Maybe some of the other Otel folks have more of an opinion or more information on that, though. + +**Rahul:** Okay, cool. Were there any learnings around managing access tokens? And did you use multiple access tokens, or does it, you know, if you're sending a lot of metrics or traces, or a single access token? + +**Jacob:** Yeah, we just used the same access token approach that we did. I don't know if I can speak to that just because it's, you know, internal security stuff. + +**Rahul:** Yeah, no worries. + +**Jacob:** I'd say that, you know, the best thing you can do for security in like a cloud-native environment is use some sort of secrets provider. The Kubernetes External Secrets Operator is great. You can hook it up to something like GCP's KMS and decrypt your secrets to then load access tokens from, though you can do the same thing with AWS or Vault or any of these other providers as well. + +**Jacob:** If you're particularly security-inclined, it's also important to use things like mTLS as well. Something like Istio can help you with some of that. The Otel operator actually provides some mTLS features in OpenShift. So, you know, there are a lot of security features out there to be used. Hopefully, that seeds some interesting investigation for you. + +**Rahul:** Yep, yep. Thanks. + +[00:32:50] **Jacob:** No problem. So moving on, you might be wondering what's next. Are we going to do a logs migration? Right now we're under a different migration, which is changing our infrastructure metrics to use the Otel first receivers like the CP cluster receiver, the CET receiver, and so forth. Because we really want to start using some of these things the community is writing. + +**Jacob:** After that, we should be able to begin migrating to use the new logging case, which should be looking pretty good in a few—they're looking good now, but I'm not sure what the state of it is for Go just yet, given that Go just released like a new standard logging library. + +**Speaker 4:** Any more questions? + +**Jay:** Hi, this is Jay speaking. I have one general question regarding OpenTelemetry. I'm pretty new to OpenTelemetry, and the reason why I was investigating OpenTelemetry was for its ability to be backend agnostic for generating metric data. The question is, however, I was interested in exploring pull-based metrics exporters, but so far, I don't—if I'm right or wrong—but the Prometheus exporter within the OpenTelemetry SDK is the only one that is supporting the pull-based metrics approach, is that correct? + +**Jacob:** So it sounds like you're going—so there are a few different types of exporters within Otel. So there's an SDK exporter, which yeah, there's a Prometheus exporter. I think there might even be a Datadog exporter. But one—what might be a better fit is to use the Otel collector and use their exporters, which are numerous, and pretty much every backend has some type of exporter. + +**Jacob:** So I think I would try and say you should, if you're using Otel SDKs, you should export an Otel to an Otel collector and then export to, you know, your protocol of choice. + +**Jay:** But when using this Otel exporter, this would rather be a push-based mechanism, right, to send data to the Otel collector? + +**Jacob:** Yeah, that's correct. + +**Jay:** Okay, okay. You can also, if your instrumentation is in Prometheus right now, though, you can have the Otel collector scrape the Prometheus metrics and then export them as Otel or export them as StatsD or, you know, whatever you want. + +**Jacob:** Good, thank you! + +**Speaker 5:** Do you use any solution to manage or, you know, update the YAML file of the fleet of agent fleet on the go? And is there any built-in solution that you guys are using to manage the collectors? + +**Jacob:** Yeah, so I am a maintainer for the Otel operator and internally and externally I recommend using the Otel operator with the Otel collector CRDs. They're pretty easy to use, easy to set up, easy to manage, and we're always developing and thinking about new features as well. And so that'll be the place to get those now and in the future, and I'd continue recommending that. + +**Speaker 5:** Okay, thanks. + +**Jacob:** Could I ask a question following on from the last question? Would you recommend using something like OpAmp in terms of managing your fleet of collectors, or would you say it's preferable to do it in an operator basis, you know, kind of the C pattern? + +**Jacob:** Yeah, so if you're in Kubernetes—OpAmp, by the way, is still pretty early alpha right now. Well, the protocol itself is stable, but the actual implementations are in alpha. + +**Jacob:** So I'm going to answer this question with the assumption that the implementations are done, if that's all right? + +**Speaker 5:** Sure, yeah. + +**Jacob:** So if you're in Kubernetes, I would recommend using the OpAmp bridge that we're developing. The bridge is a component that can connect to your vendor and will be able to manage pools of collectors rather than just having an extension that works on—or a supervisor and an extension that works on a single collector pod. + +**Jacob:** The reason for that is usually you are running a collector, and what you're running a collector in a pool, not as a monolith. And so whereas OpAmp would be—OpAmp on a using a supervisor would be very useful for, you know, a VM or just running it as a binary, doing it in Kubernetes, if you're running it as a pool, it's not a great pattern in Kubernetes to have a supervisor update a single pod's configuration to make it not uniform with the other pods in its replica set. + +**Jacob:** So what that means is, you know, if you're going to run it in Kubernetes—if you're going to run a replica set of pods in Kubernetes, you want those pods to be the same configuration. And if you're only running a supervisor, if you're running a supervisor on each of those pods but you're only making the change to a single one, that's an anti-pattern and can get you into some trouble. + +**Jacob:** So the bridge, however, can manage pools of collectors and is definitely what I would recommend to use. Again, with this stuff being completed, that's what I would recommend for Kubernetes. + +**Speaker 5:** Okay, thank you. + +**Jacob:** No problem. + +**Speaker 6:** I've got a few questions related to the work. I work at a large US bank, and I'm trying to bring in Otel and essentially have that as our main strategy to try and move away from beyond vendors specifically. The question that we're running into just now are the challenges like vanilla versus vendor. And what I mean by that is do we just pull down, if I take the collector for example, do we just pull down the collector, configure the collector the way that we want it with receivers and exporters, or with the specific strategic vendors that we work with? + +**Speaker 6:** There are obviously pros and cons in doing both. It doesn't sound like, you know, from your side Jacob, obviously you're working from the vendor side, but I have spoken to other vendors who have given me an interesting range of opinions on what area or which of those to look at. It'd be quite interesting to see what your thoughts are. + +**Jacob:** This is a great, really great question. It totally depends on your deployment model, I would say. So one option, especially, you know, if you're running thousands of pods and, you know, hundreds of clusters, the model that I would recommend you use is the Gateway model where, you know, let's say you have a collector per group of apps and then all of those collectors forward to a centralized pool of collectors that then forward to your vendor. + +**Jacob:** This means that your—those pools for your applications are vendor neutral because all they're really doing is gathering and forwarding stuff for your application teams. And then a centralized team would manage the Gateway collectors, and then that's the place where you make the decision about, "Am I going to use my vendor-wrapped collector or am I going to use my, you know, vendor-neutral one?" + +**Jacob:** The choice becomes really easy to make, you know, if there's some feature that your vendor provides that's only available in their vendor-specific collector, you could choose to use the wrapped collector there. And then in the future, if you wanted to change vendors, you still have Otel data sending to that vendor collector, and so you can just change that one out very, very easy. + +**Jacob:** The reason that this is good is because you wouldn't have to go to your application teams or any of these other, you know, orgs and say, "Hey, you know, you have to reconfigure your whole setup because we're changing our underlying vendor here," whereas you, as the centralized team, could be the one to just change a single pool to make it all consistent. + +**Jacob:** That's probably what I—that's like a pretty future-proofed approach. The configuration for what you're going to do, no matter what, is going to be complicated, but that one is probably going to do you best if you really want to use the vendor collector. If not, still doing the Gateway approach is a pretty good one. + +**Jacob:** You can centralize things like efficient sampling rates or even like, you know, requirements for telemetry things like attribute requirements can be centralized before they can egress. It also means you can have, you know, set points of egress as well, which, you know, if you're in a pretty locked down Kubernetes cluster, as you know, is really important to have, you know, only a certain amount of applications that can egress from the cluster. + +**Speaker 6:** Okay, I think I definitely follow in terms of the deployment patterns and layouts and having the multiple levels of collector. I think we're thinking and going. I think for me, being in the enterprise, what we are worried about is, again, moving stuff like this to production. So if we have a theoretical issue in a, sorry, a vanilla collector, you know, again, just a theoretical example, if we've got RTO and we've got to fix issues within a one or two hours, for example, that's probably going to be the tipping point for us on the vendor versus vanilla question. + +**Speaker 6:** And because we would obviously be looking for some kind of support from a vendor, and typically we would have that as most folks would have through a vendor. But then that certainly in my mind gives us another problem where instead of going from agent polyproliferation, where you are just now, it's almost like going to Otel proliferation. You know, it's almost like you're solving one problem and creating another. + +**Speaker 6:** So I think, like the others on the call, we're relatively early in our journey, and we're just trying to go through the not necessarily the technical questions, but the hardening questions. What would reality look like when we're in production with, you know, very high volumes of traffic coming through? + +**Jacob:** I mean, that sounds like you're on the right path here overall as far as your thinking going. There definitely is that worry of collector proliferation. You can avoid that, you know, with multiple pools to gateways if you'd like. If you're going from—I mean, usually you're doing this if you're going for like a legacy protocol, something like StatsD, which doesn't like to go over the Internet because it's UDP, or something like Prometheus, where you have all these targets and you don't want to worry about managing the scrapes for them, right? + +**Jacob:** Doing this at scale is going to be really environment and volume dependent. I think if your vendor provides their own collector, they should be able to give you some support for, you know, the vanilla collectors that you run. I, you know, with the people that I work with, do give support for, you know, whatever collectors their customers run. + +**Jacob:** And I mean, we don't have a vendor-specific collector—like, our company just doesn't give out a vendor-specific one. But I, you know, I provide support for any collectors that customers run. So if that's the fear, I would check with your vendor to see if they also will give you that type of support. + +**Jacob:** I also found that the steady state of these things is pretty—once you tune it with resourcing and autoscaling, it's pretty hands-off, I found. I actually was just working yesterday in a cluster that I touch every six months or so to do some Helm chart testing, and it's been running for six months without issue, and with like a huge varying scale of traffic because of autoscaling and sort of just because of how simple we keep—I keep those collectors. + +**Jacob:** This is for both metrics and traces too, for infrastructure and application. So, I mean, it's a much smaller example than what you're talking about for sure, but the point remains where it's like once you reach a good steady state, especially with like your Bal cycle, if that's what you have, if you're autoscaling the setup correctly and if your configuration is pretty, you know, nailed down, you should be—it should be pretty hands-off, knock on wood. But that's definitely the hope. + +**Speaker 6:** Yeah, okay. Thanks, Jacob. Appreciate that. + +**Jacob:** Yeah, no problem. Thank you! + +**Speaker 1:** So folks, we are coming up on time. So we've got about five more minutes in case anyone has any more burning questions for Jacob. + +**Speaker 1:** Alrighty, I will take that as a no, but thank you Jacob so much for joining today and sharing your migration story. I think this has resonated with a lot of folks, so we definitely appreciate you coming on and sharing this experience with everyone. + +**Speaker 1:** Like I said, this recording will be made available on the Otel YouTube channel also for anyone who has missed Jacob's Otel Q&A session that we had last month. There is a video up on the Otel channel, and Reyes, who works with Ren and me on the Otel end user working group, did a wonderful write-up of the Q&A in case video isn't your jam. So definitely be sure to check that out. + +**Speaker 1:** Okay, so if you go to this link here, you should be able to find our various Slack channels, and we love—we encourage everyone to just ask questions, share use cases. We love hearing all that stuff. And also, if you or anyone you know has a really cool Otel use case, you're just getting started or you're a more advanced user, does not matter, we would love to hear from you. We're always looking for folks for Otel in Practice, Otel Q&A, and we also have monthly Otel end user discussions, which we run those for three different time zones. So we have them for EMEA, APAC, and Americas. So be sure to join any one of those because there's always really amazing and thoughtful discussions coming out of these. + +**Speaker 1:** So yeah, everyone, thank you very much, and once again, Jacob, thank you so much for taking the time to chat with us twice. + +**Jacob:** Yeah, thanks so much for having me. I appreciate it. + +**Speaker 1:** Thank you! Bye! Have a good rest of your day. Bye! ## Raw YouTube Transcript diff --git a/video-transcripts/transcripts/2023-10-11T20:52:28Z-the-evolution-of-observability-practices.md b/video-transcripts/transcripts/2023-10-11T20:52:28Z-the-evolution-of-observability-practices.md index 3ec98ee..cd48f2f 100644 --- a/video-transcripts/transcripts/2023-10-11T20:52:28Z-the-evolution-of-observability-practices.md +++ b/video-transcripts/transcripts/2023-10-11T20:52:28Z-the-evolution-of-observability-practices.md @@ -10,98 +10,276 @@ URL: https://www.youtube.com/watch?v=zSeKL2-_sVg ## Summary -In this YouTube panel discussion hosted by the OpenTelemetry User Working Group, panelists David Win, Iris, VJ Samuel, Austin Parker, and Noika Melera explored the evolution and current state of observability practices, particularly focusing on OpenTelemetry (OTel). David initiated the conversation by sharing his background in observability and the challenges faced when transitioning to cloud-native architectures, emphasizing the need for better tools for developers to understand complex systems. The discussion highlighted how OTel provides a standardized approach to telemetry data, enhancing developer productivity and enabling smoother integrations. Panelists shared personal experiences with implementing OTel in their organizations, addressing both successful strategies and lingering challenges, such as the need for improved logging capabilities and dynamic configuration management. The conversation concluded with a forward-looking perspective, expressing excitement about future developments in OTel and its potential to revolutionize observability practices across the industry. +The video features a panel discussion on the evolution of observability practices, hosted by the OpenTelemetry and User Working Group. Panelists include David Win, Iris Dear Mishi, VJ Samuel, Austin Parker, and Noika Melera. They share their experiences and insights on the transition to OpenTelemetry, discussing the challenges and benefits of adopting this standard in their respective organizations. Key points include the importance of developer engagement with observability tools, the need for standardization across platforms, and the shift in focus from merely collecting data to making it actionable. The panel also highlights the ongoing improvements in OpenTelemetry, such as the Collector Builder tool, and anticipates future advancements that will further integrate observability into development workflows. Throughout the discussion, panelists emphasize that the real challenges are often people-related rather than purely technical. ## Chapters -Here are the key moments from the livestream, along with their timestamps: +00:00:00 Welcome and introductions +00:05:10 State of observability before OpenTelemetry +00:10:50 Challenges in observability practices +00:14:40 Evolution of observability tools +00:19:30 OpenTelemetry standardization benefits +00:24:00 Observability culture and leadership buy-in +00:29:00 Collector Builder introduction +00:33:56 Surprising challenges in implementation +00:40:06 Observability practice focus changes +00:45:00 Future of OpenTelemetry and industry impact -00:00:00 Introductions of panelists and overview of the discussion -00:02:30 David's introduction and the concept of evolving observability practices -00:05:00 Iris shares her experience as an observability engineer -00:06:30 Vijay discusses his role in observability architecture at eBay -00:08:15 Austin introduces his background with OpenTelemetry -00:09:45 Noika talks about her experience working with observability and OpenTelemetry -00:13:00 Discussion on the state of observability before OpenTelemetry adoption -00:19:30 Challenges faced with observability tools and getting developers on board -00:27:00 The importance of standardization in observability practices -00:32:15 Insights on the role of open source in observability evolution +**Moderator:** All right, I think we can go ahead and get started. Thank you all so much for being here. The panelists we have today are David W., Austin Parker, VJ Samuel, Iris Dear, Mishi, and Noika Melera. I'm hoping I got all those correct. We'll do a quick run of introductions. This discussion is hosted by the OpenTelemetry and User Working Group, which I am part of, as is Adriana. I see her there. Today we're going to just have a casual conversation. Feel free to get as opinionated as possible about basically the evolution of observability practices. -# Observability Panel Discussion Transcript +**Moderator:** David, since you kind of inspired this, I would love for you to do a quick introduction, and after that, we'll just kind of go through the rest of the panelists. Yeah, let's hear a little bit more from David, whose brainchild this was. -**Moderator:** All right, I think we can go ahead and get started. Thank you all so much for being here. The panelists we have today are David W., Austin Parker, VJ Samuel, Iris Dear, Mishi, and Noika Melera. I'm hoping I got all those correct. +**David:** Hello everyone, my name is David W. I am Principal at Edge Delta, which is in the observability pipeline space. The thing about being a startup is you just sort of do whatever needs to be done, so I flex the title appropriately as such. Ree and I were chatting about different ideas that might be fun to discuss, and one of them that seemed very apropos to the group would be sort of the evolution of observability and where things are going, how people are tackling the challenge of shifting to OpenTelemetry, and what are some of the interesting lessons learned along the way. Not only from the 10,000-foot view of like we see where the mountains will go but also at the 10-foot view of boy, this grass is tall sometimes, trying to get a little bit of feedback on all different directions of it. -This discussion is hosted by the OpenTelemetry User Working Group, which I am part of, as is Adriana. Today, we're just going to have a casual conversation. Feel free to get as opinionated as possible about the evolution of observability practices. +**David:** To continue with introductions, I'll go ahead and do it popcorn style. Iris, why don't you introduce yourself next? -**David**, since you inspired this, I would love for you to do a quick introduction, and then we can go through the rest of the panelists. +**Iris:** Hello everyone, my name is Iris, Iris Dear, depending on the country where I am. Currently, I'm based in Portugal. I'm a Platform Engineer, Observability Engineer at Farfetch. My day-to-day is building an observability platform, maintaining it, modernizing it, and offering this kind of service to the engineers in my company. I don't know, that's all about it. Go ahead and call someone else out. -### Introductions +**VJ:** Hi everyone, I'm VJ Samuel. I work at eBay. My day job predominantly revolves around doing architecture for the observability platform internally. Everything—logs, metrics, events, tracing—helping all our developers do alerting, visualization, anomaly detection, the whole shebang with regards to observability. That's pretty much what I do. -**David:** Hello everyone, my name is David W. I am a principal at Edge Delta, which is in the observability pipeline space. The thing about being a startup is you just sort of do whatever it needs to be doing, so I flex the title appropriately. Ree and I were chatting about different ideas that might be fun to discuss, and one of them that seemed very appropriate to the group would be the evolution of observability. We can discuss how people are tackling the challenge of shifting to OpenTelemetry and what some of the interesting lessons learned are along the way. +**Austin:** Hi everybody, I'm Austin Parker, Community Maintainer for OpenTelemetry. Formerly at LightStep, part of ServiceNow, and currently—it's a surprise, and you'll find out very soon what I'm currently doing. I've been a part of OpenTelemetry since it was created. I was an OpenTracing maintainer. I've been working in observability for over five years now and got a lot of thoughts from seeing it kind of grow and evolve from what it was to what it is. -So, to continue with introductions, I'll go ahead and do it popcorn style. **Iris**, why don't you introduce yourself next? +**Noika:** Hi everybody, I'm Noika. I'm at the open-source startup Signos and have been working with observability stuff from back when we called it Real's Performance Management. Mainly now working with OpenTelemetry and Kubernetes stuff. -**Iris:** Hello everyone! My name is Iris, or Iris Dear, depending on the country. I'm currently based in Portugal. I'm a platform engineer, specifically an observability engineer at Farfetch. My day-to-day involves building, maintaining, and modernizing our observability platform and offering this kind of service to the engineers in my company. That's about it. Go ahead and call someone else out, **VJ**. +**Moderator:** Excellent, thank you all so much again. We have a list of questions that are kind of intended to help guide the conversation, but once everyone gets going, I expect it to become a lot more dynamic. I think we're all totally happy to see where this takes us. -**VJ:** Hi everyone, I'm VJ Samuel. I work at eBay. My day job predominantly revolves around doing architecture for our observability platform internally. This includes everything like logs, metrics, events, and tracing—helping all our developers with alerting, visualization, anomaly detection, the whole shebang. +[00:05:10] **Moderator:** To get us going, we want to know about the state of the world before you all undertook your OpenTelemetry journey. What was working pretty well? What did not work, or what sucked? What was the moment that prompted you to change? Feel free to raise your hand. All the panelists at least are on camera, so if you need visual cues as to when you can step in, hopefully that helps, but feel free to raise your hand too. -**Austin:** Hi everybody, I'm Austin Parker, a community maintainer for OpenTelemetry. Formerly, I was at LightStep, then part of ServiceNow, and currently, I'm involved in something you'll find out about very soon. I've been part of OpenTelemetry since it was created and have worked in observability for over five years now. I have a lot of thoughts from seeing it grow and evolve. +**David:** I'll actually start because I think I have what is probably not a very unique story but an interesting one. Before I got into observability as a field, I started out in software doing QA. I was a software developer in test, and this was, you know, 2013, 2014, I guess is when I started really getting into technology as a career, or software as a career, I should say. The cloud was a thing, but Cloud Native wasn't quite a word yet. We didn't have this concept of like, oh, we're just building all these things with all these cool APIs and this idea of infrastructure on demand or whatever. -**Noika:** Hi everyone, I'm Noika, and I'm at the open-source startup Signos. I've been working with observability since back when we called it Real Performance Management. Now, I'm mainly working with OpenTelemetry and Kubernetes stuff. +**David:** I saw the company I was at go through these various transformations, and one of them was a DevOps transformation where we went from, okay, when you build your code and you deploy your CI, you write, you pull a ticket, you write some code, it works on your machine, great, you push it, and then that night someone else gets to deploy it and see if it actually worked. One of the things we wanted to do was really tighten up the feedback loop here. We wanted to get from 24, 48 hours before changes got into test to minutes or hours. -**Moderator:** Thank you all so much again! We have a list of questions intended to help guide the conversation, but I expect it to become a lot more dynamic as we go. +**David:** A big part of that was getting on-demand infrastructure rather than sort of static infrastructure. We're going through this, and we're building all this out, getting stuff into the cloud, and it's great. What we started to see, though, was it wasn't actually fixing a lot of the problems we had. There was kind of this ground truth that everyone had agreed on beforehand: the problem is that we have bad infrastructure. These servers are not properly cared for. We're just wiping stuff and recreating it rather than actually getting fresh images every time. -### State of Observability Before OpenTelemetry +**David:** So it must be some config thing; it's probably not the code. Then something would go into production, and we would make it into a patch, and then the customer would come back and say, "Hey, this is actually broken," and we missed it because we thought this was because of our testing infrastructure. When we started going into the cloud, we had fresh images, all this stuff, and we're finding all these problems that we really didn't even know about before. -To get us going, we want to know about the state of the world before you all undertook your OpenTelemetry journey. What was working well, what didn't work, or what sucked? What was the moment that prompted you to change? Feel free to raise your hand, and we'll go from there. +**David:** The question came back, "How do we know what's going on? How do we know what's breaking our product?" It was a platform as a service, so we had hundreds and hundreds of nodes, various logs in all sorts of different places. It was Windows servers, it was Linux servers, we had all these different databases, and it was very difficult; it was kind of big. It was hard to keep your head around. Someone, one of the engineers, actually came back and said, "Okay, I made a topology service topology," and it looked like if you've seen one of those nail art things where someone will make a picture by putting a bunch of nails in a piece of wood and then tying string together—it was like that, where you have just lines everywhere and things connecting to each other. -**David:** I’ll start. I think I have what is probably not a very unique story, but an interesting one. Before I got into observability, I started out in software doing QA. I was a software developer in test around 2013-2014. The cloud was a thing, but "cloud-native" wasn't quite a term yet. I saw my company go through various transformations, one of which was a DevOps transformation. +**David:** Nobody could keep this in their head. Nobody could understand how services actually talk to one another. You could look at a very small section and say, "Okay, I get this," but looking at it holistically was impossible. I brought in, at the time, we tried New Relic, we tried Datadog, we tried a couple of things. What I found was, ironically enough, that it didn't matter what tools I brought in; the developers weren't interested in using them just because it wasn't data, it wasn't information that was kind of like at their level. -We wanted to tighten the feedback loop and move from 24-48 hours before changes got into tests to minutes. A big part of this was getting on-demand infrastructure rather than static infrastructure. We found that moving to the cloud didn’t actually fix many of the problems we had. +**David:** They didn't understand how do these things correlate, what does it mean when SQL Server spikes and memory usage increases, but there's no way to really tell what was happening, and that got me into observability—trying to answer that fundamental question. -We started seeing issues we didn’t know existed before, and the question became, “How do we know what’s breaking?” We had hundreds of nodes with logs in various places, which made it difficult to understand. An engineer created a topology service that looked like a nail art project with strings everywhere, and nobody could keep it in their head. +**Austin:** Right off of that, I remember working in New Relic during my time there, and there was this really fundamental thing of like, oh, this shows how this request hit all these spots, and it shows it as a trace with a bunch of time spans including individual function calls on all these different services. Pretty cool, but we get these questions back that were like, "Well, just show me where the request went, show me what services were hit, and also like in an interpretable version." -We tried tools like New Relic and DataDog, but developers weren't interested in using them because the data wasn't at their level. This got me into observability—trying to answer that fundamental question. +[00:10:50] **Austin:** It was an example where there was a lot of focus on getting a certain piece of information back, right? But the developers wanted information that was at a different level. This was exactly it: I just want to know where the request is going, which services are involved, and how the failure on the SQL Server might come back up and affect the front end in these ways. That was very hard to tease out, whatever this was seven years ago, but it's a similar theme, right? It's really about making sense of a pile of stuff again. -**VJ:** I resonate with that, David. Our problem space was a bit different. Pre-OpenTelemetry, we had a centralized logging platform for over 20 years. Developers had to learn proprietary clients, which limited our observability efforts. +**Austin:** Trying to zero in on that moment before we get into these things, there's the point where you have everything, and then you realize that having everything was the problem, and then it's not. Then you're like, "Wow, this computer doesn't know how to draw maps," and somehow it just looks like a bowl of spaghetti. Then you're like, "Cool, how do I zone this back down again?" -With the cloud-native era and large-scale Kubernetes adoption, we needed standardized SDKs across the board. OpenTelemetry came in with the promise of standardization, allowing us to offer our developers a community-managed solution they could use across the board. +**David:** It seems like sense-making is at least one of the unifying concepts we see there. VJ, Iris, whoever wants to take it— is that a similar feeling that you guys got when you were hitting this inflection point? -**Iris:** My experience is similar to VJ's. I started with companies that had zero observability. When I joined my current company, I encountered a robust observability culture. Our challenge was that we were using multiple open-source and APM vendors, which meant we had to gather information from ten different places. +**VJ:** For us, the problem space was a little bit different in the sense that pre-OpenTelemetry, we had a pre-cloud native era and during Cloud native as well. If you take the pre-cloud native era, we had something called the Centralized Application Logging Platform inside of eBay for more than 20 years, and it had the concept of transactional logging, where you have a root transaction, nested transaction events, very similar to what we have in the tracing world today. -We decided to go the OpenTelemetry route because it centralized everything, allowing us to move from vendor to vendor easily. +**VJ:** But the problem with the pre-cloud native era is always that developers come into eBay; they have to learn proprietary clients. We had clients only for a few languages, so if they are not using that or writing their own code, you cannot observe things inside the company or you're on your own to figure out—spin up your own ELK stack or anything that you can do to monitor the system. -**Austin:** To both of your points, I think what’s transformative about OpenTelemetry is that it offers a universal idea of how developers should emit telemetry information. Many companies have had some form of transactional tracing for years, but the difference is that OpenTelemetry makes it a core part of being a developer—an essential tool in your toolbox. +**VJ:** When Cloud native came in and we had the large-scale Kubernetes adoption, we had the Prometheus endpoints scraping from log files, a little more flexible, but you do not have standardized SDKs across the board. Even for metrics, it used to be that a few people used the official Prometheus client, some used Micrometer, and for Node.js, you didn't even have an official community-supported SDK. -### Surprising Aspects of the OpenTelemetry Journey +**VJ:** I think that's where OpenTelemetry came in with the promise of standardization. It was like, "Okay, now we can just offer all our developers one standard, and it's community-managed. They can hop from any company into eBay, and they should be able to use the observability platform as long as we are OpenTelemetry compliant." -**Moderator:** As we move along, what was most surprising as your OpenTelemetry journey got underway? What were the trickiest stakeholders you encountered? +**Iris:** I would say that my viewpoint, my story, is kind of a bit like VJ. My observability experience first started with some companies that had zero observability in place, and that's how I got to know it. Where I'm currently working, I jumped into a completely different world that had a very nice observability platform, a very nice observability culture, which was a big shock. Our job was actually not to come up with ways to monitor but to just keep improving. -**David:** I often say in observability, there are no technical problems, only people problems. So, what were some of the surprising things you found either that were sticky and tricky to navigate or that were surprisingly smooth? +[00:14:40] **Iris:** Of course, we came to that bottleneck that we were using a lot of open source, we were using APM vendors, and to get the information, you have to go in 10 different places, which is not great. I would say that it is from a developer productivity viewpoint as well why we went the OpenTelemetry route, because everything is centralized, and we can move from vendor to vendor if we need. We are collecting everything, standardizing everything, so I would say we have kind of the same case here. -**Iris:** For us, there hasn’t been a challenge we couldn’t find a solution for. The challenges we’re working on now involve the OpenTelemetry operator and the collector. We have a love-hate relationship there. The presets often differ from the normal collector, which creates challenges. +**Austin:** I think to both of your points, like that to me is what is really transformative about OpenTelemetry. It offers, for the first time, this truly universal idea of how should I, as a developer, emit this telemetry information. I think your story, VJ, is not unique. Most especially in large distributed systems, companies that have large distributed systems have had some sort of transactional tracing, some sort of structured logs with transaction IDs that they can look at for a decade or more. -Logging isn’t part of OpenTelemetry at all in our case, so I would like to see more stability in that area. +**Austin:** When we think about an OpenTelemetry trace, the actual data model is very similar to what Google produced in Dapper almost 25 years ago now. There’s nothing new under the sun. What's been missing is the idea that this is actually a core part of being a developer—a core tool in the toolbox, a core part of your trade is learning how to emit good telemetry about what your system is doing. OpenTelemetry provides a standards-based way to do this that also can be natively integrated and available through your service framework, your RPC library, or whatever else. -**VJ:** I agree, Iris. One of the features we were shocked didn’t exist was dynamic reloading of configurations. This was crucial for us while scraping Prometheus endpoints in Kubernetes. If that existed in the collector, it would simplify a lot for us. +**Austin:** To me, we're at the beginning of the beginning almost of seeing how that actually impacts the industry. -**Austin:** I think everyone will be excited about the announcements we have coming up at KubeCon this November, which will address many of these concerns. +**Iris:** I just want to add a little bit to what Austin was saying. It's absolutely true. The other good thing about OpenTelemetry is that you don't have to break your whole system to implement it as well. That is one of the great things because it's compatible with almost all technologies that we currently have, especially the open source ones. You don't have to cause downtime or have your developers blind for hours and days; you can just put it there flawlessly. -### Future Focus of Observability Practices +**Austin:** I think there's really two fundamental shifts. One is we started adopting architectures that became much harder to diagnose with the stuff that you taught yourself when you taught yourself to code. Adding in some logging lines and such is not going to work on a large microservice architecture. -**Moderator:** How has the focus of your observability practice changed, or not changed, after implementing OpenTelemetry? What are you looking forward to next, either from the project or as a result of your observability efforts? +**Austin:** The other change—and this has been a while, but it's still there—is that when you have conversations about funding, when you have conversations with VCs, they're going to ask you about your observability setup. Those two changes combined have really changed where the conversation is with observability, where it went from an internal discussion about maybe developer velocity and rollback times to, “Yeah, we have to have this, and it has to be applicable across this stack.” -**David:** We’re seeing enterprises realize that the amount of control they have with OpenTelemetry compensates for some of the grit in meeting their needs. New developers are also realizing that implementing observability is part of being a competent developer. +**David:** I think there is a fair bit of standardization that gets promoted on the consumer side of things as well, given that there is a standard in how you instrument tech metrics, for example, gauges, counters, exponential histograms, whatever. Earlier, it used to be that if you picked one time series database, you would get certain capabilities defined in a certain way. If you have to switch, then the likelihood of that capability existing may not be always possible. -**Iris:** I would like to see improvements in logging and overall stability in the OpenTelemetry operator. +**David:** But now since there is a standardization, eventually there will be a good amount of standardization on what technology the providers—either open source projects or vendors—provide. The net benefits are definitely there across the board, not just on the instrumentation side. -**VJ:** I agree. More reference implementations for tracing and metrics would be beneficial. Case studies on how to use OpenTelemetry effectively would also help. +[00:19:30] **Austin:** You bring up a really interesting point, right? Historically, everything about telemetry has really been a pretty binding abstraction to the data store. If you were using StatsD, metrics worked in the way they did because of the way that StatsD, you know, stored and let you query that. Prometheus metrics work the way they do because of how you store and query them. Logging databases tend to, you know, if you're using Elasticsearch or something, the format mattered, and that influenced how the client libraries were designed and so on and so forth. -**Austin:** I think OpenTelemetry is foundational for the future of observability. We’re just beginning to see its impact, and there’s still plenty of room for growth. +**Austin:** With OpenTelemetry, we kind of break that, right? OpenTelemetry tells you, “Hey, this is what a histogram looks like; this is what a structured event of any kind looks like.” I think it's interesting that you also see this rise in popularity of column stores for storing observability data—like metrics, logs, traces, sessions, whatever. You throw a rock, and you'll hit a new column store-based or heck, new ClickHouse-based telemetry thing. -**Moderator:** Thank you all for your insights today. This has been a fruitful discussion, and I look forward to seeing how OpenTelemetry evolves in the future. If anyone has to go, feel free to jump off. +**Austin:** It's great, and I think OpenTelemetry has actually been a huge factor in this because it used to be if you wanted to go and build an observability tool or build some sort of analysis tool, you would have to come up with all this stuff or no one would use it. You'd have to come up with your API, you'd have to come up with your SDKs, you'd have to build integrations for 40 billion things, and OpenTelemetry means, actually, that's all a commodity now. You just get that for free with OpenTelemetry, and you can build really interesting workflows on top of that data. -**Everyone:** Thank you! +**Austin:** I think that's very friendly for developers, right? Going forward, we haven't even really started to see how do these tools become integral in development workflows, right? When do we get to IDE-level integration and all that? + +**Iris:** I know it's very easy to feel like OpenTelemetry is done or like that it's kind of hitting this plateau, but I think we're really, really still in the early days with what people can actually do with it. + +**David:** I have to agree on this being early days for OpenTelemetry. A little bit about my background is I was in observability before I hopped over to CL for a bunch of years, and now I'm back in the observability space. The conversations that I'm having with people today remind me a lot about conversations about Git, like 15 years ago or something like that, when it was sort of a perennial joke that everyone's like, “Oh yeah, Git looks really interesting, we're thinking about moving to Git, but we're still on, you know, insert system here.” + +**David:** It feels almost every conversation I have—and again we're in the pipeline space—it's about routing and moving and transforming all the data to the right spot. But everything that comes out of us is OpenTelemetry. If they just need to bolt on something, they want to do that. But the first thing they talk to us about is something with some sort of data not being in the right place, and the second thing is, “Oh, also we're starting to look at OpenTelemetry. Is that good?” Being the second question, I think is a good sign of climbing the curve. + +**Moderator:** Speaking of that, how did you first hear about the projects? I think it’s kind of clear like some of the things that have drawn you all to it. Something I think, too, that people might be interested in is how did you get leadership buy-in? + +**Iris:** I can say something about this because for me, it’s easy. When I started working with observability, I became obsessed with tracing. I’m still fascinated by tracing, so I was like, “Oh, tracing is so amazing,” and I started just searching online about it. Then I came across OpenTelemetry. I started playing with it for a long time and not actually implementing anything. + +[00:24:00] **Iris:** When I joined Farfetch, we were like, “Okay, we need something new, something better,” and we thought about OpenTelemetry. It was very surprising because there was immediate support. I like to say and brag that we have a very good observability culture, but maybe everyone had heard about it. We just made a small presentation, and our leadership was on board. Of course, it has a lot of benefits, and we had to present what we were lacking, the issues that we were currently facing, and how MRI can help us with. It was immediate support, so every time I tell the story, some people don't believe me, but that's exactly how it happened. It was very interesting, very fast support. + +**VJ:** I can maybe go next. At least in our case, I first came to know about it as part of the OpenTelemetry will basically be the standard, and we would eventually retire OpenTracing and OpenCensus. I think we were passively trying out OpenTracing around that time, and this announcement came, and it was like, “Hey, interesting, this is something that we need to watch out for.” + +**VJ:** We were doing passive experiments with the OpenTelemetry SDK and the Collector for a few months, and once it hit stability, that’s when we made the informed call that, “Okay, we’ll start tracing with OpenTelemetry as an offering inside the company.” So now we had the OpenTelemetry SDK for traces and the OpenTelemetry Collector for accepting all the spans, but on the other hand, we had Metricbeat and Filebeat for collecting logs, events, and metrics. + +**VJ:** At that point, we were at a crossroads of if we should support multiple agents or just standardize across OpenTelemetry Collector for everything. I wrote a very big memo internally on what it would take to migrate out of Beats for metrics, logs, and events, presented it to leadership, and I think we felt that being on a vendor-neutral industry standard would be the better thing for us in the long run. It’s best to do it now than later on, so that’s how we did the migration. + +**VJ:** The metric migration was a massive one because we, at that time, if I'm not wrong, we were already at 32 million samples per second across more than 100 Kubernetes clusters—million-plus Prometheus endpoints and whatnot. It was a very, very involved migration that we did, but the benefits are definitely there now. + +**Austin:** Were there any other challenges when you're coming from a culture that has a strong kind of build vibe? What I'm trying to get is, do you guys run your own Collector or are you taking something? How do you manage that in terms of the particular flavor that you decided to land on? + +**VJ:** We do run our own Collector internally, but I think the number of custom processors that we run is very less. It's mostly that we don't need all the exporters for all the various vendors that are packaged in, so we create only the ones that we absolutely need. For now, we export using either OpenTelemetry Protocol or Prometheus Remote Write, which are pretty much open standards right now, so it's about being lean. + +**Iris:** Are you all using the Collector Builder to make your images? + +**VJ:** Right now, no. We just craft our own Go mod and then just do it ourselves. + +**Austin:** Y’all thinking about using the Collector Builder? + +**VJ:** I think we can. We haven't really explored it, but yeah, we should. + +[00:45:00] **Austin:** I'm just trying to get more people aware of the Collector Builder, because I think it's really cool. The Collector Builder, or OCB, is a tool that we provide from the Collector repo, and what you can do is you can give it a manifest of Go modules, and it will create a custom image or a custom build of the OpenTelemetry Collector for you. So you, like eBay, can get something that has just the receivers, processors, exporters, connectors, and extensions that you want. + +[00:29:00] **Austin:** It’s also a really great way to extend the Collector because in this way, instead of having to fork contrib or fork whichever raw base you want and then bring in the code, you can simply have a Go module published in wherever GitHub and pull that in through the Collector Builder and bring in your custom extensions and things that way. + +**Austin:** It’s a really great tool. It’s also very friendly for your security teams because it gives them a manifest that they can kind of look at and say, “Hey, this is exactly what's in there,” and that way I know if you're at a larger security review can be extremely taxing, and if you're trying to use the contrib image, there's an awful lot of dependencies. + +**Austin:** Something to kind of cut that down to size is very friendly to your friends in security. + +**Iris:** There's a new user now for the Builder. I actually had missed this, so there’s actually some—there’s check out the website, OpenTelemetry.io does have a little tutorial on building with the Collector. + +**Iris:** I actually recently used it for a personal project where I was making some custom processors to talk to an API and do some data, so it was really, really fun to use and makes it really easy to use the Collector, maybe the way it's sort of intended. + +**VJ:** As long as we—oh, go on. + +**Austin:** I was just going to point out that VJ and Seth very helpfully shared a couple of links about the custom Collector Builder. + +**VJ:** Perfect. + +**Austin:** The other, sorry, go ahead. + +**Iris:** Go on first; you were talking. + +**VJ:** Yeah, I'm just going to mention we’ve found it very useful. We have forked just an exporter and are working on trying to get a whole request approved for it. But in the meantime, we continue to need to persist those features and newer versions of the Collector, so this helps us to kind of drag our single exporter along the way and keep rebuilding it with new releases, so it's been really helpful. + +**Austin:** That’s another really great use case. If you have custom stuff or modifications, I know that PRs can sit for a bit sometimes. + +**Moderator:** I think it’s going to require several moving parts. + +**Austin:** Moving the conversation along since we’re coming up on time almost, what was most surprising as your observability journey got underway? What were the trickiest stakeholders in your case? + +[00:33:56] **David:** I think I should stop saying that, even though I'm probably going to keep saying that, but there's particularly in this space, the technical problems mostly are straightforward. It's all the people problems. To the whole panel, what were some of the surprising things that you found either that were sticky and a little bit tricky and had to navigate or that were surprisingly smooth? Both of those are valid surprises because we've gone over some of the key selling points here that I think a lot of people would recognize as being useful with going in an OpenTelemetry direction, but what's the part we didn't say that was either good or that was almost good? + +**VJ:** I'm currently running a poll on LinkedIn where I ask people if they think OpenTelemetry is easy to use. I won't disclose the results of that poll quite yet. I think there's a—I’d be interested to hear from people that have gone through implementation journeys. + +**VJ:** I can say we're in a pilot for integration with applications, and the .NET library is not quite having the log exporter stable until just very recently. That’s been a large deterrent for a very long time. About two years ago, I started trying to convince the application teams to all integrate with OpenTelemetry, and there’s been kind of an appetite for some of the more cutting-edge teams in our company to try out something that still doesn’t have a stable label. + +**VJ:** That was a big deterrent, but now, with I think it’s the 16 release now that that’s stable on the .NET instrumentation library, there’s a better story there, and there's more acceptance now. Outside of that, the Collector itself, just getting lots of new features for free has been amazing. So that’s part of the reason we know about the Builder, just because we have these custom features that we really want, and we keep seeing better, newer versions of the Collector come out. It's like, “Oh wow, we better upgrade because we don't want to sit on this old stuff,” and all these other great people have been contributing a lot of good enhancements, so that's been an easy sell to say here's why part of the why of helping teams move to this instrumentation challenge that will take some time for them to work through. + +**Austin:** Actually, I have a—can we play a fun game? Can I get everyone to open up your Zoom chat and tell me what language you use OpenTelemetry in, and do you think it's good? Do you think the SDK is good in that language? This data may be used by the project. + +**David:** You don't know is that the part of this request that's hard for me is that the Zoom client on my machine is very temperamental in terms of opening the chat window, so that's the hard part. + +**Austin:** Okay, well, I can see the chat node. + +**David:** Yes, it’s good. I think JS has gotten a lot better. I think most of the problems in JS right now have a lot more to do with like ESM and CJS and various JavaScript ecosystem things rather than any problem with the language SDK. + +**Austin:** Go needs some love. I feel like Go was written by several dear friends of mine who know Go in and out left right and indifferent, and it's great, and it's actually pretty idiomatic Go, I think, but it's kind of hard to use because it's idiomatic Go, personal opinion. + +**Iris:** Exemplars not getting implemented in what? Which language? All of them? Some of them? Most of them? + +**Austin:** I think the only one where it’s actually implemented is Java, and then from what I've asked around, not really anywhere else. I've seen a bunch of open PRs around that, but no actual implementations. I think we need to get better about closing PRs. + +**Iris:** .NET, Python, Java—pretty good. + +**Austin:** Yeah, I’d say Java—Kotlin, multiplatform, nonexistent. + +**VJ:** Have you looked at the—Derek, there's—I don't know if it actually is in the repos yet, but I know Splunk donated their Android client SDK thing that I think targets Kotlin. I know they want—it either has been donated or is being donated. + +**David:** I’ve been watching that donation process. + +**Austin:** Okay, yeah. I've been looking for an SDK that could be used by Kotlin multiplatform to go both on iOS and Android clients. I know some people are using the Java SDK, you know, purely for Android, but we have to just implement that separately. + +**VJ:** That is interesting. Will you be at KubeCon, by any chance? + +**Austin:** I will. + +**VJ:** Look me up when you're there. We’ll talk about it. + +**Austin:** Cool. + +**VJ:** Go—terrible. + +**Austin:** Yeah, Go's right now a little bit of a hobby horse for me. I think it's high on my list of languages that feels like it needs some developer experience love. + +**Iris:** I'm sorry for taking over your panel, Reese. + +**Moderator:** Good. I kind of like it. It's always—I feel like it's hard to get a big group of people online to participate. + +**Austin:** So down API. + +[00:40:06] **VJ:** Kubernetes is also an interesting one. When I went to Kubernetes years ago, if you're not aware, Kubernetes has started to feature gate an alpha and beta native OpenTelemetry traces from things like CUE and a few other components, and the API server. So there is more support for OpenTelemetry coming in. + +**VJ:** The biggest thing is Kubernetes metrics are all super Prometheus out, and that is probably not going to change anytime soon. But there is some interesting stuff with Prometheus talking about should Prometheus just use OpenTelemetry libraries and stuff like that, so we might see some changes there. + +**David:** Hopefully that can lead to a more unified telemetry story for Kubernetes. + +**Austin:** Your functions as a service—I'm sure the majority of people are using Lambda, and that seems to be the main focus of the community at this point. + +**David:** I think we really just need people to—I actually—I mean, I will also plead ignorance. I don't know if they have an equivalent for Lambda layers or extension layers. Do you know? + +**Austin:** I've looked across the project, and while at the top of the OpenTelemetry site, if you look at the documentation, it says, for example, for Azure Functions, AWS Lambda, there's really not much integration, if any, with Azure itself. + +**David:** So my experience on the pipeline side with Azure Functions, I believe is the official name—spending a lot on that naming convention—is there's a dual writer that you can enable, but then you essentially have to set up something that collects all of that information. + +**David:** We recommend throwing up a small Cassandra cluster depending on how much traffic you want to throw at it. But you could be running anything in there, and it's basically a pretty good dual writing mechanism, whereas the Lambda layer stuff tends to have startup and cooldown times that I think—I think actually Azure Functions is probably a more scalable approach, particularly if you get really— + +**Austin:** Is it function to App Insights, and then App Insights split out into OpenTelemetry? + +**David:** It doesn't bounce off of App Insights; it's from the function itself, I believe. + +**Austin:** Yeah, a writer SDK that Microsoft provides to write out to wherever. Part of the challenge was like, "Well, is there a way from Azure Monitor to monitor the actual use of it rather than incurring extra cycles within the function itself?" + +**Moderator:** Sorry, very interesting discussion. Feel free to ping me on the CNCF Slack if you'd like to continue it, but I think it's going to require several moving parts. + +**Austin:** Thank you for that. We can definitely host this in another conversation around this another time. + +**Moderator:** To bring it back to this panel, I would like to know how has the focus of your observability practice changed or not changed after implementing OpenTelemetry, and what are you looking forward to next, either from the project or as a result of your observability efforts? + +**David:** I think there are two things we're seeing. One, of course, is we're seeing enterprises really see that the amount of control they have with running OpenTelemetry sort of compensates for some of the grit that exists in the gearbox as they try to make it meet their needs. And then on the other side, we're seeing new developers realize that part of being a competent new developer—and I think the Git comparison is very apt—is, you know, like, "Yeah, of course you know source control if you're going to write code of any kind for production," right? + +**David:** The same thing's happening with observability where previously, maybe, "Hey, you were just entering a few log lines, and then you were ready to go." It's like, "No, we need to know how to actually implement one of these tools from the start and implement tracing from the start." So, sort of from two directions, I think OpenTelemetry is getting a lot of momentum in the next year. + +**Austin:** I am looking forward to the day when you have to have a reason to be off OpenTelemetry instead of to be on OpenTelemetry, and to see how that affects the vendor landscape. I’m here on behalf of a vendor, but because we’re in pipelines, if I could get rid of all of our integrations that weren't OpenTelemetry, it would just be so much easier instead of having to do something really bespoke for everything that's out there. + +**Austin:** There was a great article not that long ago that you wrote, right, Noika? Isn't that right? About how OpenTelemetry, at the moment, is being treated in various tools differently? + +**Noika:** We need to talk about that. + +**Austin:** I've got some great stuff coming out next week that is not quite so mean to my good friends at New Relic, but I think we all need a little bit of a kick in the pants sometimes. + +**VJ:** One of the really fundamental things, like when I was working at New Relic on the integration with X-Ray data, right, is that it's like, "Hey, yes, we can get—we can shove these things together, and we can make it kind of work." But until we all adopt one open standard, this is just too complex a data type to just really fluidly chart in one view and see from one view and do things like what is the average amount of time consumed by this single function, by the single DB call, right? That is going to be very tough to do until you just say, "Look, we all have to be on these open standards." + +**David:** Some really weird LLM training task, yes. Okay, it's going to use some super strange system for—for this doesn't really look like tracing or time spans at all, cool. But everything else, we need to adopt an open standard to really be able to have OpenTelemetry data be a first-class citizen, so I'm looking forward to OpenTelemetry winning. + +**Austin:** Same. I think it's already won in many ways, right? Just not to toot my own horn or to toot the project's horn, but in, you know, everyone likes to refer to the XKCD, like there's 14 competing standards. I'm going to make the unifying one now; there are 15 competing standards. + +**Austin:** I think, whatever, you know, I’m under no delusion that this is the apex of innovation in the space. There will be future developments and other things, but I think in terms of consolidating the last two, three decades' worth of thinking about application and infrastructure telemetry data and how it can be used together, I think OpenTelemetry is going to be foundational for a generation of observability practitioners, for engineers, right? There will be something in the future, but I think in terms of where we are now, this is kind of the thing. + +**Austin:** Whatever comes next is going to be influenced by things that we don't even see. I've been doing a lot of research on AI and large language models and stuff, and if you look at what you're actually doing when you make a request to a large language model, you're making a trace. It's literally a trace. It is a Dapper-style trace where you have requests and responses. Even the most advanced thing we can think about in terms of software architectures right now, and whatever looks like stuff that we have a way to model in OpenTelemetry today. + +**David:** So until that changes, I feel like OpenTelemetry is, you know, long term going to be what—at least until I am retired, so give it 30 years, but like it's going to be the thing. + +**Austin:** Justin in the chat asks any thought on Micrometer’s approach of blending the signals into a single observation API from which traces, logs, and metrics are all derived? + +**David:** I think that's fine. I think it's—I do think that the thing you actually need is we need as a project for people to commit to OTL as being an output format, and we need people to respect, you know, hey, this is trace data, and it should be interpreted differently than metric data or whatever. Even if you are going to like splay those out into a generic sort of event data structure, there are use cases for keeping those things separate in terms of observability pipelining. + +**David:** Yeah, it's—there's maybe not a cut-and-dry answer to this. + +**Moderator:** I would like to point out that it is the top of the hour, so if you have to go, obviously feel free to jump. + +**Austin:** See you all. Have a good day, everyone. + +**David:** Thank you all so much. + +**Iris:** Bye-bye! ## Raw YouTube Transcript diff --git a/video-transcripts/transcripts/2023-11-21T17:59:37Z-otel-in-practice-filtering-parsing-data-with-the-otel-collector.md b/video-transcripts/transcripts/2023-11-21T17:59:37Z-otel-in-practice-filtering-parsing-data-with-the-otel-collector.md index 3346048..ca8c1e8 100644 --- a/video-transcripts/transcripts/2023-11-21T17:59:37Z-otel-in-practice-filtering-parsing-data-with-the-otel-collector.md +++ b/video-transcripts/transcripts/2023-11-21T17:59:37Z-otel-in-practice-filtering-parsing-data-with-the-otel-collector.md @@ -10,89 +10,298 @@ URL: https://www.youtube.com/watch?v=y3AAerwIFfg ## Summary -In this YouTube video, the speaker discusses filtering techniques for telemetry data using OpenTelemetry (OTel) collectors, focusing on traces rather than logging. They emphasize the necessity of filtering to handle noisy, sensitive, or excessive data generated by code-level instrumentation, which often cannot be modified directly by operational teams. The session covers various methods for data filtering, including removing spans based on attributes, transformation processors, and the importance of maintaining clean observability practices without altering the underlying codebase. The speaker provides practical demonstrations of configuring filters in the OTel collector and discusses the limitations of these processes, such as the inability to change span relationships or drop entire traces. Notable attendees included Adrian and Ren, who contributed questions and insights throughout the presentation. The video also highlights future sessions and resources available through the OpenTelemetry community. +In this YouTube video, the speaker, along with participants, discusses advanced filtering techniques for telemetry data, specifically focusing on OpenTelemetry (OTel) collectors. The main points covered include the need for filtering to manage noisy and sensitive data without altering the codebase, the configuration of filtering attributes, and the importance of transforming data to avoid sending unnecessary information. Various scenarios are explored, such as handling personally identifiable information (PII) and excessive spans from data collection. The speaker also addresses the limitations of OTel processors, including the inability to change span relationships and the nuances of head versus tail-based sampling. Throughout the presentation, practical examples and live demonstrations illustrate how to effectively implement these filtering techniques in a collector configuration. The video concludes with a discussion on best practices for testing configuration changes and managing observability tools. ## Chapters -Here are 10 key moments from the livestream transcript, along with their timestamps: +00:00:00 Welcome and intro +00:02:01 Filtering for traces +00:04:05 Data collection challenges +00:05:20 Collector level filtering +00:08:00 Filtering by attributes demo +00:12:10 Filter processor details +00:14:06 Regex matching in config +00:19:00 Limitations of collector processors +00:21:35 Testing config changes +00:36:00 Closing remarks and Q&A -00:00:00 Introduction to the topic of filtering in OpenTelemetry -00:02:15 Importance of filtering at the OPA Telemetry collector level -00:06:40 Discussion on the challenges of noisy and sensitive data -00:10:30 Example of excessive data being collected in spans -00:15:00 Demonstration of filtering data at the collector level -00:20:45 Live demo: adding filter configurations in the OpenTelemetry collector -00:25:10 Explanation of filtering by attributes in spans -00:30:00 Overview of using regex for filtering data -00:35:15 Discussion on head vs. tail-based sampling -00:40:00 Final thoughts and audience questions on configuration testing and deployment +**Speaker 1:** This is filtering the noise. I think I promoted as like better filtering for logging, but I'm mainly going to demonstrate filtering for traces. The principles are the same. I promise we'll get there, folks. -Feel free to let me know if you need any further details! +Let's talk about some level setting for considerations with—let me just turn off my age back here, sorry—considerations with why we would want filtering at the OPA Telemetry collector level. So just review these first principles. Everybody seeing the slideshow? Okay, it's got the little green line around. That's why I assume you're seeing this. Great. I'm gonna assume that's great. Give me a negative emoji react if you're not seeing it, but otherwise, we're good. -# Filtering the Noise: A Deep Dive into OPA Telemetry Collector +So OpenTelemetry instrumentation, like any code-level instrumentation, unless you send a lot of data rather easily, some of that data is noisy, some of it is sensitive, and some of it is good but needs tweaking. Once we've done the work to do instrumentation, especially instrumentation at the code level, when we're faced with a data collection problem, we would ideally want to fix that without requiring changes to our code base. -Welcome, everyone! Today, we're going to explore filtering in the context of OPA Telemetry Collector. While we may have initially framed it around logging, the principles we'll discuss apply broadly to tracing as well. Let's dive right in. +For most people who are working operationally, the thing is like, “No, I cannot edit the code base.” I don't know where these hooks are added. I don't want to message the product team and say, “Hey, turns out we're sending, you know, people's Social Security numbers to our observability backend. Please go do this code commit tomorrow.” That's just not the workflow that makes sense for a lot of people. -## Introduction +[00:02:01] We could talk all day and all night about how, well, you should have access to the code base; you should have a better connection. That's real DevOps, blah blah blah, great points. But let's talk about, like, you know, okay, it's one thing when, “Oh my God, we're sending a ton of people's PII into observability when we shouldn't.” What about the situation where it's like, “Hey, we're sending a little too much”? -First, let’s establish why filtering at the OPA Telemetry Collector level is crucial. OpenTelemetry instrumentation, like any code-level instrumentation, can generate a lot of data—some of which is noisy, sensitive, or simply excessive. Ideally, we want to address these issues without requiring changes to the codebase, especially since many operational teams lack the ability to edit the code directly. +Hey, the traces, they, like, you know, every step of a 500-step for loop they're recording is a separate span, and they shouldn't. Is that really something where you want to bother the product people about it and ask them to, like, edit the code? You know, for sure, that's like, “It'd be really nice if we had operational tools to do that.” -### Considerations for Filtering +So there are a—let's take a look at this code where we can sort of imagine where we might have gone a little far here. So we had our developer. We said, “Hey, would you add some custom code points?” Maybe they got excited about it. We did our little brown bag. They added a bunch of points. They added, “Hey, let's send the user ID as a value for this span. Let's send, like, how many rolls are we doing? We'll use that.” -1. **Sensitive Data**: We might inadvertently send sensitive information, such as personally identifiable information (PII), to our observability backend. -2. **Excessive Data**: Sometimes, we may be recording too much data. For instance, if every step of a 500-step loop is recorded as a separate span, it clutters our traces. -3. **Operational Independence**: It’s often impractical for operational teams to reach out to product teams for code edits to address these issues. +Or, right, I think we decorate the child span with that. Yeah. And then, um, let's also send their database key up as part of what we're sending. That probably was going a little far, right? We're like, “Oh no, we're shipping this data across the internet, and maybe we don't want to send their full database key.” -### Example Scenario +By the way, key is a somewhat overloaded term here, but let's just say, like, this is not a critical security issue, but it is like, “Hey, that's more than we really want to be sending around and showing to everybody in the backend.” -Let’s imagine a scenario where a developer adds multiple custom code points while trying to enhance observability. They might include user IDs, role counts, and even database keys in the spans. While this may seem beneficial, it can lead to unnecessary data exposure, such as sending full database keys over the internet. +There are a ton of—this actually is a diagram I drew for something else, but it works here. There are a ton of points where we could say, “Hey, I don't want that database key as a complete string to be stored.” -This highlights the need for operational tools that allow us to filter or scrub sensitive or excessive data without modifying code across potentially numerous services. +[00:04:05] Going in and editing these code points, right, would be editing it at the service level, which might be in a whole lot of places. We might be able to—we very often have these collector-to-collector architectures. So, what we're going to be talking about today is hitting it in one of these collector levels. -## Filtering Approaches +The last and relatively common is doing it at the data store level. This would be like, “Hey, maybe we're sending way too many spans on a single trace.” Maybe that example of, like, “Hey, we're sending 50 identical spans.” -Now, let’s discuss some filtering approaches we can implement at the collector level: +Actually, this example, I don't think—I don't have the code here. I'm not going to hop over to my IDE for this thing, but it is like, it's saying, “Hey, roll. If you roll seven dice, it's sending a span for every single dice roll.” -### 1. **Filtering by Attributes** +Like, “I don't know. I don't want all those sub spans,” right? That might be an example where instead of trying to filter at the collector level, a pretty normal step would be like, “Oh, let's just go and edit the data store and say, ‘Hey, maybe older than this amount, we want to compress our traces in this way.’” -We can filter spans based on specific attributes. For instance, if we identify certain internal test users whose spans clutter our production environment, we can configure the collector to drop spans for these users. +That's a perfectly reasonable way to do it, depending on whether or not you have access to do that. I know with Chronosphere, generally, you do. Certainly, if you have a, you know, self-hosted data store for this, that should be very doable if you're using, like, Loki or Signoz to do that. -Here’s how to set it up: +[00:05:20] But we're going to talk about filtering at the collector level. Any questions about, like, why we're doing all this, why this matters, or is a tool that we would want to have? -- **Configuration**: We add a processor section starting with a filter, naming it descriptively (e.g., `Dev users`), and include it in the pipeline configuration. -- **Live Demonstration**: After saving the config and restarting the collector, we can test it by sending requests from both allowed and disallowed users. The spans for disallowed users should be dropped, while those for allowed users should still appear in our observability backend. +**Speaker 2:** Yes, why? -### 2. **Transforming Data** +**Speaker 1:** Yeah, so in general, it's going to be sensitive data or it's be overfit data are sort of the two main reasons. But there's also just going to be this general sort of vibe, which is, “Hey, I'm looking at this trace here, and I don't like what I see for some reason.” -Sometimes we don't just want to drop attributes; we may want to transform them. For instance, if we have a sensitive database key in our spans, we might want to replace it with a generic identifier while preserving other useful information, like user roles. +So, see, change the share here. I'm saying, “Hey, I'm looking at this trace, and this isn't helpful for a reason.” I will explain, right? That really could be anything. But, you know, when we work on the operation side, work on the SRE side, like here's an example where it's like, “Hey, you roll the dice three times. I don't need these little sub spans.” -This can be done through a transform processor that captures specific patterns in the data and replaces them accordingly. +Once the request gets complex, it really starts to clog up the view. It's not so much about, like, having an elegant view that should be something that you can control on the UI side, but it's like, “Yeah, something's not looking right, and it's leading to distracting data.” -## Limitations of Collector Processors +A perfect example is, um, “Hey, maybe you have a large request that is, in fact, asynchronous, but it's getting rolled into your database time.” So it's like, “Hey, that's very deceptive, and it's causing everybody to say, ‘Hey, the database is running really slowly,’ but that's not what's going on.” -While collector processors are powerful, they do have limitations: +And so that's a pretty good time when we would say, “Hey, we don't want to go and actually edit application code or custom code calls. We want to do something at the collector level.” -- **No Changing Relationships**: You cannot redefine the parent-child relationships of spans. -- **Cannot Drop Entire Traces**: While we can drop individual spans, dropping an entire trace is more complex and generally not supported in the current filtering architecture. -- **Head vs. Tail Sampling**: We need to consider the differences between head-based and tail-based sampling. Head-based sampling occurs before a trace starts, while tail-based sampling evaluates traces after they are collected. +So just a reminder of, like, what can happen inside the collector, right? Is like we have receivers and exporters, right, which do our transport standard communication, can send stuff out to StatsD, can take stuff in from a million places. -## Testing Config Changes +Then we have our processors in the middle, and we're going to talk about data scrubbing, data normalization, and a little bit about sampling, batching. You know, it's batching; that is what it is. But that is the concepts that we care about. -When implementing changes to the collector configuration, it's critical to test them before deploying to production. Here are some recommended strategies: +[00:08:00] Okay, so let's talk about filtering by attributes. So we have some attributes on this span that we say, “Hey, if we get these attributes, we really don't want to save this.” So let's go ahead and demo that live. -1. **Test Harness**: Use a local version of the collector for testing. -2. **Canary Deployments**: Deploy changes to a subset of your collector architecture to evaluate their impact. -3. **Resource Values**: Leverage resource values in your conditions to limit the scope of your changes. +Let's do a new share. Here, so here's my OpenTelemetry collector configuration. Just stepping through what was necessary to add this. Hey, if a span has these attributes, I do not want to send it across. -## Final Thoughts +In our application, right, we take a request attribute, and we add it to that span right down here. We say, “Hey, the user for this is this request attribute user ID that we received.” Obviously, with a real application, we should have the user ID from a million other places, but we picked it up here. -As we conclude, it's important to remember that while we have the power to filter and transform data, we should exercise caution to avoid creating meaningless metrics or data that could lead to confusion later on. +It turns out that a couple of our users are like our internal test users, right? So in fact, that's so consistent because it's like an automated testing suite. So we know they're always going to be named this. We say, “Hey, don't record any spans for these people because maybe they're testing stuff. Maybe those spans get crazy long; they introduce a bunch of fuzzy unnecessary data to our production environment.” -Thank you for joining this session! If you have any further questions or need clarification on any points, feel free to ask. Also, for those interested, recordings of this session and future discussions can be found on the OpenTelemetry YouTube channel. Don’t forget to check out the upcoming Q&A sessions as well! +So we can add a processor section, which starts with filter and then has whatever name we want it to have. So this is slash; we say, “Hey, we can call this Dev users because that's descriptive.” ---- +Then critically, and I think I have a slide about this too, we also add it to the pipeline down here, which is where I usually mess up. I usually go ahead and create a very careful filter in the collector config and let me give like one stke of zoom here, and then forget to add it to this thing. -Let’s open the floor for questions! +And this pipeline is read left to right, so, you know, if you're doing things like transform filters, transform processors, which we'll talk about, you know, you want to have just this in the correct order so that your transformations will have effect when you filter. You generally want batch at the end. + +So if we save this config and go and restart our collector—come on, bud—cool, cool, cool, cool. Then we send in a couple of requests. So that was from David, and if I check my config, David's like an allowed user. But if I send one from Bob, so here's the service, right? + +It's the same dice roller that you've seen in the OpenTelemetry examples, right? It returns an array of dice rolls. You know, Bob still gets his data because it's not like we're, you know, this is all observability side, so the application works exactly the same. But we should see that this span is dropped. + +We can hop over to our UI, and I have gone so far as to—so this guy is just coming in as normal, and I believe has a user attached to it. Yeah, this is David. He's allowed. + +And this is like the Juliette child chopped onions because, no, I did not wait for this to show up in my observability backend. It would be there probably by now, but just so I didn't hit refresh and have it not show up. + +But then if we did send it in with a—um, with this guy who we didn't want, it will go ahead and drop that span. Now, in this case, that span did have child spans, so it still shows up as, “Hey, there was time spent here,” but we did drop the span. + +So we know it existed, but we're dropping all data from it, which in this case is what we wanted. What we wanted was we wanted to say, “Hey, just this span is what doesn't matter.” This is like we're doing like span-level metrics for execution time, and so then we care about dropping that single span. + +Okay, I think I have like screenshots of that back in the presentation. We'll find out—new share. Okay, so we want to go ahead and drop out the span. Yep, there's our nice little thing, and then our regular one rolls on just as we'd expect. + +[00:12:10] Okay, so some stuff to remember about the filter processor. First of all, once it hits an attribute that'll filter, it's not going to hit the later conditions. So if there's an error on a later condition for some reason, it's not actually going to execute. + +And then, you know, any matching condition will work. So, you know, the config that you just saw where it's like, “Hey, user equal to Bob, user equal to David, user equal to Alice,” filter right, that is going to work. It doesn't have to meet all those conditions; it just needs to meet one. + +Then, if we drop out events, this is a confusion for some people, depending on the model that they're using for monitoring. The span is going to stay there. So if we say, “Hey, the type is a span event, and we want to drop every single span event,” we're still going to have that span. + +A metric works not the same way. If we drop data points from a metric until there are no data points, the metric itself will not be reported, which makes sense. You wouldn't generally report, right? Like, “Hey, this data point, but nil metric,” or sorry, “this metric, but nil data points, was not something you would normally report.” + +So yeah, this is my slide to remind myself and everyone else that you want to actually add these to the pipeline. + +Okay, so now let's talk about this problematic data here where this is the key that we're getting through, which we really didn't want. So in this case, obviously, we could write something that just said, “Hey, by name, drop this attribute,” but we don't want to drop the attribute because we have this guest down in here that's useful to us. + +We want to know if it's guest, admin, whatever else, whatever other role that's useful information to us. We just want to drop the full key. For that, this got me for a minute. You can have a filter that says, “Hey, it does it by match, and we'll take do star, say accept any attribute, but then it will not accept a full regex.” + +[00:14:06] Then I read this portion of the documentation, which is, “Hey, here's some alternative config which could soon be deprecated,” and that includes using regex as your match type. I was like, “Wait, can I not? Am I not supposed to be able to use regex here? Am I supposed to just use this relatively simple direct string matching?” + +So you can. My thing is, write down this link because something that I've had on my to-dos for a few weeks is to get this document a little better linked into the other documentation because a complete list of the OpenTelemetry transform language functions is like not super well linked into the filter and transform processor. + +Don't actually write this down; I will share this. I'll drop it into the chat; you don't have to write down a whole link. But, yeah, OpenTelemetry transform language, those functions include a pattern-based replace, which we'll demonstrate in a minute here. + +So some notes on using this regex in a collector config: you want to double your escapes like this, which you can set as a style thing on regex 101 or wherever other regex test tool that you're using. + +You can use capture groups; they do work. But it will seem like they don't because you have to use this double dollar sign thing to do it. In this case, it's like, “Hey, whatever the actual—in this case, this person wants to change Cube to K8s and U. They want to change it to KES dot, but they want the ID. They want to preserve the ID.” + +So they're saying, “Hey, match this key, which is Q, but then some string of letters and numbers, and then go ahead and replace it with K8, but that include what you matched in the parenthesis.” Apologies, that's a review for you all, but regex is my home and my joy, so I always like to talk about it. + +But, yeah, so you need to escape the dollar sign for this capture group with a double dollar sign, and you have to use triple dollar sign to use a single dollar sign. Sorry, this is just one of those things you have to note because the beautiful thing about regex is that it always works, except when it doesn't. + +These are just things you want to remember about your syntax. In this case, what we'd want is we'd want a transform processor because we want to change our value, right, to just be underscore guest or just be some kind of escape and underscore guest. + +In this case, right, we say, “Okay, go and find it.” I've been a little bit greedy here where I've said, “Hey, look for this thing. It's a whole bunch of numbers, more than one number for this.” + +Like, “Don't just take anything that is a user DB key, and then go ahead and overwrite the ID with replacement ID and then your whatever group you first matched.” So then you'll get something like this replacement ID. + +I have a version of this trace in the Signal dashboard, but take my word on it, this works. So transform processors are a really effective way you can grab some other values if you need to. You can chain them together, but the big thing is just to do really smart collapse of cardinality and collapse of PII. + +Really, really nice tool to do that. So what can't you do with collector processors? You know, we've seen this beautifully impressive demo here of the stuff we can do. A lot. There's a lot that you cannot do with collector processors. + +So here I would say is probably the big one: you cannot change the relationships that spans have with the processor. You cannot just say, “Hey, you go be part of this trace now,” nor can you create a parent-child relationship like, “Oh, you actually were kicked off, and we're inside this other one, and I'm going to know that from your naming convention, and then I'm going to stick you onto this other one.” + +So you can do filtering and transformation, and you can touch trace ID. I haven't experimented with this extensively, but apparently, it is not possible to push something into a trace after the fact. This makes some sense because of the way that sampling and other kinds of working with traces happen. + +There are some computer science reasons why this is not a trivial thing at all. It would not be a trivial thing at all to implement, but you want to be aware of it. I learned this for the first time—it's sort of, you know, it's hard to notice an absence sometimes. You feel like, “Oh yeah, I'm fully in control. I can do whatever I want.” + +[00:19:00] But then, Hazel's great talk for this same user group clued me into like, “Hey, this is a limitation here.” I haven't tried to hack it to every degree, but it's definitely—yeah, there's no built-in call to say, “Hey, like, add child relationship.” Not at all. + +You also can't drop a whole trace. You notice that I had this example where it's like, “Hey, don't look, the, you know, the span is gone. No worries.” But you would sort of like to say, “Hey, I can go in and say, ‘Hey, this span looks bad. Let me go ahead and just lose this trace.’” + +That is quite a new kettle of fish called tail-based sampling, right? Where you're looking at all of the traces that you receive and you're saying, “Hey, I only want these ones to come through based on some information about the trace.” + +So this is the dichotomy between head and tail-based sampling. Head-based sampling is right before the trace even starts. You decide if you're going to run it or not, maybe based on the tiniest amount of data, like what route it's in or whatever request information you have. + +But otherwise, basically, it's just, you know, I want to say the word—I know it's the wrong word—iskoric has a completely different meaning, but you're doing probabilistic sampling in that case. + +With tail-based sampling, this thing that we all want, whether or not we can have it, is you're saying, “Hey, now that I'm looking at the trace, I decide that you're interested.” You want to send you on. Tail-based sampling has a bunch of promise for the efficiency that it's going to add to your system, and only sending interesting traces—traces are relatively expensive to transmit, store, process—so it's very promising but has its own caveats to make work. + +So this is most of the presentation. I did not pause for questions a bunch of times, but do we have questions about that, about head and tail-based sampling and why that is not supported in the simple filter transform system? + +**Speaker 3:** Love it. I have to go give this same talk—not the same talk, but a version of this talk—to CGL next week, I think. Thank you so much for joining me on this journey because there's not a ton out there about these PII, I mean, mildly. + +Like, the transform processor is in alpha, so I totally get why there wouldn't be a ton out there about it, but I'm glad we're getting to talk about it now. So do feel free to make comments or drop in the chat if you have other questions. There are 29 things in the chat, and I have not been looking at the chat at all. Sorry. + +[00:21:35] Most of it is our—but Paige is asking, “How do you recommend testing config changes before shipping them to prod?” Or monitoring to know if you messed up a filtering rule? + +**Speaker 1:** Yeah, really good question. + +**Speaker 3:** Sorry about that. I do think that a really good way to do kind of experimental stuff where you're saying, “Hey, I want to do a kind of a multi-level change to what we're doing. I want to do a complex transform.” + +The two things are, one, is to just have a test version—have some piece of your observability inside your test harness. Have your local version of a collector or somewhere else to test it. + +The other is to—this collector to collector architecture is extremely, extremely useful here. So you can do like a canary deploy to say, “I'm just going to send it out to this one sub piece instead of just our master collector architecture.” + +Then the third is to do some further limitations because you can chain together an AND here. You can use resource values, so you can say, “Hey, only do this to this service name.” That's super useful for doing this. + +I didn't show that piece; I didn't do that here, but you can say, “Hey, I want this to have X service name and also check for this span value and then drop the span.” So you can start out as being more limited before it spread out to everybody else. Really good question. + +**Speaker 4:** Is a collector restart always required to pick up new config? + +**Speaker 1:** Great question from a long time ago. I would get kicked off of Twitch so fast because I'm not looking at chat. H—embarrassing. I used to know the answer to that, Paige, and now I do not. + +I seem to recall that there was some issue that I ran about the feasibility of some kind of hot reloading of config. But essentially, yes, you have to do a collector restart, certainly in your little test bed environment. + +Yeah, there's also not like—the collector has relatively limited remote abilities to pick up config. So yeah, that's something maybe one of the Honeycomb people will know something about, has fiddled with a bit, and I can check with the rest of the Signoz team as well. + +Okay, so we dropped this replacement ID. We did that talk about what we can and can't do. Yeah, there is a tail sampling processor out there. It has its limitations, but you want to explore that separately as its own processor and its own service for dropping entire traces. + +**Speaker 3:** Words of caution with metric transformations, so conversion between data types isn't supported by the metric model. But you can do it with a transformer, but don't do it. But you can do it, but don't do it. + +It's exceedingly easy to create meaningless metrics, right? To like doing things like creating averages from max values, right? Which you shouldn't do. Yeah, I would say very much like you're in the realm of, you know, creating statistical transformations just with like Excel equations at that point. + +You are in danger, certainly, of messing up what you're doing and getting stuff that looks okay but is, in fact, extremely bad signal. What have I seen? I've seen like somebody processing and creating a new average with a single new value. So they have an average of the last 10 values, and they get a single new value, and they say, “Okay, cool. I can just add that one to the average,” which was not quite right. + +So yeah, things like that are worth a concern. A bunch of the transformations between measurement types are one-directional, and so you should not use the transform to try to go back or to try to move between these lanes because you will end up with bad data that is still data that still looks like data and will chart like data. + +So that's a concern, very similar to going in and doing just queries directly into your data store to edit your data, right? You want to be exceedingly careful with that because, of course, there is always the possibility of destroying the information that you have. + +**Speaker 1:** Slide two of cautions about metric transformations. So you can change the labels on metrics and traces. You can cause yourself data store problems by giving things the same names as two metrics, two time series. You give the same name, attributes, and scope. + +I tried to break Signoz, the ClickHouse data store, doing this because I was very excited to see something break, but I couldn't quite do it. But presumably, you can get to the point where you're having your dashboards not load or other queries not working because you have double results when there should be unique values in a column. + +So a more common outcome of that will be orphan values, will be spans that are not part of any trace and traces that are not connected to any service. + +Oh yeah, Ren, I'll put it in the CNCF collector channel. I'll ask it in the CNCF collector channel, and the first thing I'm going to do is actually search the CNCF collector channel because it's interesting. + +The question was, “Do you have to do a collector restart always to get to pick up new collector config?” I think the answer is yes, but I—yes, yeah. I thought there was an issue about it. There was a talk about exploring some support for that, but I'm certain that by default right now the answer is no. + +That was the end of the slideshow, folks, which really just crept right up on me. Do we want—we saw live demo stuff. Oh yeah, let's go do one last live demo thing. That's right. That's what I wanted to do. + +New share, new share, share. Okay, so, man, there was totally some—oh right. We have the situation where we are getting way too many components. + +This thing, like, roll it, you know, it sends you back an array of roll dice. We had a situation where when we sent a request to say like, “Hey, roll, you know, 41 dice for me,” we get back this nice array very, very quickly. + +But the problem is that when we go and look at the traces for these guys, they just have a ton—a ton of these individual single dice rolls which, you know, in this version, right, is not really a problem. Right, you know, your trace dashboard should be able to just hide those a whole bunch of rolls. + +But for whatever reason, in our example, we're saying, “Hey, this is an issue. We have hundreds of these. We have thousands of these. We know they're all identical; they always take the same amount of time. We want to get these out of here.” + +This is an older trace that didn't have 40; we just had six, but you get the point. What we want to do is we want to say, “Hey, I want to drop these spans, these roll one spans.” + +But some helpful and also sabotage-focused developer has named each of the spans uniquely, so we can't just say, “Hey, go ahead and drop any span that has this name,” because it includes—like, this is the counter on that role. + +So that is something that we want to handle. We don't need a regex for this; we just need the match tool. So if we go back into our IDE, let me—oh, I started typing the command, but I won't actually show this. + +So if we go back into our IDE, we go back into our config, we can say, “Hey, span, if you have a match on your name that is roll ones followed by anything, let's go ahead and drop that span.” + +If we save this and then we restart our collector, our app's been running this whole time. So now asking for 41 dice rolls—let's ask for 40. Who likes using a prime number? I certainly don't. + +While I politely wait for a second for these traces to hit my local collector, get to the batch moment, which is like every—I think every five minutes on my thing—and then get sent off to the backend, do we have other questions about this stuff? + +**Speaker 5:** Oh, I didn't mention it, but you can—you have the same filter functions available on metrics and logs. So we demoed this all with spans, but it is totally doable with metrics and logs by the same principles. + +But yeah, other questions, feel free to drop them in chat or come off mute. I promise I am watching the chat now. No worries. + +Okay, so yeah, now let me go ahead and start sharing this. This is just our last five minutes of requests. If we look at—we were just looking at a previous one where it was like, “Hey, we have way too many of these.” + +Now, if we go ahead and sort this and look at our most recent traces, we should sure enough be able to see—oh, I also gave it the name that's like, “Hey, drop this name.” + +So we're dropping two things here, but our actual R dice is happening. It drops the routing span because, again, it has this name it doesn't like, and then it also drops the sub-like individual roll ones spans. + +Okay, folks, that's been my stuff. Man, 38 minutes, wow. Usually when I'm doing something for the first or second time, I time it out, talk it all through, I'm like 45 minutes. Then when I actually get in front of people, it's 11 minutes. Just zipping, zipping. + +But this one, I guess there was a lot to say about this. Final stuff: the functions are all out, are all fully supported. The transform processor is still in alpha, and you're going to need your own build of the collector to make sure you include it, which, I mean, you need to do for other collector contrib stuff. + +So that's pretty standard, but just be aware of it. You wouldn't want to base your whole DIY observability stuff on transform processors. Do some work ahead of time to get your stuff labeled right and don't be completely relying on it because it is an alpha and it's going to shift a little bit. + +There are some standards conversations happening, but it's definitely worth doing for stuff like this, for like, “Hey, you have this filtering or transformation task, and you don't want to bother your product team.” + +Okay, folks, that's been my time. I don't know, Adrian, if you want to give final stuff. + +**Speaker 6:** While you're filtering out these spans not to be shipped to Signoz, can you route them to something like S3? + +**Speaker 1:** Oh, that's an interesting question. So not with this set of tooling. You cannot use like a filter processor to say, “Now go to a different exporter lane.” But you certainly can do that with, you know, with your import tools, right? + +So with your components where you're taking in data, you can obviously say on your receivers, “Oops, don't do that.” You can say on your receivers, “Hey, I want to pick up these values and send them to this other endpoint,” yeah, and then go into a separate pipeline. + +I don't—yeah, filter is like drop; filter is like drop or remove data. It's not route this data to another endpoint. Great question. Pretty sure you can do that routing by span name, but also, should you do that? Should do— that's a good question. + +That's a really good—I should write something about that. But that's really good. I'm going to look at a future blog post about that. That's a really good question. + +**Speaker 7:** Any other questions for Na? + +[00:36:00] **Speaker 1:** Ren says in chat, “Yeah, there are a lot of tools out there that help with managing your pipeline, including like especially, you know, we got three, I think, observability team people here who can talk a lot about how, like, there are times when I think all of us want to be like, ‘Hey, we're your one-stop shop to look at your data.’” + +But there's going to be reasons that you're sending it to multiple places. Examples, of course, be like sending stuff to CloudWatch but also sending it to a New Relic dashboard, sending stuff to S3 as like, “Hey, this is—we're going to cold storage this, but we want to have it.” + +Obviously, classically logs, right? Sending them to multiple places, to one place where the retention is 10 days and another place where the retention is 10 years. So, yeah, that's going to make a ton of sense. + +There's something written about that, Ren. Maybe I'll hit you up for like a—we'll do like a partner post or something on it because it's an interesting question and not one where there's a ton written about it. + +**Speaker 6:** Sure, yeah, that sounds good. + +**Speaker 1:** Paige is like, “In the last webinar I did, they asked us how many observability tools they had, and a significant portion of the groups had five plus.” + +I think if we were being honest, everyone's going to say at least two, right? Every—well, no, at least three. They say, “Hey, I have an observability tool, which is the thing called observability that says observability on its SEO filing. Then I have some kind of logging tool, and then I have a debugging tool that I use, right? I have my ClickOps tool that I use to go in and look at what the heck's going on.” + +And so that's at least three, right? If not, then, okay, well now what's—what do you use as like a pinger or other synthetic metrics, right? That can—can so, yeah, now we're at five, right, with just what I would say. + +Oh, like you list all five, you're like, “Yeah, that's normal. That's normal. I need all this.” It's like me looking in my purse. I'm like, “Uh, I need to get some stuff out of here.” They're like, “No, I need all of this. I need two batteries for my phone.” + +Okay, rid of these. Oh man, great chat. Thank you so much. And again, big shout out to R. You missed it right at the start when I was like really thanking you for doing the last-minute promo. + +I will be giving a similar but not identical version of this talk because we can all admit there were parts that dragged. I'll be giving a similar version of this talk at CGL next week, and then also we'll be doing it at another conference whose name escapes me in January. + +We'll be doing Cod Smash, which is in January or something. I'm going to be giving the same talk, and also be at KubeCon. If you want to come say hi at KubeCon, come say hi. Also, thank you, thank you so much Na for joining us today. + +**Speaker 2:** And thank you everyone who was able to make it. This was actually a really good turnout for OTel in practice, so thank you for taking the time. + +Also, for if you know anyone who wanted to attend but couldn't make it, let them know that we will be posting a recording of this on the OpenTelemetry YouTube channel. The handle is OTel Das Official. If you don't, haven't subscribed yet, we've got a bunch of previous videos from OTel in practice, OTel Q&A, and even some of it are in user discussion groups, so definitely check it out. + +Please check out the Hazel Weekly one from six weeks ago now. It was really, really key stuff. So that's really worth—if you want a reason to go check it out, that's a really good one. + +It's next level. It's very, very good content. And we've got an OTel Q&A coming up on November the 16th with Jennifer Moore, where she talks about her experiences with OTel implementation at one of her previous companies. + +So that will be also a really great session to look forward to. It'll be in this time slot, same time slot. + +Does anyone else have anything else they want to share? + +**Speaker 5:** Ren? + +**Speaker 6:** Nope, just a huge thank you to Na. + +**Speaker 1:** Yeah, this was awesome. ## Raw YouTube Transcript diff --git a/video-transcripts/transcripts/2023-12-04T23:34:44Z-otel-q-a-feat-jennifer-moore.md b/video-transcripts/transcripts/2023-12-04T23:34:44Z-otel-q-a-feat-jennifer-moore.md index 38cb1cb..4db1914 100644 --- a/video-transcripts/transcripts/2023-12-04T23:34:44Z-otel-q-a-feat-jennifer-moore.md +++ b/video-transcripts/transcripts/2023-12-04T23:34:44Z-otel-q-a-feat-jennifer-moore.md @@ -10,124 +10,208 @@ URL: https://www.youtube.com/watch?v=ztrknOWIUh8 ## Summary -In this YouTube Q&A session, host Jennifer Moore interviews Jennifer Moore, a DevOps lead who shares her experiences with implementing OpenTelemetry at her previous company, Screencastify, which specializes in screen recording applications. The discussion covers the application stack used at Screencastify, including TypeScript and Google Cloud Platform, and the initial struggles with observability practices. Moore explains how she identified the need for better observability tools and transitioned from using a proprietary SDK to OpenTelemetry for improved performance and understanding of the system. She emphasizes the importance of distributed tracing and how it facilitated better incident response and team engagement. The conversation also touches on the challenges of integrating observability tools, the need for consistent logging and metrics, and the cultural shift towards DevOps practices within the company. Overall, the session highlights the significance of observability in software development and the impact of proactive measures in improving system reliability. +In this YouTube Q&A session, Jennifer Moore shares her experiences with observability practices at her previous employer, Screencastify, where she served as the DevOps lead. The discussion delves into the company's application stack, consisting of TypeScript and Google Cloud services, and highlights the initial lack of effective observability tools. Jennifer explains her motivation to enhance the system's observability, which involved transitioning to OpenTelemetry for better tracing and diagnostics. She discusses the challenges faced, such as previous attempts at instrumentation and the complexities of integrating various observability tools. Despite initial neutrality from management regarding observability, the value became clear during critical incidents, leading to greater acceptance of the new practices. The conversation emphasizes the importance of observability in troubleshooting and operational efficiency, while also exploring the ongoing evolution of OpenTelemetry and its integration challenges. Jennifer's insights provide encouragement for teams looking to improve their observability strategies in a practical and impactful manner. ## Chapters -00:00:00 Introductions and welcome to the stream -00:02:15 Background on Jennifer Moore and her previous company, Screencastify -00:04:30 Overview of Screencastify's application stack -00:06:50 Jennifer's role as DevOps lead and responsibilities -00:09:00 Discussion on the need for observability at Screencastify -00:12:30 Jennifer's initial assessment of existing observability practices -00:15:45 Motivation for improving observability in the system -00:18:20 Introduction of OpenTelemetry for better instrumentation -00:22:00 Challenges faced during implementation of OpenTelemetry -00:25:30 The impact of observability on team dynamics and management perception -00:30:00 Future applications of knowledge gained from previous experience at Influx Data -00:33:15 Feedback on OpenTelemetry and suggestions for improvement -00:37:00 Discussion on the correlation of logs and traces for better insights -00:40:30 Closing thoughts and encouragement for adopting observability practices +00:00:00 Welcome and intro +00:00:20 Guest introduction: Jennifer Moore +00:01:21 Background on Screencastify +00:04:00 Role as DevOps lead +00:05:00 Observability needs at the company +00:06:30 Initial observability challenges +00:08:00 Implementing OpenTelemetry +00:10:00 Team involvement in observability +00:11:01 Management's perspective on observability +00:14:00 Transition to OpenTelemetry collector -# Otel Q&A with Jennifer Moore +**Host:** Welcome everyone to Otel Q&A. We are super excited to have Jennifer Moore with us today. Welcome, Jennifer. -Welcome everyone to the Otel Q&A! We are super excited to have Jennifer Moore with us today. Welcome, Jennifer! +**Jennifer:** Hi, thanks. -## Introduction +[00:00:20] **Host:** Okay, so I guess first things first. I asked Jennifer to come on because I had talked to her previously about some experiences that she had at a prior company around Otel, and I thought it was a really, really interesting and compelling story. So, I guess we'll start with, why don't you give us some background? I know this is not at your current employer; it's at a previous employer. Why don't you give us some background of, you know, for starters, like what their application stack was like, and then go from there. -**Jennifer:** Hi, thanks! +[00:01:21] **Jennifer:** Sure. This was a whole job ago, and all of this was happening like a year ago, so some of the details will have gotten fuzzy for me by this point, but that's cool. So the company is called Screencastify. They make a screen recording application. The kind of early days of a version two would have a video hosting and editing service, whereas before it had been just kind of a screen recorder, and it was saved to whatever storage you happened to have handy. And that was the whole thing. -I asked Jennifer to join us because I previously discussed some compelling experiences she had at a prior company regarding Otel. I thought it would be interesting to share her story, so let’s start with some background. +This stack was lots of TypeScript, and all running in GCP and containers. The main back end was a TypeScript, I think Next.js, I forget, some TypeScript backend server, running in Google Cloud Run, this sort of serverless containers product that they have. Everything is serverless, or many things are serverless, will be relevant. The back end handled all of the routine, you know, we're hosting a web service kind of stuff like account management and billing, etc., and also some on-demand video encoding. -## Company Background +Then there was what we called the task system, I think, or job system, something like that, which ran in Kubernetes. It was a bunch of Kubernetes microservices that would respond to videos being uploaded and edited and things to do video transcoding and processing work on that stuff. That all was powered by a BullMQ message queue, job queue, whatever, and yeah, I don't know, otherwise ran fairly self-contained. -**Interviewer:** This is regarding a previous employer, so could you give us some background on their application stack? +**Host:** All right. And what was your role then at this company? -**Jennifer:** Sure! This was a whole job ago, and everything was happening about a year ago, so some details may be a bit fuzzy. The company is called Screencastify; they make a screen recording application. They were in the early days of version two, which included video hosting and editing services. Previously, it was just a screen recorder that saved to whatever storage you had handy. +[00:04:00] **Jennifer:** I joined as the kind of DevOps lead. A lot of what I did there was to kind of drive DevOps practices and then also do the chop wood, carry water parts of DevOps. So, a lot of working on the CI/CD pipelines, a lot of work on observability, internal tooling, and developer experience and things. You know, everything that wasn't primarily focused on building a feature in the application was probably something that came across my keyboard at some point. -The stack consisted of a lot of TypeScript, running in Google Cloud Platform (GCP) within containers. The main backend was a TypeScript server, utilizing Next.js, and it was running in Google Cloud Run, which is a sort of serverless containers product. Many components were serverless, which is relevant to our discussion. This backend handled routine web service tasks like account management and billing, as well as on-demand video encoding. +[00:05:00] **Host:** Cool. That's a lot of hats to wear then. Awesome. Okay, so now you mentioned observability, and we are here. So tell us to what extent, like when did you realize or actually let's take a step back. Did the company see a need, a big enough need for observability? Like had they seen that already when you had joined? -There was also what we called the task system, which ran in Kubernetes. It consisted of several microservices that responded to video uploads and edits, performing video transcoding and processing. This system was powered by a message queue, which helped manage the job processing. +**Jennifer:** Yes, somewhat. I think they had encountered the concept and they had made a start at trying to do some observability things. There were definitely some people who recognized that it was a good idea, and like nobody was fighting them about it. But I think they didn't really know what they should expect out of it. They had kind of done some scattered starts at it and instrumented some things, but like you know some things in isolation and it wasn't in a very useful way. -## Role at Screencastify +When I got there, they had tracing, but it was very disconnected and not really the things that people were interested in. A lot of anything that anybody actually needed to get out of it before I took that on came from logs. -**Interviewer:** And what was your role at Screencastify? +[00:06:30] **Host:** Okay. So then you entered into the picture. What was at that point when you started implementing or making the systems more observable? What was sort of your motivation for that? -**Jennifer:** I joined as the DevOps lead. A lot of what I did was drive DevOps practices and handle the day-to-day operations, like working on CI/CD pipelines, observability, internal tooling, and developer experience. Essentially, anything not primarily focused on building a feature in the application likely crossed my keyboard at some point. +**Jennifer:** So my motivation throughout for making this system more observable was that I wanted to better understand what it was doing. One of the hats I was wearing was Sr. and I needed to support these things, and that's basically impossible to do if you don't know what it is or what it's doing, and certainly not how it's behaving. -## Observability at Screencastify +After a little while, I grew very uncomfortable not having that understanding of the system and decided to just start removing the false start instrumentations that were already there and start instrumenting things more consistently. -**Interviewer:** You mentioned observability. Did the company recognize a need for it when you joined? +**Host:** So you actually went into the application code and started instrumenting things? -**Jennifer:** Yes, somewhat. They had encountered the concept and had started trying to implement some observability practices. There were definitely people who recognized that it was a good idea, and nobody was fighting against it. However, they didn’t fully understand what to expect from it and had made scattered attempts at instrumentation, which were not particularly useful. +[00:08:00] **Jennifer:** Somewhat. So mostly what I did was remove the auto-instrument that were already there. There were, I don't know, DataDog's proprietary ones, and I'm sure they would work if they worked, but they didn't actually auto-instrument several of the libraries we were using, and so it wasn't helpful for us. I replaced those with OpenTelemetry auto-instrumentations, which did have auto-instrument for all of the things we wanted, and suddenly my flame graphs filled out and I could see what was going on from start to finish, more or less, and that was exactly what I'd been looking for—magic. -When I joined, there was some tracing that was very disconnected and not really aligned with what people needed. As a result, most of the useful information came from logs. +**Host:** And how was your learning curve in terms of OpenTelemetry? Is it something that you were familiar with initially or is that something that you had to teach yourself? -**Interviewer:** When you started implementing observability, what was your motivation? +**Jennifer:** So, yeah, I mean I was familiar with it conceptually initially. Back when I was on Twitter, I spent a lot of time talking to observability and resilience folks on Twitter. The goals and the concepts and everything were all very familiar to me by that point, but then the actual mechanics of how do you turn on and start using OpenTelemetry was not something I'd done before. -**Jennifer:** My motivation was to better understand what the system was doing. As a senior engineer, I needed to support these components, and it’s almost impossible to do that without understanding the system's behavior. After a while, I grew uncomfortable not having that understanding and decided to remove the ineffective instrumentation and start implementing things more consistently. +**Host:** Did that end up being like a single-handed effort on your part? Did you have anyone from your team assisting? -## Implementation of OpenTelemetry +**Jennifer:** Yeah, like I didn't really want to be the only person who knew what it was doing or how it was working. I did do the initial setup myself, but then I made a point to kind of stop there and distribute follow-up work from that to the rest of the DevOps team, at least, so that there would be other people who knew what was going on and knew what they were looking at whenever things needed attention. -**Interviewer:** Did you start instrumenting the application code yourself? +**Host:** And how was that received within your team? -**Jennifer:** Mostly, I removed the existing auto-instrumentation that was not helpful and replaced it with OpenTelemetry auto-instrumentations, which worked much better for us. Suddenly, my flame graphs filled out, and I could see what was going on from start to finish. It was exactly what I was looking for! +[00:10:00] **Jennifer:** I mean, I think that went really well. Distributed tracing has some inherent complexity to that domain, and so there’s some sort of learning and ramp-up to get comfortable with the concepts there. Everybody had to go through that, but I think after having gone through that, everybody was very pleased to have done it and very happy to have it. -**Interviewer:** How was your learning curve with OpenTelemetry? +**Host:** That's really great. So did you say then that you were starting to see favorable results? You mentioned that you yourself started seeing some favorable results, like seeing the things on the flame graphs. How would you say the rest of your team took it, and even a step further, how did management take that? Did they see that as a benefit? -**Jennifer:** I was familiar with the concepts but had never implemented it before. The actual mechanics of turning it on and using it were new to me. +**Jennifer:** I think, you know, at first management was kind of neutral, like neutral to positive about it. They recognized it was a good thing to do, but I don't think they really saw how much value it had at first. -**Interviewer:** Did you have anyone from your team assisting you? +But then we get to some stories with some incidents in them, and I think the value of it becomes a lot more obvious to them at that point. -**Jennifer:** I didn’t want to be the only person who understood how it worked. I did the initial setup myself, but I made sure to distribute follow-up work to the rest of the DevOps team so everyone could get involved and understand what was going on. +**Host:** Awesome. Circling back a little bit, you did a lot of stuff around the auto-instrumentation side of things. Do you think this incentivized the developer of the actual application to go and do some additional work and add some manual tracing, for example? -## Team and Management Response +**Jennifer:** Yes. There were definitely some engineers who were very on board with the idea of having tracing and wanted to go all in on that. Then there were others who were pretty neutral about it. No one was fighting against it. A lot of it was, you know, like sure, that seems fine, you can do that. -**Interviewer:** How was this received within your team? +A handful of boosters. But were they okay with going in and instrumenting themselves as well to kind of enhance what the auto-instrumentation was giving? -**Jennifer:** It went really well! Distributed tracing has inherent complexity, so everyone had to ramp up, but they were pleased to have gone through that process and were happy with the results. +**Jennifer:** Yes, I think so. It's a little bit hard to say because honestly, some things blew up organizationally, and so I didn't have as much opportunity for that to actually happen as I would have liked. But people were very eager to have done it. -**Interviewer:** What about management? Did they see the value? +**Host:** Right. At least they got to reap the rewards of what you had set up, right? -**Jennifer:** Initially, management was neutral to positive about it. They recognized it as a good idea, but they didn’t see the full value until we encountered some incidents that highlighted the need for better observability. +**Jennifer:** Yeah, which is always good. Now in terms of your setup, did you end up implementing an Otel collector? -## Developer Incentives and Future Work +[00:14:00] **Jennifer:** They did, yes. That was something that I had wanted to do. The company was sort of already on DataDog and kind of invested in the DataDog tooling ecosystem. Moving away from that proprietary stack and towards the OpenTelemetry libraries and collector was a process. But it's one that I tried to start, and it continued after I left. My understanding is that everything is OpenTelemetry there now, and I think they've even moved away from DataDog to another vendor. -**Interviewer:** Did your efforts incentivize developers to do additional work like manual tracing? +**Host:** Oh wow, that's really cool to be able to move to a fully vendor-neutral sort of setup. It's super awesome, and I think especially like you had set the wheels in motion around that, and that it continued even after you left. -**Jennifer:** Yes, several engineers were very on board with the idea of tracing. Some wanted to dive deeper and contribute additional instrumentation. However, some organizational issues arose, which limited opportunities for that to happen as much as I would have liked. Still, those who were interested in enhancing the observability found it rewarding. +**Jennifer:** Yes. Now how's that continued? The knowledge that you gained at your previous workplace, is it something that you've been able to apply where you're at right now? -**Interviewer:** Did you implement an OpenTelemetry collector? +**Jennifer:** Yes and no. Right now, I am at Influx Data. At Influx, all of the engineering attention has been directed towards getting the 3.0 version of the database into 3.0, which is a thing that we have done now, and that's going pretty well. That frees up some people to think about other things again. Where that intersects with observability and OpenTelemetry has been more about getting the database itself to be better suited to work as a backend for collecting spans and telemetry data rather than instrumenting our own things for traces. -**Jennifer:** Yes, I wanted to move away from the proprietary stack toward OpenTelemetry. The company was already invested in Datadog, which made the transition challenging, but I started the process before I left. My understanding is that they now fully transitioned to OpenTelemetry. +We do have a lot of things that are very well instrumented and very detailed instrumentation, and a lot of that is very manual. That's done with more of a performance objective of understanding performance more than behavior or investigating errors or things like that. -## Current Experience at InfluxData +Which is to say there's very deep small spans as opposed to wide spans, which I think are a little bit easier to work with if you're trying to ask something weird is going on and you have no idea what. -**Interviewer:** Has your knowledge from Screencastify translated to your current role at InfluxData? +**Host:** Yeah, so you have a slightly different goal compared to the last place but still using OpenTelemetry to achieve that goal. -**Jennifer:** Yes and no. Currently, most engineering attention is focused on getting the new version of our database ready. My work intersects with observability, but it’s more about making the database a better backend for collecting spans and telemetry data rather than instrumenting our own applications for traces. +**Jennifer:** Yeah, cool. -## Challenges in Implementing OpenTelemetry +**Host:** What would you say were the most challenging aspects, given all your experience around OpenTelemetry? What were the most challenging aspects in terms of implementing OpenTelemetry in your previous organization? -**Interviewer:** What were the most challenging aspects of implementing OpenTelemetry in your previous organization? +**Jennifer:** Yeah, so like I think the most challenging was that there had already been a start with a vendor proprietary SDK. That confused a lot of things. I had tried to leave that in place while I instrumented with OpenTelemetry, but then these two libraries just started reporting on each other, and it made the whole traces more confusing than they had been without it. -**Jennifer:** The biggest challenge was that there had already been an attempt with a proprietary SDK, which created confusion. I initially tried to keep that in place while adding OpenTelemetry, but it led to complications. I wish I had removed the proprietary SDK right away. Another challenge was areas without existing auto-instrumentations. I had to write my own for some libraries, which was a learning curve but also a valuable experience. +Removing that proprietary SDK and just having the one mechanism for instrumentation in any given process was something that I wish I had done right away in retrospect. I think the other thing was in areas where there did not already exist auto-instrumentations. Like IBQ earlier, which didn't have one until I wrote it because I really needed it. -## Final Thoughts and Feedback on OpenTelemetry +I can say that's a challenge but also a benefit because nobody else had one either, and with OpenTelemetry, it was a viable thing that I could go and write my own auto-instrumentation package and be able to get that for myself with libraries that otherwise just didn't have it. -**Interviewer:** Do you have any feedback for the OpenTelemetry project? +**Host:** How was that experience of writing the auto-instrumentation yourself? Was it a huge learning curve? -**Jennifer:** I think the onboarding process is steep, and if that can be improved, it would be great. At the time, metrics and logging were not as stable as they are now, so having a consistent interface and SDK for all telemetry signals would be beneficial. +**Jennifer:** Yeah, that was a pretty substantial learning curve. It was a good thing that I started early because then we got into some incidents and I really needed it, and I was able to get it functional in a relatively short time span from there so that I could start using it and actually see what was going on. -**Interviewer:** Thank you for your insights! Does anyone else have questions for Jennifer? +**Host:** That's so cool. You mentioned that your fellow DevOps engineers were leveled up in the ways of Otel. Is that something that you helped them out with? How is it that they gained the skills and leveled up so that they could also have that knowledge of OpenTelemetry that you had? -**Audience Member:** Could you elaborate on having a consistent interface in the SDK? +**Jennifer:** I think the majority of that was just hands-on doing it. Again, after I had set up basically one auto-instrumentation and gotten that configured, I stopped there and made a point to have other people take that further. I wanted them to have that experience. -**Jennifer:** With OpenTelemetry, spans are streamlined in terms of configuration, but logs and metrics often require different setups. It would be great to have a unified way to send all telemetry types to the collector without making it an application concern. +More broadly, we did also have a very strong learning culture, which was one of my favorite things about that company. We had a sort of technical book club, and we read the Observability Driven Development book for that book club. I think that was also very useful in terms of getting a lot of people familiar with the concepts and clarifying expectations about what this gets you from an office perspective and from an engineering perspective. -**Interviewer:** Thank you, everyone, for attending. Jennifer, your story is incredibly inspiring. It highlights the importance of observability, especially when things break. We hope others draw inspiration from your experiences with OpenTelemetry. +**Host:** And hopefully got them stoked too about it, right? -Stay tuned for our next Q&A on December 7th with someone from HashiCorp! Thank you, Jennifer, and we’ll let you know once the video is posted. +**Jennifer:** Yeah, I mean, the people who were inclined to be interested definitely came away from that more interested. + +**Host:** It's really cool that this is a very operational-driven Otel undertaking. I guess it kind of makes sense because when things break, it's always going to be more on the operational side of things, and it's not necessarily the developers who are going to be the ones who are going to say why is this breaking, right? Because by then, it's like, I hate to say it, but less their concern, even though it is very much their concern. But unfortunately, I think our culture is still kind of shifting towards the not-my-problem because it's in production. + +**Jennifer:** It is a little bit farther away from them, and I have some sympathy for that. I spent a lot of time doing app dev, but also only some sympathy because the whole time I was doing app dev, I was very frustrated by my inability to have any contact with production. That was also one of the things that I was trying to drive there, like beyond observability and OpenTelemetry, was to actually fulfill some of the promise of DevOps and have that not be such a bright line between the two. + +**Host:** Do you think you got closer to achieving that goal? + +**Jennifer:** I think so, yeah. One of the ways I did that was to really emphasize how the tools are used. Whenever it came up, like in doing incident retros, I made a point to ask people what they looked at that made them think that was the concern, and how did you execute those queries? Let's actually open up the tool and go through the UI and see how that's performed. + +[00:11:01] **Host:** I want to go back to our chat earlier about management's perspective on using observability. I believe you mentioned that it was kind of neutral until they realized this is useful. One of the things that I always find can be challenging in an organization is bringing something new to management, and you almost need that permission from them before executing. Is that something where you asked for permission, or did you end up asking for forgiveness at that stage? + +**Jennifer:** Yeah, like neither, honestly. The way this worked out, they had already, before I joined, whatever conversations they had about getting better observability, and somebody signed off on doing something. They started some instrumentation, so that was clearly a thing that was okay. + +I didn't need to pitch them on having it, and as a matter of fixing it, that was more of a question for my fellow engineers. Somebody put this in, and somebody probably understands it, and I need to, but it's not satisfying the goals that we had for it. I need to do something about that, but management wasn't going to say no to that. + +**Host:** Right, because you already had some sort of an observability culture. In terms of investing further in it, we launched a big part of this launch, and a big part of the reason for going to the 2.0 version of the product was to enable some account and billing features that just weren't possible with the way those data schemas existed. + +They existed in Firebase, and they were moving to Postgres to have real schemas. That had some predictable problems, and the migration kind of stalled out, and that was real bad. Things just stopped working for hours at a time because the migration was kind of killing the Postgres server without ever completing, but we didn't know that. + +After some initial investigation, instrumenting all the things to figure out what is even going on was the next thing, and it wasn't something that I needed permission to do at that point because production is down, and it's going down for hours at a time repeatedly, so it needs to be fixed. + +When the building's on fire, you don't need permission to install sprinklers anymore. + +**Host:** Absolutely. I guess at that point, even though you were using a different tool set for observability and you made the decision to use OpenTelemetry instead, that became less about putting in the tool that works for us. + +**Jennifer:** Yes, and we had already started that by the time we got to that incident, and so that was very fortunate because we were getting a better experience with OpenTelemetry. There were some parts of the system that we could fairly well understand what they were doing, but the rest of it had not been instrumented, and that had to be done quickly. + +**Host:** That's your incentive when things are burning—OpenTelemetry to the rescue. Before I turn it over to the rest of the folks on this call, do you have any feedback for the OpenTelemetry project as a whole, good or bad, in terms of your experience? + +**Jennifer:** Yeah, I'm not sure that I don't know. I think that, and this was a year and a half ago that I was doing it, the early getting set up onboarding learning curve is pretty steep. I don't know exactly how to improve that, and I'm sure I'm not the first person to have said this either. + +If that can be improved, that would be great. Also, I think that metrics and logging got to stable since I was doing this, and it would have been really nice to have had that at the time, I think, because it would be very convenient to have a consistent sort of interface and SDK for all of those telemetry signals. + +Then being able to send them to a collector and then have the collector send them to wherever they're going and not making those things application concerns would be great. I haven't had the chance to actually do that to see if it works or not. + +**Host:** Cool, thank you. Does anyone else on this call have any questions for Jennifer? + +**Participant:** I was just curious on the last piece you just said about having a consistent interface in the SDK for the telemetry signals and not making them application concerns. I was just curious if you could elaborate a little bit more on that piece. + +**Jennifer:** Yeah, so you have OpenTelemetry for your traces, and then you have some sort of logging solution for your logs, and hopefully you're doing structured logs. Again, some solution for metrics, assuming you're doing metrics—probably Prometheus or whatever. But to get those logs and those metrics to go somewhere, you wind up having to configure some vendor's logging library so that you can ship your logs to your logging backend, or you log everything to the terminal or to a file or whatever, and have some sort of infrastructure magic pick it up and send it somewhere. + +With spans, you configure your exporter and you're done. Your exporter just sends them to the collector, and the collector address or whatever can come from config after you're instrumented. You don't have to maintain much there. At the time, I would have really liked to be able to send my logs and my metrics to the collector in the same way that I was doing with spans rather than having three solutions for three different kinds of telemetry. + +**Participant:** Gotcha. I would see we're getting closer to that, right? + +**Participant:** I personally don't know. I'm asking. + +**Participant:** I think we're getting closer to it. I think metrics have definitely matured a lot in the last year, and I know a lot of observability backends are now ingesting metrics as well, so in addition to traces, which is nice. + +Then I think we're starting to see that with logs as well. Logs, I know, are a different beast because there’s no logging SDK. There’s a logs bridge SDK, so the idea is to use one of the common logging libraries that is supported by OpenTelemetry, and there's a bridge SDK for it, and then it'll convert that format to OTP so it can be ingested by whatever backend. + +So, which is kind of nice. I guess they didn't want to reinvent the wheel with logs because there's so many different things out there. + +**Jennifer:** Yeah, the momentum on the way people do logs is enormous. It's not like that's not changing. But how they get collected and organized, you know... + +**Host:** Yeah, I think the game changer is being able to correlate the logs to the traces so that you have that fuller picture, which I'm super excited about because I understand a lot of us grew up on logs, but I think the traces will help tell the full story. So when you have that correlation, then you can get magic. + +**Jennifer:** I think logs correlated with traces is the best, most ideal outcome. I know that traces have events and can do point-in-time things, but logs are just better at that, and being able to see point-in-time events correlated to where they happened within a span and within a trace is exactly what I want. + +**Host:** Anyone else have any other comments or questions for Jennifer? Shall we break early then? + +**Host:** Cool. Awesome. Well, thank you, Jennifer, for sharing your story with us. I really love the story because I think it's the things that are broken that need observability. + +I think it's such a compelling case for anybody who's on the fence on whether or not it's a worthwhile investment. It's awesome that your company already kind of had its foot in the door for that, and that you were able to take it further to where it actually needed to be so that it was useful to you and your team for troubleshooting. + +I think it's a great story. I hope others will draw inspiration from that as well as a compelling reason as to why observability with OpenTelemetry is a good combination. + +Do we have... I think we've got some upcoming events? + +**Host:** Yeah, on December the 7th, we have another Q&A with someone from HashiCorp who has attempted many times to instrument Nomad. I think that'll be a really interesting story as well. + +So anyone who is following along, stay tuned for that Q&A. Hopefully, we'll get to see you live, but if not, we will see you on YouTube. + +**Jennifer:** Sounds good. It was great to meet you all. + +**Host:** Yeah, thanks so much, Jennifer, and we'll let you know once the video is posted. + +**Participant:** Yeah, I was going to say thank you, Jennifer, for your story. That's really, really inspiring because I feel like so many people want to adopt it, but then getting that buy-in from your leadership is always the problem. I mean, technically, it's not making their money until you show that this is causing problems, and then we really need to do this. + +**Jennifer:** Yeah, I don't know why it's such a fight so much of the time. Problems are going to happen, and what this gets you is being able to point at where the problem is happening. I don't know why people don't want to be able to do that. + +**Participant:** It's so logical. In my experience, it only works either in teams that are excited about progressive engineering practices or that are feeling so much pain from vendor lock-in that they're ready to set everything on fire. + +**Jennifer:** Yes, so true. Basically, it means you need a really nasty incident or really bad vendor lock-in as your compelling reason. + +**Participant:** Well, if you get that incident, don't let it go to waste. + +**Host:** Yeah, totally. Right, quote of the day. Awesome. Well, thank you, everyone. ## Raw YouTube Transcript diff --git a/video-transcripts/transcripts/2023-12-18T23:04:41Z-otel-q-a-feat-luiz-aoqui-of-hashicorp.md b/video-transcripts/transcripts/2023-12-18T23:04:41Z-otel-q-a-feat-luiz-aoqui-of-hashicorp.md index 1dc007d..d2e459b 100644 --- a/video-transcripts/transcripts/2023-12-18T23:04:41Z-otel-q-a-feat-luiz-aoqui-of-hashicorp.md +++ b/video-transcripts/transcripts/2023-12-18T23:04:41Z-otel-q-a-feat-luiz-aoqui-of-hashicorp.md @@ -10,83 +10,270 @@ URL: https://www.youtube.com/watch?v=HRIx9gJtECU ## Summary -In this Otel Q&A session, Luiz Aoqui, a developer on HashiCorp Nomad, discusses his experiences with integrating OpenTelemetry (OTel) into the Nomad workload orchestrator. He shares insights from his four-year journey with Nomad and how he discovered the need for distributed tracing to improve observability and debugging. The conversation covers three main approaches he took: the first being an ambitious attempt to instrument all aspects of Nomad, which proved to be overly complex and generated excessive overhead; the second aimed at enhancing user applications running on Nomad by automatically injecting relevant metadata into traces; and the final approach, which focuses on creating meaningful spans for specific actions within Nomad without altering function signatures. The discussion highlights the challenges of working with legacy code and the importance of finding a balance in telemetry data to avoid overwhelming users. The session concludes with acknowledgment of the evolving conversation around OTel implementations and the shared learning experience. +In this Otel Q&A session, Luiz Aoqui, a developer from HashiCorp working on the Nomad project, discusses his experiences with implementing OpenTelemetry (OTel) for distributed tracing in Nomad. The conversation covers his journey through three approaches to integrating OTel, beginning with an attempt to create extensive tracing across the entire Nomad workflow, which proved to be complex and generated significant overhead. Luiz highlights the importance of understanding Nomad's architecture, explaining its server-client structure and job scheduling process. He describes a shift in strategy to focus on more meaningful traces by using semantic conventions, allowing for easier filtering and visibility in the traces generated. The discussion emphasizes the challenges faced when working with older codebases and the need for thoughtful instrumentation. The session concludes with Luiz sharing insights and advice for those looking to implement OTel in their projects. The video will be available on the official OTel YouTube channel for further viewing. ## Chapters -Sure! Here are the key moments from the livestream along with their timestamps: +00:00:00 Welcome and intro +00:02:30 Nomad overview +00:11:46 OpenTelemetry integration attempts +00:19:00 First approach: Distributed tracing +00:25:50 Challenges with first approach +00:32:37 Second approach: User-focused instrumentation +00:39:00 UX challenges in second approach +00:44:00 Guidance from OpenTelemetry community +00:45:19 Final approach: Instrumenting Nomad +00:54:00 Closing remarks and Q&A -``` -00:00:00 Introductions and welcome to Luiz Aoqui -00:02:15 Luiz shares his background and experience with Nomad -00:03:45 Overview of Nomad architecture and components -00:05:30 Explanation of distributed tracing and its relevance to Nomad -00:10:20 Discussion on initial attempts at implementing OpenTelemetry in Nomad -00:12:50 Challenges faced with the first approach to distributed tracing -00:17:00 Introduction of the second approach focusing on user experience -00:23:30 Discussion on environment variables and their limitations -00:30:15 Transition to a more focused approach on key processes within Nomad -00:35:45 Explanation of how semantic conventions aid in tracing -00:40:00 Conclusion and reflection on the journey of implementing OpenTelemetry -``` +**Luiz:** Welcome everyone to Otel Q&A, and I am super excited to have Luiz Aoqui join me. He is a developer on HashiCorp Nomad. Welcome. -Feel free to let me know if you need any additional information! +[00:19:00] **Luiz Aoqui:** Thank you. Really excited to be here. always fun to talk OpenTelemetry. I mean, I haven't used it yet, but I dabble a lot, and I'll share more details of my adventures with OpenTelemetry. I guess I'll introduce myself first. As I mentioned, I'm Luiz, I'm an engineer, working at HashiCorp specifically on the Nomad project. I've been working on Nomad for like four plus years now. At some point, I don't remember exactly when, I bumped into OpenTelemetry, Distributed Tracing more specifically. It felt super interesting to me as a developer in Nomad because Nomad can be so opaque to debug, but also to implement and deploy and use in multiple different ways. I was like, Oh, it might be cool to do some distributed tracing there to get a sense of what's going on internally. -# Otel Q&A with Luiz Aoqui +And then through the course of several sort of like after our release, we usually do like an internal hack week just to cool down a little bit after that stressful moment. By the way, we just released Nomad 1.7, so if you want to try that, it's out now, which means we'll probably have a hack week soon and maybe I'll do some extra OpenTelemetry work, but that's a tangent. Yeah, I've tried three different work streams with OpenTelemetry Nomad, and I'll go into more details there, but each of them had sort of their own challenges and also nice things, but also bad things. I'll explain those. -Welcome everyone to the Otel Q&A! I am super excited to have Luiz Aoqui join me. He is a developer on HashiCorp Nomad. +[00:02:30] But yeah, that's it about me. I don't know if we should do a quick Nomad intro, because a lot of the things, since the idea of distributed tracing is to expose the internals of Nomad, I think I kind of need to explain a little bit about those. -**Luiz:** Thank you! Really excited to be here. It's always fun to talk OpenTelemetry. I mean, I haven't used it yet, but I dabble a lot, and I'll share more details of my adventures with OpenTelemetry. +**Speaker:** Okay, yeah, that's helpful. Why don't you go ahead? -### Introduction +**Luiz Aoqui:** Cool. Let's start there. I was going to do like PowerPoint, but Adrienne told me that's more like a Q&A and I also tend to overdo it in PowerPoint. So if you have something you can share, if not, that's cool. We are super chill. I'll just draw live, just because feel free to mute, ask questions. I don't have any specific agenda here, but I'll start there and then maybe that will help. The general idea is like, feel free to ask questions as I go. -So, as I mentioned, I'm Luiz, an engineer working at HashiCorp specifically on the Nomad project. I've been working on Nomad for over four years now. At some point, I bumped into OpenTelemetry, specifically Distributed Tracing, and it felt super interesting to me as a developer in Nomad because Nomad can be quite opaque to debug. Implementing, deploying, and using Nomad can be done in multiple different ways, so I thought it might be cool to do some distributed tracing to get a sense of what's going on internally. +Cool. So first of all, what's Nomad? Nomad is a workload orchestrator. Think of it like Kubernetes, you give it a magic file, that file becomes containers or other things, right? But that's what the orchestration aspect of this is. Now internally, it's important to understand some details of Nomad to understand what I'm going to show a bit later, is that the general architecture is like, you have two components, very simple: like server and client. These two components have different roles. -After our releases, we usually have an internal hack week just to cool down a bit after that stressful moment. By the way, we just released Nomad 1.7, so if you want to try that, it's out now! +So the server, and you usually have three or five servers, they maintain the core room, basically. They sort of are always talking to each other to maintain state. All of your jobs that you run, everything that you're running on your cluster is stored in the server. They have an internal database, you know, in the Kubernetes world, think of this as like etcd, but Nomad has its internal database. They all have the idea that they start a state of the cluster, and then they talk to each other to make sure that state is replicated and sort of available in case of a failure or something like that. -### Nomad Overview +So maybe I'll write bullet points. They store state, that's their main goal. The second goal is that they also do the scheduling. The scheduling process happens inside the server. All servers have several, what we call workers, which is quite confusing because I know that in the Kubernetes world, workers is something completely different. But the workers here are the scheduler workers. What the workers do, and each instance normally has like one per core on your machine by default, I believe. -Now, I guess I should introduce Nomad briefly because understanding the internals is crucial for the distributed tracing aspect. +What do workers do? They're like goroutines, you know, to go into a little bit. There's like a thread running on your servers and their goal is to, given a request to run something, so that would be like a job in a middle angle, given a job, where should I place things? So imagine that, I should move this. Imagine that I have a bunch of clients here. -**Nomad** is a workload orchestrator, similar to Kubernetes. You provide it with a configuration file, and that file becomes containers or other work. Internally, Nomad has a simple architecture with two main components: servers and clients. +Oh, I didn't talk about what clients do, but clients, they run stuff. Clients are the machines that are actually running your containers. They're actually running your workloads. That's the point of the clients. Normally, you can have like from like one to like 10,000 clients. So like to infinity and beyond here. Some people really try to push Nomad to its limit, but you can have a lot of clients on your cluster. -- **Servers:** You typically have three to five servers that maintain the core state of the cluster. They communicate with each other to ensure that the state is replicated and available in case of a failure. The servers also handle scheduling, determining where to place jobs based on the requests they receive. +The job of the scheduling is given a job or like, you know, like a YAML spec in the Kubernetes world, given this file, how do I translate this into specific work for each client? So let's say I want to run like three containers here on my file. The job of the scheduler is to say, okay, like I'm going to put two containers here on this client and then one container on this client. So that's what scheduling means. It's like, given the spec, how do I make it real? -- **Clients:** These are the machines that actually run your workloads. You can have anywhere from one to thousands of clients in your Nomad cluster. +And this thing that sort of makes things real, this is called a plan. So the goal of the workers here, the scheduling workers, is to create a plan given, you know, what we call the job spec. So that's what the scheduling process does. Cool. So those are the two main things that servers do. They store state, they do scheduling, and then the clients run stuff. When the client receives the work, like after the plan is generated, it gets validated, gets stored in state. The server says, okay, like this client here is going to run two containers. This client is going to run one container. -The scheduling process involves creating a "plan" based on the job specification, which is then executed by the clients. +So periodically, this client is going to go and ask the server like, Hey, give me, which allocations I should run. To translate a little bit, an allocation is like a pod in Kubernetes world. It's like the unit of workload that's going to run. Periodically, the client goes like, Hey, what should I be running? Since we just scheduled two containers, the server is going to reply, Hey, these are the two containers you should run. -### OpenTelemetry and Nomad +Then the client is going to start a few sort of internal components here. First, it's going to create an alloc runner, and the alloc runner is responsible for setting up the allocation environment. It's going to create folders in your file system, like folders inside that machine to store files. It's going to register the service, you know, either internally or register service in console. It's going to do things that prepare the way of the land, right? Call CNI plugins, all of that happens in the alloc runner. -I have tried three different work streams with OpenTelemetry in Nomad, each with its own challenges and benefits. My first attempt was to add distributed tracing throughout the entire process. The idea was to create a single trace for the entire process, so I wanted to map the process of running something end-to-end — from submitting a job to getting visibility into what happens inside the cluster. +And then just like a pod can have multiple containers, an allocation can have multiple tasks. The alloc runner for each task inside your allocation is going to create a thing called a task runner. This task runner, kind of like the alloc runner, is going to set up the environment for the tasks, like for the specific container. It's going to create like environment variables, it's going to download files, download the Docker image, whatever. Everything for that specific workload to run. -This approach, which I refer to as "boil the ocean," aimed to trace everything at once. However, it created a lot of overhead and many tiny spans that weren't particularly useful. +And then the task runner is going to start, like, so like the end result of the task runner is like a container or a binary or whatever you want to run here. It creates like a process on the machine. From that file that is specified, what you want to run, you give that to the server. -### Challenges Faced +The server is going to send it to the... Oh, I forgot to mention the process of how it gets to the worker. Okay. So normally you have three or five servers and one of them becomes the leader. The leader is the only one that can actually write state to disk and write the final, like the global, the single source of truth is stored in the leader. The leader also has a thing called the queue. That's called the eval queue. The eval queue is like work to be done. An eval is like something that has to happen in my cluster. -1. **Overhead:** Each span created was very short-lived, leading to excessive data transfer and storage. +That something that's described in an eval goes into the queue, and then the other servers sort of dequeue from there to find work to do. The workers are dequeuing work, processing that eval, generating a plan. That plan becomes state, so like gets written into the global cluster state, and then the clients are sort of like asking for updates on that state to figure out what needs to run, what needs to be stopped, and all of that. I think that's it. That was a lot. Let me know if there are any questions on this part. Hopefully, it all made sense. -2. **Code Changes:** The implementation required significant changes to the Nomad codebase, including wrapping HTTP handlers, which made the process quite messy. +[00:11:46] So let's get to the actual stuff that I did. This is the Nomad repo. There are a few, so if you search for OTEL branches, you're going to find my previous adventures putting OpenTelemetry enrollment. I'm going to go sort of, and you can see, it's been a while since I've been trying to do this properly. I think I'll go probably chronologically here. -3. **Function Boundaries:** Distributed tracing is usually designed for microservices, but Nomad operates more like a monolith, complicating the process of tracing across function calls. +So the first attempt, when I learned about this, it was like, cool, let's put distributed traces everywhere. My goal here was for, given all of this process that happens, it should all like, in my mind, it was like, Oh, this is like one process. Everything here should be a single trace or like have a trace, a unique trace ID that, Oh, this could be a span. When it goes to the queue, that's another span. When it gets dequeued, that's like creating a bunch of spans in a huge tree. -4. **Consistency:** There was a lack of consistency in the metadata added to the spans, making it challenging to derive useful insights. +I wanted to map this process of running something end to end, you know, as a user, submit a job, how do I get visibility of what's happening inside the cluster? That's the approach that I like to call boil the ocean because I wanted to do everything at once. First, let's run this branch, I guess, to see what that looks like. Let's hope that the two-year code works. I was testing this yesterday, but I think it will work. I'm also going to start the OpenTelemetry demo repo that I found. I need some sort of setup to run, like I need Jaeger, and I need a collector, and I need an application to test. -### New Approaches +So I'm just going to start the whole demo here just to have something to generate stuff, to generate data. Okay. Deprecations or warnings are not errors. So let's assume that this worked. If you look at the branch, let's start looking at the diff. One of the things that I did, or I had to do was to have a sim config, additional configuration for Nomad. We have it on, like when you start a Nomad agent, you need to pass a configuration file. -After some guidance from the OpenTelemetry community, I shifted my perspective. Instead of trying to instrument everything at once, I focused on core aspects of Nomad that would provide the most value. +So I added like configuration for OpenTelemetry endpoint and just to explain to like, where to send all the OpenTelemetry data. I think I have it here. So like telemetry, OpenTelemetry endpoint, localhost for 3.7. That's where I'm running the OpenTelemetry collector from the demo and insecure because I'm not using TLS. I think that's all I need. -- **Increased Granularity:** The new approach involved creating smaller, more meaningful spans rather than a single mega trace. This allowed for easier filtering of traces based on standardized attributes. +If you haven't used Nomad before, Nomad Agent Dev is a quick way to start. It gives you like a fully functioning cluster locally. It starts a client, starts a server in the same process. It's a handy way to try stuff. It's also ephemeral. Once it goes down, it destroys everything created, which is nice for cleanups, but sometimes bad because sometimes you do want to keep stuff. But it's a good way to quickstart with Nomad. -- **Semantic Conventions:** I implemented semantic conventions for Nomad, allowing for better organization and filtering of trace data, making it easier for operators to find relevant information. +I'm going to read this config with the additional OpenTelemetry configuration. So let's start that. I'm actually also going to run a job. If you haven't seen it before, a Nomad job looks like this. It uses the HCL language, sort of like Terraform does. There's not a whole lot to go over this right now. I think the main thing is, it's kind of hard to map to Kubernetes concepts. -### Conclusion +I'll try my best, but like a group is kind of like a deployment. It tells you what to run, like how to run things. In the group, you can have a count, how many instances you want to run, stuff like that. The task is kind of like, or like the groups are kind of like the pod spec. It tells you what to run and how to configure stuff. The task is like a container definition inside a pod. You tell, Nomad doesn't only run containers, so you can do other stuff, and that's plugins, so those are test drivers. -This journey has been enlightening, and I hope everyone gained insights from my attempts to integrate OpenTelemetry with Nomad. +The config is how you configure the test driver. This is just like saying, hey, I want to run a Redis container. It's going to expose this port and give it these resources to run. So fairly simple job here. When I run that, it creates evals. The evaluation because something changed in my cluster. That's how I communicate changes in Nomad, is through these evals. -Thank you for joining this Q&A session! We will be posting a recording of this on the OTEL YouTube channel (OTEL - Official). Keep an eye out for the announcement on social media and Slack once it's up. +That eval created an allocation. The allocation is like the pod. It's the thing that is actually running. And it works. It's all running. If I go to the Nomad UI... Oh, this is a node version of the previous. There's a special flag that I had to run, but let's hope it's actually running. -**Luiz:** Thank you for having me! I appreciate everyone's participation. Bye! +Now, if I go to Jaeger, which is where... here, I will see a lot of services. Those are for all the demo stuff. But I also see two Nomad services here. So on this boil the ocean approach, as I mentioned, I wanted to get the whole flow end to end, from user to like the clients. I started from the CLI. So you can see here, there's a CLI, Nomad CLI, what's it called? Service with a job run action here. + +It's kind of small to see. I started putting traces everywhere I found stuff. It's one huge trace with a bunch of different spans for each individual action that is happening. You can see here, the Nomad CLI ran the job run command. I think there's like, I don't remember how much extra data I put. I was experimenting with all of this stuff. + +So I could put log entries here. There's some Nomad CLI job run command ran that CLI made an HTTP request. So they put request to this endpoint here. That put request became a job request in the FSM. The FSM, for those who haven't heard this term before, stands for finite state machine. That's how data replication works. + +Imagine you're trying to help your friend find your house. Normally, you tell them to go to Google, but imagine you live in the middle of nowhere, there's no GPS. You need to give them instructions on how to find your house. You know, drive two miles east, turn right, drive another, how many miles, turn left. You give specific instructions. Now imagine you're not just telling one friend, but multiple friends. You kind of give that instructions to all of them at the same time, so they can reach the same destination. + +That's what the FSM does. It gives an instruction. It mutates the state in a consistent way across all servers. Hopefully, that made sense. I think I got confused myself, but let me know if that wasn't very clear. + +So it creates a job request in... So now we moved away from the CLI. That's the HTTP request that was made. Now the Nomad agent is who received that HTTP request and turned it into an internal request for a register. That's a job register command. I caused the job register method internally. You can see here, I'm putting logs and stuff, trying to figure out how the OpenTelemetry library works. + +After we registered that, we have these things called emission controllers, which there are two kinds of them. They're like the mutators and the validators. The mutators are the things that we need to mutate when the job comes in. For example, if you're using vault in your job, we mutate your job to put a constraint to say, "Hey, only run this on clients that have vault." Otherwise, your job will never succeed. + +That's what the mutator does. It mutates the input. There are multiple of them. One of them kind of, I hate this word, canonicalizes the input. It sets default values that have not been specified by the user just to make sure that the job is consistent. Console connect, expose checks, and all of these are separate things that are mutating the job. + +Then we have validators. They validate the job. Then it goes through… So like, Raft is the mechanism we use to propagate changes across our servers to make sure that those changes are consistent in all the servers. So it goes to Raft. There are a bunch of methods that are called there. Then it goes to NQ. + +That's the queue that I mentioned about the eval broker queue. It's like, here is some piece of work that needs to be done. Now it's in the queue. You can see here that it got dequeued later on by another server. And what's this? Oh yeah, I don't know what this is. + +Yeah, and then like got dequeued by somebody to process and create that plan that I mentioned. So like... And then, yeah, there's a lot happening here, which is kind of neat, but also kind of overrun. After the CLI made the put request, it keeps monitoring the eval to get status updates. It makes a bunch of HTTP GET requests just to get the update for that eval. + +It starts monitoring the deployment and again, starts making a bunch of HTTP requests. The gist of it is like that whole end-to-end approach. I wanted to create a span that showed me what happens after I run Nomad job run because from the user perspective, all you see is just this output, which is quite confusing if you're not used to Nomad and quite tricky to debug. If you're an ops person and your job runs in a pipeline somewhere, you really don't have a lot of visibility of what happened or what's happening. + +That was my first attempt. It was like, how do I map this process end to end and give as much detail as possible to the cluster operators? Now why this was not a good approach is one, because it creates a bunch of overheads. If you look at all the different spans, they're like zero microseconds long or like 50 micro. That's not very useful, I guess. It's nice to note about them, but in a practical sense, it creates a lot of overhead. Each of these has a lot of data in them that needs to go through the network, needs to be stored somewhere. + +It creates a lot of overhead for every time a job runs, for example. What happens when you drill down into one of the spans? What kind of information does it send you? + +[00:25:50] I don't think I put a whole lot. It also depends on the span. Sometimes I put data. Sometimes I think I did and where was it to Q? Maybe I put some Nomad specific data. Oh, here, like the eval ID, for example. I was trying to experiment with adding metadata to different spans, but it wasn't very consistent. Some of them have additional data. Others, there's just the standard OpenTelemetry SDK data. + +So the first problem was the overhead. The second problem is that if you look at the diff, there's a lot of code changes. For example, let's look at... It's like the HTTP handlers. OpenTelemetry does have a nice wrapper. It does have a nice wrapper for like Go HTTP, but we don't quite use that. I had to manually wrap each endpoint in a function that had to use reflect to figure out which RPC should be called. It's kind of messy, but not too bad. + +The actual bad part is that normally distributed tracing, I think it was like created with the idea of microservices. There are like a metric boundary more or less between your services. Since Nomad is more like a monolith, like each component of Nomad is a big monolith, a lot of this, there's like boundaries defined by function calls. I had to modify all the functions that I needed to monitor, like that I wanted to instrument. + +Had to be modified to take a context, for example, as a new argument to keep propagating that span forward. Using function as the boundary for your telemetry is not great because it just becomes like, it's not as transparent as monitoring the metric layer, right? There's no way to do it automatically, and it just requires so much code change. + +Yeah, after I got... and I didn't even go through the whole process. If you notice, I stopped at the dequeues. After dequeues, the span sort of ends because I just couldn't keep going with all these code changes. Most of this PR is just adding context as the first argument to a bunch of functions, which is not bad by itself. A lot of functions having a context there can be helpful, especially for async work. But a lot of times it was not, it was just to keep the span going. + +So there's a lot of code changes for not a whole lot of... not necessarily not a lot of benefit, but like a lot of code changes that were not the best changes to make. + +**Matthew:** Oh, hi, Matthew. Yes, I can send. I'll send you the list of all the branches, I guess. + +**Luiz Aoqui:** Oh yeah, thanks. Also, hi, long time no see. + +**Matthew:** Hey, nice to see you. + +**Luiz Aoqui:** Yeah, so these are all the branches. Once you click on the branch, you can see GitHub will give you a link for like one commit ahead, and that's the deal. + +So this approach, I think, is very nice to have this view. Jaeger has some nice things about giving you like how the systems interact. I could make this bit like each service could be like a different server and I'll have three servers, multiple clients, and sort of have that view. But I don't think that was the right approach. It feels like it felt like going against the principles of distributed tracing, which is like you instrument your microservice and then the network becomes sort of your boundary of when to move the span. + +I was doing it at the functional level, which felt very, very fine grain to what I was doing. Just before you move on, you mentioned that you'd done some, you had to like create some wrappers yourself around some of the OTEL calls. + +**Matthew:** Yeah, all of these. + +**Luiz Aoqui:** How come you had to do that? Sorry, I missed your reasoning on that. + +**Matthew:** Oh yeah, so normally the SDK does it automatically for you if you're using a framework like net/http, or I think it does support the Gorilla Mux. Normally you just import the library and it sort of does the magic for you. But the way we do it in Nomad, it's like another thing to keep in mind is that Nomad is a very old project. A lot of people don't know how old Nomad is, but it's as old as Kubernetes. I think Nomad is like one month younger than Kubernetes. The V1 came out sort of around the same time. + +So that was like 2017, I want to say. Nomad is a very old code base. A lot of the things we do predate a lot of the new sort of code niceties. The whole HTTP layer, the whole RPC layer is something that we had to create because there wasn't a standard or a more common way to do it. + +**Matthew:** Gotcha. Gotcha. That's very interesting. I think especially hits on an interesting point for folks who are looking to incorporate OpenTelemetry into their existing projects. Oftentimes, we are dealing with very old code bases, and that's not something that we discuss enough. So I'm definitely happy that you're bringing this to light because I'm sure you're not the first and only person to encounter this. + +**Luiz Aoqui:** So yeah, no worries. But like it did, as you can see, the rep is fairly simple and does use the SDK quite a lot, but it's just like the way the project was was not the way that usually Go HTTP handlers are written nowadays. That was kind of why I had to do this. + +[00:32:37] Cool. So that was the first approach. I kind of gave up because it was getting too complex and the hack was done, so I didn't have a lot more time to work on. The second approach, I sort of shifted the perspective a little bit. Instead of trying to instrument Nomad itself, the idea was to make it easier for people using OpenTelemetry and using them to instrument their own applications. + +What I did in this branch was, here's the test runner. Going back to the task runner, it's the thing that runs your container, that actually runs your workload. You set up the environment to run your workload. What I did here was I started adding... If you use OpenTelemetry before, you know this environment variable will tell resource attributes, which I think is part of the spec. + +It defines values that are automatically added to your span context or whatever you create with OpenTelemetry. This environment variable sort of creates standardized values to or like metadata to add to those resources that you create. What I did here... + +Oops, this one is like, I say, okay, if you're running, if you're already using OpenTelemetry and you're running Nomad, could be helpful to automatically get Nomad data for your OpenTelemetry stuff. So having the alloc ID, the alloc name, the evals, like all of this data sort of gets injected automatically for you by Nomad itself. + +So this whole code, what it's doing is just setting the environment variable with a bunch of Nomad data. Let's take a look at how that works in practice. I think what you're doing here is more or less like what Kubernetes does as well now when it emits telemetry data that you have that kind of free with purchase, environment variables. + +**Luiz Aoqui:** Right. Which is quite nice. Yeah. Like I think that information about the fault information about it, like that sort of just happens automatically for you when you're using the OpenTelemetry SDK and all the collector. So let's run this. I don't think I need the config because I'm not sending any data. + +So I'm just starting Nomad and, oh yeah, I remember now. So I need... I'm running Nomad. Nomad is going to automatically instrument my application. So I need to run something in Nomad that uses OpenTelemetry. I found this project. I just Google like OpenTelemetry simple job or Docker image, so I found this OTEL gen project here that just generates traces for you. + +It runs that image and points to my Docker container here that is running the collector and generating multiple traces. It has a batch job, so it's just going to run once. When I run that... + +**Matthew:** Sorry, what does that batch job do? + +**Luiz Aoqui:** It generates OpenTelemetry data. Let me find that repo here. Yeah, so it's this project, which is very helpful. It just generates... You can tell that you generate metrics, traces, and then it just creates sample data for you in the OpenTelemetry format. + +**Matthew:** And is it supposed to be like, as you said, it's just sample data. You're not actually using it for anything other than for test purposes? + +**Luiz Aoqui:** Kind of thing. Yeah, it's just like, imagine this is like your application that is generating traces. + +**Matthew:** Got it. Got it. Cool. + +**Luiz Aoqui:** And that application is running. So let's look at the data it generated. So it created three traces here, three traces, say that fast two times. + +Now, if you look at each trace, it now has all the Nomad stuff. It has like, this was like a GitHub project, so I did not change any source code at all. But just by the fact that it's running Nomad, Nomad itself is instrumenting, like populating a lot of metadata for each trace, for each span. It gets the alloc ID, the alloc name, the eval, the group names. + +If you are a user of Nomad and if you're already using OpenTelemetry, all the things that you're running in Nomad with OpenTelemetry will automatically have all of this additional context and data for you. I didn't know how much data to go, so I put like a lot, but this is to help the data. It's not instrumenting Nomad itself, but to help people that already instrumented their applications. + +When they're running them, they get additional benefit. That was a different approach to OpenTelemetry. The problem with this approach was one, I didn't know what to put. So I just put a lot of things. A lot of this is probably not useful, like job type, who cares? Because, you know, metadata is good, but too meta, too much metadata. + +[00:39:00] It's also tricky because you increase your metric packet size. You need to store this somewhere, so it kind of balloons your store. The challenge with this approach was just to how do we... It's more like a UX problem. + +How do we allow people to control what information they want, or like, do we allow them to control or do we hard code some values? Figuring that balance was a little tricky. Looking at the code, I think this is just like, yeah, I just hard coded a bunch of stuff because again, this is a Hack Week project. + +One of the problems with the other approach is that if you are already using OpenTelemetry, you're probably doing something like this. You may already be adding your own values here, right? You may already have your own environment variable for your attributes out there. If I run this job or this version of the job, now that code that I initially wrote would kind of overwrite that. + +In this other approach, I instead of overriding, I merged the changes just to be nice for the user. What I think... Oh yeah, I actually found a bug in the SDK here, which wasn't decoding the values as it was describing the spec. + +But then we actually figured out that there are like two versions of the spec in place. Then I had to go and change this back. There's like a whole side quest that I had to go through to fix this. I think it hasn't been... Let me do a check. + +So I changed this back, which means every SDK had to be updated. And there's the link. I think there's still some open ones. People are looking for, you know, contributing job and telemetry. This might be a good first issue. They're like sort of simple. It's just changing all the... + +That variable gets decoded, and there are still some languages that have not been fixed. So yeah, for contributions, this could be a good first issue to look into. + +Yeah, so that's the second approach. Good for users, not so much instrumenting Nomad itself, but could be useful. Just the UX paper cuts were the major drawbacks there. Nothing OpenTelemetry specific, just how do you do surface text? + +Oh, one thing, another downside of this approach is that since it's using environment variables, those values are rather static. You cannot modify an environment variable to a container that is already running. If you want to put data that is not known ahead of time or some data that can change throughout the lifecycle of that container, this approach doesn't quite work because the environment variable is already created. So it's not great for dynamic values or things like that. + +[00:44:00] That was the second approach. Each approach, like a year apart. After doing all of that, I think it's sort of when I met, right after this, or like I met Adrienne and the other folks at the OpenTelemetry community. I started to get some guidance of like, you know, Hey, I'm trying to do this, you know, going back to the original approach. It's not quite working because of all of these reasons. + +Speaking with Adrienne and I think Ted was also very helpful in giving feedback on this. It's like, don't try to boil the ocean. You don't need to instrument everything at once. You can start small. Find your core, I think the exec suggested, find your core business instrument that, and then you get a lot of value out of that already. + +Translating that into the Nomad world to me, it felt like, Oh, okay, like I don't need to instrument everything end to end. We already have this eval thing that is like the unit of work. By instrumenting that flow of the eval, maybe I can get enough information out of this to be useful. + +[00:45:19] So let's in practice how that works. Go to this branch now, and I don't think I need a config for this one. Let's run this version. What this version does is like, it's going back to the first implementation, like I want to instrument Nomad itself. This is adding traces and spans and all of that into the Nomad code itself to instrument Nomad for operators. + +So going back to my traces here, nothing happened because I didn't run the job. Let's run a job to do some work. I don't need to wait. + +**Matthew:** Yeah. + +**Luiz Aoqui:** So now I have a service called Nomad. If you look at the traces now, instead of, you know, let's compare it to the previous implementation. Before, I had... I think I... Oh yeah, I think I might have deleted... but before I had like one huge trace of everything. Now instead of going and trying to keep the context around each function calls, what I did was more... + +You can see here, there's a bunch of traces instead of a single one. What I did was... I didn't try to connect all those function calls, but it was more about instrumenting the specific aspects. Like here in the alloc runner, when the alloc runner starts, you create a new span. I didn't worry about keeping the same span going. It's like, okay, this is a new action that is happening. It's going to be a new span. + +So I guess the approach was being more mindful of what you wanted your traces to be. Because you went from a mega trace to a bunch of smaller traces, where these smaller traces then have more meaningful information to you, is that? + +**Matthew:** Yeah, exactly. It's like changing the perspective a bit instead of keeping the gigantic trace, be more serious about like, okay, this function is important. This function is going to create a trace. + +What this means is I don't need to change my function signatures. I don't need to keep passing context around. I can just, whenever something important happens, I just create a trace there. The downside, as you can see, is that there are a lot of traces here. + +But a good thing, something that I learned later on is that, well, I knew this already, but I didn't know how to use it specifically, but traces have attributes. I think that was the key to this approach. There's this notion of semantic conventions in the OpenTelemetry SDK. For those who don't know, it's that of predefined attributes for your... just like Docker has, or I guess they're called containers, like your container engine has predefined keys that it's supposed to use. Kubernetes has its own, AWS has their own attributes or standardized attributes that they use. + +So the key to this approach was like, okay, what does that look for Nomad? I think it was at the bottom. Yeah. So I created semantic conventions for like, Oh, what does a Nomad region key look like? Oh, it's Nomad.region. What does a Nomad space key? Nomad.space. + +This allows each span, you know, these are all like different spans coming from all different parts of the code, but they have standardized attributes. What this allows me to do is like, I could come here and say, Oh, I actually want job ID to be example. I can use these values to filter all this mess that nobody's ever done. + +It's not like a nice single huge trace. It's just a bunch of different disconnected traces. But by using the semantic conventions, I can help find and filter things better. These are all the traces for, you know, this job here. But let's say I go to the UI now. + +This job created an allocation, for example. Let me get the allocation ID here. I said, I don't want the whole job. I just want this allocation. So now it's like all the traces for that allocation because even though they're different traces coming from different parts of the code, different parts of Nomad, they all share the same conventions. + +That allows me as a cluster operator to go and filter out all the "noise" that gets generated to be more meaningful to me. You can see here the same things that we saw in the big trace, but now they're just breaking down. + +So like the alloc runner, the state stores, like the database. I also started experimenting with like the log events as well. Instead of having... Let me find it. I think that might be more interesting data there. + +This is when I registered a job. Where did I get the ID? Okay. Is this... Oops. Again, maybe this... I don't remember how I did it. Let's remove all traces and then I'll find manually. + +I was looking for... Yeah. On... Sorry, the Zoom window is going to go. + +**Matthew:** See it? + +**Luiz Aoqui:** Yeah. Oh, maybe I'll find it. Share later. But the idea is that I started to experiment with logs a lot more. + +**Matthew:** Are you referring to just span events or actual OTEL logs? I'm just curious. + +**Luiz Aoqui:** Yeah. Yeah, events. Data. Let me run the job again. Let me take a look at this. Yeah. So like... So for example, this is the trace, the sort of like the scheduler part, the worker scheduler, all of that. + +I started putting logs that like, as it was scheduling, it's like all the scheduling decisions are like part of the trace now. So you can see like why did this change happen? You can see here as part of events in your span. Instead of having the multiple tiny traces, those function calls sort of become more events instead of a new span. + +**Matthew:** Yeah, that's a strange point. That's awesome. + +[00:54:00] **Luiz Aoqui:** We are at time, just FYI, an hour blew by really fast. + +**Matthew:** Yeah, that was what I showed. That was the last attempt that I made. If you have questions, sorry, I missed chat. + +**Luiz Aoqui:** You have to add attributes to each trace. + +Oh yeah. So the code to add the attributes, you do have to call for each trace. But you can kind of make it like helper functions to do like... Let me find an example here. Yeah. + +You can pass in like helper functions like, okay, given this allocation, this pod, give me all the attributes for that. You do have for each trace, but you can create helper functions too to ease the burden. + +**Matthew:** That is so awesome. Thanks for sharing your OTEL journey with us, Luiz. This has been super, super helpful. I hope everyone on this call got something out of it. + +I think it's so important to have these kinds of conversations because I think we're moving past the, you know, let's intro to OTEL. We're now getting into like more people doing really digging deep into their OTEL implementations, and hearing about the challenges that you've experienced and the different things that you've tried has been really awesome. + +So thank you so much for sharing this with us. For those of you on the call, tell your friends who couldn't make it that we will be posting a recording of this on the OTEL YouTube channel. The YouTube channel is OTEL-official, so just keep an eye out. I'll post on socials and also on Slack once we have the recording up and running. + +Thank you everyone for joining. Really appreciate it. + +**Luiz Aoqui:** Thank you everyone for coming and thank you for having me. + +**Matthew:** Bye. + +**Luiz Aoqui:** Thank you. ## Raw YouTube Transcript diff --git a/video-transcripts/transcripts/2023-12-20T04:44:51Z-the-humans-of-otel-kubecon-na-2023.md b/video-transcripts/transcripts/2023-12-20T04:44:51Z-the-humans-of-otel-kubecon-na-2023.md index d71667e..1f9a986 100644 --- a/video-transcripts/transcripts/2023-12-20T04:44:51Z-the-humans-of-otel-kubecon-na-2023.md +++ b/video-transcripts/transcripts/2023-12-20T04:44:51Z-the-humans-of-otel-kubecon-na-2023.md @@ -10,129 +10,166 @@ URL: https://www.youtube.com/watch?v=coPrhP_7lVU ## Summary -In this YouTube video, a group of OpenTelemetry contributors and maintainers, including Tyler Yahn, Amy Tobey, Carter Socha, and others, discuss the evolution and significance of Observability in software systems. The conversation covers the integration of logs, metrics, and traces as essential components of Observability, emphasizing the importance of having these signals connected to provide valuable insights during troubleshooting and system monitoring. The participants share their personal definitions of Observability, which range from understanding application behavior to being able to address issues swiftly in production environments. They also highlight the collaborative nature of the OpenTelemetry project, its journey from various predecessor projects, and the community's efforts to create a vendor-neutral standard for telemetry data. Throughout the discussion, the group expresses a strong preference for traces as the most effective telemetry signal, citing their depth and usefulness in operational contexts. +In this YouTube video, Tyler Yahn, a maintainer for the OpenTelemetry Go SIG, leads a discussion with several other contributors to OpenTelemetry, including Amy Tobey, Carter Socha, Juraci, and others, about the concept of observability in software systems. They explore the evolution of observability from siloed systems of logs, metrics, and traces to an integrated approach that allows for better data analysis and quicker problem resolution. The participants share their personal definitions of observability, emphasizing its role in understanding system behavior and improving application performance. They also discuss the significance of OpenTelemetry as a community-driven initiative that standardizes telemetry data collection across various platforms, minimizing vendor lock-in. Key points include the importance of integrating different telemetry signals, the role of OpenTelemetry in modern software development, and the collaborative nature of the project. Overall, the conversation highlights how effective observability can enhance operational efficiency and user experience. ## Chapters -Here are the key moments from the livestream along with their timestamps: +00:00:00 Welcome and introductions +00:04:20 Observability discussion +00:10:40 Observability definitions +00:12:30 OpenTelemetry involvement +00:15:00 OpenTelemetry definition +00:18:42 Community collaboration +00:20:30 Telemetry signal preferences +00:23:10 Traces vs. metrics discussion -00:00:00 Introductions of speakers and their roles in OpenTelemetry -00:02:30 Discussion about the OTel CLI tool and its features -00:04:00 Overview of the importance of Observability in software systems -00:06:30 Different definitions of Observability from various speakers -00:10:00 The evolution of Observability and the integration of metrics, logs, and traces -00:13:45 The significance of having unified telemetry data for better analysis -00:17:00 Speakers share personal experiences that led them to OpenTelemetry -00:20:30 The community aspect of OpenTelemetry and its impact on the Observability landscape -00:25:00 Favorite telemetry signals and why traces are favored by many -00:28:00 Closing thoughts on the future of OpenTelemetry and Observability +**Tyler:** I'm Tyler Yahn. I am a maintainer for the OpenTelemetry Go SIG. We're working on some auto-instrumentation there, and specification. -Feel free to ask for more details or specific moments! +**Amy:** I'm Amy Tobey. I am senior principal engineer for digital interconnection at Equinix. I maintain a tool called OTel CLI. -# OpenTelemetry Community Discussion +**Tyler:** Oh, you maintain the OTel CLI! -**Participants:** -- Tyler Yahn, Maintainer for OpenTelemetry Go SIG -- Amy Tobey, Senior Principal Engineer at Equinix -- Carter Socha, Product Manager and Maintainer of OpenTelemetry Demo -- Bogdan, Former Maintainer of Java and Collector -- Constance Caramanolis, Contributor to OpenTelemetry Collector -- Juraci, Software Engineer and Collector Developer -- Jacob Aronoff, Maintainer for OpenTelemetry Operator -- Alex, Contributor and Maintainer of OpenTelemetry -- Purvi, Senior Software Engineer +**Amy:** Yeah, that's my project. Yeah. So that mostly just has Traces right now. And I've been meaning to implement Logs and Metrics for a while and I think like Logs just went GA recently, so it's time to do it. But Traces have been so effective and people really like it that I haven't really had a lot of demand for them. ---- +**Tyler:** Is the OTel CLI part of OpenTelemetry? -## Introductions +**Amy:** It's not yet. I maintain it on our Equinix Labs GitHub account. It's not a lot of process. Mostly it's just me with a few folks like Alex and others that throw me a PR every now and then. But I've thought about bringing it back to the community. But I'd have to maybe be not as so far away from the standards as I am right now because doing it in the command line, a lot of the standards don't really translate very well. So I've strayed a little bit away from the standards in a few places. -**Tyler:** I'm Tyler Yahn, a maintainer for the OpenTelemetry Go SIG. We're working on some auto-instrumentation and specification. +**Tyler:** It would make sense. And I've talked to Austin about it. -**Amy:** I'm Amy Tobey, a senior principal engineer for digital interconnection at Equinix. I maintain a tool called OTel CLI. +**Carter:** Hey, I got Ted Young with me! Hello! I'm Carter Socha. I work on a couple different things. I'm one of the few product managers floating around, but I helped start the OpenTelemetry Demo, which I'm a maintainer of. I also work in the SIG security, which helps the project improve their security response process. -**Carter:** Hi, I'm Carter Socha. I work on a couple of different things. I'm one of the few product managers floating around, but I helped start the OpenTelemetry Demo, which I'm a maintainer of. I also work in the SIG security to help improve the project's security response process. +**Bogdan:** My name is Bogdan. I took a break for parental leave so I'm just jumping back. -**Bogdan:** My name is Bogdan. I took a break for parental leave, so I'm just jumping back in. I’ve done a lot of things, including being a member of the TC and GC, and I was a maintainer of the Collector. +**Tyler:** Okay, what were you doing before? -**Constance:** Hi, I’m Constance Caramanolis. I’m one of the OG contributors to OpenTelemetry. I worked on the OpenTelemetry Collector and contributed to the configuration aspects. +**Bogdan:** I done a lot of things, including member of TC, member of GC, maintainer of Collector. I was a former maintainer of Java, so I've done a lot. -**Juraci:** My name is Juraci. I'm a software engineer and have been working with OpenTelemetry or Observability for a few years now. I was a maintainer on Jaeger and part of OpenTracing. +**Constance:** Hi. I'm Constance Caramanolis. I know that you were involved in OpenTelemetry and you are one of the OG contributors. Tell us about that involvement. -**Jacob:** I’m Jacob Aronoff, a maintainer for the OpenTelemetry Operator project. +**Bogdan:** Yeah, so I worked on the OpenTelemetry Collector. I contributed to that. I did a lot of config things. I was also on the OpenTelemetry Governance Committee. So I did a lot of the start, we were doing the incubation process, starting a whole gathering process, a POC, a lot of putting processes in place, getting adoption. Quite a few talks. KubeCon talks... -**Alex:** Hi, I'm Alex, a contributor and maintainer in OpenTelemetry. I even wrote a book about it. +**Juraci:** My name is Juraci. I'm a software engineer and I've been working with OpenTelemetry systems or Observability for a few years now. I come from a Tracing background, so I was a maintainer on Jaeger. I was part of OpenTracing back in the day and I helped choose the name of the project that we have. And right now I'm a Collector developer. I help out on some components for OpenTelemetry Collector. And I'm also part of the Governing Committee for OpenTelemetry. -**Purvi:** Hey, my name is Purvi. I am a senior software engineer and have worked a lot with browsers and JavaScript. +**Carter:** And newly re-elected, right? ---- +**Juraci:** I was just re-elected, yes. -## Observability Discussion +**Jacob:** My name is Jacob Aronoff. I am a maintainer for the OpenTelemetry Operator project. -**Purvi:** So, what does Observability mean to you? +**Alex:** Hi, I'm Alex and I'm a contributor and maintainer in OpenTelemetry. I wrote a book about OpenTelemetry. I don't know what else. I do stuff with OTel. -**Carter:** To me, Observability means that when you wake up at 2:00 AM to fix a problem, you can resolve it. Ideally, you can revisit that code the next day and find a way to prevent the issue from occurring again. +**Jacob:** Cool. I am a contributor and maintainer of the OpenTelemetry Collector and the OpenTelemetry Collector Contrib repo and I have been spending a lot of time in various SIGs and specialty working groups around configuration and security. And previously I spent a bunch of time maintaining and contributing to Python. -**Bogdan:** I think Observability is the capability of monitoring your production environment and determining when something goes wrong. +**Purvi:** Hey, my name is Purvi. I am a senior software engineer. I worked over my career a lot with browsers and javascript. -**Alex:** I view it as a tool to ensure things are working as intended. It's about gaining insight into both black and white boxes. +[00:04:20] **Carter:** First question. We're here talking about Observability. So what does Observability mean to you? -**Constance:** I like to think of it as a murder mystery. You see clues and have many questions, and you use Observability to figure out what's going on. +**Purvi:** Yeah, that's a great question. Personally, I think Observability means that when you woke up at 02:00 a.m. to go fix a problem, you can fix it. And ideally, the next day you're able to look at that code again and find out a way to never have that problem exist. I think that's really what it means to me. It means being able to look at things coming out of the box and tell what's going on inside parts. Be very convenient. I like that. -**Juraci:** It allows us to understand problems in our systems without needing to know what's going wrong ahead of time. Observability is a spectrum; we won't achieve perfect Observability from day one, but we should have some telemetry to help us understand our systems. +**Jacob:** First of all, it's monitoring... But really, Observability is this nebulous term, but it did show up as part of a sort of shift in how we are thinking about monitoring our system. And I would say that shift is the way we used to do it was you had these different signals, you needed logs, so you had a logging system, you needed metrics, you made a metric system, you needed tracing, but you didn't know what that was, so you didn't do it. And instead of having these three separate, totally siloed systems, what we've been doing over the past couple of years, especially in the OpenTelemetry project, is trying to say it's really bad for these three things to be separate. Or the four things, if you include profiling. -**Amy:** Observability means understanding what's happening inside your applications, particularly in the code that matters to you. +When you're using these tools, you use them together, you're moving back and forth between them, right? Like you get an alert based off of a metric that you set up. But when that alert goes off because errors or something spiked, the next thing you want is to look at the logs that are in the transactions that are causing these alerts. You want to look at the logs that are in a particular transaction. You really want to have a trace ID stapled to all those logs, so you can actually look them up. -**Jacob:** For me, Observability is essential. When something goes wrong, I can ask questions about my system and figure out what happened without needing to know exactly what to expect. +So we want to actually use all these tools together. And in order to use all of these tools together, you need to have the data coming in, the telemetry actually be integrated, so you can't have three separate streams of telemetry. And then on the back-end, be like, I want to cross-reference. All of that telemetry has to be organized into an actual graph. You need a graphical data structure that all these individual signals are a part of. For me, that is what modern Observability is all about. It's about having all this data connected into a graph in such a way that we can leverage the machine to do what they're good at, to reduce the amount of time we need to spend investigating issues. ---- +Instead of being like, I wonder if this is the problem. Therefore I am going to collect all the logs and grep through them, try to whittle it down to something. I'm going to look at all the config files myself, try to figure out what's going on. You can just quickly get an answer to a lot of those questions and then move on to the next hypothesis. The amount of time you save with modern Observability, I think, changes how we actually practice, and that's an ongoing trend. -## OpenTelemetry Insights +But with OpenTelemetry going, effectively going GA this year, with tracing, metrics and logs now stable, yes, finally, only like two years late anyway. But the fact that we have that now, the fact that we now have telemetry that has all of these correlations baked into it, you're going to start seeing a new wave of analysis tools, all the existing ones out there, but also new ones being built, that leverage the fact that this data is available and that it's like a standard data format, kind of proprietary data format, stable data format. You can rely on it. -**Tyler:** When did you all get involved with OpenTelemetry? +So it's like okay to build your giant platform on top of this data or build some kind of like boutique analysis tool that just does one thing and does it really well. That's where I see it all going. And that's what Observability means to me. -**Amy:** I got involved in 2019. I was brought in to instrument the entire stack for the Equinix Metal product. +**Purvi:** What does Observability mean to me... It means like, an application owner can see what's going on in their environment and answer pertinent questions to them about their business and how they can improve their service. Observability is an overloaded term in our days, but it means the capability of monitoring and determining when something goes wrong in your production environment. -**Carter:** I was introduced to OpenTelemetry through my org at Microsoft, which was already doing a lot in that space. +**Bogdan:** Observability means... what does it mean to me? I use it as a tool to kind of making sure that things are working the way you want. It's getting insight into black boxes or even white boxes. I view it more as you kind of see things, but you have a lot more questions from it, and then you use Observability to actually figure out what's going on. So I like to call it murder mystery, usually. -**Constance:** I was part of the early days when we were working on the incubation process. +**Constance:** I like that. -**Jacob:** I got involved through working at Honeycomb, focusing on OpenTelemetry JavaScript, particularly on the browser side. +**Purvi:** Yeah, I want to use that on a slide. ---- +**Jacob:** That's a good question. I think... not going to be strict on a definition, I think what this really means is it is a way for us to understand what a problem... we have a problem in our system... we should be able to answer or to determine what is going wrong or what's happening. And it doesn't matter if it comes from logs or metrics or tracing, as long as we can tell and understand what's going on. I think that's when we can say we have Observability. And it's not a yes or no. It is a spectrum. I don't expect to have Observability, perfect Observability from day one, but I am expected to have some sort of telemetry that helps me understand what's going on. So I think telemetry is like a path to getting perhaps utopic place where we understand everything about our systems. -## Perspectives on OpenTelemetry +[00:10:40] **Amy:** What does Observability mean to me... I think Observability is understanding what's happening inside of your applications. Maybe what's happening in the code you care about. -**Alex:** OpenTelemetry is about collaboration across the Observability space. It provides a vendor-neutral way to instrument systems. +**Purvi:** Yeah. -**Carter:** It makes my life easier by integrating with both open-source and proprietary components, allowing me to see traces across all products I use. +**Jacob:** Oh my goodness. It means everything. Observability is life. I think Observability means that when something goes wrong, I can ask a question about my system and get a sense of what is happening without having to know ahead of time what to expect. Like I can just go and dig into my data and my services are instrumented well enough. Not like not perfectly, but well enough that I can just figure out what happened. And can I reproduce this thing that happened in probably production off in my own environment so that I can improve my code to manage it better next time. -**Amy:** OpenTelemetry is a collection of standards that brings together various telemetry signals like metrics, logs, and traces. +**Amy:** I like what you said about not instrumented perfectly. There is no such thing as perfect instrumentation. That's a lie. Just like there's no such thing as done code, right? That's also a lie. Or that the network will never break. That's a lie. -**Juraci:** OpenTelemetry is a set of tools that helps extract telemetry data out of applications, gradually leading to better understanding of the systems. +**Jacob:** Oh, that's such a good question. To me, Observability, it's really about being able to get curious with your data and be able to have a lot more confidence about your production system. So being able to kind of squash things before they arrive. Testing in production is the best way to test your system because no matter what people say, Prod is always its own different animal. And if you have really good Observability, you can test in Prod. It's a much better experience for your users and for your developers too. -**Constance:** It's a wonderful community that helps make Observability better by working across vendor boundaries. +[00:12:30] **Carter:** Absolutely. When did you get involved with OpenTelemetry? ---- +**Purvi:** I got involved in 2019, I think. -## Favorite Telemetry Signals +**Carter:** Oh, so like early? + +**Purvi:** Early, yeah, I was not at the original meeting, but yeah, I got in really early. I really love writing Go, and so that's where I started. But I was pretty quick into the specification and started working in that space and I think it was just coming from the pain point of using have to run systems. And being that person who has woke up at 2:00 a.m. I wanted a better software solution for this and I think that I saw the value in it and I jumped in. + +**Jacob:** We work in pain and trauma, right? + +**Purvi:** Yeah, exactly. When I was hired into Equinix, they hired me to instrument their entire stack for the Equinix Metal product. So that's what I worked on for my first year. This was like three years ago, before all the fancy auto-instrumentation stuff was complete, adding instrumentation to all of our systems. + +**Carter:** So you are an OG user of OpenTelemetry. + +**Purvi:** A little bit. The team I was working on at Microsoft, at least my org at least, was already doing a lot in the OpenTelemetry space, and that seemed to be where all the cool things were happening. And so that kind of got my interest. And then I got switched to working with a development team that was focused solely on OpenTelemetry, both for external purposes and internal purposes, because Microsoft uses OpenTelemetry really heavily internally. And so that's what got me introduced. And when I started looking around, wondering where I could start, I realized there was no real good example of how to use OpenTelemetry in the wild. And so that was a problem that I thought every vendor might have. And something we could solve together as a community, and we have. + +**Amy:** Cool. How long have you been working on OpenTelemetry? + +**Purvi:** Since the beginning. So like 2019 or like, pre? + +**Jacob:** Even pre. Did you start out with Ted in the... pre days? + +**Purvi:** No. Actually, there were two competing projects that merged into OpenTelemetry. + +**Jacob:** Right. So I was on the other project. + +**Purvi:** Which one, OpenCensus? + +**Jacob:** Yeah, I got involved with OpenTelemetry through working at Honeycomb. So I got involved with it, and I have a particular interest in OpenTelemetry JavaScript, and especially the browser side of OpenTelemetry JavaScript. It's really great to be involved with it. + +**Carter:** What's your definition of OpenTelemetry? + +[00:15:00] **Jacob:** I think OpenTelemetry is, I mean, it's a standard, I think it's a collaboration across the entire Observability space. And it is, I think, a path forward for all of instrumentation. The idea that you don't have any vendor lock-in, the idea that you can just take one code base and always have some way to look into a system, I think the future of how we're going to make software better in the long term. OpenTelemetry makes my life easier, because I can integrate it with open source components that I'm using, or proprietary components. And at the end of the day, all of the OpenTelemetry flows through to my Observability vendor, and I can see traces across all of my products that I use in one space. + +**Purvi:** It is my project... of soul. I think OpenTelemetry gets really biased, but I feel like it's a really good combination of a lot of different views finally coming together, actually making the previously hard advancements easier, like gathering the data. The hard part is actually making sense out of it. And so they're finally coming together. It's worked out pretty well in terms of collaboration to get metrics, traces, and logs. + +**Carter:** Oh, that's a deep question... + +**Purvi:** Yeah, it is. On a technical side, it means OpenTelemetry for me is a set of tools that would help me get telemetry data out of my typical application. Sometimes also infra. But OpenTelemetry really is the tool that I can use in a vendor neutral way, get data out of my application so that I can get into that uptopic thing. If I had perfect instrumentation, then I can get into a uptopic place. But OpenTelemetry provides me the tools that I need to gradually get into that. Now, it does stop at a very specific place, in a sense, which is as soon as you send data out, that's where OpenTelemetry stops. That's where you get to the vendor or to the open source tools that provide the database, visualization tools, and so on. But a more deeper aspect, OpenTelemetry is where I have my colleagues, people that I work on a daily basis for a few years. + +**Jacob:** Yeah. That's what OpenTelemetry is for me. What does it mean? What's the concept? OpenTelemetry is Observability backed by everybody. It's not a single vendor. It's letting you do the thing that is agnostic to where you send the data. In the same way that you don't have to relearn how to drive a car every time you step into a new car. You don't have to learn how to ride a bike based on the vendor of the bike that you buy from. You should be able to instrument your code no matter where you send that data. So that's how I sell it. That's how I think about it. + +**Carter:** Awesome. The other benefit is we as the maintainers, we have a lot of our maintainers here and approvers here, so we can collaborate and work together to figure out what's really needed in the next coming months. + +[00:18:42] **Purvi:** I described it almost like summer camp. There are some people where, oh, I haven't seen you in a few months. How you been? It's catching up. Like, I mean, OTel has been amazing. The project itself has been wonderful. It's one of the first projects to take a bunch of standards and condense them down into less standards. We took OpenCensus, OpenTracing, we brought Prometheus to the table. The Elastic Cache format is there. OpenTelemetry just is a wonderful community, that's all. Trying to make things better in the Observability landscape by working across vendor boundaries, which has just been something that I've never done in the past. I've never worked in an open source project where so many vendors are involved and so many end user communities are involved, and it's been great. + +**Carter:** Yeah. And that's what I like personally about OpenTelemetry, because everyone plays nice and I feel like it's a very deliberate, "No, we are not going to favor one vendor over another." And if a vendor tried to showboat, then it's pretty much shut down, which I think is great. We've just had a lot of really good folks at all the levels of the project trying to push everybody in the right direction, which I really appreciate. + +**Purvi:** I'm going to give a shout out to Ted Young for, especially being one of those people that always just, he's over there somewhere. I can see him just like looking around, see, waving his head. He has no idea we're talking about him. + +[00:20:30] **Jacob:** OpenTelemetry to me is really all about the community. Like communities being able to take ownership of their own telemetry data, because vendors should not be determining the type of telemetry data that gets sent to your systems. Because Observability about your system is so personal to your system. And when you have vendor lock in or lock in through, like, the instrumentation of vendors, it can be very limiting. **Purvi:** What is your favorite telemetry signal? -**Jacob:** I’d probably say traces, as they provide a deeper understanding of operational behavior. +**Carter:** That's a good question. I wish I had a good, nuanced answer there. Like, I don't know... metrics, I've known for the longest, I guess. But I think traces are probably a little closer because you get a lot more depth into operational behavior. So, yeah, I think I'd probably go with traces. It's also a little bit more automatic for you. You really have to understand what those metrics are and build them into something. Versus tracing, can show you based on just the structures they come with. So, yeah, I'll go traces. + +**Amy:** Oh, it's traces, of course. -**Amy:** Traces are definitely my favorite too; they give you context. +**Purvi:** My favorite signal... Probably be the Bat Signal. If that thing could go on every time a system goes down, that would be. -**Carter:** Traces are the cooler version of logs. They give you more depth. +**Jacob:** So I think I've heard this reference around... and I truly believe it. Traces are just the cooler version of logs. Like, it's like logs with a mustache, and maybe a top hat. Because essentially a span is just a log, but a trace-correlated log. So I'd probably say traces, but my backup answer is log. -**Juraci:** I love traces. They allow you to quickly identify issues without overthinking. +**Carter:** Signal? I think the most... I like metrics. I think we did try to change the way how metrics were done before, and we may not have been the most successful yet, but we are getting there. But it was a necessary change, and I feel like it changed something in the way how things were done. For tracing, I mean, we didn't change too much from other Dapr paper or other things, but for metrics, I think we changed. -**Constance:** Traces number one! They are elegant and useful. +**Purvi:** I love traces, especially because you could... My favorite example is when I used to do... when I was at Lyft and I would get paged in the middle of the night... One service, four deep... everything was going... Everything between that and the front was getting paged. You were able to actually figure out, like, okay, this one's the cause. Instead of overly thinking about it. That's what I love about it. It's very different paradigm than what we're used to talking about. -**Alex:** Traces are magic because they correlate metrics and logs with context. +[00:23:10] **Jacob:** Trace. Come on. They're beautiful. No, it is. Traces. Traces number one. They are the easiest to work with. They are so simple to get started, and they're just so much more useful than anything else. So traces all the way. ---- +**Purvi:** Traces, because it's clearly the elegant log, but also you can just get metrics out of it. It has, like, everything you need in a signal. It's metrics and logs correlated with context. It's beautiful. They're magic. -In summary, this engaging discussion among various contributors and maintainers of OpenTelemetry reflects their shared passion for improving Observability and making it accessible and useful across different environments and applications. Their varied experiences and insights highlight the importance of community collaboration in advancing the field of Observability. +**Carter:** Oh, that's easy. It's tracing. ## Raw YouTube Transcript diff --git a/video-transcripts/transcripts/2024-01-19T04:34:21Z-observability-panel-with-charity-majors-amy-tobey-and-adriana-villela.md b/video-transcripts/transcripts/2024-01-19T04:34:21Z-observability-panel-with-charity-majors-amy-tobey-and-adriana-villela.md index 4590167..1fd547b 100644 --- a/video-transcripts/transcripts/2024-01-19T04:34:21Z-observability-panel-with-charity-majors-amy-tobey-and-adriana-villela.md +++ b/video-transcripts/transcripts/2024-01-19T04:34:21Z-observability-panel-with-charity-majors-amy-tobey-and-adriana-villela.md @@ -10,101 +10,358 @@ URL: https://www.youtube.com/watch?v=Fuy3W5bro9k ## Summary -In this panel discussion on observability, hosts Adriana Vela (ServiceNow), Charity Majors (Honeycomb.io), and Amy Toby (Equinix) explore the evolving landscape of observability tools and practices. They discuss the transition from traditional observability, characterized by fragmented metrics and logs, to a more integrated approach that emphasizes high cardinality data and meaningful insights. Charity defines observability as a property of socio-technical systems, while the panelists highlight the importance of understanding the context of data and the need for tighter feedback loops between engineering and operations. They also address misconceptions about observability being a standalone solution, emphasizing that it requires a cultural shift within organizations. The conversation touches on the role of AI in augmenting human decision-making and the challenges of managing data volume. As they conclude, the panelists express their hopes for a future where observability becomes more ingrained in the software development lifecycle, encouraging collaboration across teams to enhance visibility and improve the overall quality of software delivery. +The YouTube panel discussion focuses on the topic of observability, featuring experts Adriana Vela from ServiceNow, Charity Majors from Honeycomb, and Amy Toby from Equinix. The conversation delves into the definitions of observability, with Charity emphasizing the shift from a traditional metric- and log-based approach (observability 1.0) to a more integrated and explorative method (observability 2.0). Key points include the challenges of data overload, the need for better communication among teams regarding observability, and the importance of understanding the context of data collected. The panelists also discuss the role of AI in enhancing observability, stressing that while AI can assist in data correlation and reduce cognitive load, human insight remains crucial for addressing complex, novel issues. The discussion concludes with reflections on the future of observability and the importance of fostering a culture that values continuous feedback and understanding in software development. ## Chapters -Here are the key moments from the livestream along with their timestamps: +00:00:00 Welcome and introductions +00:03:00 Definition of observability +00:05:22 Observability 1.0 vs 2.0 +00:10:30 Meaningful data in observability +00:13:40 Challenges with open Telemetry +00:18:00 Transitioning to observability 2.0 +00:22:10 Importance of feedback loops +00:27:30 AI's role in observability +00:34:00 Current challenges in observability +00:40:00 Misconceptions about observability -00:00:00 Introductions of the panelists -00:03:30 Adriana shares her background in observability -00:05:00 Charity discusses her definition of observability -00:09:45 Amy talks about her experience with observability at Equinix -00:14:00 Discussion on the evolution from observability 1.0 to 2.0 -00:21:00 Charity explains the importance of meaningful data in observability -00:28:30 The panel discusses the challenges of onboarding open Telemetry -00:35:00 Amy mentions the need for observability features in network services -00:40:00 Discussion on the role of AI in enhancing observability -00:50:00 Panelists share their thoughts on the biggest challenges in observability -00:55:00 Closing remarks and future discussions on observability +**Host:** Here we are. Hello, hello, hello everyone! We're very excited to have a panel today around the topic of observability. Plan for the observability community. Here with us today we have Adriana, Amy, and Charity joining us. We hope to have a fun, billed conversation. Would you mind starting out, Adriana, and introducing yourself? Who you are and what you do? -This format highlights the main topics covered during the livestream, making it easier for viewers to navigate through the content. +**Adriana:** Oh yeah, so I am Adriana Vela. I work with Anna at ServiceNow Cloud Observability, formerly known as Lightstep. Say that three times fast. I, uh, yeah, I'm a senior staff developer advocate. I've been doing the dev rel thing for, what, almost two years? But before that, I was oscillating between, like, IC engineering and management roles. So yay! And I love observability. -# Observability Panel Discussion Transcript +**Charity:** I knew you before you were a dev rel, before you were in observability! -**Moderator:** Here we are! Hello, hello, hello, everyone! We're very excited to have a panel today around the topic of observability. We hope to have a fun and engaging conversation. Would you mind starting out, Adriana, by introducing yourself—who you are and what you do? +**Adriana:** Yeah! Yeah, when I was just, like, trying to figure out what the hell this stuff was all about. Time definitely has passed by. -**Adriana:** Oh yeah! So, I am Adriana Vela. I work with Anna at ServiceNow Cloud Observability, formerly known as Lightstep. I’m a senior staff developer advocate and have been doing the dev ops thing for almost two years. Before that, I oscillated between individual contributor engineering and management roles. Yay! And I love observability. I knew you before you were a developer in observability! +**Charity:** What about you? -**Charity:** Yeah! When you were just trying to figure out what the hell this stuff was all about! Time definitely has passed by. +**Charity:** I'm Charity Majors, co-founder and CTO of Honeycomb.io. I identify as an Ops engineer and probably always will, even though it's been a long time since I held the pager. But I feel like I started, I feel like I picked up my first pager rotation when I was 19, so I feel like I've got still like half my life on call. So I still, I can still call myself an operations engineer. -**Moderator:** Charity, what about you? +**Adriana:** You paid your dues! -**Charity:** I’m Charity Majors, co-founder and CTO of Honeycomb.io. I identify as an Ops engineer and probably always will, even though it's been a long time since I held the pager. I feel like I started picking up my first pager rotation when I was 19, so I feel like I've got half my life on call. +**Charity:** I think so. What about you, Amy? -**Moderator:** You paid your dues! What about you, Amy? +**Amy:** Hi, I'm Amy Toby. I am a senior principal engineer at Equinix, which is the largest data center company in the world. I've been doing observability since pretty much my first job as well, starting back on Solara Systems in, like, '98, '99. And then into NIO stuff. I'm responsible for some things in NIO, I'm sorry. And, like, I have been doing it all along. These days I do a little bit more kind of work in the soot technical space, but initial work at Equinix was bringing OpenTelemetry into our entire Equinix Metal operational stack and pumping those metrics out to Honeycomb. -**Amy:** Hi, I'm Amy Toby. I am a senior principal engineer at Equinix, which is the largest data center company in the world. I’ve been doing observability pretty much since my first job, starting back at Solara Systems in the late '90s. I’ve been responsible for some things in networking and have been working on bringing OpenTelemetry into our entire Equinix Metal operational stack, pumping those metrics out to Honeycomb. +**Adriana:** Nice! Very exciting work that all of you are doing and creating community around observability and OpenTelemetry. And I get to call you all friends, which is super exciting to have you all today on the panel. -**Moderator:** Nice! Very exciting work that all of you are doing and creating community around observability and OpenTelemetry. I get to call you all friends, which is super exciting to have you all today on the panel. +[00:03:00] **Host:** Sweet! So, I think we're going to start with one of the spicy questions, and I'm going to kick it off with Charity. What is your definition of observability? The correct one, of course. -Let’s kick it off with one of the spicy questions. Charity, what is your definition of observability? +**Charity:** We'll see. Yes, spice! You know, okay, so back in the days when nobody else gave a [__] about observability, the Halcyon days of... I'm saying that even though we almost went out of business many times. You know, I think we really tried to push this idea that there was a technical definition for observability. You know, that it was high cardinality, high dimensionality, explorability, and all these things. And nowadays, you know, like everyone and their cat does observability. It's like if you have any telemetry, you do observability. And you know what? Actually, I’ve come around to this. I'm all right with this. I feel like it's actually a great definition to say that observability is a property of a socio-technical system. So it exists in a continuum, right? -**Charity:** The correct one, of course! Back in the days when nobody else cared about observability, we really pushed the idea that there was a technical definition for it—high cardinality, high dimensionality, explorability, and all these things. Nowadays, everyone and their cat does observability. If you have any telemetry, you do observability. You know what? I’ve come around to this. I think it’s a great definition to say that observability is a property of sociotechnical systems. It exists on a continuum. +I'm totally down with that because I think that it gives a lot of grace to people who are, you know, in the early days of their journey. Like, I don't want to be the person who, like, "Well, actually, you're not really doing observability." Nobody likes that guy, so I don't want to be that guy. -I think it gives a lot of grace to people who are in the early days of their journey. I don’t want to be the person who says, “Well, actually, you're not really doing observability.” Nobody likes that person. +That said, I do think that there's a real kind of step function difference in, let's call it observability 1.0, that built off the primitives of, you know, metrics, logs, and traces. Where every time you gather the data, you have to gather it again in a different format and pay for it again. So like you're gathering it once for metrics, again for logs, again for traces, again for APM, again for profiling, again for security. You know, like talk about a cost crisis in observability tooling. Like, no wonder, right? And you're very limited in each one of those by the format that you happen to gather in, and none of them can be connected to each other. The only thing that connects them is the poor human sitting in the middle guessing, right? Like guessing eyeballs, you know? -That said, I do think there's a real kind of step function difference between what I'll call observability 1.0 and 2.0. Observability 1.0 is built off the primitives of metrics, logs, and traces. Every time you gather data, you have to gather it again in a different format and pay for it again. You gather it once for metrics, again for logs, again for traces, again for APM, and again for profiling. Talk about a cost crisis in observability tooling! +So let's call that 1.0, and then observability 2.0 tooling, I think, is based on a single source of truth. Right? These arbitrarily wide structured data blobs you could, you can derive metrics, you can derive traces, you can derive all these things, but you've got one source of truth. It handles high cardinality, it handles, you know, it's a more explorable sort of interface. So you don't have these static dashboards; you have an interface where you can ask questions and follow breadcrumbs. Go, and then what? And then what? -You're very limited in each one of those by the format you gathered it in, and the only thing that connects them is the poor human sitting in the middle guessing. So, let’s call that observability 1.0. +[00:05:22] So that's my very long-winded answer. I think that there's kind of two generations of observability tooling out there right now, and I think that they lead to two very different outcomes. Like one of the, I know I'm taking a very long time here, I'm almost done, I promise. I think that one of the most, one of the soot technical factors that you really associate with observability 1.0 is it's like a checklist thing. You know, you add it at the end before you put stuff in production. Like, cool, I can monitor this. -Observability 2.0 tooling is based on a single source of truth. You can derive metrics from a structured data blob, and it handles high cardinality and provides a more explorable interface. It’s dynamic, where you can ask questions and follow breadcrumbs. +And for observability 2.0, I think like the fundamental truth of it is it underlies the entire software development life cycle, and your ability to hook up these fast feedback loops rests on how well you can, you know, ask questions and explore your data and have this sort of constant conversation going with your code. -So, that’s my very long-winded answer. I think there are two generations of observability tooling out there right now, leading to very different outcomes. +**Amy:** Awesome! What about you, Amy? -**Moderator:** What about you, Amy? +**Amy:** I think I agree with most of what Charity said, but I've been learning a lot from my work lately in kind of developing a new networking product, and one of the feature sets that we have to include in that today in a modern Network as a Service kind of product is observability features, and customers demand this. And we've been scratching at this, like, what, how much do they need? Like, what, how much should we build? How much should we hand off to either, you know, ServiceNow or Honeycomb and let those tools do what they're best at? -**Amy:** I think I agree with most of what Charity said. My work lately has involved developing a new networking product, and one of the feature sets we have to include is observability features. Customers demand this. We’ve been figuring out how much observability they need and how much we should build versus handing off to services like ServiceNow or Honeycomb. +And so what we've been finding with customers is, like, that operational part that Charity mentioned, right? That they are tired of this world where they have to go hire very expensive people who are often cranky, who can basically carry that mental burden of tying all the context together with the metrics and divining what the heck is going on with the network. And so what they're asking and what they're demanding from us and all of the other cloud vendors is how do we get the observability telemetry, which is mostly what they ask for? Because when we talk to customers, like, most of the time when folks we're talking to don't want to talk about cardinality. -What we've found is that customers are tired of hiring expensive people to carry the mental burden of tying context together with the metrics and figuring out what’s going on. They want observability telemetry because they need insights about these hyper-complex systems to fix them and get back to business. +So they're asking for, like, how do we get all the stuff we need so we can feed it to our AI Ops tools, feed it to our other observability systems that we already have, so that we can start to leverage the skills we already have to get those insights? But it always drives to how do I get insights about this hyper-complex system so that we can fix it and get back to business? And sometimes folks don't even want to do the heavy lifting. They're just like, do it for me! Nobody does, right? Because almost none of our customers, any of us here, their primary business is not telemetry. It's undifferentiated lifting. They want, they got a business to run; they don't want to be messing with this stuff. So they want me to provide really high-quality telemetry, and then they want you all to make that all super easy to consume. -**Adriana:** I think I started out with Charity's OG definition of observability, and I heard one recently from Hazel Weekley, which I quite like: "Observability is the ability for you to ask questions and get meaningful answers." It's up to you what is meaningful. What data are you collecting that’s meaningful to you? +**Adriana:** Yeah, that's very true. I love that. It makes so much sense. What is your definition, Adriana? -I believe in trace-first observability, but we need to focus on meaningful data. You can instrument the heck out of a system, but if it’s emitting garbage, you won’t be able to figure out what’s going on. Observability is a design problem. +**Adriana:** So I think I started out with, you know, Charity's OG definition of observability, which carried me well throughout the years. And then I heard one recently from Hazel Weekly, which I quite like, which is, "Observability is the ability for you to ask questions and get meaningful answers," which I really like because it's, you know, it's up to you as to what is meaningful, right? So what data are you collecting that's meaningful to you? And I think that's really important in our world. -I feel like we spent a long time figuring out the underlying plumbing, which is just a prerequisite. The real issue is that there’s so much data, and we need to equip ourselves to understand it. +I will still say that I think trace-first for observability no matter what. But I think, like, we really need to focus on the meaningful data aspect of observability because, like, sure, you can, like, instrument the crap out of a system, and it's emitting a bunch of useless garbage, right? You know, garbage in, garbage out. So if you don't have good data that you're instrumenting well, good luck trying to figure out what the hell is going on. You can have the best observability tool out there; it won't do you any good. -**Charity:** I think we’re seeing that with OpenTelemetry. It’s an awesome standard, but the challenge will be making it easy for onboarding. Many organizations are scared of instrumenting their own code. +So you're going to need, um, to have... It takes a bit of, you know, there's that learning curve of what's important to you, what should you be collecting to be able to do the thing. Observability is a design problem, like in a huge way. Like, once you've s... Like, I feel like we spent a long time, like, just sort of like figuring out the underlying plumbing, which is just a prerequisite to what really matters. -**Amy:** Exactly. The metrics matter, and we often see folks reaching for tools to make sense of the noise they’ve built for themselves. They’re overwhelmed with alerts, and it’s challenging to know what’s important. +[00:10:30] This is a design problem because there's so much data, and I think Amy and I share very similar thoughts on AI Ops. But, like, there's two kinds of tools, right? There are tools that pave over complexity, and there are tools that try to help equip you to understand it. And I will always be in the second camp because at the end of the day, we are legally liable for the software that we put out into the world. Somebody somewhere is going to have to understand that [__], and the harder you've made it, the more magical you've made it, the harder that day of reckoning is going to be. -**Moderator:** That leads us to the next question: What do you think is the current largest challenge in observability? +**Charity:** Definitely interestingly enough, I think we're seeing that a little bit with open telemetry, right? OpenTelemetry is awesome. It's, you know, this awesome standard that pretty much everyone has opted vendor-wise. But then now we're looking beyond, like, that initial adoption of OpenTelemetry and, like, people really using it out in the field and making sure. I think OpenTelemetry's biggest challenge, I think, in the near future is going to be making sure that it's something that is easy for onboarding because I think that can be really off-putting for organizations, and that can be really scary. -**Amy:** Sampling! We generate a wealth of data, and while we're not sampling a lot today, as we turn on tracing, we find that it tips over our collectors. We need to get control of the data we're generating that might not provide value. +And then that leads to the, you know, "You instrument code? I don't want to instrument my own code!" kind of mentality. I think that the place that the metrics go really matters, though. Because we talked about AI Ops, where I see folks really reaching for these tools. Charity called it paving over. So what we've done in the network world for decades now, and it's still the case at most organizations, is we have all these devices generating gobs of alerts and metrics. Nobody knows what any of it's doing. There's a sense that we should be able to correlate all these events. -**Charity:** I think people don’t realize how badly they have it now. Many have spent their careers managing metrics without understanding how much better it could be. It’s a mental model shift that they have to make. +So, like, some port fails on a core router, and all the stuff behind it disappears. We should be able to roll all that up, and I'm using the "should" word the way my therapist calls it, the "S" word, right? Like, we believe these things that should happen, but the real world is much harder than that. So, like, these alerts just flood through, and people fight these alerts. Sometimes they've been fighting their whole career trying to get on top of this alert stream, and so they reach for these tools to make sense of the morass that they built for themselves because they're not really empowered to go back to the telemetry and, like, dial back the cardinality or re-transform how they generate those metrics to get higher, you know, higher cardinality that they can process at the end. -**Adriana:** I agree. Organizations often get in their own way because they don’t fully understand observability and how to implement it effectively. +Often they don't even have choices about what to collect because they're buying network devices that offer you get your SNMP or your G, and that's what you get. Oh my God, it's a world out there of legions of people that are just fighting through the noise still. And it hasn't changed for them. Whereas in the software world, we keep advancing the state of the art, but it's going to be a challenge still to bring a lot of the classic sysadmin and network administrators who are used to this world along on that journey. -**Moderator:** What would you say is the biggest misconception around observability right now? +[00:13:40] I feel like one of the characteristics that I often think about when we're talking about, like, transitioning from the 1.0 world to the 2.0 world, which is going to take [__] forever, no doubt. But I feel like it's part of it is generating from a push world to a pull world, where we're used to, like, the only way you could make sense of, like, a NaaS-based system is by looking at the cluster of things that just alerted you and going, like, "Ah, it's probably that." -**Adriana:** People think it will solve all their problems without putting in the work. There’s a misconception that observability is magical fairy dust. +And so you rely on, you know, all of these alerts, but that's so noisy, and it does not scale. Right? But, like, as an ops person, I grew up thinking you only look at production when you get alerted. You don't have to look at it otherwise. And I feel like the trade-off that we have to learn to make is, okay, we're only going to alert you when users are impacted, right? We set SLOs, these are our agreements with ourselves, each other, and the world. This is the level of service we are going to give you, and we alert when the outage is bad enough that users are being affected. -**Amy:** I would extend that. The need for understanding doesn’t disappear; it just gets more efficient. +But in order to get to that world, you have to agree that you're going to go look at your telemetry affirmatively because most problems will never rise to the level of waking you up! God, I mean, I hope, right? Like, most of the things, most of the codes you write have subtle bugs that affect a few people, or it's a small thing, right? -**Charity:** I think the biggest misconception is that observability is its own standalone project. It’s interwoven with software development and requires a holistic view. You can’t just check off an observability project; it requires continuous improvement in how we build software. +So, like, in order to have a system, in order to have a hygienic system that is not just like a hairball that the cat coughed up, you know, we can't just keep... like in the old days, we were shipping code every day that we didn't really understand these systems. We've never really understood, and then we wonder why it's a giant trash fire, right? Like, and the way that we start improving that is by, you know, not just making the telemetry richer and better, but also by committing to closing that loop. -**Moderator:** As we wrap up, what do you think is in store for observability in the upcoming year? +As a software engineer, my job is not done when the tests pass. My job is done when I have looked at my telemetry in production and gone, "It's doing what I expect it to do. Nothing else looks weird," right? So, like, that's asking... it's like I like the technical debt metaphor here because it's asking you to, like, put in a little bit of effort upfront in order to avoid the... you know, there's this great graph that shows that the cost of finding and fixing bugs goes up exponentially from the moment that you write them. -**Amy:** My wish for the year would be that more people grasp SLAs and SLOs and realize the power they have in understanding their systems. +So, like, you backspace? Good for you, good job, right? You find it in your test? Good for you! The next best time to find it is right after you deployed that [__]. After that, God knows how long it's going to take: days, months, years, who knows? But, like, the accumulation of that [__] is what makes this job a nightmare for so many people. -**Adriana:** I want to see more people understanding that observability is everyone’s problem—not just the ops team. It should be part of the language across all teams involved in software delivery. +**Charity:** Yeah, it's so true. Like, I think back to, like, you know, my old days of having to hand off deployment instructions to, like, our ops folks who, they had no [__] clue what was, you know, what it was they were deploying. And, you know, I had to pray that my instructions were correct, and then I had to pray that they read my instructions correctly, and then they would deploy it to prod. And these quote-unquote successful deployment-like things went correctly, was considered a successful project. The machine, we're done, right? -**Charity:** I hope we can drive a deeper understanding of observability into our industry so that we don’t have to start at zero every time we discuss it. +**Adriana:** Right, yeah. -**Moderator:** Thank you all for the amazing insights today on our panel around observability. This group will be coming back later in June for another discussion, so stay tuned for those details. Thank you for watching, and stay warm out there! +**Charity:** And so, I love the idea of, like, no! Like, because otherwise it becomes a hit-and-run deployment, right? You know, or drive-by deployment, rather, is a better way of putting it, where, like, you know, it's like, "Deployed! All right, great! Let's just leave it to fate." But I think, like, you know, paying attention to what's going on as soon as you deploy, like, there's something to be said because otherwise, because you, as the person writing the code, you have context that nobody else in the world has or will ever have. -**Everyone:** Yay! +You know exactly what you're trying to do, why you're doing it, what you tried that worked, what didn't work, what the variables were named, what the [__]. You know all this stuff, and if you can close that loop before you have to concept switch another, it's so powerful. Like, these feedback loops are what socio-technical systems are all about. + +Like, this is the one job of technical leadership, in my opinion, whether you're an IC or a director or VP. One job is these feedback loops to make them as tight and short and functional as possible. I think that, like, nailed down of, like, engineering responsibility has been so important, and observability really brings it to a front. But a lot of people are just not paying enough attention or, like, not willing to do it. They're just like, "My job is to code, and that's it, and maybe I'm on call twice a year, but it's someone else's problem." + +[00:18:00] **Amy:** There's not much we can do about that, though, like, a lot of times, right? Like, if you are an engineering leader and you're trying to build a dynamic, powerful engineering organization, you have to either lead those people to care or lead them out, right? Like, because, like, the reality is, is like, there are the customers that y'all want that have that engineering leadership and culture of, like, giving a crap. + +And then there's the vast majority of organizations out there where most of the people who are writing the code that runs our world are trying to put bread on their table. And so as soon as they are done with the specification that they were given for their job, they e that code into Git, and they run because, like, why should they care? Why should they metrics? But this is a self-fulfilling prophecy because you know what makes people, like, check out, tune the [__] out? Not being connected to the results of their labor! + +Like, there's something that's, like, what is it? Dan Pink says we all want in our job is autonomy, mastery, and purpose. The purpose of your work, if you're so disconnected from it, yeah, that's all there is to it. I'm like, I'm not trying to suggest that this is easy or can be done in a one-stop, you know, whatever, but I do think that every step we make along this path that tightens up the feedback loops that connects people more with the consequences of their code is good and valuable and pays off. + +**Charity:** Interesting, oh, sorry, go ahead. + +**Adriana:** I was going to say it's even letting engineers understand customers. Like, I've worked at many shops and with so many engineers that, like, that customer impact doesn't matter to them or they're unaware of how the customers are properly using their software, or they themselves are not even using the software that they're building. That's why, like, dogfooding is so important. + +I go in to help teams all the time where they're, you know, they're just churning. This is, like, the most common thing that I go work and help teams with, right? They just churn. They're not making progress. Like, usually I can guess that without even talking to anybody. I can just, like, look at their JIRA for two minutes and be like, "Yeah, they're churning, so let's go figure out why." + +And then you go look, and I lost my train of thought. I'm sorry! + +**Charity:** People are not connected to their work; they're churning, right? + +**Adriana:** I still forgot where I was going to connect that. + +**Host:** It's okay! It's okay, we can move on. I wanted to add one thought to the mix because, you know, the overarching theme here is, you know, like they deploy, and then they go, or they write the code, and then that's it. And I mean, ultimately, we are failing the promise of DevOps at the end of the day, right? + +**Charity:** Exactly! + +**Host:** We did nothing to solve or to alleviate the problem because we are still continuing to do that. Yes, we've got CI/CD pipelines; that's awesome! We've got automation, but we still have not fulfilled the promise of DevOps at the end of the day. And I think it's interesting, though, because I think observability is kind of, you know, forcing us to face that once again, right? + +Because now there... like, these are the consequences of... now we know. I think the opportunity here is now that we have the data to show the business how the consequences of bad engineering management are harming our customers. Like this, we didn't have back when we started DevOps, right? Like, we had no freaking idea how our failed deploys or any of this were impacting our business, right? + +We were just guessing. We were throwing dark birds. And, like, some of us were fairly accurate, like, "Yeah, this is what's happening," and then go talk to a customer, and they would confirm it. Great! Now we can go do DevOps. But there's tons of shops where they just never close that loop, right? + +And then you could bring the data in, though, and this is one of the things I've been slowly doing, right? Is I go talk to a team and be like, "Well, the first thing you need to do is go talk to your product manager, and I'll help them define some freaking SLOs so you know what the acceptable abuse of your customers is," because most people don't even know. + +[00:22:10] Yeah, let's start observing that! Let's put in SLOs. We don't have to alert or nothing. Just start looking, getting the data. Now I can go back to the business, and I can go tell your director of engineering or your VP or even your SVP and go yell at them, be like, "Look at this data! You need to go invest in this team and change how they work because this is what your customers are going through." And I have actual data to show it, and that's a game changer! That changes the discussion. + +**Adriana:** It really does! + +**Charity:** I always get so much energy talking to Amy because, like, all these things, I feel like I'm just sitting here thinking about, like, she's out there in the field just doing like over and over and over, and I'm just like, "Oh my God, you're so cool!" + +**Amy:** Talk sounds cool, but the day-to-day is like meetings and waiting for the opportunity to go like, "Hey, here!" + +**Charity:** Yeah! + +**Amy:** But the impact! I mean, I just think it's so cool. But, like you say, we're failing the promise of DevOps. I actually feel like DevOps is like the Band-Aid that we have on a self-inflicted wound that never should have happened. + +Like, we never should have separated them from Ops! Like there's no possible world in which having one set of people write the code and another set of people understand the code makes any [__] sense! And, like, I get it. We were doing the best we could at the time. You know, we're like, "This is too complex; we got to split this up somehow." + +But, like, I feel like the entire idea of having different people do these jobs, and it's a little bit unfortunate. I think that we've kind of settled on going, "Oh, well, it's because Ops sucks, and we're all going to be developers." Like, whatever! + +Like, okay, fine. I mean, most of the time when this happens, it's cultural inertia because most of the corporate world, they live in this world where they think about, "Okay, I have some people who make decisions about the product, and then I go and encapsulate that in some kind of requirements or design docs or whatever, and I ship it off overseas to be manufactured." + +So some other people do the labor, some stuff shows up in a warehouse I probably never even see, and it goes out to distribute, blah, blah, blah, blah, blah. Like, this is like most of the businesses in the world today are some kind of form of this kind of dis... breaking the labor away from the design of it. + +And this is actually, I think, changing radically out even outside of the software world where now product design is speeding up. You see kind of things coming faster, so now they're starting to look at what we're doing in the DevOps and going like, "Oh, maybe there was something there. Maybe expertise matters. Maybe the idea that there's kind of work that isn't knowledge work is [__]!" + +**Adriana:** Yeah, I like that! + +**Charity:** I think one thing that is, is to some effect a consequence of this whole idea of separation of concerns, right? Which, you see especially in, like, large enterprises. Like, when I worked at a small to medium-sized startup, you know, I had access to, like, the freaking PR database. And then, like, I joined a bank right after I wrapped up that job, and they're like, "Oh yeah, we've got like a DBA for UAT and a DBA for prod." I'm like, "Huh? You don't even want you to log into your laptop if..." + +**Amy:** Yeah, that's right! + +**Charity:** ...to do your job! Yeah! That's changed a lot! You know, we borrowed so much of that stuff from the accounting world where things like, you know, the concept of you shouldn't have the same person file the expense report and approve the expense report totally makes sense. + +You know, and I think that there are some concepts that, you know, translate nicely, and most of them really don't. I actually, the most recent talk that I wrote was called "Modern Engineering is Not Incompatible with Highly Secure or Compliance Environments" or something like that. Yeah, and it's just like, because it's been... and it was super... like, I've been wanting somebody to write this talk for like 10 years, and I finally was like, "All right, this is not my area. Just somebody's got to do this!" + +So I sat down and wrote it, and I did a bunch of research, and it was so interesting because, you know, this is all probably old news to y'all, but I learned a lot about how just, like, you know, it's really engineers should not have to think about security and compliance in their day-to-day work. + +Like, if you have to think about it all the time, something is very, very wrong. It should be baked into the architecture so that by default, you do the right thing, and you can use the accounting and, like, logging and, like, auditing principles of, like, your CI/CD system. And, like, engineers should just be doing the work. You really only when you need to, like, spin up a new data source or, like, build something new, then you pull your security folks in; you make those decisions from the get-go. + +But, like, you know, it's just like the entire frame and what's so unfortunate is I think that security has this really unfortunate, you know, the security through security they laugh about, but they all do it. But it's like they think that talking about how they do things well is going to be bad! + +**Adriana:** And it's not! + +**Charity:** Like, I think that the only way we're going to make progress in this as an industry is if people start to realize that all their competitors are doing this well, and they're about to be outcompeted by the fact that everyone else in the world is so able to, like, write off for, like, a modern team without being bogged down in the security theater. + +[00:27:30] **Host:** I know you all touched about AI, and I wanted to ask a question about it. How do you think AI can help teams adopt observability? Or how is it that AI can actually have some benefits to the observability world? + +**Charity:** I think I've been saying about AI for a few years now, right? Like, I think the real opportunities lie in joint cognitive systems research, where the way we're using AI isn't necessarily to replace humans in the pipeline. It's to augment humans in the pipeline. And so, for observability, I really feel that the strongest opportunities for AI are in kind of automated attention on doing correlation, basically like auto-correlation, and surfacing that to humans. Not this idea that I'm going to eliminate all my cascading alerts through this because that's a fool's errand that we've been chasing for at least my entire career. + +I don't think AI really changes that game, but if we start to think about it instead of this crazy, like, we're going to make all the alerts go away, but instead I'm going to help your people do more with less, right? Less context, less expertise, maybe less work. Obvious! Now it gets super-duper powerful because that's the hardest problem to solve, is how do I reason about this hyper-complex system and draw out insights from it? That's where I think AI can help by augmenting the human, not replacing it. + +**Adriana:** I couldn't agree more! It's really about the inputs so much more than the outputs. Like, we are all familiar with, like, the problems of spam filters, right? Like false positives are brutal, you know, very high possibility for failure. But, like, there's so much we can do, you know, Honeycomb, we've been talking about bringing everyone up to the level of the best debugger, right? + +Increasingly, like, the hard part is getting detail. Like, these socio-technical systems are made of just as much of our brains as they are the data. Like, an example I often give is like The New York Times and The Washington Post. If you just switch their people overnight, nothing would work because so much of the system is in those people's heads. And the more we can bring it out of the people's heads and put it somewhere that other people can add to it and ask questions of it, you know, then it pools our knowledge and our resources. + +Like, one of the biggest things that blew my mind after becoming a CTO was how many engineering executives out there trust their vendors more than they trust their employees. And they're constantly susceptible to vendors who come up and like, "So give me $10 million and your people will never have to understand your systems again. We will tell you what to look at and we'll tell you what it means." + +And that is... because people come and go and vendors last forever! But, like, come on! Like, that is just not going to work! But there are absolutely things that we can do to make your people more productive, you know, better, you know, to get better data in, to like pull your knowledge. + +**Charity:** I like I wrote that down: joint cognitive research because I think that that's so true. You know, at Honeycomb, we actually built a little AI thing, Bob, which is using ChatGPT to, when you deploy some code, you can ask questions about it using natural language. Just like, "What's slow?" You know, which is super useful because using a query explorer is hard for everyone. + +But, like, just being like, "What's slow? What changed?" is super valuable, and it helps those engineers that, like, just go around the room and it, like, ask a question to the person next to them. And it's like, you're now winning back time in a way! + +**Adriana:** So much time! + +**Amy:** Yeah, you know, I agree with everyone on the stance on AI because I think there's this overarching fear out there where they're like, "Everyone's like AI is gonna take our jobs!" No, I hope so! I don't think we're at Skynet levels yet, but also I think, like, we, like, personally, I like the idea of having, you know, something take away some of the cognitive load so I can focus on the cooler things and also take away some of the biases that I might have, assuming that the AI is trained without the biases, right? I guess that's the caveat. + +But, you know, I like the idea of like something saying to me, "Hey, this is where you look further. You might be interested in blah." So you look and you're like, "Sorry, AI, you're wrong, but thanks for pointing it out anyway!" And I think that that can be hugely helpful to just, I don't know, just take a load off, right? + +**Charity:** Definitely! Or like you're getting paged about something, it's a database you're not super familiar with, and it's like, "Hmm, something like this happened last year around this time, and here's what they did to resolve it." Would this be useful to you? Yes! Exactly! + +It's just your little help on your shoulder that's just telling you all the goodness that could be going on in your system anytime. + +**Charity:** I think about something that I, or people, spend a lot of time trying to get information out, whether that's like combing through data or trying to find stuff in the history or something, that's an AI problem! Like, they're so good at that [__]. Let computers do what computers do best so that humans can do what we do best! + +And I know that, like, there's this fear that we all have about, you know, change and stuff, and some jobs are absolutely going to be lost. But many more will be changed. And I think that the way that we make this good for people is not by resisting it, fearing it, but by being thoughtful about how we involve it and how we adopt it, and, you know, making a change gracefully. + +**Charity:** Obviously, now we're gonna start talking about government policies, which is... anyway, but like, yeah, like Amy said, I hope part of my job gets automated away because I could be using the cycles on something else! + +**Host:** Well, there’s something else I want to maybe shift to. From when you were talking, Charity, one of your favorite topics, which is the unknown unknowns. Because I think AI can eat up most of the known knowns, right? Like, we go and ask the chatbot and be like, "Hey, this seems like it's happened before. Has this happened before? And, oh look, here's all incidents reports and what happened before." Those are known knowns. We kind of know what to do. + +[00:34:00] I think the place where today's efforts, anyway, we'll see if it changes, but I don't think it's going to change drastically in the next decade, um, are still not aimed at figuring out the unknown unknown, right? This is where I think humans are still going to have an edge for a very long time. + +Basically, like we'll use the AI to collate the data, to do all the undifferentiated lifting, but it's still going to be people that peer into that space between the data and go, "I wonder if there's a packet loss somewhere in, you know, deep in this network that's never happened before. We've never seen it before, and now you have a novel insight." + +And that's the place where today's AI really, like, it can kind of make things that seem like novel insights because it's approximating what a human would do, but those actual really tough novel insights, I think are going to remain in our domain for a long time. + +They always come out with these demos that show these, like leaps of things, you're like, "Oh, I never would have thought to look at that." But, like, those are impressive because they're rare. And at the end of the day, if it was the right thing, you someone still has to go and... + +Like, the thing that I keep coming back to that humans are good at is attaching meaning to things, right? Like, any computer can tell you if there's a spike or how big it was, but like only a person can come out and go, "This one really means something," because of that context that the computer won't necessarily always remember. + +**Host:** What do you think is the current largest challenge in observability? + +**Amy:** I'll start with Amy. Sampling! Ask me again in five years; it'll probably still be sampling. Like, it's just, you know, there's a few places where we could probably tune things up, and generally we're not sampling a lot today, but some of the systems that we want to turn on, we're going to have to because they do a bunch of this is kind of circling back on like that tech debt under the covers. + +Like, as soon as we turn on tracing in some of these systems, we see that they're doing a bunch of busy work that doesn't make any sense. But the first thing that happens is it tips over our collectors and overloads our upstream account, and then we got to go turn it off. And so we don't actually get to figure out what the heck is going on. + +So I think, like, sampling and really just kind of getting control of all of the data we're generating that might not ever have any value is super important because it's by no means an easy problem to solve. It's probably the hardest problem in observability right now. It's like just how do we get this to a rational amount of data so that my cost is rational so I can do more observability, get deeper insights into my systems? But you got to pay the bill, and so, like, this is continually gonna be our struggle. + +**Charity:** That's the answer of a super user! + +**Host:** True! What do you think is the largest challenge, Charity? + +**Charity:** Um, the people don't understand how badly they have it now and how much better it could be. People who have spent their careers, like, slugging it out in the salt mines using metrics, managing the overhead of metrics, not having high cardinality data, dancing around between metrics and logs and [__]. They genuinely have no idea that the way they're doing it is the hard way. + +And they don't, they don't actually... nobody's optimistic in computers because why would you be optimistic about computers? But like until people see their data in our world, it doesn't... they don't understand how much easier it could be, how much better it could be. + +And this, there's this mental model, you know, and observability 2.0 is dramatically easier than 1.0 because it's just interesting, shove it in, interesting, shove it in. There's no, like, custom metrics to measure or, like, manage and farm over time. There's none! But, like, the difficulty is that it's a mental model shift that people have to... and I am confident that 10 years from now, 15 years from now, people will be, you know, using the systems that, you know, we kind of sketched out a few years ago for observability to understand their systems because the rate of complexity is just going up. + +You know, too high! We might be out of business by then! Like, we might have been way too early! But I am extremely confident that there will be no other way to understand your systems 10 years from now but using something like this. + +**Adriana:** That's fair! They are definitely not getting any simpler or not. And, ironically, what that leads to is this increasingly, this increasing reliance on heroes and martyrs because if there's nothing knitting together all of your sources of the data except the people with the intuition and the scar tissue who can make good guesses, you have to... you don't have any other choice! + +I call them load-bearing humans in my God because, like, when I run into that, that's how I talk to the management tree about it. I'm like, "You have a load-bearing human. This one person is doing this task, and if they go on a vacation or win the lottery, we are going to have a bad time!" Right? + +And that's the way to surface the risk is like, you wouldn't put a single, you know, firewall. You would always do redundant, right? Like, so we need to make at least make that pun, and that's usually a good way to start getting people realizing that load-bearing humans are holding the world together. + +**Adriana:** Yes, all over the world! And even as our rhetoric has gotten better about this, and our understanding has gotten better about this, there's still these tech... it's like the finger in the dyke. So to speak, you know? + +**Charity:** The little Amsterdam! + +**Adriana:** Yeah, dude, just like plugging the dyke with his... + +**Charity:** God, I got to stop using anecdotes! So terrible to say out loud! + +**Host:** But like these people exist, and then they're doing a hero's job, but they shouldn't have to, and it's bad for everyone! It's also putting a lot of trust and retention... + +**Charity:** Oh my God! Very fragile! + +**Adriana:** Yeah! + +**Host:** What about you? What do you think is the biggest challenge? + +[00:40:00] **Adriana:** So I think the biggest challenges are organizations getting in their own way when it comes to observability. Either because they don't... they kind of get observability in much the same way that everyone kind of got, like, Agile and DevOps is, like, important but they kind of totally missed the point at the same time. + +And so we're, you know, we've made like a little bit of progress, but also not. And on a similar vein, like with OpenTelemetry, again, it's like, "Oh, I understand the benefits of observability. Let's get OpenTelemetry into the org!" And then again, like not understanding, like, what they should do with that. + +Either because, you know, of vendors who oversell, right? Where we have a tracing tool, we can totally take your traces and put them on a little island by themselves with no food or water, and nobody ever freaking looks at them because they're a little island. + +**Charity:** Yeah! + +**Adriana:** Yeah, we'll totally do that for you. We'll take your money too. Thank you! They'll be safe! + +**Host:** True! Exactly! Never use them! + +**Charity:** Yeah! + +**Host:** I, you know, we get, especially in large organizations, we get very tempted or drawn in by the shiny, shiny new thing that seems too good to be true, and you're sold on the vaporware, or like, some reads an article about blah, and they speak to a salesperson about at blah company, and then all of a sudden they walk in all starry-eyed. + +So I feel like these are such huge barriers to actually getting [__] done for observability! + +**Host:** What would you say is the biggest misconception around observability right now? And I'll start with you, Adriana. + +**Adriana:** Biggest misconception? I think right now I would say, like, people thinking that it'll solve all their problems without putting in the work. I think there's a lot of, like, people assuming there's a lot of, like, magical fairy dust that comes in with observability, and then when they realized that it's extra work, it was like, "Oh crap!" + +**Amy:** What about you, Amy? + +**Amy:** I think I'll extend what Adriana said, right? Like, it's that extra work of, like, the need for understanding doesn't disappear. Ideally, as we get the data presentation into a better place, it gets easier to achieve, but the need for people to understand the world around them and be able to synthesize new information from the observability data, um, that doesn't go away. It just gets more efficient. + +**Charity:** What do you think, Charity? + +**Charity:** I think the biggest misconception I see is just people thinking that observability is its own sort of standalone project that you can buy or do or check off when, in fact, it's so interwoven with the development of software, iterating on software, owning, deciding what features you're going to build, you know? + +I mean, deciding, figuring out how your product is working, figure out what your users are actually doing, why you should care about it, where you should spend your time. Like, it has its tentacles everywhere, and there are lots of things that we do that make observability better or worse have nothing to do with anything that's marked observability. + +And this is both, I think it should be both encouraging and reassuring to people and also a little bit depressing and demoralizing because, but like it just... it sees for a more holistic view of like this is not... I don't think you could have, like, an observability project that you could just check off. + +It's... it requires us to get better at understanding the mission of building something, which I think we're doing bit by bit over the years. Like we're understanding more about how to build software that is better for our users, that is better for our engineers, that is more linked to the business. + +And, like, you were talking earlier about, you know, there was this great discussion that was kicked off by McKinsey, of all people, about measuring the work of software, whether or not it's measurable, you know? And of course, they're completely wrong, which meant that K Beck gave him a smackdown, and Gade gave him a smackdown, and there's a great discussion about how you measure software development. + +But in one of the pieces, someone said that people have got to start realizing that building software is not like building construction. It's like, it's more like the design phase, and you would never say that a blueprint is better because it has more blue lines, right? + +Which gets to your point earlier, Amy, about how this all work is knowledge work, right? And so I... where was I going with all this? I don't know! I feel like it is indivisible from the act of getting better at delivering software. + +**Host:** So, if I think that a lot... God, I keep... I swear I've taken up 50% of the airtime in this, and I really apologize! I may have just had too much coffee today, but I feel like so many software orgs out there have kind of given up on saying that they're trying to get better at delivering software. + +And so instead, they set these goals themselves, kind of nibble around the edges. We're going to get better at observability this year; we're going to get better at CI/CD; we're going to get better at blah, blah, blah, blah, blah. But, like, the great leaps forward that you make that you have to make in order to keep up with complexity because you can't just hire your way linearly out of this. You have to make these big leaps, and that requires taking a much, you know, a much more bird-eye view and being like, in order to deliver software better this year, it requires that we make these investments in observability, we make these investments in testing. + +And I feel like, in conclusion, in conclusion, building software is an amazing job! I thank my lucky atheist stars every day that I get to be in this industry, not like picking weeds and getting dirt under my fingernails, but like sitting here at a computer solving puzzles and playing with my friends. And I feel like I wish that everyone in software could have as magical of an experience and career as I have had. + +And I think that observability is a big part of how we get there. + +**Host:** That's true! That actually brings me to my next question as we're getting ready to close up. What do you think is in store for observability for this upcoming year? + +**Charity:** The first person to jump in is... + +**Adriana:** Okay, I'm not gonna pick on one of you! + +**Charity:** Oh, I haven't been keeping up on all the roadmaps and stuff, but I guess... So what is your wish? What is your wish that this year brings for observability? + +**Adriana:** I guess the thing that I'm working on is a lot of folks still don't really get SLOs and SLIs as much as I think it will help both engineering management and engineers on the ground just communicate better about what they're trying to accomplish. + +Like, if I had a wish, like, it would be to drive kind of that understanding deeper into our industry just so now, so I can go into a room and talk about the power that's available here and just, like, having the metrics to understand and not just get stared at like I just nailed a fish to the wall, right? Like just, you know, it would just be nice to not have to start at zero over and over. + +**Charity:** That's fair! + +**Adriana:** I personally want to see more people, like, besides like, you know, we often associate, I guess, observability with more of the Y side of things, and it is so not just that, right? I mean, it's everybody's problem, and so I want to see more conversations where we get more developers giving a crap about observability and not just developers, like folks who are planning tech projects, like software releases, whatever. + +Like, make sure it's part of their language. Make sure that we empower our QAs to use... everybody misses that! Give it to your support team! Gosh! Like, what an opportunity! + +**Amy:** Yeah! Well, even your QA, right? Like, they're testing your software, and they could use observability to figure out what the hell is wrong and tell developers, "Hey, this is why it's wrong!" + +You know? So I want to see more of that, and I think it puts us closer to, you know, Charity's observability 2.0 where it's like more part, more of a holistic part of the SDLC, right? + +**Charity:** Yeah! And I think that's where it needs to be so that it's, you know, part of... it's baked in, baked into how we deliver software and how we code software and all that good stuff! + +**Amy:** Definitely! Everybody does a better job when they can see where they're going and see what they're doing when they are getting feedback back from the system for sure. + +**Host:** I had this sticker that says that SLOs are APIs for our engineering teams, and I completely agree! Like, if you don't have an agreement that you've all agreed upon, you're down in the weeds negotiating over every single decision you make about how to spend your time. + +Like, it's absurdly inex... it's absurdly expensive! + +**Charity:** I don't know! I expect that what's going to happen in 2024 is that everything's going to get even more muddled, and people are getting even more confused, and, um, it's fine! It's entropy! It's how it goes! + +I guess my wish for Christmas would be that... my wish would be that more people just have that feeling in their gut that you get when you see telemetry feedback to you about what you've done and just go... that feeling that you get, this, like, you feel more grounded, you feel more certain, you feel more confident about what you've done. + +Observability is the way that we can move swiftly and with confidence, and I feel like this is something we all know in our heads we need to learn in our bodies. And the only way that we can learn it in our body is by seeing it and experiencing it. + +**Adriana:** I would wish that everyone had that experience! + +**Host:** Most definitely! I love that! To keep that in store for 2024 as, like, find that fuzzy feeling when you start seeing your logs and metrics start showing up on your vendor. + +Definitely! Well, thank you all very much for the amazing insights you've been able to provide today on our panel around observability. This group is going to be coming back later in June to have a little other chat around observability. Stay tuned to hear about those details later! + +With that, we'd like to sign off and say thank you all for watching. Stay warm out there! + +**All:** Yay! ## Raw YouTube Transcript diff --git a/video-transcripts/transcripts/2024-02-23T19:09:30Z-otel-in-practice-opentelemetry-orientation-how-to-train-your-teams-on-observability.md b/video-transcripts/transcripts/2024-02-23T19:09:30Z-otel-in-practice-opentelemetry-orientation-how-to-train-your-teams-on-observability.md index 3bec09d..f6fab4a 100644 --- a/video-transcripts/transcripts/2024-02-23T19:09:30Z-otel-in-practice-opentelemetry-orientation-how-to-train-your-teams-on-observability.md +++ b/video-transcripts/transcripts/2024-02-23T19:09:30Z-otel-in-practice-opentelemetry-orientation-how-to-train-your-teams-on-observability.md @@ -10,81 +10,272 @@ URL: https://www.youtube.com/watch?v=o-1UE67l9q4 ## Summary -In this presentation for OTEL in Practice, Paige Cruz, a developer advocate at Chronosphere, discusses her extensive experience in observability and shares insights on training developers to implement tracing effectively. Having transitioned from a Site Reliability Engineer (SRE) to focusing on open-source observability, Cruz emphasizes the importance of hands-on workshops tailored to an organization's specific tech stack to bridge knowledge gaps among developers. She explores the complexities of observability and the rising significance of OpenTelemetry (OTEL), noting that many developers lack fundamental knowledge about tracing and the tools available. Cruz also shares strategies for creating effective training programs, including identifying learners' needs, structuring content to promote engagement, and leveraging existing resources. The session concludes with discussions on the challenges of ownership in observability, highlighting the need for collaboration between operations and development teams. +In this YouTube video, Paige Cruz, a Developer Advocate at Chronosphere, shares her insights on observability and the importance of effective training in this emerging field. With a background as an SRE at observability companies, Paige discusses the challenges her team faced in leading observability initiatives, particularly in migrating systems and re-instrumenting applications. She emphasizes the need for hands-on workshops tailored to specific tech stacks, noting that successful training should bridge knowledge gaps and engage developers. Paige also highlights the complexity of observability concepts, the significance of understanding OpenTelemetry, and the necessity for developers to take ownership of instrumentation. Throughout the talk, she provides practical advice on creating effective training programs, measuring their success, and fostering a culture of observability within organizations. The discussion includes audience engagement, where participants reflect on their training experiences and share strategies for improving ownership and responsibility in observability practices. ## Chapters -Here are 10 key moments from the livestream with the corresponding timestamps: +00:00:00 Welcome and introduction +00:01:47 Observability initiatives overview +00:02:38 Importance of hands-on workshops +00:03:50 Training development discussion +00:05:30 Observability complexity and knowledge gaps +00:08:00 Tracing as a valuable telemetry type +00:10:40 Effective training strategies +00:12:50 Identifying learner gaps +00:15:30 Tailoring training to specific needs +00:18:30 Creating effective training materials -00:00:00 Introductions and Paige Cruz's background -00:05:15 Overview of observability initiatives and challenges -00:10:20 Importance of training in the observability space -00:15:00 Discussing the complexity of introducing tracing -00:20:45 The significance of hands-on workshops for developer engagement -00:30:10 Strategies for effective training development -00:35:00 Identifying learners' knowledge gaps and tailoring content -00:45:30 Delivering training in various modalities for better reach -00:50:15 Sharing resources and further learning materials -00:58:00 Discussion on ownership boundaries in observability and training outcomes +**Paige:** Thank you so much for having me, OTEL in Practice. I have been attending the last few sessions and excited to present a little bit about my world. I am Paige Cruz. I work at Chronosphere. It is my third observability company. When I wasn't working for observability companies, I was an SRE at customers of those observability companies working on observability. So if you can't tell, I have thought about this stuff for a very long time and from lots of different perspectives. -# Observability Training Workshop +And just a kind of a quick tale from the field. I have retired from SRE today. I'm a dev advocate that really focuses on the open source side of observability, but in my day when I was practicing, I had to lead. My team was always tasked with leading big observability initiatives, whether that's migrating from one vendor to another, off of self-hosted open source onto maybe a managed service from one of our cloud providers, rolling out tracing. Oh, my God, the logging bill is so high, you got to get on it. There's just a lot of things that kind of fall under the observability umbrella that were kind of targeted for my team to be the stewards of. -Thank you so much for having me, OTEL in Practice. I have been attending the last few sessions and am excited to present a little bit about my world. +[00:01:47] And what I found when we had to do these initiatives that touched every team, like re-instrumenting either to add traces or away from a proprietary protocol onto beautiful open source hotel, is that yes, technically, if the dev teams were too busy, my team could do heads down work for like three or four quarters and yeah, we could deliver something and we could be migrated by the end of that session. However, then we were really seen as like the owners of observability and we were kind of treated as like Ask Jeeves. And it just really felt like the line of responsibility between ops and dev kind of went out of whack. And that's totally fair. We moved everything. It was a new platform, new UI, new libraries. Of course, there was going to be a lot of knowledge gaps to cover. -I am Paige Cruz, and I work at Chronosphere. This is my third observability company. Before working for observability companies, I was an SRE at customers of those companies working on observability. If you can't tell, I've thought about this stuff for a very long time and from lots of different perspectives. +[00:02:38] And so I have steered quite a few migrations and I can say I've got to try lots of different tactics to be able to delegate that re-instrumentation work or to really pass that baton of ownership rightfully from solely on ops over to app devs as well. And the best tactics out of everything I tried, linking the docs, sending conference talks, the best was hands-on workshops that were specifically tailored to my org's tech stack, what we were trying to get developers to do, the value that it brings at the end of the day. -### A Quick Tale from the Field +So this is why I wanted to bring train the trainer, specifically we're talking about tracing training today, and that just requires participation from devs across the stack up and down. And to quote the seminal work High School Musical, we are all in this together. So let's talk about why training matters. Yes, let's talk about how are you actually going to develop a training? These aha moments, unless you were a teacher in a previous career, you probably don't have a strong background in learning theory or delivering trainings and teaching. So let's just get into what you need to know to train folks. -I have retired from SRE today and am now a developer advocate focusing on the open-source side of observability. During my time in SRE, I led my team through big observability initiatives, whether that was migrating from one vendor to another, transitioning from self-hosted open source to a managed service from one of our cloud providers, or rolling out tracing. The logging bill would often get so high that we had to take action. +And then I'll kind of walk through some really specific decisions that I made when developing the KubeCon workshop. And then probably the favorite part of anything that I put together is the further resources. So please, I will share these slides, books, talks, things that can help you along your journey. -There are many components that fall under the observability umbrella, and my team was tasked with being the stewards of these initiatives. I found that when we had to implement changes that touched every team, like re-instrumenting to add traces or moving away from a proprietary protocol to open-source tools, there were significant knowledge gaps. My team could do heads-down work for several quarters and successfully deliver a migration, but we were then seen as the owners of observability—almost like an “Ask Jeeves” for developers. This blurred the line of responsibility between operations and development. +[00:03:50] So why does training matter? Well, there are a few layers of complexity that we're talking about when it comes to, say, introducing tracing. The first thing is like, full stop, observability is still emerging. It is something that today we have learned basically on the job pretty informally. It kind of depends on what tech stack your company has, what vendors or protocols that your company is using, how many companies you've worked for, what you've seen. -While it's fair to say that we moved everything to a new platform with a new UI and libraries, there were many knowledge gaps to cover. I've steered numerous migrations and tried various tactics to delegate the re-instrumentation work and pass the baton of ownership from operations to application developers. +So even two developers with the same amount of years of experience could have totally different sets of knowledge about this space. And so, you know, a baby is not sitting there thinking about the milestones that we have been reaching. That is just not happening. We have to teach this stuff. And while it is done, and while I think the informal approach is important for developing operational skills and sort of how do you use this data in the heat of an incident to decide on mitigations, you still have to find some sort of way to bridge that gap between the formal patchwork knowledge and a strong foundation that you can build on. -### Effective Training Techniques +So that your developers can say, add tracing or maybe bring down the bill or understand how sampling affects like viewing aggregate trace data. There's like a lot of complicated stuff in this space. And so you really got to think about where your learners are, what they know, what they might have seen. -Out of everything I tried, the best tactic was hands-on workshops tailored to my organization's tech stack. This approach demonstrated the value of observability and the changes we wanted developers to implement. This is why I wanted to introduce the concept of "train the trainer," specifically focusing on tracing training, which requires participation from developers across the stack. To quote *High School Musical*, "We are all in this together." +[00:05:30] And specifically, so we've got observability, kind of new emerging domain, we're kind of bringing some things from monitoring, but we're also discovering a lot of new stuff. Within observability, we've got OpenTelemetry, which totally used to just be known for tracing. I'm so happy that OTEL has now, like, subsumed, like, I think the collector is the more popular, like, most well-known component at this point, which is kind of cool. But unless you have been really interested in observability or specifically tasked with bringing OTEL into your org, like a lot of my developer friends that I talked to don’t, they're like, "Oh, that's cool. OTEL exists, you know, we're on some vendor, it's not really something I'm looking into." -#### Why Training Matters +So while we are like, "Oh my God, exponential histograms, logs are GA," like, all this stuff is happening, it's real. There's still a whole segment of our colleagues that maybe haven't even heard of OTEL. And if you were to just say, "Hey, add tracing to this service. Here's a link to the OTEL docs. It's got everything you need to know." If they don't know what a span is, if they don't know how to navigate or the different components of OTEL, like maybe y'all are using a vendor's collector versus the OTEL, there are just a lot of ways that you could get lost within the OTEL project. -Observability is still an emerging field. Most learning occurs informally and depends on the tech stack, the vendors or protocols used, and individual experiences. Even two developers with the same number of years of experience can have completely different knowledge sets. +And so observability, new, OpenTelemetry, new and rapidly evolving. And finally, the nugget is for tracing, in particular, the problem that I have seen is that it really has this reputation as a tool for power users, the super set of engineers. And to be fair, like, if I reflect on how I got to my tracing knowledge, I literally worked building the tracing product at New Relic and then went to LightStep to go be an SRE and support the tracing infrastructure there. -While informal learning is beneficial for developing operational skills, we must find ways to bridge the gap between patchwork knowledge and a strong foundation. This enables developers to add tracing, manage costs effectively, and understand sampling effects when viewing aggregate trace data. The complexity of observability means we need to consider where our learners are, what they know, and what they’ve experienced. +[00:08:00] And so I've had to think about tracing a lot, I've had to work on these systems, and that is just not a repeatable experience for everybody, nor should it be. We need all types of people to develop all sorts of things. And so this was the problem that I really honed in on for my workshop was like, I think tracing is really a super valuable type of telemetry. I want more people to use tracing. One of the barriers that I see is that yes, partial traces still can be helpful, but the best case is that you've got a full complete trace end to end throughout your system, which requires every service owner along the way to do their part in instrumenting and then using that data. -We also have to recognize that OpenTelemetry (OTEL) is rapidly evolving. Even if developers are familiar with OTEL, they may not know how to navigate its components or the differences between vendor implementations. Simply linking to the OTEL documentation may not suffice if they don’t understand basic terms like "span." +So I said, this is the problem that I want to solve, and I want to help people both know how to send trace data, oh my God, and how to use trace data. Because just having the information alone of how to instrument isn't enough if you don't know what to do when you get to a trace waterfall. So the counterpoints that I've heard, the doc site has everything they need to know. I wrote the exact instructions down. They just have to follow them. -### Training Development +But while I will say the OTEL documentation site is absolutely fabulous, I rely on it all the time, I still see a lot of questions like, is this library instrumented with OTEL? Is this, is the Ruby log spec, like, where is that at? And those are like very easily answered questions if you know where to look at the doc site. Just linking to OpenTelemetry.io is not going to necessarily help someone navigate through our set of projects. -To develop effective training, we need to find those "aha" moments of discovery. After all, the goal of training is to change behavior and encourage learners to take action. Just providing rote instructions is insufficient. +Yeah, so yes, plug for the OTEL doc site, it is fabulous, but people need more than just the knowledge, they need the skills. So very quickly, who here, you can maybe raise your hand or throw it in the chat, who here has taken a training course at work? You don't have to tell me where or what, but you've taken a training course that just felt like a total waste of time. You really could have used those extra two hours and actually done something productive. I will raise my hand. I have sat through a couple of those. It's okay. I know it's been out there. Even our mandatory trainings that we've got to take sometimes are like eye roll. -We need to consider what our learners need to know and the skills they need to develop. This requires understanding the problem we’re trying to solve, whether it’s better visibility or deeper insights into a profitable system. +So bad training is very different than good training. It could be that the training course you took, you were already an expert, and it was just covering stuff you already knew. It could be it wasn't tailored to your use case. Like, your company signed up for generic Kafka training, but actually you needed to know how to tune and specifically level up the operations for your cluster with your configurations and everything within the context of your org. -#### Identifying Learners’ Needs +So the generic 101 Kafka training wasn't going to do anything for you. It wasn't tailored to your use case. And then also it could just be the format and the delivery was totally mismatched. Like for me, I'm not a morning person. So anything that's before 9 am, good luck. My brain is not firing at full capacity. -If you're unsure of what your learners know, consider sending a survey or having informal coffee chats. It’s essential to include a variety of roles, as observability impacts everyone. +[00:10:40] And so there's a lot of if you've encountered bad training, this is not what I'm advising you to do. We're going to talk about how you avoid some of those things and have effective training. So to answer with kind of this section, why does it matter? Why does training matter? Well, I came across this quote that made me so sad, but like is totally exemplifies this. "I don't even know what I don't know, which makes it pretty hard to get started answering my own questions." And this was some anonymous user on Reddit who's trying to find out things about OpenTelemetry. -Understanding the consequences of missteps can also clarify the importance of training. If developers fail to instrument their applications correctly, it can lead to increased costs or loss of visibility, which highlights the need for ownership. +And that is the case. Like this is where we need to start. This is where our learners are. There are some people for whom OTEL is this mystical black box that they don't know anything about. And if you're on this call, I'm pretty sure you have, you know, your way around the projects, you know the architecture, you know what's up, but we've got to keep in mind that we are kind of a select group here and the observability and OTEL knowledge needs to be very widely distributed, democratized. -### Scoping Training Content +And that is why training is important because not everybody is here where we are yet. And we need them here for full traces, for better observability across companies, across industries. That's the why. Let's just talk about the how now. We don't want to replicate the bad, awful trainings that we have been through, but what I kind of shortened this as, you want to find these aha moments of discovery. You want to have these learners engaged, and really the whole point of training is that you are trying to change behavior or get learners to take actions. -When creating training material, focus on what is necessary for your specific audience. For instance, if you want attendees to leave the workshop able to instrument traces independently, ask yourself whether each piece of content contributes to that goal. +So just giving them a rote, so for my workshop, if I had just given them, here's the step by steps of how to instrument specifically a Python Flask application with traces. Cool. They could leave and they would know, for that tech stack, like, where to go, where to find that information, and how to get traces out of that. Maybe they work at a place that has Ruby or works on Ruby on Rails. If I didn't do the work to explain the higher-level concepts or here's the OTEL registry, you can use it to check any language instrumentation library. If you don't include the actionable pieces in the takeaways, people can still get really lost. -Additionally, consider the existing resources within your organization. Don’t reinvent the wheel; leverage the available tutorials, documentation, and community knowledge. +[00:12:50] Even if your instructions were so beautiful and so detailed. Believe me, I have tried that route. So when really where you want to start with training is finding the gap, which is between where your developers are today, what they know, what they understand, what they've seen, and where you want them to be, what actions do they need to take? Do they need to re-instrument from proprietary format to OTEL metrics format? Do they need to add spans? Do they need to take away spans? Because maybe the auto instrumentation they turned on is like totally blowing up the bill. -### Delivery Methods +Think really think concretely about what should happen after this training. What are you expecting to see the results and the change? And then that gap between where they are and where you want them to be, that is where your training needs to fill. And we're not going to get deep into learning theory, but really two things. What do they need to know? Knowledge, what info? And then skills. What do they need to be able to do with that knowledge? Together, that is the recipe for a successful training. -In delivering training, utilize various modalities, such as live sessions, self-guided learning, and recorded sessions. This approach accommodates the diverse learning styles and schedules of your audience. +And so when you're identifying the gap, get clear on the problem you're trying to solve. What is the main driver of the initiative? Better visibility? Are there a problem area in the profitable part of your system and you really need deeper level visibility there? Do you want to own your data? If you think about the motivations, that can help you really scope down. Because for us, observability experts, it is very tempting to include the whole world, be like, well, let's start with OpenTracing and OpenCensus and, you know, before OpenTelemetry, and like, you will lose people. I promise you. I promise you, you will lose people if you do that. -### Conclusion +So getting really crystal clear on these, answering these questions for yourself will set you up to really hone down the exact things you need to train on. And if you don't know your learners or you don't know what they already know or what they're exposed to, send a survey, do some informal coffee chats, and make sure that you're including a variety of roles. Grab front-end engineers, grab security engineers, back-end, edge, network, you know, because it does take everybody. You need a representative sample to really be able to understand your learner base. -In summary, effective training in observability requires: +And sometimes it helps to think about the flip side. So, like, where do I want them to be? If you're having trouble answering that, think about what is the consequence if they get this wrong? What is the consequence if we don't reduce the bill or add tracing in this next quarter in order to evaluate, you know, companies or whatever? Sometimes you can just flip the script because you can get a little bit lost in this wide world. -- Clearly defining the problem and identifying learner needs. -- Bridging the gap between current knowledge and desired outcomes. -- Tailoring content to the specific tech stack and organizational context. -- Utilizing existing resources and encouraging ownership among developers. +[00:15:30] So as a small, small exercise to think about who our learners are. And so I can make the point that learning and knowledge gaps are totally separate, understandable and expected, no matter your seniority when it comes to observability and monitoring. Let's maybe consider a few learners. We've got like a new grad, they are probably like maybe first job, probably were exposed to logging at some point in their education and maybe if they did an internship or something, but excited to learn doesn't know where to start. Needs both the knowledge and the skills. -Thank you for your attention! I'm looking forward to hearing your thoughts and discussing your training needs. +Kind of best case to have somebody like excited to learn. Because you will encounter resistant adult learners sometimes. Even thinking about a senior developer thinking about the background, did they work at a mega corporation where there were so many teams that handled developer productivity and platform engineering that they are used to the amazing capabilities and benefits of observability, but going to a smaller org. Oh, I like I've got to add instrumentation. We don't have a wrapper library that covers that is always up to date and covers what I need to know. + +Even with senior engineering, it's tempting to say, oh, they should know what to do. Send them the docs, like, send them the ticket, let them do the work. But it really depends on where you've worked and what you've been exposed to. If you've worked at a small startup, you know that you've touched almost every part of an observability stack, from the bill to setting up collectors, to doing instrumentation within applications. + +So really think through, yeah, what people's backgrounds are. That's why this is so individualized and contextualized. It's why a generic 101 basic training won't meet everybody's needs because everybody's kind of coming and bringing to the table like a different set of knowledge and, you know, by the time you're a staff engineer, I guess I would assume maybe you shouldn't assume you've seen a lot of systems. You've seen what works. You've seen what doesn't work. You've seen, you know, you've seen the rise of observability within monitoring, and you probably got a lot of opinions on how things should go. + +That is awesome. And developing a training to meet both the staff engineer's needs and the new grad's needs is probably a stretch. So that is why we want to think about our learners. However, I will say one of the tips is to have once you've got a training kind of sketched out, you've got your first draft, having both a novice walk through it and an expert walk through it and kind of watching what they do, kind of seeing where the stumbling blocks are. That is a very powerful exercise. So tap into that expertise when it exists in your org. + +[00:18:30] Okay, so when it comes to actually, like, let's create our material, like, what is this course going to be on? Again, the reason I keep saying you've got to scope it to, like, what do they actually need to do with this information? Because what I was making the 101 instrumentation workshop, oh my God, all I had to cut so much out. I did not know how much I wanted to include in there. And so I had to get really crystal clear on at the end of this workshop. + +I want attendees to leave and be able to bring OpenTelemetry tracing instrumentation into their own projects. I wanted them to be independent. I wanted them to be able to take this knowledge and use it in their daily life. So that meant that was my guiding star. Anytime I was like, should I put this in? This feels like it's too long. I went back to say, would this help them independently instrument traces in the future, yes or no? If no, cut it out. + +So grounding experiences in the real-world context, that is where you will have a huge leg up if you're developing it for just your organization or business because you are sharing common organizational goals of selling the product or the service or whatever it is that you do. So you already have kind of a shared context and community. You're all working within the same tech stack. + +So while I had to make a choice of like, what is going to be the most easy to read language, minimal framework that I could kind of take any developer and walk them through instrumentation, you would get to say, "Oh, we do Go and Java, and these are our frameworks," and you can really create the dev environment and the exercises to mimic what your developers would be doing in the real world. + +That being said, if you're developing it for an org, I would say your best bet is to host an instrument-a-thon where you have folks actually bring their real service that you're asking them to instrument and working through it with them. Sandboxes and tutorials are great when you have sort of a wider, more generic use case, but always get as specific as you possibly can be and scope things down. + +So, you have a leg up if you're working on one for your org. And then, again, like, what is the absolute minimum, minimum setup and dev environment? I mean, I think I talk about it later, but I really was torn on whether or not I should introduce the collector. Because you think about the collector is this big component of OTEL, surely people are going to be running into this in the future. Wouldn't it benefit them to know, kind of, really that full path that telemetry takes? + +However, when I saw that the Jaeger all-in-one image had a native OTLP endpoint, I was like, oh my God, Collector, bye-bye. We only have two things to run. That is only two things to have to debug if something goes wrong during the workshop. That is much better than three, and it cut out a whole, you know, part of like configuring the collector, running the collector. And it wasn't, my workshop wasn't about production, production ready tracing system infrastructure. It was, how can you get traces today? And do you know what to do with them? + +So you, again, like, major benefits if you're working with a group of people with a shared mission, working on the same system and tech stack. And then when it comes to delivering, really tap into what already exists in your network. Like, yes, a standalone training can do the job. But if your organization has communities of practice or like sometimes they're called chapters, like chapter back-end or chapter front-end times and places to exchange ideas and learn on the job, tap into that and ask the organizers if you can host the next meeting and do the workshop or get their help in advertising. + +Because with these organization-wide initiatives, just sending one Slack about a training is not going to get many people showing up. Maybe you've got people who can make it mandatory, that's cool, but forcing learning is also kind of comes with its own battles. So the second thing you want to consider is the modalities. Lots of people learn different ways. A lot of us work in distributed remote organizations. So you can't even count on being in the same time zone as somebody. + +So make sure that you provide both like a live guided session for people to, you know, that mimics more of a classroom, self-guided, but make sure that you've got office hours for people to check in with you. And then, you know, you can always record a live session and make that available as well. But having these multiple ways means you're going to have more luck getting more people actually getting through this content, learning the things they need to learn, and doing whatever you need them to do to increase system observability. + +So, I guess, like, to recap, you want to like, really think about what is the problem you're trying to solve with this training. You've got to learn, you've got to figure out where your learners are today, where you then you've got where you want them to go and then just really narrow that training gap and scope it as small as possible for the best success. + +So we'll take a quick look at some of the choices that I made for this workshop. And the first is do not reinvent the wheel. Like, I did not start with a blank page and I said, okay, instrumentation, let's go. Like, no, the beauty of working with open source and working with OTEL is that my friends, like, Reese has fabulous, like, I look at Reese's metrics workshops all the time. And the talks that she's done and Adriana, like, everybody across the vendors and our open source community members have already contributed a ton of interesting tutorials and knowledge, and it's really about shaping and scoping that down for your use case. + +And yeah, making sure that it fits and so you're not starting from scratch. You weren't hired to be a teacher or instructional designer. Please borrow. Particularly the way that we, that folks organize information. You could take my Python Flask workshop and totally rewrite it in whatever language and framework you want, while keeping the same structure and sort of curriculum and lesson flow. Because that's where I spent a lot of my time. + +So it doesn't need to be a cut, copy, pasta thing, but it's really like, use what exists. This is the beauty of having an open and sharing environment. The other thing is vendors often have training on their specific platform and products. What I will say is sometimes if you take those trainings, you're introduced to all of the features and all of the products on the platform that your org maybe doesn't even pay for or use or has interest in. + +And so that can contribute to learners being like, "Is this synthetics turned on or whatever?" And so I think borrowing the curriculum, borrowing the phrases, the links, the resources, how concepts are mapped out one by one, great. And if you're all in on one platform, go ahead and use the vendor training, but that is just my word of advice there. + +So, what was the problem I was trying to solve? I believe tracing is an underutilized telemetry type. And I think a major barrier to implementation is purely getting useful data out, getting to that complete trace as a starting point. And so in my opinion, the best way to tackle that was to teach application developers, because a lot of us in ops and a lot of us in SRE have been talking about tracing forever. It's the app devs we've got to reach. They're the ones we need to do the instrumentation. + +So that was my guiding light, getting app devs to instrument traces and actually use trace data. My learners were app devs interested in open source who are running cloud-native systems. I think tracing is useful. Always? Well, I shouldn't say always. I think tracing is very, very helpful no matter the size of your system. However, specifically working at KubeCon and CNCF, I was targeting cloud-native systems. + +And what did they already know? I did not have the benefit of having access to my attendees ahead of time. And so, the assumptions I made were that they were likely familiar with metrics and logs. That they might have had exposure to APM, application performance monitoring, and that they were curious about tracing in OpenTelemetry. Because no one signs up for an OpenTelemetry workshop if they don't know what it already is, even at like the highest level. + +So that's what I started. How did I scope down? What was our minimum local dev environment? I chose Flask and Python. Python for its very welcoming community on just like as a language as a whole. Flask because it is a framework that doesn't do magic stuff behind the scenes. You don't really need to know the ins and outs of Flask in order to just look at it. It's a very basic web app. And if learners wanted to continue on and maybe like extend the workshop, the Python, specifically the Python OpenTelemetry cookbook section and the documentations for the Python Instrumentation Libraries or Tracing are excellent. + +So I knew if people wanted to go further, that those resources would be available to them. Obviously, OTEL APIs and SDKs for instrumenting. And then, yeah, where did I want to store and visualize these traces? Well, this was a workshop focused on local development of traces. So, the Jaeger all-in-one where you're holding those traces in memory was a fine, totally acceptable use case for this. We were only going to be generating a few requests at once and kind of, it was mostly building up the feedback loop of adding instrumentation code, running the services, verifying that the data matched what we wanted, and like over and over again. + +So, we didn't need to hold stuff long-term, so JaegerFIT. And because this was an open-source for open-source conferences, it needed to be all open source up and down the stack. So JaegerFIT, although I'm really excited to see some other, I think is it, hotel desktop viewer, I hope is getting revived. I see murmurs of that. I would like to have lots of options, but this is what I went with. + +And then I kicked off, it feels silly, but you, I always kick off with a definition of observability because I think if you asked like five developers to define it, you would get 10 different answers. Actually, I think you'd get more than five different answers back. And so when you're conducting this training, it's important that you start with like, these are the terms we use, this is what it means for this org, or for this project, or this is what they mean here. You may have a different idea, let's talk about that, but like, for this session, here's where we're grounding. + +Doesn't need to be long, literally one sentence definitions, we don't want to overwhelm or bore people, especially at the beginning of a training. And then from there, I had talked about my big debate, do I include the collector, do I not? Here's what I decided to do. I thought it will serve people to know about it. I'm going to include the big OTEL diagram from the docs. Again, doc site showing up here. Love it. It's not a knock on the doc site. We just got to help people learn where to go to find the info. + +So I talked about the collector briefly. I kind of said, here's where it sits in the overall OTEL, like kind of architecture and the project landscape. Here's when you would use it. Today, we're going to be using a more simplified version where you don't, it had, the purpose of putting the slide is they know what it is. They can connect it to some other ideas within OTEL. And the next time they hear it in the wild in a conversation, they go, "Oh, okay. It was like the pro, you know, we can send telemetry data through that." That is a fine place for beginners to be, especially app devs who are likely not going to be like tuning collector configs. + +And so the hardest thing I think to do is to introduce new concepts and skills one at a time. Because if you've been working for this, with this stuff for a while, you have like the cursive knowledge where you just don't even know what it's like to be a beginner. You don't even know what would be a fire hose of information versus like this beautiful, nice breadcrumbs. + +So, like, if you think back to your first day on the job, you probably had HR benefits, then IT laptop setup, and then maybe like a session with the CEO about the history of the company. And then you're like with stuff to lunch. And then by the end of the afternoon, you finally get to your laptop, you get to your desk and you've got like a hundred emails and you're like missed meetings you didn't know about. And like, all of that is just so overwhelming when you're getting started. + +So really, really focusing on just one little concept at a time should be a guiding principle. So the way I broke instrumentation up, I looked at a lot of different tutorials and I even got, I will admit I got a little bit confused because I would be looking at the same tech stack that they were trying to instrument and the code samples would, would kind of vary. And I looked at a bunch and I said, what is this missing concept here? What is this missing link? + +And what it turned out to be was people were up front deciding whether they wanted to do automatic, programmatic, or manual instrumentation without explicitly saying, "Hey, this is what I'm doing and why." And so that meant if you were to just take one of those tutorials that maybe did all automatic, you might be really confused when someone asks you to manually add a span attribute because you were like, I, the OTEL agent is like wrapping my service call. Like it's doing everything. What do you mean? + +And so that for me was the gap that I wanted to fill is to know when and where to use automatic, programmatic, and manual. And there's no like bearing the lead, like show everything up front. And this is in module one. I'm like, here's automatic, programmatic, manual. Here's the differences. No surprises. + +And one note on the automatic, it is really nice to get a quick win in very early for learners. I decided to use for automatic instrumentation, just the console span exporter. Because for a lot of these folks, it was their first time even like, thinking of a span, knowing what a span is, and the textual representation I thought was kind of nice to look at and become familiar with what I could have done was like, put up an example span and like, blah, blah, blah, lectured through all the trace ID and parent ID. + +Or we could set up auto instrumentation, they could get their first span in front of them, and then I could say, "Hey, do you notice this thing?" And then they're looking at their laptop, they're engaged, it's their data. This is sort of the aha moment I'm talking about. We don't want to lecture at people, but we want them to discover on their own because that just goes so much further. We do learn by doing. + +Programmatic manual. So the other thing I did was break up how and why. So after we do the different, we do automatic, we move on to programmatic. And once we have traces that have multiple spans, like the textual representation, like techno we got to start looking at this in a waterfall. So I took the time to do a quick tour of Jaeger, of each of the different visualizations and kind of talk through, here's how you analyze them, here's what you're looking for, here's how to search and filter, and I ended that how section with this slide on why because traces I think are very different from metrics and logs. + +And so I was like, "Why do we have so many visualizations? Why did I just show you like 10 slides of different ways that you can interact with and manipulate this data?" And it's like, well, the power of tracing, you know, the zoom out, the zoom in, the telescope, the microscope. But combining, separating, but following up the how to with the why was a really key component for people's light bulbs to go off. + +So with that, I wanted to share a few more resources and then kind of dive into hearing what training needs y'all have. These two books, highly recommend, "Design for How People Learn." It is hilarious, it is funny, it is a book on learning that, like, it is not boring at all. She talks, she walks the walk in this book as well as "Better Onboarding" from a book apart, which is also a fantastic publisher. + +When with all of the talk about internal development platforms and platform engineering and treating your platform like a product, like, let's just borrow the good books they're using already, like, that are onboarding and think about how to bring that into onboarding to observability. Open observability resources. I love that the OTEL Docs have a specific section that's like, "Are you a developer? Like, and you want to learn more? Start here." Versus, "Are you ops? Start here." Like, that really speaks to who, knowing who the audience is. + +The CNCF Glossary. It's helpful to have bookmarks. Sometimes people ask you about acronyms and maybe you don't remember or you want sort of an official resource to turn to that one is nice, plain text, like small, like it is just very plainly defined. And then finally, while my workshop is not real-world, right? It was a very small Flask app. It had three endpoints. If you wanted something more realistic to look at, the OpenTelemetry demo is your choice. It's got, I think, I believe all the languages. + +It's, it exercises almost every part of OTEL that is like GA now. Super helpful resource. That would be the one that I would use if I knew what hardware my learners had. Cause it was a little bit of a mouth breather last time I ran it. Okay, and then learning experiences. + +So there is the tracing workshop that I gave at KubeCon is self-guided. It's up, it's on GitLab but you can, it's set up as like a slideshow, so you just kind of work through it at your own pace. There's a recorded session, which is sort of where I'm teaching the workshop, and I'm able to use more words than I put on, on the slides. And then sort of related, I developed a different learning experience for on-call onboarding when I was at LightStep, and I presented on how and why to do your own on-call onboarding at SREcon last year. + +That is a great talk, and it is a practice that has evolved to multiple companies past LightStep, so it has kind of proven itself out in the world. So I believe we are, yes. Gotta do my attributions. Thank you to SlidesGo for the slide templates, FlatIcon, Freepik, and Storyset for all of the beautiful things. I am not an artist. I'm not a teacher, I'm just a developer advocate who wants people to trace more things. + +So with that, I'm really curious to learn. Does this resonate with folks? Do you have training needs today? Do you want to talk through some tactics or ways to learn your, about your learners? Let's talk. + +**Audience Member:** I actually have a couple of questions, Paige. The first is, are you able to share, do you have a shareable copy of your slide deck? + +**Paige:** Yes, let me. Our Google workspace is really locked down, so I will need 5 minutes after this meeting to get that over to you. But yeah, no worries. + +**Audience Member:** And then I was also curious, how do you measure results or, like, what are some strategies to measure results from your workshops? + +**Paige:** Yeah, so my workshop in particular, I looked at the attendee feedback and I talked to folks afterwards and especially the folks that had raised hands and had questions or hit a stumbling block. I wanted to make sure because somebody had an issue PIP installing something in a virtual environment and it was like, not something I'd seen before and it was not something I could troubleshoot on the spot. So I walked them through. + +Okay. This is what I was trying to teach you here. Here's another way that you can get at this. What else would you, what do you need to go forward? So there's a little bit of that personal one on one and then they send out a survey. And so I always look at survey results mostly not the numbers. Not a social scientist. I know people like the Likert scale, but I really care more about what people took the time to type and say. + +And so folks had said, like, this was a great introduction. This was my favorite tutorial. Like, thank you so much. And so I'm like, okay, we did good. If I was developing this for my company, so say you have an initiative that you need to re-instrument away from proprietary onto hotel metrics by the end of next quarter. Okay. So what you where you want your learners to be is to, well, what you want at the end of the day is maybe all of your services instrumented with OTEL metrics. Cool. + +What's standing in the way? Where are your learners today? Well, half of them are using Vendor A, the other half are on Vendor B, and then maybe the ops team is, like, secretly running Prometheus somewhere, and you're like, oh my god. So, in that case, I would say, like, okay, well, for the Prometheus folks, there's like a bit of education to do with the interoperability between Prom and hotel metrics and kind of talking through which approach do we want. + +Do we want to use the pull-based or do we want to do the push-based? That is kind of the training and the knowledge and the skills that we need there. For each of the vendors, like I do think it is helpful to have this is how you would do it in vendor A and this is how it needs to be, this is how you need to change your behavior. Sort of a use this library, not that library, the contrasting from where they are in the context of what they know to here's the hotel metrics like cookbook for, oh my God, the cookbook for Python. + +Here's the, it'll go away. Here's. And even a really basic like primer on metrics again, because I, even with like, yes, observability is great when we talk about monitoring, really, we talk a lot about metrics and thresholds and alerts, and that is still an area that I see is like under serves as we like through the page at the developers and ran away, we like kind of forgot to teach them all the stuff we learned after years of like fighting with incidents and trying to understand our systems better and measure them. + +So I think in that case, yeah. Yeah, I would just really get as specific as you can on what these groups would have in common, and then map from there to the goal. So I would measure the success of like, okay, two weeks after the workshop, or one week after the workshop, how many PRs did I see get through? How many services either started their re-instrumentation, finished their re-instrumentation? I would kind of use that to track the progress of the project, which is sort of the lagging indicator of like did they learn the thing or not. + +Yeah, but those are great questions. Thank you. + +**Audience Member:** Do you have a favorite training that you've held or you've led? Do you want to share how, like, I love your metric stuff. So you want to talk a little bit about how you made choices there? + +**Paige:** I mean, so it's interesting because I did the metrics. I did a metrics talk twice, and so I changed up the second, change it up the second time just based on like, you know questions that people had and stuff that I had wanted to kind of put in but didn't have time or hadn't quite figured out how to put into it. But, you know, I think what you're talking about is not applicable just for workshops, but like really anything that involves like a wider audience, because, you know, I think I frequently am like, oh, you know, what kind of level do I need to assume that people are at so that I know. + +Yeah, and you know, you can make like general assumptions about, you know, okay, well, this event, you know, whatever, but sometimes it's. Yeah, I don't know. Sometimes you just guess and hope if it's the most people, you know, sometimes, yeah, there'll be like one or two people that were like, oh, this was like way too beginner or like two people that would say the opposite. Like, this is, you know, way over my head. + +And yeah, that's I think scoping like, does it need to be novice? Does it need to be mid-level where I assume some things or am I talking to the expert that probably wrote a system like this in the past that it is you can't stretch one training to fit all those groups for sure. + +**Audience Member:** Yeah, I know. And yeah, I'm, I'm always like, I feel like on the verge of, oh, I want to, you know, because I always appreciate, you know, people who explain things very well. And so trying to, you know, fit that for the people that are there, but also like not wanting to piss them off, but like, you know, kind of have to be like, oh no, this is like, this is more basic than I wanted. + +**Paige:** Yes, absolutely. Yeah, it's hard. And I, when I write blog posts and stuff that people who edit are always like, you don't need to include the whole history of observability. You could kind of just like, start with your topic. I'm like, well, people still don't know. You know, I work a lot of booth duty and I can go, oh, you know, are you happy with, you know, observability at your company today? And they go, what's that? And I'm like, yeah. + +Okay, all right. Let's start up. Let's start at not the beginning, but let's start at do a monitoring. Oh, okay. And yeah, just finding those ways to meet people where they are because we do use a lot of insider academic terms telemetry. There's just no shortage of them in this space. So yes, I and I do think the quality of the questions that you get asked maybe in your Slack help channels can really tell you if your training was a hit or a miss because if you're continuing to get, "Hey, is this random JavaScript library instrumented with OTEL?" It's like, I don't want to be rude and like point you to the registry, but also like, did you take the training? + +Making sure you track who takes the training is also huge because I have been on tiny teams that have kind of manpowered through migrations and we would get developers kind of treating us like we were the owners of their own CI pipelines. And we would say, "Oh, so and so we noticed you haven't taken the training yet. I totally get why, you know, this is a confusing thing. Why don't you check out lesson two? Here's the link." And then it's like, Okay, this is, we're negotiating the boundaries of responsibility between operators and developers. + +I think we're still, even though like DevOps movement has, we're like many years past that, it does still feel like we are negotiating these boundaries when it comes to monitoring and observability. No one team should be in charge of this. You should be the, as experts, you should be the advisors, the consultants, the guides, but not the like, total tyrannical, like, owners of the thing. It's exhausting to be responsible at that level. + +So, yeah. Or does anybody have good trainings that they've taken that they want to share or ideas for courses? I'm, yeah, like Rhys said, sometimes we can get in our head about, like, what do people even want to know? + +**Dan:** Go ahead, Dan. + +**Audience Member:** Hi, thanks. And thanks for the talk that resonated very well with me. Sort of just on your last point, specifically, I was going to ask. So, one of the things I think we struggle with is like this ownership boundary of, we provide some training or maybe not even training. I don't know if we do that well, but we teams are like, we need observability in our app or not even our app. + +Like, I'm a team that owns a framework that dozens of teams that my org uses and we help them implement sort of, you know, tracing let's say at the root of that. So the app developer doesn't even need to know. But then like, you know, a month passes, three months passes and they, something's not working properly, they want to add something and they're always like looking to us to be the ones to like do the work or, or, or, you know, again, advising them is my role. + +So it's, it's like, that's okay, but like, I just feel like they don't really end up ever taking ownership. Like it's a, it's a common pattern. And I guess what I'm asking for, or opening a discussion on is just, like, how would you drive the connection between, like, the training and the knowledge transfers, et cetera, and then like, defining where that boundary lies because I always feel like people treat observability as like a third class citizen when I feel like it needs to be their first class citizen. + +Like, we get all these incidents, the directors talk about it, et cetera, but they don't take the ownership at the end of the day. So I'm just curious about your experience in that relationship. + +**Paige:** I'm hoping somebody on the call has experience working at a midsize or larger companies because unfortunately I have been at startups where I didn't I have not had like a director of SRE or or really even like, I do think it comes down to engineering leadership needing to explicitly like, say, here are, here is what platform or ops or SRE provides. + +And here's what you're responsible for sort of like the, what is it? The cloud security, the shared responsibility model between the cloud provider and you like, and yes, there's like the fuzzy boundary in the middle, which is where you would do some consulting. But I have not been able to successfully win the owner, like get ownership successfully established because of the kind of the nature of the companies I worked at, but I'm really curious. + +Like Adriana, I know you've worked at some big shops, you've seen a lot in the past. + +**Adriana:** Yeah, yeah, I mean, I can say that like from personal experience, like I worked at an organization that was like, they hired me in to do observability, like to lead an observability practices team. And yet when it came time to like, okay, let's get the developers to instrument their code, they're like, huh? Why can't you do it? + +So it really, and then similarly, like or, or I should say an opposite story to that. We had last year Iris De Mirschi who she was at Farfetch. I forget where she's at now, but she was talking about how she came from an organization that was an observability first organization. + +And they're like, because executives mandated observability and that, you know, their team wasn't responsible for instrumenting code, even though, like, she was, she was part of an observability team, like, they were helping with practices. They were like managing infrastructure, like the collectors, but application instrumentation was to be done by the application teams. + +And so it was a very different sort of outcome as a result, right? Because it's almost like, what are you going to say? Like, leadership has basically said that this is the way you're going to do it. So that's the way you're going to do it compared to where I was at before, where I was at before. I was trying to shape what observability for the org looked like, but unfortunately, leadership was like, they kind of half dipped their toes into the observability water. + +So they're like, eh, not really. + +**Dan:** Oh no. Does this mean we all need to get promos to engineering leadership to make observability happen? Oh no. + +**Adriana:** Or be very persuasive. + +**Dan:** Do you feel that like, I hear what you're saying, but like, I also feel like in the spirit of the goal of observability and the goal of like pushing these responsibilities downward. It's like this. These top level mandates are really like, sure, maybe the team will do it at the end of the day, but they're never really like, caring about it. + +Like, do you think those two are like competing? You have to have a convergence at the end of the day, right? Cause like you need the mandate from the top but like, you have to be, you have to have the folks willing to do it. + +And I think like to Paige's points in her talk, like, you know, having people who are excited about observability and, and having a proper training program in place as well to get the, you know, the individual contributors, the ones who are, who have an actual stake in this who are, who are going to be doing the work. + +Like because if you don't, if you don't have like that convergence from top and bottom, like it's just going to be a half-assed thing. + +**Adriana:** Yeah, go ahead. + +**Audience Member:** Yeah, I was just going to chime in quickly. So the one experience I had or the recent experience that when, when, when teams start facing issues, like when they are trying to figure out one particular issue, they are fixing like the incident. I guess like even in not in each day, but they are trying to find a particular issue with the application and then they need help, right? + +Like they don't know they haven't done the training or I don't know how to use the observability tool. I think that I kind of use that as a leverage whether then have you done your, have you instrumented your application correctly? Have you added the correct labels on your telemetry data? And then having to like having used that kind of work as well. + +Like, okay, you want something that you want to have with identifying or figuring out this issue. I also want you to do your part as well. Make sure that you have done your part. Then it will make it easier. So I think that was one tactic used in the past. + +**Dan:** Yeah, I think, like, from what I'm hearing so far we I had similar experience in my previous org. We were building our brand new platform, and luckily it was all written in Java, which meant that, like, we just kind of utilized the Java Auto Instrumentation, built out the kind of generic dashboard and had all that kind of semantic convention baked into it which meant that like all of the telemetry was easily identified. + +And we went just like we just went in all the application and then because it was not that much effort on our part to then make that change rather than like manually instrumenting everything and then we had other teams which are like all the services were returning lambdas and goes in that case, like, yeah, I had like, nine months project with them trying to kind of pursue them and slowly, like kind of building set of all kind of POC and like, based on the water. + +The extension layer building out like like custom version of that all configured. They like just add this layer. At least that part is sorted for you. But then you just have to do the instrumentation part on your. + +So, yeah, I guess it depends on the situation, but it's a mixed bag. + +**Paige:** Yeah, Dan, it sounds like you've already got you're working within sort of that entrenched pattern where you're already seen as the owners and that maybe like you've been able to be able to do these tickets for them in the past, but now you want to change. I remember two things I've tried. One, we'll do this first one together. We'll pair on this. Then you will know and this will be great. + +Or I find an example PR that's like exactly like instrumenting the, whatever the counter or the gauge you need. And like here, use this as a guide. So it is hard to come out of a already established pattern like that, but DoorDash did a talk at KubeCon about how they rolled out service mesh and sort of the challenges with giving people out of the box dashboards. + +I can find that and put it in the slide deck they would be great to talk to because they kind of had a similar problem of being the owners. But we're out of time. Oh my gosh, that went by quick. If you have any more questions you can reach Paige on the socials that she shared, or you can just search Paige Cruz in CNCF Slack. + +We will have this recording edited and up, I think, hopefully in the next couple of weeks it'll be on the OpenTelemetry YouTube channel. And yeah, it's called OTEL Official. Yes, so, you know, it's official real. Yes. Give it a follow. Give it a follow so that you can get all the latest updates. + +And yeah, thank you all so much for being here Paige. Thank you so much. This was like really great. I love your slides. I screenshotted a couple. And it's all, I all work out in the open, so take, borrow whatever you need, adapt, extend. Let's bring more people to tracing. And please, yeah, email me and get in touch if you want to talk about this stuff more. I have spent a lot of time, a lot of time thinking about it. + +Awesome. We will see you all next time. All right. Thank you. ## Raw YouTube Transcript diff --git a/video-transcripts/transcripts/2024-03-05T19:49:24Z-otel-collector-user-feedback-panel.md b/video-transcripts/transcripts/2024-03-05T19:49:24Z-otel-collector-user-feedback-panel.md index 2f2747b..2d4e0db 100644 --- a/video-transcripts/transcripts/2024-03-05T19:49:24Z-otel-collector-user-feedback-panel.md +++ b/video-transcripts/transcripts/2024-03-05T19:49:24Z-otel-collector-user-feedback-panel.md @@ -10,133 +10,148 @@ URL: https://www.youtube.com/watch?v=LL8v_B417ok ## Summary -In this YouTube video, a panel discussion was held featuring practitioners of the OpenTelemetry (OTEL) Collector, including Adriana Villela (moderator), Iris, Juraj Michalik, Greg Eales, and Joel from various companies. The panel focused on gathering feedback about the OTEL Collector to inform its development roadmap. Each panelist shared their experiences with the Collector, discussing its applications in their respective organizations, challenges faced during deployment and debugging, and the use of custom builds. Key topics included the need for better handling of metrics, issues with scaling and memory management, and desired features such as profiling and cumulative metrics processing. The panelists expressed a desire for the OTEL community to address these challenges and enhance the Collector's functionality. The discussion concluded with announcements about upcoming events at KubeCon in Paris, where further user feedback sessions will take place. +In this YouTube video, a panel discussion featuring four OpenTelemetry (OTEL) collector practitioners—Adriana Villela, Iris, Juraj Michalik, Greg Eales, and Joel—explores their experiences and challenges with the OTEL collector. The panelists discuss their roles in various companies, the usage of the collector for telemetry data collection, and the importance of community feedback in shaping its future. Key topics include deployment strategies, debugging difficulties, challenges in scaling the collector, and desired features for improvement, such as the need for profiling, better integration with Prometheus, and handling metrics more effectively. The session highlights the practitioners' collective learning experiences with the OTEL collector, emphasizing the balance between rapid development and stability in the ecosystem. The discussion concludes with an invitation to the audience to engage further with the community and participate in upcoming events at KubeCon in Paris. ## Chapters -Here are the key moments from the livestream along with their timestamps: +00:00:00 Welcome and intro +00:01:40 Panelist introductions +00:05:00 Discussion on Collector usage +00:09:10 Metrics and traces handling +00:12:40 Collector distribution types +00:15:36 Contribution experiences +00:20:01 Debugging challenges +00:25:00 Collector scaling feedback +00:30:00 Wishlist for improvements +00:48:25 Closing remarks and KubeCon info -00:00:00 Introductions and event overview -00:02:30 Panelist introductions - Iris -00:04:00 Panelist introductions - Juraj -00:05:30 Panelist introductions - Greg -00:06:50 Panelist introductions - Joel -00:08:30 Discussion on how teams are using the OpenTelemetry Collector -00:10:10 Iris discusses current usage of the Collector -00:12:00 Juraj shares experiences with past and current implementations -00:14:00 Greg talks about migrating from legacy systems to the Collector -00:16:30 Joel explains their multi-layered approach to using the Collector -00:20:00 Challenges faced with debugging and troubleshooting the Collector -00:25:00 Panelists share experiences with the Collector Builder -00:30:00 Wishlist for improvements and desired features in the Collector -00:35:00 Discussion on scaling the Collector and experiences with auto-scaling -00:40:00 Wrap-up and information on future events, including KubeCon +**Adriana:** Thank you everyone for joining. This is a really great turnout and we're excited to have assembled this panel of OTEL collector practitioners. One of the missions of the OTEL and User Working Group is to collect feedback on OTEL to deliver back to the SIGs. I think this started because Judah C, who works in the OTEL collector SIG, reached out to us to put together a survey to collect some information on the OTEL collector, to see how it's being used to sort of drive what the roadmap and vision is going to be for the collector in the next little while. Then we thought, well, okay. You know, let's extend this and also have a panel and have a nice lively discussion with a handful of OTEL collector practitioners. So here we are. -Feel free to adjust any of the descriptions or timestamps as needed! +[00:01:40] So, why don't we introduce the panelists? I'll introduce myself first. My name is Adriana Villela. I work with Reese and Dan in the OTEL end user working group. Our groups are, our numbers are getting a little bit bigger, so I'm very excited to see that. Yeah, so we're hosting this event. Let's go over to our panelists. I'm going to pick names. Iris, how about you go first? -# OTEL Collector Panel Discussion Transcript +**Iris:** Hello everyone. My name is Iris. I'm a senior observability engineer at Miro and I've been working with OpenTelemetry for a little bit more than a year now, in two companies. I have a lot of experience and use cases that I've worked because my two experiences are completely different from each other, and I'm happy to be here. Nice to meet you. Thank you. -**Adriana Villela**: Thank you everyone for joining. This is a really great turnout, and we're excited to have assembled this panel of OTEL collector practitioners. One of the missions of the OTEL and User Working Group is to collect feedback on OTEL to deliver back to the SIGs. This started because Judah C., who works in the OTEL collector SIG, reached out to us to put together a survey to collect some information on the OTEL collector and see how it's being used. This information will help drive the roadmap and vision for the collector in the near future. We thought it would be great to extend this and also have a panel for a lively discussion with a handful of OTEL collector practitioners. So, here we are! +**Adriana:** Okay, next we have Juraj. Did I pronounce that correctly? Please correct me if I'm wrong. -Let’s introduce the panelists. I’ll introduce myself first. My name is Adriana Villela. I work with Reese and Dan in the OTEL end user working group. Our numbers are getting a little bit bigger, so I’m very excited to see that. We’re hosting this event. Let’s go over to our panelists. I’m going to pick names. Iris, how about you go first? +**Juraj:** Hi, my name is Juraj Michalik. I'm based in Madrid, currently working very recently for a company called Swissery in the insurance space. Before I worked for like two different companies where we used, in the last one, very sparingly, and in the previous one, we basically fully migrated to OpenTelemetry for collection of logs, metrics, and traces. I'm a Senior Logging and Monitoring Engineer in this current role. -**Iris**: Hello everyone! My name is Iris. I'm a senior observability engineer at Miro, and I’ve been working with OpenTelemetry for a little over a year now, across two companies. I have a lot of experience and use cases since my two experiences are completely different from each other. I’m happy to be here. Nice to meet you! +**Adriana:** Awesome, thanks. Next we have Greg. -**Adriana**: Thank you, Iris. Okay, next we have Juraj. Did I pronounce that correctly? Please correct me if I'm wrong. +**Greg:** Hi, can you hear me? My name is Greg Eales. I work for Ocado Technology, which started out as an online supermarket and now sells online supermarket technology to other supermarkets. We've been using the collector for a bit over a year, my team, but I've only been on the team for a bit less than a year. -**Juraj Michalik**: Yes, you did! Hi, my name is Juraj Michalik. I’m based in Madrid and currently working for a company called Swissery in the insurance space. Before that, I worked for two different companies where we used OpenTelemetry. In the last one, it was used very sparingly, but in the previous one, we fully migrated to OpenTelemetry for the collection of logs, metrics, and traces. I’m a Senior Logging and Monitoring Engineer in my current role. +**Adriana:** Oh, awesome. And finally we have Joel. -**Adriana**: Awesome, thanks. Next we have Greg. +**Joel:** Hi, I'm Joel. I'm from a company called Open Systems. We're based in Switzerland, but we have a presence globally. We have offices in San Francisco, actually. I've never been there. Also in London and Germany. This is where we're based. We offer managed SASE, so managed connectivity and VPN for our customers. I'm on the observability team, so I'm responsible for making sure that we get telemetry from our devices, which we have 10,000 of, all across the world. We're now having to deal with data coming from Kubernetes, so Prometheus metrics, logs, all this nice telemetry, which we need to handle. We've been using the collector since about a year now, I think just under a year, I would say probably April last year was when we started. We really decided very quickly that we want to fully migrate everything to it because at the moment we kind of had a big mass of different service teams doing things in their own way, so we're kind of consolidating everything now and we found that the collector and OpenTelemetry in general was a very nice way which we could do that. -**Greg Eales**: Hi! Can you hear me? My name is Greg Eales. I work for Ocado Technology, which started as an online supermarket and now sells online supermarket technology to other supermarkets. My team has been using the collector for just over a year, but I’ve only been on the team for a bit less than a year. +[00:05:00] **Adriana:** Awesome, well, thank you again all of you for joining us. So let us get into the questions. First things first, we'll start, let's start in the order in which folks were introduced. Iris, how about you start with telling us how you and your team are primarily using the Collector? -**Adriana**: Oh, awesome! And finally, we have Joel. +**Iris:** Okay, so, I could talk a little bit about how we're using it now and in my previous company as well, since I am very new at my current position. We are using the Collector mostly in my current company mostly as a transport layer at the moment. We already migrated traces. We're in the process of migrating logging and metrics. In my previous company, we were a bit more advanced than that. We were using the collector itself and the operator. Currently, we are only using it as a deployment, basically all it's doing is transporting from our legacy components and we're slowly moving to collect everything through OpenTelemetry Collector. In the past, we actually used the presets a lot, the presets that are offered by the collector itself. So we were running it as a daemon set and the collectors were collecting everything, especially in our Kubernetes cluster. That is the goal right now, to have a collector take as much of the collecting of the information as possible and get rid of the legacy ones like Jaeger, for example, Prometheus. -**Joel**: Hi! I’m Joel from a company called Open Systems. We’re based in Switzerland but have a global presence with offices in San Francisco, London, and Germany. We offer managed SASE, which includes managed connectivity and VPN for our customers. I’m on the observability team, responsible for ensuring we get telemetry from our devices, of which we have 10,000 across the world. We’re also dealing with data coming from Kubernetes, including Prometheus metrics and logs. We started using the collector about a year ago, probably in April last year, and we quickly decided to fully migrate everything to it since we had various service teams doing things in their own way. We found that the collector and OpenTelemetry in general was a great way to consolidate everything. +**Adriana:** Awesome. Same question, Juraj, please. How is it that you and your company are using the collector? -**Adriana**: Awesome! Thank you again to all of you for joining us. Let’s get into the questions. First things first, Iris, how about you start by telling us how you and your team are primarily using the Collector? +**Juraj:** In my current role, which I just joined this month, they're just starting with OpenTelemetry and traces in general. But in my previous role, we basically, it was a SaaS startup where early on, there was some concern about the costs. One of the things we started to work on was migrating from a vendor's proprietary collection pipeline and client libraries to OpenTelemetry, which then enabled us to do a POC of other alternatives, where we basically just used the concept of the pipelines to take the data we were already gathering and just send it to a different vendor and also a self-hosted solution. It was mostly, we ended up settling on the self-hosted solution, which was basically the Grafana "looks good to me" stack. So hosted in Kubernetes, the way we are on OpenTelemetry. There were a couple of modes. There was the typical Kubernetes deployment where we had the daemon set for collecting logs, metrics, and traces from everything running on the nodes. There were a couple of stateful sets, one dedicated to collecting Kubernetes related metadata. We also had a couple of auto collector deployments with ingresses for either ingesting the data from the sidecars in the ECS into Kubernetes, or we even had a hybrid mode of our product where the execution layer would run in customers VPCs. This was written in Java and had the Java agent instrument included in, and that would also send back metrics and traces to our ingress behind which we had the OpenTelemetry deployment and then in front of the "looks good to me" stack, we had another layer of OpenTelemetry collectors to do some unified transformation to ensure some labels were common, things like that. -**Iris**: Sure! I can talk a little bit about how we’re using it now and in my previous company as well since I’m very new in my current position. We are using the Collector mostly as a transport layer at the moment. We have already migrated traces and are in the process of migrating logging and metrics. In my previous company, we were a bit more advanced. We were using the collector itself and the operator. Currently, we are only using it for deployment, basically transporting data from our legacy components, and we’re slowly moving to collect everything through the OpenTelemetry Collector. In the past, we utilized the presets offered by the collector itself, running it as a daemon set, collecting everything, especially in our Kubernetes cluster. The goal now is to have the collector take as much of the information collection as possible and get rid of the legacy solutions like Jaeger and Prometheus. +**Adriana:** Awesome, and how about you, Greg? -**Adriana**: Great! Same question for you, Juraj. How is it that you and your company are using the collector? +[00:09:10] **Greg:** We use the collector for traces. We had a sort of legacy, kind of trace-like thing called request viewer, which was based on logs, which was very heavy. We're migrating now, I think we've just started to delete the cluster for the old request viewer. So now everybody should be sending traces via the collector. We do some processing, do some sanitization, add some attributes, and do tail sampling as well so that we can sample by whole traces. We don't do anything with metrics apart from our own metrics, so we gather the metrics from the collector and forward them, Prometheus remote write them out to Grafana. We use Grafana Cloud as the backend for all the telemetry. We deploy it in ECS, not Kubernetes, and that's kind of been interesting. We have one central cluster. We don't have a sidecar. We have one central cluster that scales in and out with the load of the collector. -**Juraj**: In my current role, which I just joined this month, we're just starting with OpenTelemetry and traces in general. But in my previous role, I worked for a SaaS startup where there was some concern about costs. We started migrating from a vendor’s proprietary collection pipeline and client libraries to OpenTelemetry, which enabled us to do a POC of other alternatives. We settled on a self-hosted solution based on the Grafana stack, hosted in Kubernetes. We used typical Kubernetes deployment with a daemon set for collecting logs, metrics, and traces from everything running on the nodes. We also had several stateful sets dedicated to collecting Kubernetes-related metadata and pulling data from AWS CloudWatch. +**Adriana:** Okay. Interesting. I definitely have some follow-up questions on that, which we'll save for a little bit later. And then finally, Joel? -**Adriana**: Awesome! And how about you, Greg? +**Joel:** We have collectors deployed all over the place. We've run collectors on all of our hosts. They are kind of egress gateways for telemetry. Primarily, the data which we're shipping is actually fully, well not fully, but it's mainly log data. We have pipelines enabled for metrics and traces, but currently it's just the collector's own metrics, which we ship. There are very few applications using traces at the moment. So primarily we're using it as a kind of log telemetry mesh. We're using that as a way to sort of lay the groundwork for the full migration in the future. We have the egress collectors running on our hosts. We have an ingress collector running in our central Kubernetes cluster. Then we have multiple layers of collectors behind that. We have a routing collector. Then we have a collector which is specific for each backend which we're interested in. For example, for Loki, for Thanos, and for Tempo. Those are the main backends we're shipping to at the moment. We self-host everything. The reason we've done it like that is to sort of try and keep the configuration overhead simple because those pipelines can get quite complicated when you try and stuff everything into one collector. We have this sort of egress and ingress concept. The goal really that we have is we're trying to ensure that all telemetry in the future will have a uniform set of labels of metadata attached to it. So we're starting with logs, and then we're going to roll that out also to metrics and traces in the future, that you can really actually start to correlate between signals in the backends. We also, if I can quickly say, we also do some custom processing. We actually build our own collector. Some of our teams, for example, have built processes that do things to proxy logs. For example, with proxy logs, which come through, they need special processing. It's been quite well adopted also by the service teams when they see the power of building a custom processor. -**Greg**: We use the collector for traces. We had a legacy trace system called Request Viewer based on logs, which was quite heavy. We’re migrating everyone to send traces via the collector. We do some processing, sanitization, and tail sampling to allow sampling by whole traces. We don’t do anything with metrics apart from our own metrics, which we gather from the collector and forward them to Grafana using Prometheus remote write. We deploy it in ECS, not Kubernetes, and we have one central cluster that scales in and out with the load of the collector. +[00:12:40] **Adriana:** That's great. Next question is, are you... So starting with Iris, are you using a vanilla collector, collector contributor, a custom-built one, vendor distro? -**Adriana**: Interesting! Finally, Joel, how do you use the collector? +**Iris:** We're using the vanilla one. So far we haven't had the need to use any custom that is already not there, no new, or a vendor one. We're currently fully open source, so yeah, the vanilla one worked great in our case. I'm assuming that's like collector contrib as is kind of thing. -**Joel**: We have collectors deployed all over the place, running on all our hosts. They act as egress gateways for telemetry. Primarily, the data we’re shipping is mostly log data. We have pipelines enabled for metrics and traces, but currently, it’s just the collector's own metrics that we ship. Very few applications are using traces at the moment. So primarily, we’re using it as a log telemetry mesh to lay the groundwork for full migration in the future. Our egress collectors run on our hosts, with an ingress collector in our central Kubernetes cluster and multiple layers of collectors behind that for different backends like Loki, Thanos, and Tempo. We self-host everything and aim to ensure all telemetry has a uniform set of labels for better correlation between signals in the backends. +**Iris:** Yeah, exactly. Of course, we have our own configurations internally, but yeah, it's the same base image. -**Adriana**: Thank you all! Let’s move on to the next question. Starting with Iris, are you using a vanilla collector, a custom-built one, or a vendor distribution? +**Adriana:** Okay. Perfect. Same question, Juraj. -**Iris**: We’re using the vanilla one. So far, we haven’t had the need for any custom builds or vendor distributions; the open-source version has worked great for us. +**Juraj:** I honestly do wonder if anybody's actually running the not the contrib one, because the main one just does only old DLP. I think I've seen very few workloads that produce LPLP and can also ingest LPLP on the other side. Anyway, we use a variety. We have multiple custom distributions built. One of the reasons behind that is being, for example, the OTEL collectors that are sort of exposed to the Internet. We wanted to minimize, well, first of all, we wanted to minimize the size of the binary, but also minimize the number of components just to have the only necessary ones, even from a security perspective, especially around exposing it to the Internet. We had another special distribution, which contained the gold debugger in case we needed to debug some issues, like for example, an issue with the Datadog exporter, which was not exporting properly certain labels and logs, things like that. All of it is based on the upstream OpenTelemetry collector contrib with just custom configuration. -**Adriana**: Perfect! Same question for you, Juraj. +**Adriana:** Since you're talking about having a collector with just the things that you need with the reduced binary, I'm assuming then you use the builder then? -**Juraj**: We use a variety. We have multiple custom distributions built. One reason for this is that we wanted to minimize the size of the binary for the OTEL collectors exposed to the Internet. We only include the necessary components for security. We also created a special distribution with a gold debugger for debugging issues. All of this is based on the upstream OpenTelemetry collector contrib with custom configurations. +**Juraj:** Then we have like renovate bot to submit pull requests with updates. We also had some, I mean, we also contributed a bit back to the total collector. So we also used it, so we could test changes before they were merged, changes on processors, on other components before they were merged into upstream. -**Adriana**: That’s interesting! Greg, what about you? +[00:15:36] **Adriana:** Awesome. And since you did, so since you've done contributions back to the Collector and since you've used the Collector Builder, what have your experiences been around both of these things? I guess let's start with, like, has the OTEL Collector Builder, like, did you find it a useful, intuitive tool to use when you were building your own distribution? -**Greg**: We build our own. We forked the contrib repo because we needed a feature to add AWS Cloud Map service discovery to the load balancer exporter. We’ve contributed that back and have an open pull request that’s still pending. We’ve discovered some bugs and have been able to debug the code locally, which has been beneficial for learning more about Go development. +**Juraj:** I think the one, like, once we got the initial configuration going, then it's really easy to, that's why we have like six or seven different builds, because it's just, you copy-paste the config once, edit it a bit, remove, add what you need, and then just run the builds in parallel, basically, and just give them different names. It was also quite easy to then just point at like your fork of a specific processor to test it, so we can test things even before they get merged. Also, with the one thing I think the bit where there's that like replaces at the bottom, that's something we ran into a couple of times where we needed to update that because the builds broke, but that was just looking at what the upstream was doing and updating that. -**Adriana**: Great! And finally, Joel? +**Juraj:** With that information, yeah, so the second question was on the contribution process, or? -**Joel**: We’re running a custom build. We use the builder, and it was not a problem to get it running in our build pipelines. We’ve had a very positive experience with it. However, there’s always a concern when maintaining your own distribution, especially with the fast release cadence, as you can run into deprecated features that can blindside you during deployment. +**Adriana:** Yeah, yeah, how did you find the contribution process to be? -**Adriana**: Thank you all! What’s been the biggest challenge or challenges that you’ve faced with the Collector? Iris, let’s start with you. +**Juraj:** I think it's okay. It's kind of sort of nice that things get, because of the pretty frequent releases, you don't have to wait that long for your changes to go live after you submit them. I think there's decent documentation around like how you can contribute, like things out, like how to format your code, things like that. There’s a world of pretty timely response to like well-written issues where they tell you like, “Oh yeah, that’s cool. If you have time, you can contribute and fix a certain thing.” The only slight frustration I found was, at times it can take a bit for like, there’s a label on the pull request, which is like ready to merge. Even after that gets assigned, it can take up to like a week or two for somebody to finally merge it, which can be frustrating because then like the pipeline starts failing because something broke upstream and somebody merged upstream into the branch and it broke the pipeline. There are things like that, but other than that, it feels pretty good. -**Iris**: Deployment has been straightforward, but debugging has been a challenge. I recently had an issue enabling ingress. I spent more than two days figuring out that I needed to pass the right paths in the receiver. Even after enabling debug logs, there wasn’t enough information to help us troubleshoot effectively. +**Adriana:** All right, great. Thank you. Next, Greg, what collector distribution do you use? Do you use contrib? Do you build your own? -**Adriana**: Juraj, how about you? +[00:20:01] **Greg:** We build our own. We use the builder. We forked the contrib repo and based off that. We do that because, well, initially we had a feature that we needed, which was to add ECS, well, AWS cloud map service discovery to the load balancer exporter, so we could get a list of targets for the load balancer. So we forked that, did that locally, and we've actually contributed that back, or there's an open pull request at the moment that's taken quite a long time, it still hasn't been merged. I think it's getting there. I think it's near now, but that's taken quite a bit of time. We've also discovered quite a few either missing features or bugs. It’s been quite nice to both be able to debug the code locally and actually step through and find out is this how it's supposed to work? And also then to be able to go and fix it. Although we've had about five or six different bugs that we've got to the point of almost fixing them and then we've seen that it's been fixed in the upstream repo. Somebody else has worked on it and has parachuted in and has fixed it, which is obviously nicer because presumably they’re doing it better than us. None of us are Go developers; I’ve got a Java background and we have various other developer backgrounds, but nobody's a Go developer. We’re kind of a bit nervous about contributing and a bit nervous about making changes and quite slow about that as well, but it’s been a very good opportunity to learn more about Go. -**Juraj**: Debugging has been difficult, especially when bugs only occur under heavy loads. We built our own distribution with the gold debugger in the dev environment to find the root causes. Also, late-breaking changes have caused incidents, like stopping data from being sent to Datadog due to label changes that we didn’t catch in lower environments. +**Adriana:** And then same question around the builder. How did you find your experience in using the collector builder to build your own distros? -**Adriana**: Greg, what’s your experience? +**Greg:** The builder’s fine, no complaints. It’s quite, quite, I mean, as I’m new to the Go ecosystem or, you know, I don’t really know how any other Go project, large project works, but it seems to work quite well. My one complaint made is it about the builder or about the repo. There’s like two or three components in the core project right now, not the contrib, like the OTLP receiver, the batch processor, the memory limiter. I’ve been wanting to work on those, and it’s really painful that they are not in the contrib repo. It’s really painful to be able to make changes to them and get them even running locally. I don’t even know how I’d manage to build another distribution of the main thing, and I don’t know how that would work, but like, so with the builder, you just do the replace and point it to your local directory. As long as the component is in that repo that we’re building from, it works fine. But for these other components, I would be much happier if they were in the contrib because it would make my life much simpler. -**Greg**: Our biggest challenge has been the quality of the platform we provide. When users look for requests they’re trying to debug and can’t find them, it creates anxiety. We didn’t know how to monitor the collector properly, which led to doubts about where the problem lies in the pipeline. We’ve improved on that, but we still have areas of uncertainty. +**Adriana:** Okay, that's definitely a good piece of information. We'll be sure to pass that on. And then finally, Joel? -**Adriana**: Finally, Joel? +**Joel:** We are running our custom build. The reason is actually because we have, like I mentioned, service teams who built their own processes. That was actually one of the big draws for why we should adopt OpenTelemetry. That has been pretty well adopted, I would say. We use the builder; it was not a problem to get it running in our build pipelines. Very, very good experience there. It’s always quite fun. We, you know, I think, Yudai, you’d mentioned, it’s quite a fast release cadence, which is a good thing, but if it’s very easy to let a few versions slip by, and when you’re maintaining your own distro, sometimes you get some nasty surprises when, you know, if you were to just build and deploy, some config with deprecated or completely removed without, you know, you have to chase down those errors and be very kind of on guard. Even when you do a product deployment and you’ve checked everything in dev, it’s always a little bit on your mind, “Hey, something could have changed which is going to blindside me.” But I mean, that’s just kind of, it comes to the territory because a lot of the processes and modules that we’re using are in, you know, alpha or beta stability level. I think it’s kind of expected, and it’s one of the sort of things we’re happy to pay for with the flexibility of the solution. So this is good. Yeah, but overall, really, really positive experience. -**Joel**: The handling of cumulative metrics has been a challenge. We ran into issues with the count connector, which didn’t work as expected. It’s disappointing to inform users that a proposed solution isn’t going to work, and we need to rely on upstream development for fixes. +**Adriana:** Awesome. Next question, starting with Iris, what's the biggest challenge or challenges that you've faced with the Collector? It can be stuff like deployment, configuration, monitoring it, or debugging it. -**Adriana**: Thank you all for sharing your experiences! What’s on your collector wishlist as a feature improvement? Iris? +**Iris:** Debugging, for sure. Yeah, I think deploying it is pretty straightforward, but debugging, I had a situation recently. It’s kind of embarrassing to say now after finding out what the issue was. It was the first time that I was actually using the collector with the ingress, enabling the ingress. Usually, it was all taken care of inside the Kubernetes cluster. Of course, having to pass the right paths and, yeah, I kept sending spans to try and see if it’s going to work, but nothing. I just got an error message and nothing else. I did try something else, just not an error, but like a status code and that’s it. It took me and my coworker more than two days to figure out what the issue was because, yeah, we tried to enable debug logs, debug exporter, everything, but there was just not enough information for this. In general, it has happened like this. After a lot of time spent, we figured it out, but yeah, it could have been a bit easier for us. What did you end up doing for troubleshooting? Did you have to go deep in the code or? -**Iris**: I’m looking forward to profiling. I’ve seen some promising developments and am excited about its potential. +[00:25:00] **Iris:** Yeah, deep in the code. At this point, it became like a brainstorming session. Everything that we ever knew about IT, we’re like, “Okay, this could be, this could be, this could be that,” you know, and we finally got to it after a long time. Checking the code definitely helped. We realized, okay, you actually need the path in the receiver itself to explicitly add it. That’s why I say that after it’s been solved, it’s such a silly thing, but at the moment of debugging, it can get pretty overwhelming. Now, did you have a Go background to aid in the troubleshooting or like what? -**Adriana**: Juraj, what’s on your wishlist? +**Iris:** I didn’t work with Go before actually working with OpenTelemetry, but now I’m learning. That’s the thing. OpenTelemetry and debugging, checking everything has become my experience in learning Go, and I’m also learning independently. So, yeah, it was a good excuse. -**Juraj**: I’d like to see the Apache Arrow project mature, as it could reduce network usage. Additionally, there are still silent ways to lose data with metrics. I’d also like OpenTelemetry to adopt a more unified approach to metric production to avoid compatibility issues. +**Adriana:** Oh, that’s great. How about you? What’s your biggest challenge that you faced around the collector? -**Adriana**: Greg? +**Juraj:** I think when we ran into a couple of bugs, it was kind of hard to debug because they were sort of like happening only with large loads of data at times. So, that’s when we ended up having to build our own distribution to deploy with the gold debugger attached to it. At least a lot of decently used dev environment to actually see, “Okay, why is this happening?” And then we only figured out, “Oh, this is like a bug in the Datadog exporter,” for example, where it was not doing something properly. I think that like, if you have a bit more complex pipeline, especially if you manipulate a lot with like the research attributes, it’s kind of hard to test locally or even, you know, like when you basically just sort of need to roll it out to some lower environments and see if it works properly. But then if you have your own local testing environment, we don’t have that much data going through it, and it’s probably maybe slightly different than what like the actual business applications produce. There’s a, it would be, it might be nice to be able to have some sort of capability to test that bit better without having to actually deploy to like full-blown environments. In the later stages of the migration, we had an incident, for example, where we stopped sending some data to Datadog, or like some of the labels changed. We didn’t code it in the lower environments because we were no longer sending those to Datadog. We realized that in production, and then like SRE team started to scream at us, like, “Where’s the data, where’s the data?” Oh, this label changed. Why did they change things like that? I mean, one other thing we ended up doing was because we had multiple sources of the data going, like different types of auto collectors doing different types of things. We ended up attaching the label to all the metrics, logs, and traces coming through it basically telling us like which auto collector type it came from. So like, daemon set for the daemon set, something like Kubernetes for the one that collected Kubernetes metrics. We had like internal slash external interest, which was for some of it, the internal force for ECS, the external was for metrics and traces coming from external customers, just so we could easily track down, okay, where is this data even coming from? -**Greg**: I’d like the platform to mature more. I want to be able to assure users that features will work as expected. Additionally, I’d like to see a dead letter queue integration point for exporters to handle errors more effectively. +**Adriana:** Right, right. And just out of curiosity, you mentioned debugging issues in the collector also was a pain point for you. Do you have a Go background? -**Adriana**: And Joel? +**Juraj:** I’m not really a software engineer. I’m more on the ops side, but I have been using things like Prometheus and Kubernetes for years before adapting OpenTelemetry. So a bit. -**Joel**: I echo many of these points. A Prometheus remote write receiver would be on my wishlist, as well as a more generic connector for building metrics from logs. +**Adriana:** Gotcha. Okay. Same question, Greg. -**Adriana**: Great insights! Lastly, let’s discuss scaling the collector. Iris, any feedback or pain points around scaling? +[00:30:00] **Greg:** I would say our biggest challenge has been like the quality of the platform we’re providing to the rest of the organization, especially kind of coming from this other system, which kind of worked to rolling out the collector and maybe our configuration being wrong or maybe there being bugs or whatever for some reason. Traces and spans not being available in the backend when our users were looking for them. Having gone out and said, “Hey, this is the future. This is the great new technology. It’s much better than this thing we made in house.” Their actual experience of them going and looking for the requests that they were trying to debug and it’s not there. That probably initially was just naivety on our part, like we didn’t know how to monitor the collector properly. We didn’t know at which point along the pipeline from the trace or the span being created in the application to it arriving in Grafana, where we should be looking. I think we’ve improved on that, but there are still kind of areas of doubt. We’ve got a couple of things which we’re struggling to know how, in principle, we can even monitor it, or how we should do it. It just doesn’t seem to be possible at the moment. That would be our biggest thing that’s caused us anxiety. The second thing would be just the sort of bugs or the things not working as expected in the collector. I joined the team a little bit later, a bit more senior than some of the other people. When they adopted it, they kind of drank the Kool-Aid. They thought this is great. It’s going to work. It’s going to solve all of our problems. For a long time, they were struggling thinking that they were doing something wrong. Whereas as I’m a bit more senior, I’m very cynical. I’m like, “Oh, there’s probably rubbish code. Let’s go and have a look at the code. Oh, look, it doesn’t work.” They didn’t like think to ask those sorts of questions. But you know, once you’ve been around for a while, you realize that all code is rubbish code and everything’s got a bug somewhere. Some examples of that, service graph. At the moment, retry. I don’t know if you’re aware, but I think the gRPC receiver will now return a retry code, but the HTTP receiver, OTLP receiver will not return a proper retryable code. It will just return a server, a 500, no matter what it is. Whereas a transient error should be like a 503 or something, so the client retries. We turned on retries in our agent, and we’re like, “Hey, everything’s going to retry, we’re not going to lose these spans anymore.” And like, we’re still losing them, what’s going on? It’s just not implemented in the collector, right? Retriable, transient errors. Stuff like that, it’s like, “Oh, right, okay, don’t assume that it works.” We’ve just switched from using the OpenCensus metrics, internal metrics, to the sort of native OTLP, or whatever you would call it. That all seemed great, but then some of them have disappeared for the tail sampler because the tail sampler has not yet been migrated to OTLP and it’s only publishing OpenCensus metrics. Which is fine, and we’ll probably be quite happy to go and fix that because it’s quite easy and submit a pull request for it, but it’s just like somebody spent a week thinking that they’ve done something wrong and actually it’s just not. -**Iris**: We’re currently using a horizontal auto-scaler, which works great for us. We’ve also tried KEDA, and it worked well for auto-scaling based on queue size. +**Adriana:** And you mentioned like, yeah, you could submit a pull request to fix it. Have you submitted any issues around these items that you’ve identified? -**Adriana**: Juraj? +**Greg:** It’s a similar story to earlier that like I’m maybe not as good at my job as I should be. What I do is I go and try and mess around with the code and fix it. Then I go and see if there’s an open issue. Then I find out there is an open issue and somebody’s already working on it. I should have been doing something else with my time. Pretty much everything we’ve seen so far. There was one thing; we use the count connector and that produces a stream of delta data points. We needed to export them to a Prometheus remote write, so I actually created a delta to cumulative processor and opened up a merger, an issue for that and got sponsored. But then somebody else had the same issue and now they’re working on it. I haven’t had time to actually, mine was very much a proof of concept and we’ve got it running in production and it works, but it only meets our very particular use case. It’s not like a generic and it’s probably written wrong and everything. So yeah, we opened up that issue and it got some attention. That’s how it should work. We’re happy with that. -**Juraj**: We’ve also used HPA based on memory and CPU. It worked nicely, and we were looking into using VPA for better resource management. +**Adriana:** Awesome. That’s great. And finally, Joel. -**Adriana**: Greg? +**Joel:** It’s really interesting you mentioned that. I was going to be my point that I was going to make is that I think the handling of cumulative metrics is really missing. That’s something which we kind of miss with very sort of Prometheus focused. All of our metrics are in Thanos. We want to ship them through, which I think is working. For example, like we ran into exactly the same issue with the count connector, and we just figured out, yeah, that’s going to be easy. Just plug it in and Bob’s your uncle, you can ship that to Prometheus, and it didn’t work like that. Once you’re so deep into the topic, you kind of know. Sure, that’s not going to work. But you have to get quite deep into it to get to that point of understanding, which I think was disappointing to our users when we turned around and said to them, “Ah, yeah, by the way, that solution we proposed last week definitely isn’t going to work. We sort of, we need to wait on some upstream development to go on there.” I would say, yeah, that’s the hardest part at the moment actually is planning how we’re going to bring metrics into our unified pipeline. Because at the moment, it’s just a tricky part. We also have an outstanding use case, I would say, which is, you know, generic, I mean, we have, like I mentioned, a lot of logs. So we’re very logs-based at the moment, kind of a very old school, which trying to change it internally, but that takes a long time. We have like a lot of use cases where we want to build metrics out of logs. Currently, there’s no, you know, general way to do that. We’re actually working on that as a sort of internal use case currently. To build like a logs to metrics connector. There are a few functionalities I would have expected to be there which aren’t there yet. It’s also very good to see, I mean, I see Dan is here. Dan is sponsoring, I think, the cumulative to Delta or Delta to cumulative processor. It’s very good to see that the community is working on it upstream. It just can be a surprise sometimes when you think, “Ah, for sure, that feature must be there,” when actually, oh, no, you can’t generate metrics in the collector and remote write them to Prometheus. That’s not there yet. A few surprises like that. Basically, that’s, let’s say the hardest thing. You can never be too sure when you’re talking to other teams in your organization that what you’re promising is actually achievable until you sort of dive in and get your hands messy with the code. -**Greg**: We’re in the early stages of our journey with scaling. We’ve over-provisioned for now to avoid losing data. We’re also exploring exporting internal metrics to CloudWatch to inform our auto-scaling policies. +**Adriana:** All right. Well, thank you. I did have a final question for everyone because I think especially as we see OpenTelemetry pick up more in the community, I want to talk just very briefly in the 8 minutes that we have about scaling the collector. Any feedback pain points around scaling the collector, starting with Iris? -**Adriana**: Finally, Joel? +**Iris:** So, so far about scaling, we have used, at my current company, because we only are using for traces, we’re using just normal horizontal auto scaler. It works great. In the past, we were also after KEDA, and I remember we even opened an issue in the OpenTelemetry, my colleague opened an issue to support it on the original chart, but I don’t think it is supported right now. But we did do it out of the box for our own obligation, and that worked very well for us, especially considering that the, okay, so HPA works because most of the queue is memory based. Using KEDA based on the queue size was a great move for us to do the auto scaling. But yeah, even the HPA works great and depending on the load that you have. -**Joel**: We run everything on one cluster per environment and have found that HPA with memory limits works well. I need to investigate gRPC scaling issues further, as it sometimes leads to parts not receiving telemetry. +**Adriana:** All right, thanks. Juraj? -**Adriana**: Thank you all for your valuable insights. We’re at the end of our time. Thanks to our panelists for sharing their thoughts on the OTEL Collector and how you use it in the wild. Thank you to everyone who joined us today. We appreciate it! +[00:48:25] **Juraj:** For all the deployments, we were using HPA, also just based on memory and CPU. That worked very nicely, I think. For the daemons that we were looking at doing, VPA, just because they’re like, yeah, if you have a single node with a bit higher load, you need to bump the resources and there’s nothing you can do about that. -For those who couldn’t make it, a recording will be available on the OTEL YouTube channel, called OTEL Official. If you’re attending KubeCon in Paris next month, many of us from the OTEL end user working group will be there, and we hope to see you at the OTEL Observatory. Thank you so much again, and see you next time! +**Adriana:** All right, great. And Greg? + +**Greg:** We’re kind of like quite relatively early in our journey still with this, and we’re at the moment happier to spend money than to lose data. So we’ve got like quite a conservative, we over-provisioned basically, and we’ve got a simple auto-scaling policy based on memory because the biggest thing is the tail sampler, which holds all the memory. We do scale out quite far and then back down again, especially in our load test environment, obviously. That goes to almost nothing overnight, and then during the load test, we get like 30, 40 instances sometimes. But we haven’t put a lot of effort into trying to optimize that because we’re much more worried about making sure that we’re properly processing all that data. Because we’re not sure what we’re supposed to be monitoring, we’re kind of very nervous about changing anything. But we’re in AWS, so I would like to see exporting the system metrics that we’ve got, the internal metrics, to you can write them to CloudWatch logs and then turn them into metrics somehow. It’s called EMF or something because you need the metrics to be within CloudWatch so that you can hit the AWS autoscaler. We can’t use any Prometheus metrics, which is what we’re normally doing. There’s quite a lot of stuff in my head that we could do there that I’m interested to look at, but apart from getting it working on my machine, I haven’t had anything deployed, anything actually running. But yeah, that’s because we’ve been a bit burnt by losing data, so we’re just like very cautious about that at the moment. + +**Adriana:** Fair enough. And finally, Joel? + +**Joel:** Essentially, we run everything on one cluster per environment, and these are all deployments. We found that just HPA with the memory limited processor has been no problems at all. I think it’s even just the out of the box config for the processor. I’ve never seen a collector OOM actually, which I find quite impressive. That seems to work very, very nicely. Actually, I need to talk to Juraj after this call because I was suspicious of gRPC on Kubernetes because I have run into the issue where parts will scale out or deployment will scale out, but some of those parts just don’t receive any telemetry, and we are using gRPC on Kubernetes. So that sounds like an issue I need to dig into. + +**Juraj:** If you can find me on the CNC Slack and if you ping me there, I can send you the link for the, I don’t know if there’s an open issue about it, but I discussed it already with another person. I can send you the link for the thread. + +**Joel:** Yeah, that’s great. Thanks. That’s something very useful I take away from this panel at least. Thanks for that. Can I mention, you mentioned that somebody mentioned the memory limiter. I was enthusiastic to my team. Another one of these things where like I’ve read the docs and I tell everybody, “Let’s just do this.” The memory limiter there is going to give you transient errors, right? If you get near when you’re scaling out, it’s fine because the client will just retry, right? The fact that retries don’t work, we started losing this telemetry. That was a scaling problem. Now we never hit the memory limiter. We always scale well before we’re going to get anywhere near that because it basically doesn’t work, right? I mean, it’s going to stop it out of memory, but it’s not going to, the client isn’t going to retry and resend that span. That’s a much bigger problem for us than spending money on memory. + +**Adriana:** Yeah, that’s a really great point. Well, I guess we are at time. We’ve got two minutes before we wrap up. Thank you to all of our panelists for showing up today and sharing their thoughts on the OTEL Collector and how you use it out in the wild. This is super valuable, and thank you everyone else who joined to give this a listen. We definitely really appreciate it. Tell all your friends who were not able to make it today that we will have this recording up on the OTEL YouTube channel. Our channel, in case you’re not aware, it’s called OTEL Official. Also, for anyone who is going to be at KubeCon in Paris next month, a number of us from the OTEL end user working group will be there, including Reese, Dan, Hope, and Rynn. I don’t think Rynn is on this call, and I will be there as well. We’re going to be hanging out in the OTEL Observatory, which, if you were at KubeCon North America last fall, it was the happening place for all things OTEL, and we’ve got a lot of cool things lined up at the OTEL observatory. We’re actually going to be doing some more user feedback sessions. We’re going to be recording Humans of OTEL. There are going to be SIG meetings, OTEL demos are going to be running. We’re going to have signups also for some of these things. If you want to participate in a different feedback session for another SIG, or if you want to be interviewed for Humans of OTEL, we’ll have the schedule posted up. I think there was a blog post on the OTEL blog, but just came out with the sign-up links for various things. Now our feedback sessions are still TBD, but keep an eye out for that post in the OTEL blog as we finalize the details. Hope to see a number of you in Paris, and thank you so much for joining us here today. Appreciate everyone taking the time today. + +**Iris:** Yes. Thank you so much. And we will see you on Slack. + +**Juraj:** Yep. Thank you so much. + +**Greg:** Bye. + +**Joel:** Thanks everyone. Bye. ## Raw YouTube Transcript diff --git a/video-transcripts/transcripts/2024-05-01T19:33:21Z-otel-prometheus-interoperability-user-feedback-panel.md b/video-transcripts/transcripts/2024-05-01T19:33:21Z-otel-prometheus-interoperability-user-feedback-panel.md index a8d6700..e4e9076 100644 --- a/video-transcripts/transcripts/2024-05-01T19:33:21Z-otel-prometheus-interoperability-user-feedback-panel.md +++ b/video-transcripts/transcripts/2024-05-01T19:33:21Z-otel-prometheus-interoperability-user-feedback-panel.md @@ -10,102 +10,154 @@ URL: https://www.youtube.com/watch?v=9a3ctZhJj-o ## Summary -The YouTube video features a panel discussion on Prometheus and OpenTelemetry (OTEL) interoperability, moderated by David Ashpole from Google. The panelists include Iris, Nikos Takas, Vijay Samuel, and Dan Bromitt, who share their experiences with using Prometheus and OTEL in their observability stacks. The discussion covers various topics, including challenges faced in integrating these tools, the importance of maintaining compatibility to avoid breaking changes, and the need for consistent metric collection practices. The panelists express a desire for improved features in future versions, such as enhanced OTLP support and standardized naming conventions to streamline the user experience. Overall, the conversation highlights the complexities of observability in large organizations and the ongoing efforts to enhance the integration between Prometheus and OpenTelemetry. +The YouTube video features a panel discussion about the interoperability between Prometheus and OpenTelemetry (OTEL), moderated by David Ashpole from Google. Panelists included Iris from Miro, Nikos Takas from Corelogs, Vijay Samuel, and Dan Bromitt from Home Depot. The discussion focused on how these technologies can work together effectively, sharing insights into their respective observability stacks and the challenges they face, such as configuration issues, performance concerns, and ensuring seamless integration. Key points included the desire for consistency in metrics collection across different environments, the advantages of OpenTelemetry SDKs for tracing, and the potential for future developments in Prometheus, such as native OTLP ingestion and improved remote write capabilities. Overall, the panel expressed a strong interest in aligning the two ecosystems to improve user experience and facilitate smoother transitions for large organizations. ## Chapters -00:00:00 Introductions and panelist overview -00:05:00 Dan discusses his observability stack and challenges with store environments -00:08:30 Vijay explains instrumentation and Prometheus/OpenTelemetry setup -00:12:00 Nicholas shares current use of OpenTelemetry and challenges with resource consumption -00:14:30 Iris talks about using Victoria Metrics and experimenting with OpenTelemetry -00:20:00 Discussion on the challenges of adopting OpenTelemetry and Prometheus together -00:26:00 Panelists share feedback on upcoming features in Prometheus 2.0 and 3.0 -00:30:00 Discussion about the impact of breaking changes in large organizations -00:37:00 Panelists provide thoughts on the benefit of OTLP support and configuration simplicity -00:42:00 Audience participation and final remarks from panelists +00:00:00 Welcome and introductions +00:03:40 Observability stack overview +00:06:28 Challenges in store environments +00:09:40 Metric instrumentation discussion +00:12:20 OpenTelemetry and Prometheus integration +00:14:40 Challenges with performance and UX +00:18:00 Struggles with legacy systems +00:20:50 Scrape parity concerns +00:24:00 Future Prometheus features discussion +00:27:30 Feedback and closing remarks -# Prometheus OTEL Interoperability Panel Transcript +**Speaker 1:** Everyone who's able to join for this Prometheus OTEL interoperability panel, which David reached out to us to see if we could get some end user feedback on making sure that Prometheus and OTEL play nice, which is always awesome. So we've got, I believe, 4 panelists, correct me if I'm wrong, who will be sharing their thoughts. And David is our moderator who will be asking questions, and why don't we get folks to introduce ourselves, introduce themselves, start with David, and then we'll have our panelists introduce themselves. -**Welcome and Introductions** +**David:** Sure. I'm David Ashpole. I work for Google, and right now I'm working with the Prometheus OTEL work group, and we're trying our best to make sure that the protocols, the exporters, all the pieces that integrate OpenTelemetry and Prometheus together work well and consistently and help users that are using both to have a good experience. Awesome. Next panelist. Anyone want to hop in? -Everyone who's able to join for this Prometheus OTEL interoperability panel, David reached out to us to see if we could get some end-user feedback on ensuring that Prometheus and OpenTelemetry (OTEL) work well together, which is always awesome. We have four panelists and David is our moderator who will be asking questions. Let's have everyone introduce themselves, starting with David. +**Iris:** I can start. Cool. Hello everyone. I'm Iris. I work as a senior observability engineer at Miro. So my life is surrounded around Prometheus and OpenTelemetry recently. So yeah, it's nice to be here. -**David Ashpole:** -Sure. I'm David Ashpole. I work for Google and right now I'm working with the Prometheus OTEL work group. We're trying our best to ensure that the protocols, exporters, and all components that integrate OpenTelemetry and Prometheus work well and consistently, helping users who use both to have a good experience. +**Nikos:** Hello everyone. Pleasure to meet you. My name is Nikos Takas, and I'm an engineer at Corelogs, currently working on observability units. I'm a Prometheus Operator Maintainer and also a Persis Maintainer. I've been working with trying to get some codes into Prometheus to help with the Prometheus 3.0 and OTLP ingestion as well. My daily base is around Prometheus, OpenTelemetry, metric collectors, and so on. Pleasure to be here. -**Iris:** -Hello everyone, I'm Iris. I work as a senior observability engineer at Miro. My life revolves around Prometheus and OpenTelemetry recently. It’s nice to be here! +**Vijay:** Nice to have you. Let's see, my name is Vijay Samuel. I help with observability. I think I've met most of you, and nice to see you all again. -**Nikos Takas:** -Hello everyone. My name is Nikos Takas, and I'm an engineer at Corelogs, currently working on observability units. I'm a Prometheus Operator Maintainer and also a Persis Maintainer. I've been working on getting some code into Prometheus to help with the Prometheus 3.0 and OTLP ingestion as well. My daily focus is on Prometheus, OpenTelemetry, metrics, and related topics. Pleasure to be here! +**Dan:** Yeah, good to see you. And then our last panelist, I think, is Dan. Is this Dan? I hope so. I don't think we've actually met. -**Vijay Samuel:** -Hi, my name is Vijay Samuel. I help with observability. I think I've met most of you, it's nice to see you all again. +**Dan:** Hi, yeah, there's so many Dans. Hey, I'm Dan Bromitt, Senior Manager of SRE at Home Depot for the Store and Payments Systems. Yeah, we use a lot of the OTEL and observability adjacent tools. Cool. Great to have you. -**Dan Bromitt:** -Hi, I'm Dan Bromitt, Senior Manager of SRE at Home Depot for the Store and Payments Systems. We use many OTEL and observability-adjacent tools. Great to have you all here! +[00:03:40] **David:** All right. So let's see. Why don't we go person by person and just give an overview of roughly what your observability stack looks like and how you're using Prometheus and OpenTelemetry in that stack. And why don't we start, we'll go in reverse order. So Dan, why don't we start with you? -**Observability Stack Overview** +[00:06:28] **Dan:** Okay. So we observe services across a wide range of compute environments. That includes the data center, which are on-prem virtual machines, the cloud, which is GCP, primarily GKE, primarily Kubernetes. And also, in our stores, those are primarily small clusters inside of our stores where we are also running OpenTelemetry collectors. And we are also running in parallel to them, Prometheus installations as well. So the usage of OpenTelemetry as regards to Prometheus is in some cases just the typical, hey, it's Kubernetes in the cloud, and we want to have multiple different projects that we can share our metrics across all these different GCP projects. But the really, and that's fine. I think that's largely the, I don't think that's any groundbreaking stuff happening there. For us, we have challenges in our store environments where it's these things are miniature data centers. Our stores are, we have a miniature data center in the back of every single store with a lot of racks of network and server equipment. And we want to be able to make sure that stuff is actually working and our customers are happy. The network connectivity is spotty. Some of them are offline every single day. Some percentage of them are offline. Being able to backfill metrics, which is a challenge. Hey, the metric was collected at the right time at the store level, but then shipping it up to a central Prometheus level, sometimes that backfill just doesn't work because of timestamps and stuff. I'm sure that y'all are aware of. And the collection of it is the configuration across a fleet of devices, a fleet of services is, being able to configure all that is also a challenge for us. We're getting better at that on the Kubernetes side, but it's still, that's a challenge that exists. -Let's go person by person to discuss what your observability stack looks like and how you're using Prometheus and OpenTelemetry in that stack. We'll start with Dan. +**David:** Great. Thank you. Vijay, tell me about what your setup broadly looks like and which pieces of Prometheus and which pieces of OpenTelemetry you're using together. -**Dan:** -We observe services across a wide range of compute environments, including on-prem virtual machines in our data centers, primarily using GCP and GKE, and within our stores, where we run OpenTelemetry collectors alongside Prometheus installations. +**Vijay:** Sure. Starting with the instrumentation, right? We made a form for the most part. We standardized on a mix of the Prometheus Java client and Micrometer for Java applications. We use the community supported Prometheus client for JavaScript, and basically these expose everything else that's not considered managed. So Java and JavaScript are managed languages, which means that there is a framework that people can build on top of. Everything else is the wild west in the sense for Go, people use the standard Go client, and there's Python and a few other languages that are there as well. All of these expose Prometheus endpoints today and are scraped by OpenTelemetry collectors that are deployed on every Kubernetes cluster that we host internally. This is the breadth of all our metric use cases that are there, more than 90-95%. There is a smaller group of metric use cases that send in directly into our gateway APIs, which can understand more proprietary formats, but they go through sanitization and whatnot before entering into our metric store. And finally, there's also the newer breed where we are trying to directly encourage some of the use cases to use the OpenTelemetry SDK to require to instrument and then use gRPC into the collector, enrich, and then write into the backend. The backend is predominantly Prometheus first, meaning we use the Prometheus SDK to build clustering and replication on top of the Prometheus storage engine. All the characteristics of Prometheus still apply because the same headlock and everything stay the same, and we use object store backed by Thanos for long term store. Over time, we want to get to a place where we know people from things like Micrometer and from this client to just use OpenTelemetry SDK because on tracing, we are a hundred percent on OpenTelemetry today, and we are starting to look at logs as well. So over time we want metrics to be within the same fold. -Most use cases with OpenTelemetry and Prometheus involve Kubernetes in the cloud, which is fairly standard. However, our challenges arise in store environments where we have miniature data centers. We need to ensure everything is functioning correctly, as network connectivity can be spotty, resulting in some stores being offline daily. This makes backfilling metrics challenging due to timestamps and other issues. +[00:09:40] **Nikos:** Our setup is quite simple. At the moment, we are leveraging OpenTelemetry Collector for traces and recently starting with logs. In metrics, we are evaluating because at the moment we are using a mix of Prometheus servers and Prometheus agents to write metrics from many different places. And our main idea is replacing the Prometheus agent by the OpenTelemetry collector and leveraging all the OpenTelemetry pipeline capabilities. But at the moment, we know it's a lot of OpenTelemetry collector consuming more resources than the Prometheus agent in different aspects, and this is what is blocking us a bit because we are running a very large amount of metrics we are collecting. Replacing the collector by the agent at moments blocking because of the resources, but we are pretty busy. That maybe this might be solved when we have Prometheus ingesting OTLP natively; you don't need more of the conversions between OpenTelemetry and the Prometheus format, at least during the remote writing process. The same idea of replacing maybe Prometheus client by the OpenTelemetry SDK and implementing the OpenTelemetry from the instrumentation up to the storage layer is mainly this at the moment. -**Vijay:** -Starting with instrumentation, we've standardized on a mix of the Prometheus Java client and Micrometer for Java applications. We use community-supported Prometheus clients for JavaScript, which expose metrics for everything not considered managed. These endpoints are scraped by OpenTelemetry collectors deployed on all our internal Kubernetes clusters. +[00:12:20] **Iris:** For us, it is pretty simple, the architecture as well. We're using mostly Prometheus clients; we are not collecting via Prometheus most of our metrics, especially custom metrics, but we're using Victoria metrics. Everything is in Prometheus format because it is the open-source Victoria metrics. We're also leveraging the Node Exporter, for example, that is our main exporter currently that we're using. And we are at the moment experimenting with OpenTelemetry, especially with host metrics, because it is more convenient for us to run OpenTelemetry collector and use all its capabilities for metrics collection, especially in the host level than to use a node exporter. We also are experimenting in some cases because we are also running a monolith, except for Kubernetes. So we do use the Prometheus receiver, adding script jobs to OpenTelemetry to reach in some places that it is more difficult for us through Victoria Metrics. But yeah, currently everything is Prometheus mostly, but we are thinking very hard to move towards OpenTelemetry as the case, because tracing is already in OpenTelemetry, partly still open tracing. But the goal is to move everything there. So we want to have a uniform way of collecting all our data. -For about 90-95% of our metrics, we send data directly to our gateway APIs, which can handle proprietary formats. We're also encouraging the use of the OpenTelemetry SDK for metrics to streamline our processes. The backend is predominantly Prometheus-first, utilizing the Prometheus storage engine with Thanos for long-term storage. Over time, we aim to have a unified system for metrics, traces, and eventually logs. +**David:** Cool. Sounds like there's actually some similarity between most everyone. Sounds like I believe everyone's using primarily Prometheus libraries and instrumentation but is thinking of moving to OpenTelemetry. I think Iris, you touched on that a little bit. Is it just that you want consistency between the way you're doing tracing, which is OpenTelemetry, and the way you're doing metrics? Or is there something about the OpenTelemetry instrumentation that's drawing you to migrate? -**Nikos:** -Our setup is quite simple. We're leveraging OpenTelemetry Collector for traces and are starting to explore logs. For metrics, we currently use a mix of Prometheus servers and agents but aim to replace the Prometheus agent with the OpenTelemetry Collector to utilize its pipeline capabilities. However, resource consumption is a challenge that is currently blocking us from fully transitioning. +**Iris:** We really like the OpenTelemetry SDKs for tracing. We've seen that it goes a long way better than any other SDKs before. Considering that we are using this for tracing, why not use it for the whole parts of the application for metrics in this case? And why not logging? So this is the main reason. Also, the infrastructure, not just as the case, but the infrastructure makes it very easy the way that it is very flexible for us to use the OpenTelemetry infrastructure. So that really helps in this case. -**Iris:** -For us, the architecture is straightforward. We're mainly using Prometheus clients and Victoria Metrics for custom metrics, with everything in Prometheus format. We're experimenting with OpenTelemetry, especially for host metrics since it’s more convenient for collection. We also run a monolith alongside Kubernetes, using the Prometheus receiver and adding script jobs to OpenTelemetry for areas challenging to cover with Victoria Metrics. Our goal is to have a uniform way of collecting all data, especially since tracing is already handled by OpenTelemetry. +**Dan:** Hey, and David, I do want to call out that we are using the OpenTelemetry Java agent in production, in stores, and some of our legacy stuff is still using different libraries for collecting metrics for emitting metrics, exposing metrics. But the majority of our new services are using the OpenTelemetry, either Go library or Java library to expose metrics. -**Challenges in Using Prometheus and OpenTelemetry** +[00:14:40] **David:** Very cool. All right. Let's talk about challenges. Nicholas, why don't you start us off? I know you mentioned performance issues with the Prometheus receiver and the collector. What else during your journey of adopting Prometheus and OpenTelemetry together has really been a struggle for you? Either that's still unsolved or you were able to work through. -**Nicholas:** -My struggles primarily revolve around user experience and understanding how to set up metrics properly. It took time to grasp the nuances of configuring the collector and managing attributes. The biggest issue was switching from the Prometheus receiver to the OpenTelemetry operator and figuring out how to run the target allocator without it. +**Nikos:** That's a good point. I think that what I struggled with the most in the beginning was mainly UX, DX. I don't know how to properly call that, but it took me a little while to understand. Okay, we are the metrics, we are not pushing the attributes, like these are such roots and something like that to the metrics. And then we need to enable on the remote writing to send the metric info, so one or configuring the collector to expose these attributes as metric labels. This took my while to understand that, and after that, since I didn't start using the operator, the OpenTelemetry operator, to make the like switching from the Prometheus receiver to the, I just forget the name of the service. The target allocator helps a lot on the problems to discovering targets using a lot of services. But if you're not using the OpenTelemetry collector, you need to figure out yourself how to run the target allocator with the OpenTelemetry without the operator. This is like easy to do, but I did need a lot of research and reading docs, issues, and so on to make it work outside of the operator. I think that mainly the issues that I found were related to UX at the moment. UX in terms of getting it all set up or UX after you have it all set up and you're trying to use the data that you've ingested. -**Dan:** -One major struggle has been the lack of support for certain legacy systems. We tried to leverage the Java agent for observability on older microservices that ran on unsupported versions of Java and Tomcat. The configuration of OpenTelemetry for telemetry over a messaging framework was also a challenge for our engineers. +**Dan:** Yeah. To make the thing set up, set up and configure the receivers and understanding the little needs regarding the writing informs or not, and then the issue that I mentioned regarding the setup of the Target Allocator, part of the OpenTelemetry collector. Also, you have everything there. It's quite okay to use this, like at least for us because we already know about the difference between the metrics would not happen the units if we decide to not have the reboots. Is there such reboots as metrics labels? We need to join with the info metric and so on, those kind of things we were aware of. So we're pretty easy to get used to these little niches at the moment. -**Vijay:** -Our biggest struggle was scrape parity. Initially, there were significant differences in how Prometheus and OpenTelemetry Collector handled scenarios like label names starting with an underscore. Thankfully, many of these issues have been addressed through PRs we filed. +**David:** That's helpful. Dan, what's been a struggle for you? Either was a struggle or still is a struggle using Prometheus and OpenTelemetry together. -**Iris:** -Our challenges mostly stem from the size of our organization. With 700 engineers using our observability platform, any change can disrupt their dashboards or alerts. We’re moving slowly to ensure that transitions don’t negatively impact user experience. +[00:18:00] **Dan:** So first of all, the agent, the Java agent, it only supports, so we tried to leverage it to add observability to legacy microservices that had no provisions for observability at all when they were created. They were running on a version of Java that the agent didn't support and a version of Tomcat that the agent didn't support. So I'll tell you, adoption was real rough for the first six months because of that. I think it was Java 7, which is frustrating. So I don't know if that's something OpenTelemetry fixes, but it was a problem. And the other big one, I think, just the massive challenge is the configuration of OpenTelemetry to send any sort of telemetry over a push like a pub/sub, like any sort of messaging framework like a pub/sub or something. Configuring that was a little bit challenging for our engineers to get it so that we could actually, hey, we have the apps that the telemetry is collected from the apps locally, and then we're going to push it over some sort of messaging framework to get it to a central location. That was a little bit rough. And then the last one is the timestamps, and I know that it's not an OpenTelemetry only challenge; I know it's related to Prometheus, but if we have a location that's offline for a while, it comes back online and tries to send out the metrics that it was unable to send out while it's offline, those timestamps get totally borked. -**Looking Ahead: Future Developments** +[00:20:50] **Vijay:** Our biggest struggles were scrape parity, and a lot of those through PRs that we filed on the collector, we have had addressed. Some of them are like, what if a label name started with an underscore? What if we wanted to disable sanitization for whatever reason, if it started with the colon? So these are all scenarios where, initially, Prometheus greatly differed from how OpenTelemetry collector was handling it, specifically on the exporter, the remote write exporter. So those are all solved now, so we don't have a problem, but that's one of our biggest concerns as we try to marry these two, OpenMetrics and OpenTelemetry together. It's like we cannot afford any breakages. It should largely be rather than, if, going back to the document that was shared upfront, it should relax the requirement but still have ways to keep them as is so that when we try to move across versions, we don't have something break. And at the scale at which we operate, if something breaks, then it's a catastrophic breakage. Outside of that, I think forward looking on the collector itself, we are more of in wait for a few features to come out, being able to scrape native histograms and then report them into a Prometheus-compatible backend and things like that. But we don't have complaints per se right now. Certain things could be better, like Attribute processors could be a lot simpler to configure. They're very verbose right now, but those have nothing to do with the discussion we have right now. In summary, please don't break us. -**Iris:** -I'm excited about the upcoming features, especially the Prometheus 2.0 exporter and receiver. We’re testing it heavily to prepare for our next move. The support for dots in metric names and the uniformity it brings to our engineers is also something I’m looking forward to. +**Iris:** Our struggles come mostly from the size of the organization. Our organization is quite big. So we, just for, to envision it, we have at least 700 engineers that are using our observability platform. And we have to say most of them, of course, are front-end engineers, back-end engineers. So they're not, I'm not going to say fully proficient. They know their metrics and everything, but of course, if we make a big change and it's different, it could ruin their dashboard. It could ruin their alerting, and it is a very unpleasant experience. So that has been our challenge so far and why we are moving very slowly. Because if we change the way that we're collecting information and then the metrics, even if the formalities are different, it could be a big change. And 700 people to get adapted to this change needs time. So we are going very gradual with it. Prometheus is known territory for everyone that works on the team and for all the engineers that have had experience with their metrics at the moment. So it's safe. Of course, the OTEL collector needs a lot of digging into the documentation, finding out more, testing, and trials, so it makes it a bit slower, but mostly our biggest challenge that we're overcoming slowly, but still we are not there, is the experience of our engineers, so we don't ruin their dashboard, their observability performance in general. -**Nicholas:** -For me, the remote write 2.0 is significant as it promises better resource consumption. I'm also looking forward to the OTLP ingestion in 3.0, which will facilitate more seamless integration between OpenTelemetry and Prometheus. +[00:24:00] **David:** Great. Thanks. So I know we just touched on this a little bit, but I think there was, I think it was actually a Prometheus blog that came out maybe a couple months ago about 2024 and some of the cool things that are coming. Some of the things I can think of are Prometheus remote write to OTLP support and the Prometheus receiver, Delta support for Prometheus, supporting dots, woohoo, stability, or even support for OTLP and some of the Prometheus exporter ecosystem that already exists today. Iris, we'll start with you again. Which of those are, or is there anything else that really stands out as something that you're really looking forward to having? -**Dan:** -The OTLP support will simplify our configuration as we centralize our metrics. We’re also curious about handling late data with OTLP and how that will integrate with our existing infrastructure. +**Iris:** Can I say all of the above? I'm very excited for all of them, but if I had to distinguish maybe the exporter and receiver, the version 2.0, it's going to be very nice. We were using it heavily on testing heavily with it right now for our next move and the support of dots. Why not? Again, it comes from the organization and having things more uniform. It helps us a lot and the experience of our engineers. -**Vijay:** -I'm interested in the OTLP ingestion and how it will allow for a seamless experience where the instrumentation matches the backend metrics without unnecessary transformations. It’s crucial that we preserve names and avoid breaking changes. +**Nikos:** Cool. Wow. I guess it's a mix of the remote write 2.0, I guess it's going to be massive. People are working hard to have a better protocol consuming less resources. Since most of the people are using OpenTelemetry, they are emitting writing for some Prometheus-based system. I have in this remote writing 2.0 will help people saving resources, networking, CPU, and memory. Apart from that, I guess for the 3.0, of course, the OTLP ingestion on GA is going to be nice. Of course, there is a lot of work that people are doing on the Prometheus team, making out of order samples ingestion be enabled by default. This is something very important to supporting auto-allopinate. I guess the next one, just to be a little bit different from we, I guess it's the UTF-8 support and the metric label and name. I guess it's going to be nice. It's going to have some proximity between OpenTelemetry and Prometheus to use metric names and labels. I guess it's going to be useful for usability and joining the different communities. -**Audience Interaction** +**Dan:** Yeah, so the OTLP, that is a very cool thing. It will simplify some of our config. It was we get to this, the central GMP, where everything's stored. I shouldn't use acronyms to the central Google Managed Prometheus that we run where all of our metrics are ultimately stored. One of the things that we're not really clear on is, hey, if we have data that is late. So if we have OTLP bit of telemetry that has a timestamp that is five hours late, is that still going to get stored? And we haven't really been able to find the answer to that question. So we're just assuming that we'll test it and see once it comes out. -If there's anyone from the audience who would like to share feedback or thoughts, the floor is open. +**Vijay:** Yeah, we can think. Prometheus is going to support out of order writes. I'm not sure about the... -**Audience Member:** -I would like to emphasize the importance of keeping conventions consistent across both ecosystems to minimize the learning curve for engineers. +**Dan:** It's already supporting, actually. It's already supporting. It's behind a feature flag and you can just enable it and configure how much behind you want in gesture samples. -**Moderator:** -Thank you, everyone for your valuable insights. This discussion has been incredibly beneficial. We look forward to more community discussions in the future. +**Vijay:** Yeah, so the out of order writes thing, that's good. That's cool. It's the stuff that has a timestamp. That's super behind is the one where it's really unclear. -**Thanks and Farewell** -Thank you for joining us! +**Dan:** Cool. Yeah, we can sync up afterwards. + +**Vijay:** And what are you excited about? It's coming soon. + +**Nikos:** Actually, a lot of things. The ODLP ingestion is interesting. The part that we would really hope for is that when we support it, it should be in a way that, okay, I instrumented this way on my application and I see the same way on Prometheus. No addition of underscore, total or milliseconds or whatever and keep the dots as is and things like. But the part that's still somewhat unclear and we hope to see more clarity is how do the resource attributes and the metric attributes play along inside of Prometheus. What if there are name coalitions and things like that? That is somewhat unclear, but if the fundamental thing is that we want the instrumentation and what they query to perfectly match, especially when they are just using gRPC to send out from the client directly. Delta Temporality is something that some of the folks who started off directly on OpenTelemetry SDKs felt that, oh, this is nice. It is nice to have, but it's not supported on Prometheus because you drop it on the floor in the collector right now. That's there. But outside of this, this list, I know one of my wishlist items is like exemplars persistence. That's something that's not there yet on Prometheus. It's still the circular ring buffer that's there. Hopefully someone can get to it by 3.0. It's a nice thing to have, which we rely quite heavily on exemplars today. + +**Nikos:** Oh, cool. Actually, I learned something. I didn't know exemplars weren't persistent. + +**Vijay:** Yeah, so they're not persistent and recording rules don't support it. For the latter, there's a PR which I never got to finishing, but someday it will be there. But the persistence is something that no one has picked up yet. + +**David:** Oh, cool. Thanks, Nicholas. Okay, so one of the things that we've talked about a little bit so far, it seems like most people are in agreement that breaking changes are really hard. You have big organizations, big deployments, and like when we mess with things in terms of how things are translated, that can really have a big impact. And at the same point, it seems like there's at least some agreement within this group that we are excited for UTF-8 support, and we really want to be able to preserve the original OpenTelemetry data that is coming from the application. VJ, I'll put you on the spot. What do you think the right way to strike that balance is for us? + +**Vijay:** On your doc, I think I had a discussion with some of our leads internally as well. Like option one seems pretty appealing to us in the sense that at least today what we do is we separate installations between things that are going after Prometheus endpoints versus things that are emitting through gRPC directly. Okay. So it still gives us the opportunity to say that, hey, receiver, Prometheus receiver and remote write exporter, make sure that everything that Prometheus mandates today, enforce them on the scrape endpoints. But OTLP receiver and whatever is writing to our Prometheus backend, make sure that you are passing things the way that OpenTelemetry standards mandate. So it gives us a balance in keeping things as is, but things that we are moving to the next generation, we are moving it in the right way. So we have done this in quite a few evolutions where we used to run a modified OpenTSDB first before we moved into Prometheus. That's how we bridged the gap initially. This would be like a second generation where moving from, say, Prometheus semantics into OpenTelemetry semantics. I would hope everything goes behind either a configuration or a feature flag to keep the standard, the current standardization configurable to the specification that's there today on OpenMetrics and allow us to disable them as we need it. + +**David:** So to be clear, do you think there's a difference between push and pull? Or is it that it should keep or that we should make it preserve the names by default, but always give people an escape hatch? I think you said that you like that like OpenMetrics and Prometheus scrape endpoints still follow the Prometheus conventions, but for pushing Prometheus remote write and OTLP, you want to be able to just preserve it as is. Do you think there's a distinction there? + +**Vijay:** Only for the OTLP. Prometheus remote write, however, we feel it needs to be, it can be done whichever way. But for things that are coming in through OTLP, it should pass through the pipeline exactly the way that the sender intended it. + +**David:** That's what we're particular about. And on the scrapes, through either content negotiation or whatever, the behavior can be decided one way or the other, but we want to keep the scrapes as is because we have like more than 4 million scrapes that happen, and if something changes, then everything blows up at that point. + +**Vijay:** So don't break client side. + +**David:** Yes, exactly. Okay, that's very good feedback. Do you see these push and pull use cases being in the same pipeline and the exporting side needing to be automatic, needing to automatically decide whether it does that translation or not based on the source, or would you expect to have separate pipelines configured for those use cases? + +**Vijay:** I would vote for simplicity in the sense that we are okay to run two pipelines. At least the way that we are deployed right now, I don't see a way in which both push and pull can coexist, so we would anyway run them as two separate pipelines. So it's probably fine. So no, no additional complexity required on the exporter side is what I'm doing. + +**David:** Yeah. Okay. So just the configuration to decide which of those options you want and then you'll set up a pipeline for each of them. + +**Vijay:** Correct. + +**David:** Sounds good. Thank you. + +**Iris:** Iris, I'll ask you a similar question. So you have a very large organization, right? You've said, 700 engineers. + +**Iris:** Yeah. + +**David:** Are you using PromQL? You said you mostly are using Prometheus and are familiar with Prometheus. Are you then using a lot of PromQL? + +**Iris:** Yes. All PromQL for metric query. Have you seen the UTF-8 PromQL proposals read? Sorry to put you on the spot. + +**Iris:** No, I actually did skim through it right before this meeting. But I think that most of our issues, maybe I misunderstood, while I was going through the proposals is the first one to change open metrics to allow OpenTelemetry name would probably solve this. What makes me say that is that, yes, there are differences, but usually when you are making queries, especially if you are offering that to a wide organization, you are using an engine that is very intuitive when coming to finding the attributes, the labels, the metric names. So as long as we, for example, a Prometheus backend is able to support OpenTelemetry metrics, UTF-8, that should solve the problem of a common engineer that does not know a lot about observability to build a PromQL query, for example, so that is a very good first solution for this. + +**David:** I remembered my question now. Thank you. Thank you, Iris. Nicholas. + +**Nikos:** Oh, shoot. I remember it again. Nicholas, my question for you. Part of what we're talking about here is a difference between Prometheus best practices that have been codified as the OpenMetrics standard and the OpenTelemetry semantic conventions that have come and given new ways of, say, writing metrics or naming things. Are you happy with the way that these two ecosystems have evolved? And what do you think the best approach is for users to try and give them a reasonable experience? Should we let them both be, or would you like to see either OpenTelemetry or OpenMetrics change such that they both work? They align in terms of how things should be named or that sort of thing. + +**Nikos:** That's a really good and tough question, actually. Personally, I feel Prometheus and OpenTelemetry are communities that stayed away for a little while, and we developed things very separately. Personally, what I would like to see, I think that maybe the OpenTelemetry entry, we should look more at how things are working when we have high adopted systems, like we have with Prometheus, different from the tracing and the logging, telemetry data type where we have many different data stores, like in the CNCF space. For the metrics, we actually only have Prometheus-based projects. Maybe we should look more at how Prometheus is doing things and avoiding changing too much. How this is working because most like in the end what users want to do is to use the new things without from needing to change or rewriting everything. You know if we look to how the metrics how we are trying to unite the metrics between OpenTelemetry and OpenMetrics, we are trying to get some, okay neutral moment here where we agree with each other. But in the end, I guess we're going to introduce some change to the end user. And I guess that we can unite efforts between the different communities to avoid these in the future. For example, oh, we think when we decide if when OpenTelemetry user groups start discussing how metrics, we would like the feeling that I had is like we didn't get close yet to the Prometheus ecosystem and so on and trying to, oh, let's try to solve most of the issues that we have in Prometheus, but without introducing a totally different metrics naming convention just to be easier with the end users. + +**David:** Cool. Thank you. That makes sense. Cool. Thank you everyone so much for coming. Those are the questions I had prepared, and I really appreciate the feedback. I know we're ending a little early. I guess since we're done a little early, we could, if anyone here who's in the audience wants to share their feedback, we can definitely open up the floor. Any takers? + +**Iris:** Oh, I'd like to mention just one more thing if it's okay. On the querying side as well, if we can keep the conventions fairly alike, there are a few options that try to distinguish things with double quotes and whatnot. That would be new things that we need to teach folks. If we are just using, introducing UTF-8 conventions into label names and metric names by keeping the style exactly the same, that's still perfectly fine because people are just going to assume that we are just introducing new metric names and new label names rather than new conventions that they need to abide by. So if that can be taken into consideration, that would be great as well. + +**David:** You're talking about having to put UTF-8 names inside of quotes. + +**Iris:** Yeah, exactly. + +**David:** Yeah, I think that was considered. I can give you pointers afterward if you want to know the details of like why it's a struggle. + +**Iris:** Okay, sounds good. I was at the Prometheus Dev Summit earlier today, and one of the things they were discussing was introducing OTLP export support into the Prometheus client libraries. Is that something that would be useful to this group here? And if so, one of the things they weren't quite clear on where they wanted to land was whether that OTLP support needed to be part of each Prometheus client library, or if it would be okay if there was a third-party or separate repo bridge that had to be combined with the Prometheus client library and an OTLP SDK to make it function. Does anybody here have thoughts on that? + +**Iris:** Can I chime in? Is that allowed? + +**David:** Sure. + +**Iris:** So when, in our environment, we use a lot of different metrics solutions, and we are trying to reach standardization when metrics are moving from one location to another, it's a mess when we have to have a bunch of different infrastructure set up to facilitate moving metric data for Datadog versus moving metrics for Prometheus, right? It's just a mess. So the as much as we can get into OTLP, so that way we can have everything moving through the same infrastructure for observability, that just simplifies our entire ecosystem. So the more we get into OTLP, the happier everybody on my teams will be. In our case, if this proposal came out a couple of years ago, we would have probably bounced on it because at least the problem that some of our use cases, especially things that run like cron jobs, folks use the Prometheus client and use the push gateway. Those use cases we would have ideally liked to replace it with the non-push gateway based implementation because the push gateway semantics don't necessarily work always for us. We just want to be able to send it into the gateway and put it on storage as quickly as possible. For those kinds, it's useful, but in today's world, if someone is trying to achieve such a use case, we'd rather recommend them to move to the OpenTelemetry SDK because there is a defined pattern for us to go after in that. + +**David:** Any other thoughts or feedback from anyone? + +**Iris:** Thank you everyone for joining. This was a really great discussion. Really appreciate it and looking forward to having more community discussions. + +**All:** Great. Thanks, Adriana. Thank you. Thank you for the opportunity. Thank you very much. Bye-bye. + +**David:** Bye. ## Raw YouTube Transcript diff --git a/video-transcripts/transcripts/2024-06-05T21:36:59Z-the-humans-of-otel-kubecon-eu-2024.md b/video-transcripts/transcripts/2024-06-05T21:36:59Z-the-humans-of-otel-kubecon-eu-2024.md index 728928b..b74019a 100644 --- a/video-transcripts/transcripts/2024-06-05T21:36:59Z-the-humans-of-otel-kubecon-eu-2024.md +++ b/video-transcripts/transcripts/2024-06-05T21:36:59Z-the-humans-of-otel-kubecon-eu-2024.md @@ -10,136 +10,172 @@ URL: https://www.youtube.com/watch?v=bsfMECwmsm0 ## Summary -In this YouTube video, various experts in the field of observability share their insights and experiences related to OpenTelemetry, a project aimed at improving observability in software systems. The participants include Iris Durish (Meo), SE Nman (Cisco), Kayla Riel (New Relic), Morgan Mlan (Swun), Henrik Reet (Dan Race), Vijay Samuel (eBay), Daniel Gomez Blanco (Skyscanner), and others, each discussing their roles and contributions to the observability landscape. They explore the significance of observability, its evolution from traditional monitoring, and how OpenTelemetry serves as a vital tool for gathering telemetry data across diverse systems. The discussion highlights the importance of community collaboration, the future of open standards, and the integration of various signals like metrics, logs, and traces to enhance system understanding and performance. The participants express their enthusiasm for the advancements in observability and the transformative impact of OpenTelemetry on engineering practices. +In this YouTube video, titled "Observability and OpenTelemetry Insights," a group of professionals from various tech companies discuss their experiences and perspectives on observability and the role of OpenTelemetry in the industry. Key participants include Iris Durish from Meo, SE Nman from Cisco, Kayla Riel from New Relic, Morgan Mlan from OpenTelemetry, and others from companies like eBay, ServiceNow, and Honeycomb. They explore the evolution of observability, emphasizing its importance beyond traditional monitoring, and how OpenTelemetry serves as a standardized solution for collecting telemetry data across different systems. Key points include the significance of metrics, logs, traces, and profiling in understanding system behavior, the community aspect of OpenTelemetry, and its future potential in making observability accessible to all developers and organizations. The speakers express enthusiasm for the project’s growth and its impact on improving system insights and performance management. ## Chapters -Sure! Here are the key moments from the livestream with the corresponding timestamps: +00:00:00 Welcome and introductions +00:03:00 Observability definitions +00:06:30 Observability evolution +00:09:40 OpenTelemetry significance +00:12:27 Community and collaboration +00:15:10 OpenTelemetry project involvement +00:19:30 Challenges in observability +00:22:41 Signals in observability +00:24:00 Metrics, logs, and traces +00:27:30 Favorite signals discussion -00:00:00 Introductions of participants -00:02:15 What is observability? -00:05:30 The evolution of observability from APM -00:09:00 Observability as peace of mind -00:12:45 Observability versus monitoring -00:15:00 How observability impacts user experience -00:19:00 The role of OpenTelemetry in modern observability -00:23:30 Community and collaboration in OpenTelemetry -00:27:15 Personal journeys into OpenTelemetry -00:31:00 Discussion on favorite telemetry signals (traces, metrics, logs) +**Iris:** Well, I'm Iris Durish. I'm a senior observability engineer at Meo, and my professional life is all about observability. I build an observability platform that provides the tools for engineering teams at Meo to monitor, to observe, and get the best of their obligations. -These moments encapsulate the main ideas and discussions shared during the livestream. +**SE:** My name is SE Nman. I'm working at Cisco at the open source program office, and I'm a member of the open Telemetry governance committee. I'm one of the co-maintainers of the open Telemetry documentation. -# Observability Engineers Panel Discussion Transcript +**Kayla:** My name is Kayla Riel. I work for New Relic, and I'm contributing to the open Telemetry Ruby project. -**Iris Durish**: -I'm Iris Durish, a senior observability engineer at Meo. My professional life revolves around observability. I build an observability platform that provides the tools for engineering teams at Meo to monitor, observe, and get the best out of their applications. +**Morgan:** My name is Morgan Mlan. I'm a director of product management at Swun. I've been with open Telemetry since day one. I'm one of the co-founders of the project. I'm on the governance committee. Wow, what do I work on with open Telemetry? A bit of everything. I mean, early on, it was literally everything. Myself, Ted, and various others were doing many, many jobs. More recently, I've been involved in the release of traces and then metrics 1.0, logs 1.0 last year. Right now, I'm working on profiling, as well as open Telemetry's expansion into mainframe computers. -**SE Nman**: -My name is SE Nman, and I work at Cisco in the open source program office. I'm a member of the OpenTelemetry governance committee and one of the co-maintainers of the OpenTelemetry documentation. +**Henrik:** My name is Henrik Reet. I am a cloud-native advocate at Dan Race, and I'm passionate about observability, performance, and I'm trying to help the community by providing content on getting started on any solutions that help. -**Kayla Riel**: -I'm Kayla Riel, and I work for New Relic. I contribute to the OpenTelemetry Ruby project. +**Vijay:** My name is Vijay Samuel, and I help do architecture for the observability platform at eBay. -**Morgan Mlan**: -I'm Morgan Mlan, a director of product management at SWUN. I've been with OpenTelemetry since day one and am one of the co-founders of the project. I'm on the governance committee. Early on, I worked on a bit of everything, but more recently, I've been involved in the release of traces, metrics 1.0, and logs 1.0. Right now, I’m focusing on profiling and expanding OpenTelemetry into mainframe computers. +**Daniel:** I'm Daniel Gomez Blanco. I'm a principal engineer at Skyscanner and also a member of the open Telemetry community. -**Henrik Reet**: -My name is Henrik Reet, and I am a cloud-native advocate at Dan Race. I'm passionate about observability performance and am trying to help the community by providing content on getting started with observability solutions. +**Doug:** My name is Doug Odard. I'm a Senior Solutions Architect with ServiceNow Cloud Observability, which is also formerly LightStep. I'm also a previous customer of using open Telemetry for several years prior to that. -**Vijay Samuel**: -I’m Vijay Samuel, and I help with architecture for the observability platform at eBay. +**Adnan:** Hey, I am Adnan. I work at TraceTest as a developer advocate, which you can guess better than me what that is. I pretty much do a bunch of everything regarding open Telemetry. I'm one of the contributors for the documentation, for the blog, and the demo. -**Daniel Gomez Blanco**: -I'm Daniel Gomez Blanco, a principal engineer at Skyscanner and also a member of the OpenTelemetry community. +**Ren:** My name is Ren Manuso. I work for Honeycomb IO, and I am one of the maintainers of the end user. -**Doug Odard**: -I'm Doug Odard, a senior solutions architect with ServiceNow Cloud Observability, which is also formerly Lightstep. I was a previous customer using OpenTelemetry for several years prior. +**Observability Discussion:** -**Adnan**: -Hey, I’m Adnan, and I work at TraceTest as a developer advocate. I handle various tasks related to OpenTelemetry and contribute to the documentation, blog, and demos. +[00:03:00] **Ren:** What does observability mean to me? Observability to me is the biggest passion of my life and also my professional career. It is one of those areas that you are not very interested in when you start your career because you don't know anything about it. It's not taught in school. It's not preached by the tech communities a lot, but then you discover it, and you say, "Wow, this is amazing! We're actually making a change and helping the teams make the best of their products." -**Ren Manuso**: -My name is Ren Manuso, and I work for Honeycomb.io. I am one of the maintainers of the end-user documentation. +**SE:** I think observability is a big game changer, right? It's an evolution from what we have done, especially APM, over the last few years. I worked for a very long time at EP Dynamics, and we sold APM agents to customers. We gave them a lot of the things that observability is promising today as well. But the big change I see with observability is that it's coming down, let's say, to everybody. This is making the things that we did there available for everybody and even more. We're moving away from this, "Hey, let's add a post-compilation agent into your application," to "Yeah, let's make native observability. Let's make this a thing that developers and operation teams are using across all organizations." ---- +**Morgan:** To me, observability means having peace of mind. It means having something that you can rely on in order to see what happened and what went wrong. I think observability is also a way to feel more technically connected to your customers and your users so that you can see the ways that they're interacting with your software instead of just the ways that you might interact with it. -### What Observability Means +**Kayla:** I mean, observability to me transcends just the computing industry. It's the ability to peer into something and understand how it works, what it's doing right now, and thus, if it breaks, how to fix it more quickly. Certainly, when we think of open Telemetry in this industry, what observability classically means to me is visibility to back-end infrastructure and applications. -**Ren Manuso**: -Observability means a lot to me; it’s a significant passion in my life and professional career. It’s one of those areas you may not initially be interested in when starting your career because it’s not taught in school or heavily discussed in tech communities. However, once you discover it, you realize how amazing it is to help teams get the best out of their products. +**Doug:** Kind of excitingly, I think it's expanding right now, right? With open Telemetry, we're pushing into client applications. We're pushing into mainframes. -**Morgan Mlan**: -I think observability is a game changer. It represents an evolution from what we’ve done, particularly in APM over the years. My background is in selling APM agents, which provided many of the benefits observability promises today. The big change is that observability is becoming accessible to everyone, moving away from the need to add a post-compilation agent to applications towards native observability that developers and operations teams can utilize across all organizations. +**Adnan:** Usually, when people mention observability, they say, "Hey, it's about a replacement of the old name monitoring." But for me, it's more than monitoring. Monitoring is like you just look at something, and observability, for me, is like having enough information to understand a given situation. If you just look at metrics, then you have a guess that something is going wrong, but you don't understand. So having the options to get more information like logs, events, exceptions, traces, profiling, then at the end, combine all those dimensions together, then you say, "Okay, I got it. This is my problem, and I can resolve it." -**SE Nman**: -To me, observability means having peace of mind. It’s about having a reliable system to understand what happened and what went wrong. It’s also a way to feel connected to your customers and users, seeing how they interact with your software. +**Vijay:** What does observability mean to me? I belong to what is called the site engineering organization inside of eBay, and our goal is to make sure that we can observe everything that's going on in the site and ensure that we have high availability. So basically, observability means knowing if the site is running fine or not because that's why I'm there. -**Kayla Riel**: -Observability transcends just the computing industry. It's the ability to peer into something and understand how it works, how it’s performing right now, and how to quickly fix it if it breaks. +[00:06:30] **Henrik:** What does observability mean to me? It's a way for us to understand what's happening within our systems because we run quite a complex system. We need to understand what goes on inside of them so we can deliver a good experience for our end users at the end of the day. -**Henrik Reet**: -In the context of OpenTelemetry, observability means visibility into back-end infrastructure and applications. We’re expanding into client applications and mainframes, providing visibility into any systems that impact your business. +**Adnan:** Oh my God, observability is to me—I've been a full-stack developer for years, and as we observed, I actually ended up on an incident response team doing tracking of incidents but also trying to figure out what was wrong. It pointed out to me how much we need this, how hard it was to look at so many different screens. Observability, for me, is the way to actually see what's happening in your system. It's the pinnacle of not being up the whole night trying to figure out what went wrong. With open Telemetry and with the rise of tracing in the last couple of years, it has hit an all-time high with regards to the possibilities that we have right now. I'm just really, really happy to be part of the project. I'm also really happy that it's growing at the pace it's growing right now, and I can't see how that's going to evolve within the next couple of years. -**Vijay Samuel**: -For me, observability is about understanding everything happening on the site to ensure high availability. It’s knowing if the site is running fine or not. +**Kayla:** For me, observability is about being able to ask deeper questions of our systems, being able to demand more than just alerting on things that are emergencies, things we've seen before, but actually being able to go out into the unknown and understand how complex systems are performing. Open Telemetry is the tool that is making observability great again. -**Doug Odard**: -Observability allows us to understand complex systems so we can deliver a good experience for our end users. +**Morgan:** I would say that observability is seeing the surge now that open Telemetry is becoming so popular. It's allowing centralization of telemetry signals, it's allowing semantic conventions, and it's generally helping observability teams and the engineering teams take more attention to observability and building it and making it better. ---- +**Henrik:** What does open Telemetry mean to me? I think it's the vehicle for observability, right? I mean, it's enabling that. I joined the open Telemetry community a few years back because I was curious about this idea to bring observability to everybody. I think we are doing a really good job. -### The OpenTelemetry Community +**Adnan:** What it also means to me now is that it's an amazing community, right? We're at KubeCon here, and I meet so many people I just know from those digital conversations, and now I can talk to them in person. We talk a lot about open Telemetry, but we also talk a lot about other things than open Telemetry. We talk about observability, of course, about what we think about is going to happen in a few years. -**Adnan**: -OpenTelemetry means a community effort to take the best practices from existing instrumentation and consolidate them so that everyone can benefit. +[00:09:40] **Kayla:** Open Telemetry, to me, seems like it's a community effort to take the best of what's already been out there for instrumentation and collect it in one group so that everyone can benefit from it. I think that we've learned so much as different engineers, but there's also so much to learn from users of products themselves, and open Telemetry does a great job of bringing both people who are experts in observability and experts in languages to make something really great and meaningful for everyone. -**Morgan Mlan**: -OpenTelemetry is the vehicle for observability. I joined the community a few years back out of curiosity. It’s been an amazing experience, with a great community where we not only discuss OpenTelemetry but also share thoughts about the future of observability. +**Morgan:** I mean, open Telemetry is my baby. I put so much effort into creating this project. What does it mean to me? I mean, there's the boring answer, which is it extracts signals, metrics, traces, logs, profiles, everything else from your infrastructure, from your services, from your clients, and makes those observable, processable on the back end. But I think to a lot of us who've been in this community so long, I mean, open Telemetry is also just a really nice open-source community to participate in. It's a thing I just enjoy working on. -**Kayla Riel**: -OpenTelemetry has a unique ability to bring together experts in observability and programming languages to create something meaningful for everyone. +**Henrik:** Open Telemetry, for me, means the future. Because at the end, by having an open standard, we have the luxury, I would say, to have a common standard, a common format for all the solutions of the market. And having that common format for all the industry and all the vendors and all the solutions will just open use cases. -**SE Nman**: -OpenTelemetry is my baby. I’ve put a lot of effort into creating this project, and it means a lot to me. It enables us to extract metrics, traces, logs, and profiles from our services and infrastructure, making them observable. +**Vijay:** I think testing used to rely on feedback from users, and now with observable data, we can be so much more efficient in the way we test. We could be so much more efficient in replacing marketing tools, business analytics tools. I think it's the future. ---- +**Henrik:** One thing that also a lot of people talk about is AI everywhere, machine learning, blah blah blah, but I think it's the same thing as observability. I mean, Tesla, when you drive your car, it takes decisions based on the sensors that you measure. If you don't have those sensors and those measurements, then you can have the smartest systems, but without the data you cannot take the right decisions. -### Future of Observability +**Morgan:** I think it's an enabler also for the future implementations of modern applications. Open Telemetry is the standard for observability going forward, and it's very important because as we have gone through the journey of observability over the past few years, we have had to hunt for open standards in Prometheus and a few others. Now, at least with ingestion and collection, it's a single standard for everyone to adopt, and I think that's pretty powerful for the long run. -**Henrik Reet**: -OpenTelemetry represents the future of observability. By having an open standard, we can ensure a common format across the industry, which opens up new use cases. +[00:12:27] **Doug:** What does open Telemetry mean to me? I think it's bringing people together, bringing everyone together under one single language and one single way of thinking about telemetry. I think human languages are difficult enough for us to understand each other, and I think open Telemetry is bringing the technology together under one single way of thinking about telemetry, thinking about how we observe our systems. -**Doug Odard**: -OpenTelemetry is becoming a standard, allowing for centralization of telemetry signals and helping observability teams focus on building better observability practices. +**Daniel:** To me, open Telemetry is bringing the ability to have product teams, infrastructure teams helping their jobs make it easier, and also just improve the customer experience and just make it an overall better experience to do our jobs. -**Morgan Mlan**: -OpenTelemetry allows us to ask deeper questions about our systems and go beyond just alerting. It’s about understanding complex systems' performance. +**Adnan:** Open Telemetry is, I'm going to say, the future of observability. We've seen so many companies, so many vendors move to an open Telemetry first mindset. The way that you can use open Telemetry to generate, to actually gather all telemetry signals with one set of libraries, with one tool, is just the way it was supposed to be. You're not locked into one vendor or one cloud provider anymore. You can do basically whatever you want, and you can use both the metric logs and traces for basically anything you want to do. -**Vijay Samuel**: -OpenTelemetry is the future because it enables product teams and infrastructure teams to collaborate more effectively, improving the overall customer experience. +**Morgan:** Open Telemetry is an instrumentation protocol that helps us ask more detailed questions about observability because it collects multiple signals from many flexible types of systems. Folks monitor everything from the control plane in Kubernetes all the way up to physical on-prem systems. It's a really flexible language, and it's a beautiful community of humans that came together over the pandemic to build something really special. ---- +**Kayla:** I was working in a very fast-paced observability team, and we were maintaining a lot of tools. We really did not have conventions there. We did not have centralization, and we really were not flexible when it came to being vendor-agnostic in general. So we discovered this amazing tool called open Telemetry. We said, "Okay, let's give it a try." It worked great for us, and here I am today, more than a year later, and pushing the migration to open Telemetry in my second project. -### Favorite Signals +[00:15:10] **Morgan:** How did I get involved in open Telemetry? I mentioned that I got curious a few years ago. I was at EP Dynamics working as a so-called domain architect. I was an expert for Node.js, Python, and a lot of those other languages, and there was always this conversation around, "Hey, there's this thing now called open Telemetry. Should we not integrate this into our product?" I was like, "Okay, I want to learn more." Then I was like, "What is a good way to learn something new about an open-source technology? Get involved in that." -**Morgan Mlan**: -My favorite signal is profiling because it closes a significant gap in observability. +**Kayla:** I got involved in open Telemetry last spring when New Relic asked me to take a look at what the current status was of the open Telemetry Ruby project. I also work as an engineer on the New Relic Ruby agent team, and that gave me an opportunity to start to contribute to the project. -**Kayla Riel**: -I love traces because they tell stories in meaningful ways. However, I also have a soft spot for logs. +**Doug:** I was working at Google on Google's observability products like tracing, profiling, debugging, that whole thing. One of the challenges we had with tracing was getting data from people's applications. It was really, really hard. You needed integrations of hundreds of thousands of pieces of software. No one team, no one company is going to maintain that. It's just infeasible. -**Doug Odard**: -I’m partial to distributed traces since that’s where the project started, but I’m also excited about logs and their potential. +**Morgan:** We wanted to do something open source. There were other open source standards. There was one that had started, I think, roughly around the same time we were doing this called open tracing, and at some points, especially amongst the more socially media savvy members of the team, which I am not one of, there was some contention between those projects about where people who maintain databases and language front-end things should actually spend their integration efforts, and it was limiting the success of both projects. -**Adnan**: -I enjoy all signals, but I’m particularly excited about profiling. It helps us understand performance optimizations. +**Kayla:** I was leading open census. Ted and Van and others were leading open tracing, and in late 2018, early 2019, we finally sort of brought things to a head and decided to merge those into what is now called open Telemetry. I've been involved since then. I now work at a different company but still on the same things. -**Henrik Reet**: -I’m a fan of metrics because of their power in anomaly detection and machine learning applications. +**Adnan:** When I started the adventure in open Telemetry, of course, I joined a company that has their vendor agent called OneAgent. I saw this movement of open Telemetry and coming from the performance background, I looked at it and I said, "Wow, an open standard? That sounds quite exciting because I had a performance gig for a customer where I implemented collecting logs and processing it and putting machine learning." I told myself at that time, "It would be wonderful to have one common standard." ---- +**Vijay:** When I looked at the definition of the project and the things behind the project, I was so excited. I said, "Oh gosh, I want to be involved in a project." That's where I started to build content to help the community get started. I used to be a developer, but I'm a bad developer for sure, so that's why I'm trying to help the project in other ways and other directions. -This concludes the panel discussion on observability and OpenTelemetry. Thank you for joining us! +**Morgan:** My goal is to increase the adoption of the open standards, making sure that it's being adopted everywhere so that we can move forward by trying even more exciting implementations. I started a few years ago for two reasons: one, we were looking to introduce open standards inside the company, and at that time, open tracing and open census was converging into open Telemetry. + +**Doug:** We started evaluating open Telemetry for that, and given that we were moving into open Telemetry for tracing, I also went through the journey of migrating our metrics collection into open Telemetry. That's basically how I got involved. + +[00:19:30] **Kayla:** How did I get involved in open Telemetry? I got involved through working as a Skyscanner end user. I was driving adoption of open standards, open Telemetry. During COVID, there was a need for simplification in how we approach our infrastructure, how we collect, how we process, how we export data, and also basically to lead the adoption of open standards and that simplification. + +**Adnan:** I got involved in the community at Telemetry, decided to interact with all end users and meet people who want to solve the same problems. They want to find a solution that works. + +**Morgan:** I actually, for several years in my previous position, was hired to develop observability software. I was writing my own thing. We were doing a lot of alert management and various things. It was so much work, and I thought, "This has got to be easy." Plus, I wanted to make sure that it could be future-proof, but also extensible. + +**Daniel:** When I discovered open Telemetry, I was just like, "Oh, thank you," because it's something that the company could carry forward, and we didn't have to worry about storing the data as much. + +**Kayla:** It really provided an excellent platform so that we can focus on the task at hand versus how to do the job. + +**Adnan:** How I got involved in the project was actually first as a customer. It was about three, close to four years ago, kind of the infancy of open Telemetry. I would go online, look at the documentation, or I would be in the code a lot, but I wanted to learn more. + +**Vijay:** I would go to a call, and there would be someone from Google and Microsoft and other companies, and then there was this guy from this small fintech in the US. At first, it was a little awkward, but they were so excited to have me in the call because I was an end user. It was a wonderful experience to begin that way and realize that I could contribute to this rather than simply be a consumer of it. + +**Morgan:** It was great. Then I transitioned my career into working for a vendor, and we implement these systems now for customers like myself that I was years ago. It's kind of a pay-it-forward, give-back type of thing. + +**Adnan:** How did we get involved in the open project? We started contributing more to the blog with you guys, started contributing a bit to the docs as well. It's just been a wholehearted effort in the team to always dedicate a few minutes of each day to check out the open project and find a way to contribute. + +[00:22:41] **Doug:** I got involved in the open Telemetry project. Honestly, I was working at one observability company in marketing, and they didn't see the point. They didn't want me to get involved. I really believed in open source. I'd worked at Mozilla and Wikipedia and really believed that this was the way forward strategically. + +**Morgan:** So the second I could switch to a company that did let me get involved, that's what I did. Now I'm at Honeycomb, and I'm glad to say within the first three months, I became a project member and started working with the end-user working group and worked to grow it into a SIG and into all the programs that it has today together with others. + +**Kayla:** Tracing is my favorite signal. My favorite signal now is profiling because I think this is really closing a big gap that was missing in observability. I come from the APM space, and now for me, APM and observability, it's very hard to make a difference here. + +[00:24:00] **Henrik:** But one thing that when I talk with people using APM products right now is they're like, "Hey, where's the code-level visibility?" With open Telemetry, my commercial agent is giving me that line of code that is breaking something, and this is what we get with profiling, and that's why I'm really, really excited about it. + +**Vijay:** To decide a favorite signal is kind of difficult for me. I really love the power of traces. I think that traces can tell stories in ways that are very meaningful. But on the other hand, I've been so immersed in logs and trying to allow logs to have more connections to spans and traces, but I definitely have a soft spot for logs as well. + +**Morgan:** I mean, I'm partial to distributed traces because that's where this project got its start. I think early on, that's where a lot of the value was. No one else was really doing standardized distributed trace collection. There were some open-source examples of it attached to like Zipkin and Jaeger, but I think the reason open Telemetry got so much traction so quickly is that it was providing that. + +**Kayla:** I'm also partial to logs, which we launched last year just because that's one where I've been involved in a lot of parts of open Telemetry. + +**Henrik:** But that's one where I was involved in a lot of the core specifics early on and driving that, and so it was really exciting to see that ship. Also, logs are just a thing that throughout my career before working on any of this, I just got frustrated with because they're never standardized, they're slow to process, they're expensive. + +**Morgan:** Open Telemetry is going to bring a lot of changes there for the better for everyone who uses logs. And finally, I guess profiles, because I'm working on that now, and it's new. When I was at Google many years ago, I launched what I think was the world's first distributed continuous profiling product, at least publicly available, which was Google Cloud profiling stack profiling. They still support it. I still think it's free; it's very powerful. + +**Henrik:** Profiling has always been a bit of a niche thing. I know that Splunk and other companies support it, but it's not as well known as metrics and traces. I think with open Telemetry starting to later this year, we're going to launch full support for profiles. That's really going to change things. + +**Adnan:** We had customers at Google who would spend an hour on our profiler and save 20-30% of their aggregate CPU because they found some really poorly optimized code really quickly. For more people to have that ability and speed to speed things up and for developers to actually get insight on how things work, that's super exciting. The tech has been there a long time, and open Telemetry bringing this mainstream is huge. + +**Morgan:** When people ask me who is your favorite kid, usually I say I don't have a favorite kid. You know, all my kids are wonderful. They all have a great thing out of it. I think I love traces because sometimes they help you to understand where it's slowed down. I love metrics because as a performance engineer, I used to use metrics a lot. + +**Kayla:** I love logs because at the end, there's no single answer. If you just do analytics on logs, wow, you are so much more precise. I don't think I'll have a favorite signal. I'll just say that depending on what I need, I can pick and choose. + +**Daniel:** There's clearly one signal that will help me more. There's one thing that I'm very eager and waiting for since Valencia: continuous profiling. I love profiling and I think traces are great, but if there is a problem somewhere, profiling would be so much helpful. + +**Adnan:** So I think, yeah, I don't answer any questions, but I say, yeah, I love all the signals provided by open Telemetry. + +[00:27:30] **Vijay:** I am thoroughly biased towards metrics. I feel metrics are the most powerful signal as long as you are thinking through your instrumentation and making sure that you have the right granularity and cardinality being sent into the platform. You can do powerful, powerful things with regards to anomaly detection, machine learning, and many other things. + +**Kayla:** I have to say traces because they give you the content. Traces give you the backbone correlation for all the other signals. Right? But I do think that the current design of the API design of metrics is so powerful that I'm falling in love again with metrics because of the way that we decouple instrumentation and measurement from the aggregation of those metrics. + +**Daniel:** It's powerful and so much richness to basically give us a way to describe our system that I'm calling back again in love with metrics. + +**Morgan:** My favorite signal, I have to say, I'm partial to traces because I've been doing software development for so long. That was the first thing that really turned me on to it was the ability to see that, especially because I know what it's like to debug. + +**Doug:** But I also know what it's like in an incident to have to focus in very quickly. So yes, traces are my favorite, but I do also like to send that Trace ID and Span ID into the logs. Now it's kind of becoming my next favorite. + +**Kayla:** My favorite signal is traces. I'm going to say traces definitely. + +**Adnan:** My favorite signal is Ed Sheeran. + +**Doug:** What is my favorite signal? I mean, I work for Honeycomb, so I am constitutionally obliged to say traces are my favorite signal. ## Raw YouTube Transcript diff --git a/video-transcripts/transcripts/2024-06-17T19:00:22Z-otel-q-a-feat-daniel-dias-and-oscar-reyes-of-tracetest.md b/video-transcripts/transcripts/2024-06-17T19:00:22Z-otel-q-a-feat-daniel-dias-and-oscar-reyes-of-tracetest.md index ae9dc74..8caa55c 100644 --- a/video-transcripts/transcripts/2024-06-17T19:00:22Z-otel-q-a-feat-daniel-dias-and-oscar-reyes-of-tracetest.md +++ b/video-transcripts/transcripts/2024-06-17T19:00:22Z-otel-q-a-feat-daniel-dias-and-oscar-reyes-of-tracetest.md @@ -10,155 +10,142 @@ URL: https://www.youtube.com/watch?v=KR8j11FECgc ## Summary -In this OTEL Q&A session, hosted by Adriana, Daniel Dias and Oscar Reyes from Tracetest discussed the integration of trace-based testing into the OpenTelemetry (OTEL) community demo. Daniel, a Brazilian software developer and CNCF ambassador, and Oscar, a lead software engineer from Mexico, explained the concept of trace-based testing, which utilizes emitted traces to assert system states and validate interactions between microservices. They outlined Tracetest's implementation process, which involves triggering a system, generating traces, and then testing those traces using assertions. The duo shared their experience of transitioning from traditional integration tests to trace-based tests in the demo, highlighting the advantages of traceability in identifying issues and improving developer experience. They also noted the positive reception from the OTEL community and the increased reliance on trace tests in CI/CD pipelines. The session concluded with a discussion on the challenges faced during integration and the ongoing evolution of trace-based testing practices within the community. +In this OTEL Q&A session, hosts Daniel Dias and Oscar Reyes from Tracetest discuss the integration of trace-based testing into the OpenTelemetry (OTEL) community demo, which simulates a telescope shop with several interacting microservices. Daniel, a software developer and CNCF ambassador from Brazil, and Oscar, a lead software engineer from Mexico, explain trace-based testing, which uses traces emitted by systems to make assertions about their state, particularly in complex, distributed services. They elaborate on how Tracetest implements this testing through a two-step process of triggering the system and pulling traces for validation. They also recount the challenges faced during integration, such as ensuring context propagation across service boundaries and adapting existing integration tests into trace-based tests. The integration has led to improved community practices, such as incorporating trace tests into CI/CD pipelines, enhancing testing efficiency and reliability. Overall, the discussion highlights the benefits of combining trace-based testing with OpenTelemetry to improve software quality in distributed systems. ## Chapters -Here are 10 key moments from the livestream, along with their timestamps: +00:00:00 Welcome and intro +00:01:10 Guest introductions +00:02:20 Trace based testing explanation +00:04:00 Tracetest implementation overview +00:06:18 Automation and YAML usage +00:07:39 OTEL community demo introduction +00:09:04 Tracetest integration story +00:12:00 Traceability tests discussion +00:16:30 Challenges during integration +00:20:30 Benefits of trace based testing -00:00:00 Introductions and welcome to the livestream -00:02:30 Introduction of Daniel Dias and Oscar Reyes -00:04:00 Explanation of trace-based testing -00:06:15 Importance of trace-based testing for distributed services -00:08:00 Overview of how Tracetest implements trace-based testing -00:10:45 Discussion on creating trace-based tests using YAML -00:12:30 Introduction to the OTEL community demo -00:15:00 Explanation of the architecture of the OTEL demo -00:18:30 Integration of Tracetest into the OTEL demo -00:24:15 Benefits and challenges of integrating trace-based tests in the demo -00:30:00 Audience Q&A about context propagation and baggage in traces +**Host:** Welcome everyone who has been able to join us today. This is super exciting to have. We have both Daniel Dias and Oscar Reyes from Tracetest joining us today for OTEL Q&A, and we haven't had an OTEL Q&A for a while. So I'm so glad that both of you were able to join us today. For those of you who are here today, thank you for joining, and also tell your friends that we will be posting a recording of this on the OTEL YouTube channel. So that's OTEL official, for anyone who couldn't make it. And we'll also provide the links on socials once the recording is made available. -Feel free to reach out if you need further details or additional information! +So with that, let's get started. Why don't I get you to introduce yourselves? -# OTEL Q&A with Daniel Dias and Oscar Reyes +[00:01:10] **Daniel:** I can go first. So hello everyone. I'm Daniel Dias, a Brazilian living and complaining in Argentina right now because of the weather. I am a software developer at Tracetest, and while I'm not helping the team evolve with new features, I'm bragging about the features somewhere. That's all. And Daniel, he's also an ambassador, so he's really proud to have one ambassador in CNCF ambassador too. CNCF ambassador, right? Yes. Newly minted, right? Just, yes. I became my ambassador last week, last month. Yay. -Welcome, everyone, who has joined us today. This is super exciting! We have both Daniel Dias and Oscar Reyes from Tracetest joining us for an OTEL Q&A. We haven't had an OTEL Q&A for a while, so I’m really glad that both of you could make it today. +**Oscar:** For my part, I'm Oscar Reyes. I'm Mexican. I live in Mexico. And I'm a lead software engineer here at Tracetest. And yeah, I'm really happy to be here and talk about Tracetest and OpenTelemetry. -For those of you who are here, thank you for joining! Also, feel free to tell your friends that we will be posting a recording of this on the OTEL YouTube channel (OTEL Official) for anyone who couldn't make it. We will also provide the links on social media once the recording is available. +**Host:** Awesome. Okay. First things first, for folks on the call who might not be familiar, can you tell us what trace-based testing is? -With that, let's get started! Why don't I get you to introduce yourselves? +[00:02:20] **Daniel:** When we call, say about trace-based tests, the main idea is that we want to, we use traces emitted by systems to assert how the screen state of the trace, the state is. So we call some component on the system, wait for a while until the trace is generated, and we start to make assertions to this trace to guarantee that everything is right. -## Introductions +**Oscar:** Yeah, and this is really helpful for distributed services where you have a lot of moving pieces, and if the entire system is instrumented, you can get a lot of, you can have a lot of assertions around the generated OpenTelemetry data or the traces, right? You can validate that certain spans exist, but you can also go deeper and validate things like the attributes or the execution time of the spans, those sort of things. So it's a multi-layer level of testing and the entire spans or traces that your system generates. -**Daniel Dias:** -Hello, everyone! I'm Daniel Dias, a Brazilian living and complaining in Argentina right now because of the weather. I am a software developer at Tracetest, and while I'm not helping the team evolve with new features, I'm bragging about the features somewhere. I'm also proud to be a newly minted CNCF ambassador! +**Daniel:** That's awesome. And the thing I love about trace-based testing is we're already emitting traces in our code. So why not take advantage of that? So I feel like it's a two-for-one deal. Which is very awesome. -**Oscar Reyes:** -For my part, I'm Oscar Reyes. I'm Mexican, I live in Mexico, and I'm a lead software engineer at Tracetest. I'm really happy to be here and talk about Tracetest and OpenTelemetry. +**Host:** The other thing that I wanted to ask, so how does Tracetest implement trace-based testing? -## What is Trace-Based Testing? +[00:04:00] **Oscar:** Maybe I can answer this one and you can complement what I say, Daniel, if you want to. What trace-based, the way that trace-based does it is that we have a two-step process. The first one is triggering your system, the system on their test. Currently, we support things like HTTP, gRPC requests. We also support Kafka, that was actually integrated by Daniel. And then we have other types of integrations for other types of systems like Cypress or Playwright. So that's the first half of what Tracetest does, triggers their system on the test. -For folks on the call who might not be familiar, can you explain what trace-based testing is? +And then the system address will need to generate or will generate those instrument, the traces and the spans and put in somewhere, in a tracing backend. Tracing backend, it could be something like Jaeger or Tempo or any other providers that you will store the traces. And then the second half of what Tracetest will do is pull for those traces, based on the configuration that you have. So we will wait, depending on the way that system works, that system that generates all of the entire trace in one, a couple of seconds. You have other ones that generate that have long-lasting processes that could take five minutes. So we support all those variables. -**Daniel:** -The main idea behind trace-based tests is that we use traces emitted by systems to assert the state of the trace. We call some component in the system, wait for a while until the trace is generated, and then start making assertions to guarantee everything is correct. This is particularly helpful for distributed services with many moving pieces. If the entire system is instrumented, you can perform various assertions around the generated OpenTelemetry data or the traces. +And so we wait for the trace to be ready. We pull that into the Tracetest system. Based on the trace ID that we generate on the first step, that will work as a parent ID, and then the assertions on the spans, the sessions, and the test space will be triggered against that generated traces or telemetry data. So I guess, there will be a three-step process: trigger, trace, or getting the trace, and then test. That's how we're calling our three-step process. -You can validate that certain spans exist and also go deeper to validate attributes or execution time of the spans. It's a multi-layer level of testing for all the spans or traces generated by the system. +**Daniel:** Cool. Do you have anything to add on to that or you're good? -**Oscar:** -What I love about trace-based testing is that we’re already emitting traces in our code, so why not take advantage of that? It feels like a two-for-one deal! +**Oscar:** All right, cool. And if I remember correctly, because it's been a while since I played with Tracetest, but you can create your trace-based tests using YAML, right? Everyone's favorite thing that they love to hate. -## How Does Tracetest Implement Trace-Based Testing? +**Daniel:** Yeah, I think one of the things that we have pushing in the past few months is not only allowing things to have these three steps, but also how they can automate and simplify the way to introduce trace-based testing in their systems. Either if you use it through a system of Git actions or CI/CD processes, sorry, or you have them in a, I don't know, in a cron job. -**Oscar:** -So, how does Tracetest implement trace-based testing? The way that trace-based testing works is through a two-step process. +[00:06:18] With the automate way that we provide, you can use our CLI, you can use the different tools that we provide, like our TypeScript library, so you can pretty much graph the specifications and the YAML files, as you said, even JSON files from the system. If you go to the UI or either use the CLI to export them, you can put it in your report and execute everything from there. So it's like you have multiple ways of interacting with the system and with the test definitions, either through the UI, through the CLI, or the libraries that we also support. -1. **Trigger the System:** The first step is triggering the system under test. Currently, we support things like HTTP and gRPC requests, as well as Kafka, which was integrated by Daniel. We also have integrations for systems like Cypress and Playwright. +**Host:** Cool. Awesome. Before we go on, I do want to mention to folks who are watching, if anyone has any questions, do feel free to pop them into the chat and we will also have a Q&A portion at the end. I just wanted to mention that quickly. -2. **Pull the Traces:** After the system generates the instrumented traces and spans, they are stored in a tracing backend, like Jaeger or Tempo. The second part of what Tracetest does is pull those traces based on your configuration. We wait for the trace to be ready, depending on how the system works—some generate all the traces in seconds, while others might take longer. We pull the data into the Tracetest system based on the generated trace ID, which acts as the parent ID for our assertions on the spans and telemetry data. +[00:07:39] Okay. I guess the main reason why we called Daniel and Oscar to do Q&A is that they did something super cool last fall, which was they integrated trace-based testing, a la Tracetest, into the OTEL community demo. So first of all, can one of you explain what the OTEL community demo is for folks who aren't familiar with it? -So, in summary, we have a three-step process: trigger the system, get the trace, and then conduct the tests. +**Oscar:** I can answer that. The main idea of the OpenTelemetry demo is that it emulates a telescope shop, but with one major thing. Under this shop, we have a bunch of microservices that interact with each other, each one with one responsibility on the system. So we have one API to make payments, another one to do shipping and everything else. And the main idea of this demo is just to show, with a different technology, a different language, how you can integrate OpenTelemetry to your system and how you can see this data integrated across a bunch of mobile services. -**Daniel:** -Do you have anything to add to that? +**Daniel:** Awesome. Yeah. And it's, it's a great way for getting started with OTEL, especially if you're like, Oh, this is so overwhelming seeing it in a relatively complex example, just kind of goes to show the power of OTEL, which I always think is so cool. And it's not terribly awful to get started with the OTEL demo. I definitely recommend to folks who haven't tried it out to give it a go. -**Oscar:** -No, I think that covers it! +Okay. So now that we understand what the OTEL demo is, let's talk about how you got Tracetest integrated into the demo. -## Creating Trace-Based Tests +**Oscar:** Maybe I can start with the first half of the story and then Daniel can continue after that. So almost a year and a half ago, we were looking for different partnerships or different ways to contribute with the community and just push OpenTelemetry further. We discovered the OpenTelemetry demo. And I was on a task on let's try to just contribute and see how we can help and grow the community. -**Oscar:** -If I remember correctly, you can create your trace-based tests using YAML, right? +And the first thing that I was, that I worked on, it was to switch or migrate the existing front-end application that was built entirely in Go, Golang with a front, with a server-side rendered front end. Just templates, HTML templates, things like that to an actual front-end application. So we, I switched that to, from Go to Next.js. So Next.js on the, having React involved and also introducing the actual front-end instrumentation of the application by using the tools and the web, the browser site, OpenTelemetry libraries. So in that case, it is continuously connected across, starting from the browser all the way to the backend services. And that was pretty much my contribution, my contribution on the OpenTelemetry demo. -**Daniel:** -Yes! One of the things we have been pushing in the past few months is not only allowing users to have these three steps but also automating and simplifying how to introduce trace-based testing into their systems. You can use our CLI and different tools, like our TypeScript library, to graph the specifications in YAML or even JSON files. You can export them from the UI or CLI and execute everything from there, which gives you multiple ways to interact with the system and the test definitions. +**Daniel:** And then, after Daniel joined and did other stuff. So you want to go next? Continue, Daniel. -## Integration with the OTEL Community Demo +**Oscar:** After a while, we started to see the OpenTelemetry demo because it is great. Every time that you need to see, oh, I will start programming in Ruby and I need an API. If you go to OpenTelemetry demo, you will see an API there and you can see, oh, this is how you integrate things there or Python or everything else. But looking to the tests that we have in OpenTelemetry demo at the time, they had integration tests, but just on the API level. -**Oscar:** -One of the reasons we called Daniel and Oscar for this Q&A is that they did something super cool last fall: they integrated trace-based testing into the OTEL community demo. Can one of you explain what the OTEL community demo is for folks who aren't familiar? +[00:12:00] So we could test some features of each API, but we could not see how they act together. And sometimes this is painful, mainly when you have background processes. For instance, if you have an API that emits messages to a Kafka queue, and we have another API that reads data from this Kafka queue, how do you guarantee that this second service got the message correctly and is doing something correct? We did not have this at that time. Looking to this test site, I thought, what if we do a trace-based test? So we started discussing a little bit. One of the members of the OpenTelemetry team, also demo team, also asked us, because they say, oh guys, we developed some new things on the OpenTelemetry demo, but we did not notice that we changed the slide a little bit, some traces, and something is wrong. It's weird to understand, we need to manually inspect everything. We started to do traceability tests on that. -**Daniel:** -The main idea of the OpenTelemetry demo is that it emulates a telescope shop with a bunch of microservices that interact with each other, each serving one responsibility in the system. We have one API for payments, another for shipping, and so on. The demo showcases how you can integrate OpenTelemetry into your system and see the data integrated across various microservices. +**Oscar:** We started looking at each one of the integration tests, and we started writing a counterpart traceability test, but for a small difference. Instead of just seeing the API response, we also are seeing what are the APIs that we are calling behind the scenes, and we are testing these APIs too. And so my thing as well was that for us, part of that section, I don't know if it was a month or a couple of weeks, there was a specific service that was, the telemetry was failing or was broken, right? And the team didn't find out until that time passed and someone, I don't know, saw that problem. And then based on that also pushed the idea on using Tracetest to catch those problems as well, right? -**Oscar:** -It's a great way to get started with OTEL, especially if you find it overwhelming. It's not too difficult to try out the OTEL demo, and I recommend it for those who haven't yet. +And recently, we had another situation where the Kafka link, the Kafka SDK for Go changed it, changed the ownership from Shopify to IBM. And some things changed on this API. They use Tracetest to change the telemetry, update this library, and assert that everything was fine. This was another thing that was good too. -## Integrating Tracetest into the OTEL Demo +**Host:** Cool, that's awesome. Just to make sure I understand correctly, when you started writing the trace-based tests, was it, your initial process was basically converting the initial integration tests into the trace-based tests? So you already had, you didn't have to worry about coming up with the tests. It was essentially a translation and then you did not just checking for the response, but additional stuff basically. -**Oscar:** -Now that we understand what the OTEL demo is, let's talk about how you integrated Tracetest into it. +**Oscar:** Exactly. One difference on this approach is that first I started with the language that we all love that is YAML. So I did a copy and paste on the test. But when you see traces, you need to think a little bit of what types of traces this API calls does. What I did was first write a MO test, run a little bit, and then I swapped it to Tracetest UI because there we can see how is the shape of the traces that we are generating? And I could play a little bit, see, oh, I saw that the checkout service emits this, the trace that I can assert. The shipping service emits another trace, another span, sorry. And I could tweak a little bit and start building more complex tasks with that same. -**Daniel:** -About a year and a half ago, we were looking for partnerships and ways to contribute to the community and push OpenTelemetry further. We discovered the OTEL demo, and I started working on migrating the existing front-end application built in Go to Next.js, which uses React. I also introduced front-end instrumentation using browser-side OpenTelemetry libraries, which connected the browser all the way to the backend services. +Oh, I want to test the service A and see if the service B, D, and E are working as well. And I think that's a really interesting part of telemetry in general, right? And producing traces and having all of the services instrumented, because for someone that is not familiar with the system, they can, it's easier to look at a trace and understand the process and the steps that are taken for a specific use case, instead of going line by line, looking at the code and trying to understand what is going on. -**Oscar:** -After Daniel joined, we started looking at the testing side of the OpenTelemetry demo. Initially, the demo had integration tests, but they were only at the API level. We realized we needed to test how the services worked together, especially for background processes. So we discussed the idea of implementing trace-based tests to enhance the existing integration tests. +**Host:** And how long did it take to create all the trace-based tests, the initial set for the OTEL demo? Because I think it was, the announcement was made, I want to say, last KubeCon North America in Chicago, last fall kind of thing? Is that correct? Correct me if I'm wrong, please. -**Daniel:** -We saw that the integration tests were useful, but we needed a way to ensure that one service's output was correctly processed by another service. By creating trace-based tests, we could validate both the API responses and the background processes. +**Daniel:** Yes, it was fast. I believe that also to that we discussed it with our team about what was happening and checking, inspecting the integration codes. We were able to, in two or three days, to build all the tests for each one of the services and start discussing with the team, opening a PR and start discussing with the team, changes and other things that we could do. -## Challenges Encountered +[00:16:30] **Host:** Awesome. That's so cool. And, so once you implemented, in implementing the trace-based testing into the OTEL demo, what were some of the big challenges, the gotchas, the unexpected things that you encountered as part of this? -**Oscar:** -What were some of the big challenges or unexpected things you encountered during this integration? +**Oscar:** The first thing that we noticed that we thought is how we could do this test without impacting the entire ecosystem. And the first decision that we took is, we know that OpenTelemetry demo uses Jaeger. So instead of, we will see the system outside of it. We will just check Jaeger and do every other operation on Jaeger. With that, it was pretty easy to do everything. The only thing that besides that is just to understand the services, understand how they are working to make our tests, it's generated some interesting things like, one service that I noticed that, first I thought that, the JSON that we sent for the service was in camel case and later we discovered during the execution of this case that this YAML, this JSON was in a snake case. So we started to tweak a little bit and start seeing how the things are and do proper tests for it. -**Daniel:** -One of the first challenges was figuring out how to run these tests without impacting the entire ecosystem. We decided to use Jaeger since the demo already utilized it. This made it easier to test without affecting the overall system performance. +**Host:** When you were writing the trace-based tests, you said you were looking, you're looking at your, you were using Jaeger, as part of your basis. Because I know, Tracetest, I guess there's a couple of approaches to pulling the traces, right? One is either you pull it like in the collector. So you add a config to the collector, or you can pull it directly from any of the supported observability backends. So what was the approach? So I guess my question is, was the approach that you took then was pulling it directly from Jaeger, like the traces? -Another challenge was understanding the services and how they worked. For example, we discovered that the JSON sent to one service was in snake_case, not camelCase, which required some tweaking. +**Oscar:** Yes, and to that, one thing that we noticed is that, since the example sent all the telemetry data to Jaeger, Prometheus, and everything else, we were secure to use Jaeger as it is. Because we know that sometimes if you have a huge system, you might use sampling, for instance, to just send a small portion of the tests of the traces for a tracing back. It was not the case. So this was important because we know that, oh, if you have sampling or you have some specific configuration on the collector, maybe you want to do a specific configuration on OTEL collector to send some of the just traces for Tracetest to do the things, but it was easier. The job was pretty easy that I thought. Since we had everything we ever needed. -## Benefits of Trace-Based Testing +**Host:** Yeah, that's an important distinction, too. Yeah, because I hadn't even thought of the fact that if you're pulling directly from the backend, you're basically pulling the traces that were emitted there versus if you're using the collector, if I recall correctly, you create a different, you create basically a second tracing pipeline, which then intercepts the traces, and then you use those to create your trace-based tests, and then you can put whatever restrictions you want on that for your trace-based test, which I guess gives you a little more freedom as well, like to play around and configure things. -**Oscar:** -What benefits have you seen since integrating trace-based testing? +[00:20:30] **Oscar:** Exactly. Cool. That's awesome. And then on a similar vein, what are some of the benefits that you started seeing after the integration was like with trace-based testing? Did they just start receiving high praise from the maintainers in the community demo where they're like, Oh my God, this has changed my life? -**Daniel:** -One of the significant changes was when the OpenTelemetry demo maintainers started integrating trace tests into their CI/CD pipeline. Previously, they had to run manual tests, and sometimes they would forget to test specific services. With trace tests in place, they could validate PRs and catch issues earlier, ensuring that everything was working correctly. +**Daniel:** One of the game changer things that I think it was, when I, when we saw the OpenTelemetry demo, maintainers starting to integrate trace tests on their CI/CD pipeline, because this, this was, this is something difficult. And I remember at the time that discussing with them, every time that they needed to change something, they did a bunch of final testing. And since we are humans. Sometimes we might forget testing some service or thinking about a user case. But, what they did was, oh, we will create a pipeline. We will build the entire system to see if everything is right. And after building the system, we will run trace tests and check if everything was right. -We also received a shoutout from Josh Lee, which was great! His support for trace-based testing and its importance to the OpenTelemetry community helped raise awareness. +And by doing that, they started to validate some PRs and discover, for instance, that, oh, this PR updated a component that is breaking some traces. And with that, they could start to evaluate and say, oh, okay, let us fix this PR and guarantee that everything is right. And this was painful before because sometimes you could approve a PR, merge it on the database, on the code base, and just two weeks later discover that something was missing. -## Future Changes and Improvements +**Oscar:** Also want to call out the shout out that we got from Josh Lee. So I remember that when I first joined Tracetest, I saw one of his talks about trace-based tests, actually, something similar to quite what we're doing. And just having someone like him sharing out Tracetest or what it is and what it means for the OpenTelemetry community. It's just great. So I think that's part of what we, the work that we. -**Oscar:** -Now that trace-based testing has been integrated into the demo, have there been any significant changes in how you approach it? +**Daniel:** That's so great. That's very cool to hear. And now that the trace-based testing has been integrated in the demo for several months, at this point, what changes have you seen since that initial integration, even to how you approach trade? Have there been any massive changes as to how you approach trace-based testing? Or has it mostly stayed the same? Have there been tweaks that you've had to make along the way? -**Daniel:** -One significant change was that the maintainers began building their own traceability tests. This was a big step, as they realized that with trace tests, they could validate their systems more effectively. They even decided to simplify their testing approach by reducing reliance on other testing libraries, focusing instead on trace-based testing. +**Oscar:** The one thing that was great is that they started to build some traceability tests, as they, I remember that I talked with Juliano, that is one of the maintainers, and when this Kafka thing happened, he told me, Daniel, we noticed that the Kafka SDK changed and we wanted to create a basic, to do some changes on the system and create a basic traceability test in it. And it was amazing. -**Oscar:** -If you had a redo, would you change anything about how you integrated trace-based testing with the OTEL demo? +**Daniel:** And another thing that we loved was that before we had three types of tests running on OpenTelemetry demo, a front-end test, using, I forgot the name of the library. -**Daniel:** -At first glance, I wouldn’t change anything because it felt like a perfect match! However, I would like to add more tasks and think about more interesting use cases to showcase. +**Oscar:** Cypress. -## Audience Questions +**Daniel:** Cypress, they had an API call, API test with AVA, and they had trace test, and they noted, oh, we don't need AVA and Cypress anymore. Since we are focusing on seeing traces and guaranteeing that the telemetry is right and the system is working. Three steps is enough for that. And it was amazing and a huge responsibility for us because now we know that they rely on us. We need to have a good API to everything. -**Daniel (from the audience):** -One of the biggest pains for distributed systems is ensuring correct context propagation across service boundaries. Can trace tests help identify where context is being broken? +**Host:** That's so cool. That's such a success. And I love the fact, the great thing about the OTEL community demo too, is that it showcases OpenTelemetry all with open source tooling, including Tracetest, which I think is so cool and really speaks to the power of open source and the open source community. And you end up with this like really nice symbiotic relationship too, right? Because then, I assume you would get some more use cases for improving Tracetest as a whole based on how you see trace-based testing acting in the wild through the community demo. -**Oscar:** -Yes! We have all felt that pain. Trace-Based Driven Development (TDD) allows users to create assertions based on expected traces. If a span doesn’t exist as expected, it indicates that context propagation may have failed. This technique can help identify those breaks. +**Daniel:** During the time that we started to write tests for OpenTelemetry demo, we detected that some of the tests were huge because we needed to embed a protobuffer file inside of each test. And when we started to see that, we noticed that, oh, the developer experience is bad for that because you cannot see what is happening and this moved us to change the CLI and think, let us simplify the test to guarantee that you can see the test and see what matters. And it was good. It was a two-way road. We were able to help them, the OpenTelemetry team, but the feedback that they gave us helped us to improve Tracetest too. -**Daniel:** -Another question we received was whether Tracetest can make assertions on baggage. +**Oscar:** That's awesome. And as you mentioned, you've got a few folks now writing their own trace-based tests. Have you felt then that, like just getting into that mindset of people writing, like other developers writing trace-based tests, has that, have you noticed, was that a major shift or was it organic once they saw the initial examples that were, that you both added to the repo? -**Oscar:** -Currently, we can test custom attributes in spans. We are continuously looking at the OpenTelemetry specification to integrate more metadata, like error codes and span statuses. +**Daniel:** Believe that, that both things happened. For some developers that were used to OpenTelemetry, they noted that they could use this telemetry to test it, and it was. Sometimes it's way easier to test. For instance, I cannot think in an easy way to test a Kafka consumer without doing a bunch of code magic behind the scenes, and traces are good for that. -## Conclusion +So we had some examples of people using serverless knowing that it is difficult to communicate with several components and using Tracetest to help them. And we saw another developer saying, oh, we are starting to implement OpenTelemetry. We noticed that, with OpenTelemetry, we can test our system quickly. Tracetest helped them to drive and implement more things in OpenTelemetry and start doing things there. I believe that these two cases happened. -Thank you both, Daniel and Oscar, for joining today! It’s been great to discuss trace-based testing and its integration with the OTEL community demo. I'm a big fan of the demo and see how this integration adds a powerful and tracing-native approach to integration testing. Thank you, everyone, for participating! +**Oscar:** Oh, that's so great. Final question. If you had a redo, would you change anything about how you integrated trace-based testing with the OTEL demo? + +**Daniel:** First glance, I believe that nothing because I believe it was a perfect match. So we could do everything that we needed there. The only thing, the only thing that I think that I could do, and I think that I might do, if you are watching me, Pierre, Juliano, and company, I will haunt you in the future for that, is to add more tasks and start thinking in more interesting use cases that we can do and thinking more telemetry that we can show that. + +**Host:** Amazing. This is great. We do have a question from the audience. Daniel asked, one of the biggest pains for distributed systems is ensuring correct context propagation across service boundaries, especially when async operations require in-service context propagation across threads and context may be instrumented properly. Can trace tests help identify where context is being broken when it shouldn't? + +**Oscar:** I think I can answer that question. First of all, I think we have all been there. We have all felt the pain of why is my trace not being propagated to this backend system or this part of the app. And yes, actually, a great thing that, or a good thing that we are, one, one kind of standard or thing that we're pushing is TDD, but in this case, Trace-Based Driven Development. + +Where users can create assertions and test specs based on what they would expect the trace to look like. So if you already know that after a queuing system, there should be a worker that would process that message, you can pretty much, from the beginning, from the get-go, create a definition that would match. I would expect that span to exist. So if that span doesn't exist, it's because the context propagation is pretty much not, didn't work or the span, that's why the span doesn't exist, right? It's not there. So with this kind of technique that you can use, you can validate that the expected spans should be there and help you with the problem of your context propagation problem as well. + +**Host:** Awesome. So you've answered Dan's question. Thank you. And then there was a follow-up question, can Tracetests make assertions on baggage? + +**Oscar:** It depends. If you are saying baggage, so for some specific metadata of OpenTelemetry, perhaps not. But what we can do today is that if you write custom attributes in your span, you define a bunch of attributes, you can test them. You can test them. Also, we are every time looking to the OpenTelemetry specification, seeing if there is more metadata that we should integrate to the API and doing it like error codes, span statuses, and everything else. I think we have something about span links, right? I want to do as well. I remember. Exactly. + +**Host:** Nice. That's awesome. Does anyone else, who's listening have any additional questions? This has been really great. Thank you both, Daniel and Oscar for joining today. This has been really awesome. I think it's really cool. I'm a big fan of the OTEL demo and I think having trace-based testing integrated really, it's like a very tracing native approach to integration tests. I'm a huge fan, so it's super cool to see that integration in place and to see other folks, outside of both of you actually writing trace-based tests. Thank you. + +**Daniel:** Thank you, everyone. And thank you, Adriana, for putting this together and for having us. It was great. ## Raw YouTube Transcript diff --git a/video-transcripts/transcripts/2024-08-01T05:59:23Z-otel-in-practice-otel-for-mobile-apps-with-hanson-ho-and-eliab-sisay.md b/video-transcripts/transcripts/2024-08-01T05:59:23Z-otel-in-practice-otel-for-mobile-apps-with-hanson-ho-and-eliab-sisay.md index fd2c526..296cf76 100644 --- a/video-transcripts/transcripts/2024-08-01T05:59:23Z-otel-in-practice-otel-for-mobile-apps-with-hanson-ho-and-eliab-sisay.md +++ b/video-transcripts/transcripts/2024-08-01T05:59:23Z-otel-in-practice-otel-for-mobile-apps-with-hanson-ho-and-eliab-sisay.md @@ -10,115 +10,193 @@ URL: https://www.youtube.com/watch?v=mTcdwdWIFgI ## Summary -In this episode of "Otel and Practice," hosts Hansen and Eliab from Embrace discuss the challenges and opportunities of implementing data modeling and application performance (AP) design in mobile apps using OpenTelemetry (OTel). Eliab, a product manager at Embrace, introduces the topic by highlighting the significance of mobile observability, emphasizing that mobile apps are not distributed systems but rather installed software that interacts with distributed services. Hansen, an Android architect at Embrace, delves into the unique challenges of mobile observability, including the dynamic runtime environment, issues with session tracking, and the fragmentation of mobile devices. He explains how OTel's assumptions may not always hold true in mobile contexts, stressing the need for tailored instrumentation and user experience-focused metrics. The presentation concludes with a call to action for developers interested in mobile observability to engage with OTel working groups and contribute to developing robust standards for mobile applications. +In this edition of "Otel and Practice," hosts Hansen and Eliah from Embrace discuss the challenges and opportunities of implementing data modeling and observability in mobile apps using OpenTelemetry (Otel). Eliah introduces Embrace's focus on mobile observability and the transition to Otel, emphasizing the unique complexities mobile apps face compared to backend systems, such as diverse hardware environments and the need for real-time performance monitoring. Hansen elaborates on these challenges, including the limitations of current telemetry systems, the importance of user experience, and the adaptation of Otel's protocols to better fit mobile applications. They also encourage audience involvement in the mobile observability community to help shape standards and best practices. The session concludes with a Q&A, addressing various topics such as instrumentation for different types of apps, golden signals for performance, and the readiness of Otel for production use in mobile environments. ## Chapters -Here are the key moments from the livestream along with their timestamps: +00:00:00 Welcome and intro +00:01:20 Introduction of Eliah +00:02:53 Overview of mobile observability +00:05:00 Challenges in mobile observability +00:06:30 Call to action for involvement +00:07:00 Hansen's presentation on mobile differences +00:09:56 Unique challenges of mobile environments +00:12:30 OpenTelemetry challenges on mobile +00:19:00 Ongoing challenges and future considerations +00:22:00 Q&A session begins +00:44:00 Closing thoughts and wrap-up -00:00:00 Introductions and Overview of the Session -00:01:30 Introduction to Eliah and Hansen from Embrace -00:03:40 Importance of Mobile Observability in App Performance -00:05:30 Challenges in Implementing OpenTelemetry in Mobile Apps -00:10:00 Unique Characteristics of Mobile Runtime Environments -00:15:00 Fragmentation and Device Variability in Mobile Ecosystem -00:20:00 The Role of OpenTelemetry in Mobile Observability -00:25:00 Key Differences Between Mobile and Backend Observability -00:30:00 Discussion on User Experience and Golden Signals in Mobile Apps -00:35:00 Q&A Session on Mobile Instrumentation Challenges +**Speaker 1:** Well hello everyone, welcome to another edition of Otel and Practice. Today we've got Hansen and Eliah from Embrace. They're going to take us through some of the challenges and some of the opportunities as well of doing a data modeling API design in mobile apps. In OpenTelemetry, we talk a lot about the backend, but we'll see how we can apply OpenTelemetry to the client side as well. -These timestamps capture significant moments in the presentation and discussion, providing a clear overview for viewers. +We will do a presentation first, and then there will be time for Q&A. I'll drop a link in the chat, and then I've created an Agile Coffee board. You can go there, you can add your questions, and then at the end of the talk, we'll go through the Q&A. You can vote for your questions as well, so feel free to add your questions as you see in the presentation, and then we'll go through them at the end. -# OpenTelemetry in Mobile Apps: Challenges and Opportunities +So without further ado, I'll introduce Hansen. I think you're going to be kicking off, so take it away. -**Welcome everyone to another edition of OpenTelemetry in Practice!** Today, we have Hansen and Eliah from Embrace with us. They will discuss the challenges and opportunities associated with data modeling and API design in mobile apps. While OpenTelemetry is often focused on backend systems, we will explore its application on the client side as well. +**Hansen:** Great, I'm actually going to pass it off to Eliah to kick it off. -We'll start with a presentation, followed by a Q&A session. There’s a link in the chat where you can drop your questions, and I've set up an Agile Coffee board for you to add your queries. Feel free to vote for the questions you’d like us to address at the end. +[00:01:20] **Eliah:** Nice. Hi everyone, yeah like it was mentioned, my name is Eliah Cisi. I am a product manager at Embrace. We are a mobile observability company. We provide developers with SDKs that they can integrate into their mobile apps to monitor performance in real time. Our main goal really is to help developers keep their apps running smoothly by proactively identifying and resolving issues, really with the objective of improving the overall user experience. -Without further ado, let's kick things off. I’ll hand it over to Hansen. +We're also heavily invested in OpenTelemetry. We began that transition about nine months ago, and it's really helped us provide our customers with a more unified and comprehensive view of app performance from what's happening client-side on the user's mobile device all the way to the backend services that power those experiences. ---- +In doing that migration, we discovered that while the benefits of OpenTelemetry are numerous, there are some specific challenges that occur when trying to implement it in a mobile environment, and that's kind of what we're here to talk about today. I'll be honest and say that the term "we" is very generous. Also on the call is Hansen, who we talked about; he's an Android architect at Embrace, formerly a mobile performance engineer at Twitter. So I'm just here to do kind of a brief intro, and then I'll hand it off to Hansen, who will be doing a majority of the presentation. -## Introduction by Eliah +[00:02:53] **Hansen:** Next slide, thank you. One more. I want to start by just setting a little bit of context. In 2022, the average mobile user spent a little over four hours a day on their phone, and of that, over 90% of it, or about 90% of it, was within native mobile apps. By the end of this year, mobile apps are expected to generate over 900 billion dollars in revenue. If you think about your own mobile use, I'm willing to bet that the brand or companies that you interact with most have a native app that you use consistently. -Hi everyone! My name is Eliah Cisi, and I’m a Product Manager at Embrace. We are a mobile observability company providing developers with SDKs to integrate into their mobile apps for real-time performance monitoring. Our main goal is to help developers keep their apps running smoothly by proactively identifying and resolving issues, ultimately improving the overall user experience. +Now, as we think about the observability ecosystem, we've spent the last 5 to 10 years really trying to figure out how we do observability better for backend systems. There are a variety of standards that were created, most notably OpenCensus and OpenTracing, which form the foundation for where we are today with OpenTelemetry to provide the visibility into the health of distributed systems. -We are heavily invested in OpenTelemetry, having begun our transition about nine months ago. This has enabled us to provide our customers with a unified and comprehensive view of app performance, from what's happening client-side on mobile devices to the backend services that power those experiences. +Mobile apps are not a distributed system; they are installed software running on distributed compute resources that interact with distributed systems. There are a lot of unique challenges with that environment. Take session tracking for instance. In the mobile world, a user session can be anything from several seconds to several hours, and it can be interrupted by calls or network changes or the app running in the background, which is really quite different from what you see server-side. -However, we discovered that while the benefits of OpenTelemetry are numerous, several specific challenges arise when implementing it in a mobile environment. That’s what we’re here to talk about today. +Mobile apps themselves are also extremely different. A gaming app, for example, needs to track things like frame rate and rendering times, while a financial app might focus more on transaction speeds and security events. Add to that the fragmentation in the mobile ecosystem, where you've got multiple operating systems and countless device manufacturers, and it's really easy to see how complex that becomes. -I’ll pass it off to Hansen, who will cover most of the presentation. +[00:05:00] We were looking at the number of unique devices for just one customer, and we saw that there were like 42,000 plus different combinations of device models and chipsets that could potentially exist, which is crazy. For most of its life cycle, observability and monitoring in the mobile ecosystem has been mostly proprietary systems designed by vendors. The typical starting point is Firebase Crashlytics, which is free but highly sampled and extremely limited in its feature set. So if you're a serious app developer, you're not going to use that, and that's led to a bunch of vendors building their own proprietary standards and trying to convince people that they should be serious about observability with their solution, but mostly it's just crash reporting and error tracking. ---- +The result is that the paradigm looks something like this, and the unfortunate thing is that a lot of mobile engineers actually consider that to be kind of acceptable. When we talk to customers who truly care about the user experience that their customers are having on their mobile devices, they tell us that all of their customer-impacting SLOs are directly tied to the mobile device, but they don't have a good source of information that correlates that data to the work they're doing to build reliability and resiliency in their backend systems. -## Presentation by Hansen +[00:06:30] That's what Hansen's going to be talking about, which is some of the challenges in this paradigm that we're facing today, and some of the opportunities and how we're working to address that. More than anything, I think today is really a call to action for the people in the Zoom, for the people watching. If you yourself, or if you have coworkers or people that are interested in the mobile ecosystem, we would love for you guys to get involved. Hansen's involved in the Android SIG and the client-side SIG. We have people working in the Swift SIG. We're working on OTEL contributions for React Native and Flutter, and we're really committed to making mobile observability as robust and standardized as it is for backend systems. We'd love to involve as many of you in that effort as possible. -Thank you, Eliah. I’m here to discuss how mobile is different in terms of observability. The crux of the problem lies in moving tools onto mobile platforms and making them run effectively. There are fundamental differences embedded in the assumptions we make regarding durability and the questions we ask of our tooling. Without identifying these differences, we cannot improve specs or tooling. +With that, I'll hand it off to Hansen. -### Key Differences in Mobile Observability: +[00:07:00] **Hansen:** Thanks, Eliah. So I am here to talk about how mobile is different in terms of observability. I think the first thing to approach this is that the crux of the problem is actually about simply moving tooling onto these mobile platforms and making them run because there are fundamental differences that are embedded in the assumptions that we make when we have durability and the questions that we ask of our tooling. Without first identifying and acknowledging these differences, we can't start improving the specs or tooling because we don't know what it should look like. -1. **Runtime Environment**: Mobile devices have dynamic and heterogeneous runtime environments. There are 42,000 unique chipsets and device combinations, which can lead to unpredictable behavior. Factors such as losing network connection or background activity can affect app performance. +To make the front of the horse look like the back of the horse, we've got to first figure out what the face should be. Here is what I'm trying to do: I'm just going to go over a little bit what the differences may be. I can spend a couple hours here talking about this, but hey, we don't have that much time, so I'm just going to concentrate this in three general areas where mobile is different. -2. **Capture and Transmission Pipeline**: Data capture in mobile environments is fragile. Data can be lost at multiple stages before being transmitted to servers due to crashes or unstable network connections. Even when data reaches the server, it may be delayed or out of order. +First is the runtime environment. We're talking about dynamic heterogeneous runtime environments running on devices that are unpredictable—42,000 unique chipsets and device combinations. Not only that, the environment that they run in is extremely dynamic as well. You walk into an elevator and you lose your mobile connection; you lose your network connection. You background the app to answer a notification, and that affects how the operating system runs the app. Not only that, the actual hardware that is running your app is quite severely limited. Phones that existed 10 years ago still even today, PHs are released last year that cost $99. The hardware mirrors that cost, and the OS not only limits what you can do with that already limited hardware, it also does it in a very unpredictable way. -3. **User-Centric Data**: The data we capture must center around user experience. Observations from millions of independent app instances should provide insights into individual user experiences rather than just system health. +What you have is a dynamic environment where you don't know what is actually executing your app. Objective performance, which is typically what we measure, is only part of the equation because what makes an app slow differs in terms of where I'm using it or whether somebody else is using it. Perceived performance is also part of the equation, and SLOs have to somehow take that into account. -### Challenges with OpenTelemetry: +[00:09:56] Another aspect of mobile that makes it challenging and different than backend is the fragility of the capture and transmission pipeline. How to get data from these devices onto servers, I could do something with it. Data could be lost in multiple stages before it gets captured because it was a crash, before you persist the data or after we persist the data, because the network connection is unstable and we can't get the data over to the server. Even when the server gets the data, it could be delayed or out of order, so what you get may not be the full picture until several hours later when more data comes in. -OpenTelemetry generally works well as a lingua franca of observability, but it was designed to solve specific backend problems. Here are some challenges we face when adapting it for mobile: +Lastly, the data we capture has to center the user experience because operational runtime and device state is useful only in how they provide insight into individual user experiences end to end. Otherwise, it's just trivia. On mobile, we're observing millions of independent app instances, but each one of the measurements we get maps back to a user. It is not merely a state of health of the system. If something is slow, somebody is looking at a very slow phone. It's like a P99—what does that mean? Well, P99 is 1% of all measurements are that slow or slower, so if you think that's an outlier, well, 1% of interactions being that slow means it is more than an outlier. It is something that we have to capture. -- **Spans**: While they are great for measuring durations in predictable environments, they fall short in mobile where operations can be interrupted or lost entirely. -- **Protocols and APIs**: Assumptions that telemetry is recorded and transmitted reliably do not hold true in mobile environments. Tools must account for unreliable data transmission and the strain telemetry puts on system resources. -- **Contextual Understanding**: Mobile engineers may not be familiar with low-level concepts like threads or context propagation. Therefore, APIs must be adapted for broader understanding. -- **Execution Boundaries**: Mobile operations can span multiple modules owned by different teams, making instrumentation brittle without proper infrastructure. +Now let's talk about OTEL. In general, OTEL works really well as a lingua franca of observability. Its backend roots, though, mean that it was designed to solve a problem with a very specific context, and when we change that context to mobile, some parts start to not fit very well. -Lastly, mobile devices represent millions of independent instances, which complicates how we approach metrics. Metrics need to be grounded in the context of the system to be useful. +The first point I want to talk about is spans. Spans are great in OTEL if you want to measure the duration of operations of applications that have a very fixed runtime environment, predictable runtime code path, so you can actually measure and calculate deviations based on some baselines. They are not so great if, say, duration is not an indicator of performance. If you want to measure a period of time, well, OTEL signals look like spans, but is it because duration is not an indicator of performance? ---- +[00:12:30] When you start aggregating and saying, "Hey, what's P95?" that starts to not make sense. Also, operations run for a long time in OTEL, but unfortunately you don't get to know what happened until it ends, either in a failure or success. What if it gets interrupted in the middle through a crash that we don't know about? Mobile apps could be killed without actually alerting the app, so operations like that are gone or lost, and with OTEL itself, we wouldn't know about it because it's not done, and that's kind of challenging for mobile. -## Call to Action +Also, operations that need to be contextualized with a lot of mutable state that are expensive to obtain, well, that's a challenge because in OTEL you write the data into the spans of span events or attributes, but sometimes the act of getting these types of state takes a while for the mobile app to actually get from the OS, and for us to basically block the ending of a trace, just doesn't work well. -We encourage everyone involved in the mobile ecosystem to contribute. Hansen is involved in the Android SIG and the Client-Side SIG, and we are working on OpenTelemetry contributions for React Native and Flutter. We want to make mobile observability as robust and standardized as backend systems. +The second point I want to talk about is that the protocols and APIs of OTEL make certain assumptions that are just not true on mobile. For example, assumption one is that telemetry is recorded and transmitted reliably, and you could trust that data that gets recorded will make it to the server, the collector, in a reasonable amount of time. But as we mentioned before, that is not true in mobile, and it could also not be true in a number of fascinating ways. ---- +So tooling that is usable on mobile has to take that into account and build resilience, like persistence, for instance, before export. Thanks to Cesar for doing that on OpenJava or the OpenCry Android extension; it's extremely helpful in production. In the future, perhaps there will be a place for ways for us to automatically transmit a group of related telemetry so that we don't get into this weird state in the backend wondering if more data is coming. That would be nice. -### Questions +Another assumption is that recording and transmitting telemetry doesn't put a strain on system resources. The SDKs are well-written, and they perform well, so on the backend, use them, it's no problem. However, on low-end devices and metered networks, it means that every signal captured potentially reduces performance or costs users money. We have to be a lot more careful in how we record data and how we transmit data. Even simply the act of getting the data to record can be expensive, so the consideration that we have to go into what to record is a lot greater and depends on the use case. -**Q: In web, Core Web Vitals have become a standard to measure user experience. Do you see an emerging standard for mobile apps?** +Another assumption is that engineers that use the API understand low-level concepts like threads or context propagation. Even the idea of tracing and what spans are may not be universally understood if you change the audience to mobile engineers, simply because the background, the variety of background of folks building mobile apps changes quite a bit. It could be somebody who's just done a six-month boot camp who is building a mobile app for the local grocery store, and they want observability too. They want to know why their stuff is slow, but they're used to higher-level constructs like coroutines when you're using threads but not really. They don't know about it, and how do you explain propagation when they don't understand the existence of a thread? -Yes, we are working on semantic conventions for mobile that can capture metrics like ANRs and sluggishness. Once we figure out the best approach, we'll collaborate with the SIGs to submit conventions. +Adapting the APIs to be ematic is one thing, but making those concepts understandable by those without CS backgrounds is another challenge right there. Lastly, another assumption is that traced operations have a clear execution boundary and code ownership. You have a service that creates a span, and the runtime, generally a team owns that front to back, and you transmit the context via context propagation, etc. Unfortunately, things are not as clean on mobile because a single span for an operation can go through several modules owned by several teams, and not all of them may be aware of all these problems. -**Q: What about applications using Kotlin Multiplatform? What instrumentation options are available?** +Instrumentation can be extremely brittle if you don't have the right infrastructure in place to catch regressions where implementation changes but the instrumentation doesn't. It's not as modular, nicely connected, cleanly connected as you would for distributed tracing, just because of how mobile apps are architected. The last OTEL thing I'm going to talk about is that mobile devices are millions of independent instances. OTEL generally assumes that the system being monitored and observed is one connected system. -Currently, there is no native SDK for Kotlin Multiplatform that emits OpenTelemetry telemetry. There is potential for a native SDK or a bridge to existing SDKs. We are evaluating how best to approach this. +It could be made of dozens of microservices deployed in various ways, but effectively they all roll up, and metrics in that context make a lot of sense—OTEL metrics. But for mobile, when we have disconnected app instances running, that starts to break down because metrics need to be grounded in the context of the system that emits them in order for the baselines to be created and compared. Munting together metrics from phones of various models into one limits how useful those generated metrics are. What does P95 of heap size of an app at one minute mean when you look at the entire fleet of devices? I don't know. Did it change when it increases or decreases? Is that good or bad? Well, I don't know because we don't know the reasons because these are different systems. -**Q: Is the variability of runtime environments an open problem?** +[00:19:00] Not that metrics aren't useful on mobile; they're extremely useful, but you tend to need to have like-to-like comparisons, apples to apples, and it tends to involve dimensions that are high in cardinality, and unfortunately that just doesn't work fantastically well with OTEL metrics. Simply the strict time-aligned aggregation is not really suitable for apps that have operations that have variable duration and have data come in at any time. It just wasn't meant to do what mobile wants it to do. But that's okay because these are not insurmountable challenges. These are ongoing things that we could work with the protocol and the SIGs and the folks to kind of improve. -Yes, it's an ongoing challenge. We are working with the entities working group to help capture external mutable states related to apps. We have some workarounds, but we're still exploring the best approaches. +We're working with various SIGs to try to get some of these problems surfaced and addressed. We are also, at Embrace, having workarounds to go around OTEL or perhaps use OTEL in non-standard ways in order to get the data to do what we want it to do. But that's not the end state. The end state we want to see is a diverse ecosystem and tooling that builds on not only what folks in OTEL have done before but also extends support to the plethora of use cases that we're bringing up. -**Q: How closely related are mobile instrumentation and browser instrumentation?** +Frankly, this is just the beginning because mobile apps that I'm familiar with run on very small, restricted sets of circumstances and phones and tablets are what I'm used to using. I haven't worked on cars or TVs, IoT, but those apps deserve observability as well. I'm sure as we take the tooling and the specs forward, more use cases will come on board and say, "Hey, I want to connect my data from my TV to my backend data." What additional challenges are there when I don't even know how a TV works in terms of how it uses Android? I'm sure it's different, and I'm sure there are new things. -There are many similarities between mobile apps and web apps. We should consider web instrumentation when designing mobile instrumentation due to overlapping environmental factors. +At this point, we're just exploring ourselves. We're trying to ask questions; we're trying to figure out if our assumptions are correct. Are there things we could change about how we used to do things in order to fit more into OTEL? But at the end of the day, we want everyone to come up with their use cases and help ask these questions of mobile and client use cases for OTEL. Great work has been done in the Android and Swift and client SIG already, but I think there could be more going forward. -**Q: Do different types of mobile apps require different data models?** +Folks listening, maybe they're converted to OTEL, but hopefully other people also watch this and say, "Hey, I looked at OTEL, it didn't really fit, but after this presentation, maybe I can make it fit. Maybe I can use it in a way that is productive and actually truly have this become the lingua franca of observability for both the backend and the frontend." -Yes, different apps have unique characteristics that require tailored semantic conventions. For instance, gaming apps may focus on frame rates, while financial apps prioritize transaction speeds. +That's enough of us talking. Any questions? I'm going to end the presentation if I can. There we go. -**Q: Are there golden signals for mobile irrespective of device type?** +[00:22:00] **Eliah:** Thank you very much, Hansen. For those that have joined a bit later, maybe haven't seen this message, we've got an Agile Coffee board. You can add your questions in there. I think we've got a couple of questions. We'll start with one related to web CWV. Vitals have become a standard to measure user experience for better awards. Do you see an emerging standard to measure equivalent concepts in mobile apps for sluggishness or speed? -The ultimate golden signal is user happiness, which can be assessed through user engagement and retention metrics. Tracking success rates and abandonment rates for key operations is also crucial. +**Hansen:** Yes, so you know, with OTEL, everything is defined effectively as semantic conventions built on the existing signals. At Embrace, we've kind of modeled some of the TRec capture for things like ANRs on Android and various other kinds of mobile slowness or sluggishness. We did it in a certain way that works, but is it the best? We don't know. Once we figure it out, we'll want to work with the SIGs to submit conventions to model that. I believe especially with the introduction of these emerging specs of profiling and even entities, a lot of these problems that we have are going to be addressed. So it's a matter of figuring out what we have and then defining them. If you're interested in defining certain slowness metrics or telemetry for mobile, let's talk. -**Q: Is OpenTelemetry mobile instrumentation ready for production use?** +There's one that I'm in the middle of putting together with the help of a lot of folks from the client SIG, a crash convention that spans platforms that's different from exceptions. We have many more down the pipe and it's limited by the abandonments that we have, but anything that can be should be standardized as semantic conventions. So it's a very long way of saying yes, and please help. -Yes, various SDKs are available that you can integrate into your mobile apps. Depending on your needs, you can roll your own implementation or use existing ones. +**Eliah:** Good stuff. Let's go on to another one. There are some options available for OTEL instrumenting iOS or Android code. What about applications using Kotlin Multiplatform? What instrumentation options are available? Any additional considerations when instrumenting KMP? ---- +**Hansen:** Kotlin Multiplatform is interesting. For those unfamiliar, it's similar to, you know, React Native or things like that, where you write it once and it kind of generates native apps for the various platforms. I'm sure I'm getting something wrong in the technicality there, but the idea is that you have one codebase for multiple platforms. There isn't an SDK that I'm aware of that works in such a way that is built natively into Kotlin Multiplatform that will emit OTEL telemetry. Whenever we have iOS or Android, we use a Java SDK at the core, and iOS uses the Swift SDK. -**Closing Thoughts** +For Kotlin Multiplatform, either there has to be a native SDK for Kotlin Multiplatform that will transpile into the native platforms, not only iOS and Android, but web and a whole bunch of different platforms, or there has to be a way of getting a bridge built to the other SDKs. We are evaluating, at Embrace, how best to approach this. We think the native way is the best way, but we don't know. We're working through some of these problems, not specifically Multiplatform, but with React Native, which is I think why it's more well-known and more well-adopted. -We want to get more people involved in the working groups and SIGs for mobile observability. The ecosystem is still emerging, and we need diversity in our approaches. If you're using OpenTelemetry or interested in mobile, please join us in shaping its future. +There's definitely opportunity in Kotlin Multiplatform, but first, we got to have an SDK before we got to have a way of getting OTEL telemetry working on that platform, like recording first before anything else can happen. If you're working on it, you know, you should start talking to people and think about that because that would be really interesting as well. -Thank you for joining us today, and we hope to see you in the next edition of OpenTelemetry in Practice! +**Eliah:** Thank you. I think the next question is about one of your challenges. It sounds like one of the core challenges for mobile observability is the variability of runtime environments. Software and hardware— is this entirely an open problem or are there suggestions on how to start tackling this? + +**Hansen:** I think to do it natively in OTEL, entities will have to, for those who may not be familiar with that particular working group, entities is a way of capturing external mutable transition of states that are independent from but related to apps. I think for us, we could see it capturing a lot of this variable state without having to directly—well actually the implementation is not yet, I don't know, maybe it could work. + +We have done certain things to work around this that may not be accepted or enatic. We're using spans to log durations of interesting things happening, and then on the backend kind of merging the stuff all together. That's not great, but it also allows us to capture the stuff independently and not have telemetry recording be blocked and also not have to encode every change of network condition onto every piece of telemetry sent and have race conditions that will, especially on mobile apps, make the edges blurry. + +So we have something working, but we're not sure if it's the best quite yet, and we're really looking for entities to be able to do that or help us. Does that answer your question? I think there were two parts; I might have only answered one part, I'm assuming so. + +**Eliah:** I think we've got a few questions, so I'm going to move to the next one. I know that you're part of the client-side instrumentation SIG, so this was quite an interesting one. How closely or not is mobile instrumentation to browser instrumentation as far as tracking user journeys? For example, if the client-side instrumentation work gears towards one or the other, or are the data models for both browser and mobile considered the same or similar enough? + +**Hansen:** I think there are differences, but certainly there are a lot more similarities between web apps and mobile apps than I would say between mobile apps and backend distributed traces. I think the differences are more nuanced, and I think there's enough similarity that anything that we should consider for web we should consider for mobile and vice versa. There may be cases where it is immediately, you know, unfit, but honestly, you can run a mobile application on a mobile browser, on a mobile device. + +All the environmental changes are effectively similar that they have to deal with, so I think in that respect, they are very, very similar. So yeah, we work very closely with the web folks. + +**Eliah:** Moving on, what different types of mobile apps require different kinds of data models? For example, an online game like Magic the Gathering versus a social media app like Instagram or a messaging app. + +**Hansen:** The question is, do they require different kinds of data models, those different types of apps? Certainly different semantic conventions, I would say. There are certain characteristics about the runtime that are more interesting to high frame rate apps like games, for instance. If you're using Salesforce Online, scroll jank is bad, but it doesn't deter from the experience that much, and also it's not as sensitive to mobile device capabilities. + +But if you're running a game and your frame rate drops by half in critical instances, you want to know about it. Having more detailed data for that kind of stuff will be applicable to certain domains and not others. But I think that becomes more of a challenge of tooling and instrumentation rather than the spec. I think the spec as it is, you know, with logs and spans, well, events actually specifically, and spans—hopefully a way of classifying spans in the future—give us enough of the building blocks to model these different use cases. + +The same challenges that we have in terms of transmission and things like that, as long as those are dealt with, I think a good portion of that stuff is going to be taken care of. Now there may be additional things that I'm not aware of that will certainly need to have different types of consideration. But I think before we run, let's just crawl, and I think getting unity and others, those different apps that we may not typically think about, console apps for instance. You have McDonald's, and you have that thing that opens for 24 hours—that's a different use case than a mobile app that you background all the time after seconds. + +I think with the work that we're doing now, hopefully, we'll move things forward enough so that we can start looking at some of these more challenging issues like frame rates on Unity and things like that. + +**Eliah:** I think this next one relates back to some of the earlier questions, but you mentioned that duration is not an indicator of performance. Are there golden signals for mobile, irrespective of device type and OS version, or do we always need to take these factors into consideration? + +**Hansen:** I think the golden signal is whether the user is happy or not. On mobile, there are indicators of whether the user is happy in terms of whether they actually come back and use the app again. Things like whether the user comes back, a usage rate, because just because one operation is slow, the effects could be multiplied if you have many slow operations. You basically get fed up with the app. + +At the very end, the very highest level, whether the app is being used is a good indicator. But going lower level, because that level is almost too late sometimes, whether an operation succeeded—users tend to give up on operations if things take too long to load. They background the app or hop to another page or whatever it is, so looking at the abandoned rate of operations is useful if you're tracking whether a particular operation is taking too long. + +Looking at duration in those cases, even if duration is important, the abandon rate is actually super important as well because you can have a case where your P50 or P95 goes down because more people give up. So your success rate reduces, and your performance increases. You're like, "What's that?" Well, it's because people are giving up. The population is different underneath; you're losing people already, so all the people remaining are the fast people. + +In fact, this is actually a key problem in mobile performance: people look at the current state and say, "Oh yeah, P99, that looks great." Well, you haven't considered people for whom your app is way too slow and you can't even use it, or they use it, they install it, it's too slow, they uninstall it. This kind of self-fulfilling prophecy of, "Oh, I don't have to invest in apps because people experience high-quality fast," well, it's because you've lost all the people, and the inability to track those you've lost is a huge miss in mobile observability. + +I think OTEL has facilities to track it. We can end a span with an unsuccessful ending, so data can be collected to take care of this use case. I think that to me is the most important thing to track: it's not just duration; it's whether or not it actually succeeded. + +**Eliah:** Can I add just one other thing there? Just from my product hat here, I think the golden signals that are asked in that question really depend on the type of app that you have. Hansen talked about startup and abandon rates and slow frame rates—all of those can have different impacts on your users depending on the intent of your app, whether it's, like we talked about this earlier, whether it's a gaming app or a financial services app or whatever that may be. + +If I were a PM or working on a mobile app, I would go back to what are the primary key workflows that we need our users to do to make this a successful transaction, and I use that term very broadly for them. Then you go instrument those based on whatever that is. For a gaming app, your golden signal may be frame rate because you know if your frame rate is as high as possible and your users are having a really frictionless experience, you see gameplay time increase. You see all these other second-order effects really turn into positive impacts on those second-order effects. Whereas for a retail app, it's really about checkout—how fast is that checkout flow for me? Are the operations that are part of that checkout flow performant? Are they reliable, etc.? + +It really depends on what your app is trying to do, and also the audience is important too. Are they using your app because they have to, or are they using your app because they choose to? If you're a game and you're slow, I can just play another game. If your workplace uses Microsoft—oh, no, I shouldn't say—uses certain apps that you have no choice but to use, you know, it's slow, but you don't have a choice, so the golden signal there may be a little bit less than if folks have more options to go. + +**Hansen:** Next question? + +**Eliah:** Do you know of some use cases of OTEL working on video streaming apps like Netflix? + +**Hansen:** Not that I'm aware of, and if they exist, I think the instrumentation would be fairly bespoke because—no, not that I'm aware of, but it would be a very interesting use case. + +**Eliah:** I think we've got two more. Resource utilization like CPU or memory is normally represented as a gauge in a time series format on a backend system. Do time series make sense in mobile apps at all? + +**Hansen:** Certain time series, I would say utilization less so because the app is not the only utilizer of resources. You could be running very expectedly, and something higher priority, the OS decides to schedule. Somebody starts a video in the corner of their tablet; they have multi-screen enabled, and suddenly the fast cores are now going to the mobile tablet video. Well, your app is running, and suddenly things are slow, so suddenly you are getting issues that you didn't get before. + +It's good to track the utilization of the CPU, but there are probably other things you would want to know about that the utilization is supposed to tell you. On mobile, there are just so many different things that could affect utilization that you almost need to capture so much other contextual information for that to be useful. If you're going to do that, you may as well go direct and say, "Hey, are we seeing lag? Are we seeing failures? Are we seeing unexpected occurrences in the app in certain instances?" + +Knowing that your app is not being prioritized is important, but how big of a heap, for instance, you know, those changes that on you so much. It'll GC you out of, in the most inappropriate places, simply because it wants more resources for other apps. It'll kill your app in the background because another app is running and needs it, and it's no fault of yours that your thing got killed faster. + +Resource utilization—there may be use cases where it's useful, but I think for me, it's harder to use the data and make sense of it. + +**Eliah:** This is the last question we've got. Do you have some idea of when OTEL mobile instrumentation might be ready for use in production? Is it months? Is it years? + +**Hansen:** It's ready now; it depends on what you want to do. There are SDKs out there, directly the Swift SDK and the various JavaScript SDKs. If you want to kind of roll your own, it works. The Android Open project is something you can drag and drop into your app, and it'll kind of do some monitoring for you. The Embrace SDKs, you can actually use it without using Embrace. You can just drop it in, configure your exporters to go to your own collectors, use our implementation of the tracing API to have instrumentation libraries, like data sessions, and have it all sent to your own servers without Embrace being involved. + +There are, I'm sure, other implementations out there as well, but it depends on what you want and what you need. There are things off the shelf you can use for free, and you can also roll your own with the SDKs. All the challenges I was talking about really is to build a platform that is encompassing of all the corner cases that we want to support for all our customers. + +If you have a specific app with a specific use case, with a specific thing you want to measure, you may not need any of this other stuff. If you're not sharing that code, if you're just using it on your own, well, who cares about semantic conventions if no one else is going to look at that data? You're quite locked into your own instrumentation, which is not a good thing, but if it works for you, it works for you. So I would go ahead and fork some of these repos and just try it out. They should work; they do work. There are apps that are in production with all of these SDKs. + +**Eliah:** That's all we had, so thank you very much, Hansen and Eliah. Is there any closing thoughts, anything you would like to add to finish off? + +**Eliah:** I would just go back to kind of where we started this. I think as Hansen's been talking about through this at all is that we really, really, really want to get more people involved who care about mobile in the working groups, in the SIGs. We are opinionated and we come to it with our own experiences and the work that we've done, but we're by no means the final arbiter of what is right and wrong in mobile. + +The only way we can work through those is by having a variety of use cases. I think that Netflix question is a good example of one that we don't deal with a lot, and so we may not be close to all the intricacies that come with running a video service at that scale or other services at that scale. If you care about mobile, even if you're not working directly on it, or if you have teams that work directly on it, we would love for you guys to get involved in the SIGs and provide your perspective and input. + +[00:44:00] **Hansen:** The ecosystem is just emerging, I think, for mobile. There isn't enough diversity there; there aren't enough folks leveraging the SDKs and building instrumentation for mobile use cases and mobile libraries that are popular. The closing thought is that if you're using OTEL or if your backend is using OTEL and you're not, you could actually get a lot of mileage just by talking the same language and using the same signals. + +I used to not think there was a ton of overlap in terms of, "Well, just mobile folks can just have that data, and then backend folks, all you have to do is encode all your conditionals and conditions and your requests in the back." There’s all that data, right? Well, it's not so easy to ingest all that in a performant way at every request. I've come around to understanding the utility of having two separate sets of data that can be connected. + +If you're a backend person and your company has mobile apps that you know use unnamed observability companies that may be free or may be very not free, but that don't really talk to your backend signals, well, have a look at OTEL and see what you can get in terms of things that are frankly even better, especially if it's provided by folks who only do mobile. There are a lot of things that are not captured if you just use folks that—well, I forget it, I'm not going to say it. + +Yes, join us. I guess that's the thing: join us on the CNCF LA. I'll post the link in the channel for OTEL client-side telemetry. We also have the OTEL SIG and user channel if you're an end user and you want to discuss more things about how you're approaching OpenTelemetry. + +Thank you again, both Hansen and Eliah, and we'll hopefully see you in the next edition of Otel and Practice. Thank you. Bye-bye. ## Raw YouTube Transcript diff --git a/video-transcripts/transcripts/2024-08-08T05:55:11Z-otel-q-a-feat-eliab-sisay-and-austin-emmons-of-embrace.md b/video-transcripts/transcripts/2024-08-08T05:55:11Z-otel-q-a-feat-eliab-sisay-and-austin-emmons-of-embrace.md index 6abb826..331e035 100644 --- a/video-transcripts/transcripts/2024-08-08T05:55:11Z-otel-q-a-feat-eliab-sisay-and-austin-emmons-of-embrace.md +++ b/video-transcripts/transcripts/2024-08-08T05:55:11Z-otel-q-a-feat-eliab-sisay-and-austin-emmons-of-embrace.md @@ -10,102 +10,157 @@ URL: https://www.youtube.com/watch?v=A-t1hZdh7JY ## Summary -In this YouTube video, hosts Elli and Austin from Embrace discuss mobile observability and their integration of OpenTelemetry into their SDKs. Embrace, a mobile observability company, focuses on providing developers with tools to monitor app performance in real-time, ultimately enhancing user experience. They explain their architecture, which utilizes Swift for iOS and Kotlin for Android, and discuss the transition to OpenTelemetry, emphasizing its importance in creating a unified observability framework. The conversation covers challenges with mobile environments, the necessity of education on observability for developers, and the collaborative nature of contributing to the OpenTelemetry community. Key points include the benefits of a common language in observability, the unique challenges of mobile app development, and ongoing efforts to improve crash reporting standards within the OpenTelemetry ecosystem. +In this YouTube video titled "Otel Q&A," Elli and Austin from Embrace discuss their company's mobile observability solutions, focusing on the integration of OpenTelemetry into their SDKs for iOS and Android. Embrace aims to help developers monitor app performance in real-time, identifying and resolving issues to enhance user experience. They emphasize the importance of OpenTelemetry in creating a unified observability framework, enabling developers to understand both front-end and back-end performance. The conversation touches on technical challenges in mobile environments, the significance of community contributions, and educational resources for understanding observability in mobile applications. They also highlight the need for better integration between mobile and backend observability practices, stressing the importance of user experience and performance in mobile development. ## Chapters -Here are the key moments from the YouTube livestream along with their timestamps: +00:00:00 Welcome and introductions +00:01:10 Overview of Embrace +00:03:40 Embrace architecture and tech stack +00:05:30 OpenTelemetry integration discussion +00:08:02 SDKs as core product +00:10:10 Internal telemetry dashboards +00:11:30 Challenges implementing OpenTelemetry +00:14:50 Rewriting SDKs for OpenTelemetry +00:16:30 Advantages of Embrace SDKs +00:18:00 Community contributions and collaboration +00:21:30 Audience Q&A session -00:00:00 Introductions and team roles at Embrace -00:03:15 Overview of what Embrace does as a mobile observability company -00:05:30 Description of Embrace's architecture and tech stack -00:09:15 Discussion on the integration of OpenTelemetry into Embrace's SDKs -00:13:00 Explanation of the challenges faced when adopting OpenTelemetry -00:17:30 Overview of how Embrace uses OpenTelemetry for mobile observability -00:21:45 Benefits of using OpenTelemetry, including common language for observability -00:26:00 Embrace's contributions to the OpenTelemetry community -00:30:00 Discussion on resources for learning OpenTelemetry -00:35:00 Audience Q&A session on mobile observability and crash reporting +**Elli:** Welcome to Otel Q&A! We've got folks from Embrace here. I guess let's why don't you introduce yourselves. I can start. My name is Elli. I lead product for our data collection and ingestion team here at Embrace. My team is responsible for a suite of mobile SDKs across iOS, Android, Unity, Flutter, and React Native. -Feel free to use these timestamps to navigate through the livestream! +**Austin:** And I'm Austin Emmans. I'm a developer on the iOS team, part of the iOS SDK development. I talk to the other teams about OpenTelemetry in general, but my main focus is Apple platforms. -# Embrace Q&A Transcript Clean-Up +**Elli:** Awesome! Can one of you tell us what Embrace does? -Welcome to the Embrace Q&A session! We have a team from Embrace here to discuss their mobile observability products and integration with OpenTelemetry. +**Austin:** Yeah, totally! We are a mobile observability company. We provide developers with SDKs that integrate into their mobile apps to monitor performance in real time. Our main goal is to help developers keep their apps running smoothly by proactively identifying and resolving issues, really with the objective of improving the overall user experience. We're heavily invested in OpenTelemetry, which I know we'll dig into a little bit later. We began that transition about nine months ago, and it's really helped us provide our customers with a more unified, comprehensive view of app performance—from what's happening client-side on the user's mobile device all the way to the backend services that power those experiences. By doing that, we enable developers to understand the full impact of any issues and optimize their apps more effectively. We've open-sourced all of our SDKs that I talked about earlier, and we encourage everyone to check those out in our GitHub repo, play around with them, and we'd love feedback. -## Introductions +[00:01:10] **Elli:** Awesome! It's really cool that, as a mobile observability company, you are basically dogfooding, I guess, by using OpenTelemetry. It sounds like you didn't start in that direction but moved in that direction, and I can't wait to dig into that a little bit more. Before we get into that, can you describe the architecture that Embrace uses? Any of the programming languages that are being used? What's your tech landscape look like? -**Elli:** -Hi, I'm Elli. I lead product for our data collection and ingestion team at Embrace. My team is responsible for a suite of mobile SDKs across iOS, Android, Unity, Flutter, and React Native. +[00:03:40] **Austin:** I could take that one. Being an SDK developer, I focus mostly on the iOS side of things. As we took on this journey, we kind of rewrote the iOS SDK with a Swift-first approach. That takes on a Swift Package Manager project structure. The Android side is similar; it takes on a Kotlin-first approach. There's a lot of existing Java in that SDK, and they do a really good job at making sure that the interface on the Android side feels familiar if you're a Java developer. Our backend, though, is standard microservice architecture. We deploy using Argo CD and primarily run in a Kubernetes cluster. We have a couple of different data stores depending on how we're accessing the data, which include ClickHouse and Cassandra. The code for the backend is mostly Go, with some Python sprinkled in here and there. -**Austin Emmans:** -I'm Austin Emmans, a developer on the iOS team, focusing on the iOS SDK development and communication with other teams regarding OpenTelemetry. My main focus is on Apple platforms. +**Elli:** Cool! Awesome. Now that we've got some of the background, I'm really excited to hear about your OpenTelemetry integration. First off, how did you learn about OpenTelemetry, and why did you decide to integrate OpenTelemetry into the product? -## What Does Embrace Do? +[00:05:30] **Austin:** I think as recently as five to six years ago, there were a variety of open standards that led to OpenTelemetry, most notably OpenCensus and OpenTracing. Once those combined into OpenTelemetry, we saw the community really coalesce around that as the standard for modern observability practices. We decided to use OpenTelemetry because it fit perfectly with our vision of modernizing observability through open standards. OpenTelemetry provides a transparent, portable, flexible way to collect data that we felt was really essential for creating a unified observability framework that ties both the front end and the back end of applications. As we were talking to customers, we kept hearing that one of the biggest challenges that SREs and developer teams consistently faced was marrying insights from their user-facing web and mobile applications into their observability practice. In an ideal world, your front end teams are collecting data about what's happening to the end user experiences, and your backend teams are collecting data about the health of infrastructure and services. Today, it's common for those to be entirely separate tools. They don't share a common set of telemetry, aren't interoperable, and really prevent engineering teams from speaking the same language. Companies want to work on what matters most, and it requires understanding where to invest their engineering resources to deliver the best user experiences. We saw that as an opportunity. We wanted to help solve that challenge by collaborating with the OpenTelemetry community to drive the future of open standards for observability, specifically within our expertise of mobile. Our goal is to provide developers with a comprehensive view of their apps' performance. Today, we collect the full technical and behavioral details of every user session with our OpenTelemetry-compatible SDKs. Users can even extend that instrumentation to any custom library in their app using OpenTelemetry signals and leverage our platform to contextualize the added instrumentation. Because we're using OpenTelemetry at the core of our SDKs, they can easily integrate with any backend of their choice or other observability tools, really giving users of our SDKs more flexibility to help them avoid being tied to one vendor. Of course, we think our product has some pretty cool stuff for mobile development teams in terms of workflow and helping them understand data and resolve things quickly, but we really think contributing to the community and helping mobile engineering teams modernize—and helping SRE and DevOps teams take that data and make sense of it as part of their entire system—will ultimately empower them to build better, more resilient mobile apps. -Elli: -We are a mobile observability company providing developers with SDKs that integrate into their mobile apps to monitor performance in real time. Our main goal is to help developers keep their apps running smoothly by proactively identifying and resolving issues to improve user experience. We are heavily invested in OpenTelemetry, which we've transitioned to over the last nine months. This has allowed us to provide our customers with a unified view of app performance from the client's mobile device to the backend services. We've open-sourced all of our SDKs, and we encourage everyone to check them out in our GitHub repository, play around with them, and provide feedback. +**Elli:** As a follow-up question, you mentioned that you have OpenTelemetry built into your SDKs. Do you have OpenTelemetry built into your core product as well? -## Embrace Architecture and Technology +**Austin:** I would say the SDKs are our core product that our customers use. We also have a frontend dashboard. -Elli: -I can describe our architecture. I focus mostly on the iOS side, where we rewrote the iOS SDK with a Swift-first approach, using a Swift Package Manager project structure. The Android side has a Kotlin-first approach, with a lot of existing Java. We ensure that the interface is familiar for developers transitioning from Java. +**Elli:** Yeah, the front end is what I'm thinking of. Austin, can you speak to that? -Our backend follows a standard microservice architecture, deploying using Argo CD and primarily running in a Kubernetes cluster. We use various data stores depending on how we're accessing the data, including ClickHouse and Cassandra. The backend code is mostly written in Go, with some Python sprinkled in. +[00:08:02] **Austin:** I'm more involved on the SDK side. There are some features that we have, like Data Destinations, where we would collect metrics. Some of the metrics that we aggregate in our backend, we then export as OpenTelemetry signals. If you're not using the SDK and exporting the OpenTelemetry signals directly from a user's device, you might be sending data to us and then getting signals sent from our backend to wherever you've configured—whatever other provider you might be using—to hopefully get the mobile data and your backend data in the same place. -## OpenTelemetry Integration +**Elli:** Gotcha! On a similar vein, how do you interact with the telemetry that's coming from the applications and services in your organization? -Elli: -I’m excited to discuss our integration with OpenTelemetry. We were inspired by earlier open standards like OpenCensus and OpenTracing, which led to OpenTelemetry. We saw the community coalesce around this standard for modern observability practices, fitting perfectly with our vision of modernizing observability through open standards. +[00:10:10] **Austin:** We have a bunch of internal dashboards. We host our own Grafana instance, and we can run reports and monitor the standard things that you would on a backend: uptime, response times, throughput. Our DevOps team and backend teams have those monitors. As a front-end developer and SDK developer, looking at how our customers use the product, I can pull reports and generate my own visualizations just to see if this feature is being used, how it's being used, and how clean the data is. If they're adding custom properties to their telemetry, we can see if they're using it in the correct way or the way we'd expect. That is really insightful in terms of what documentation we need to provide and how we should help people because sometimes, if you're not looking at this data, then you're kind of at a loss or just guessing. For internal visibility, it's mostly internal Grafana dashboards and running our own custom queries against those. -We decided to use OpenTelemetry because it provides a transparent, portable, and flexible way to collect data, which is essential for creating a unified observability framework that ties the frontend and backend of applications together. We noticed that one of the biggest challenges developers face is integrating insights from user-facing applications with their observability practices. +**Elli:** Awesome! You touched upon something that's so interesting and that we don't hear too much about in the context of observability, which is that observability helps us with troubleshooting, but it also has that added benefit of understanding how users interact with the system. It's really cool that that is one of the things you're doing with the telemetry data that you're gathering. I wanted to call that out because I think that's very interesting and not often talked about. -By collaborating with the OpenTelemetry community, we aim to drive the future of open standards for observability, specifically within mobile. Our goal is to provide developers with a comprehensive view of their app's performance. Today, we collect detailed technical and behavioral information for every user session with our OpenTelemetry-compatible SDKs. +Now, on to some meatier stuff. When you first decided to implement OpenTelemetry in your SDK, what were some of the challenges that you encountered? -Austin: -One of the significant benefits we’ve seen is that by using OpenTelemetry, we can help developers avoid vendor lock-in. Our SDKs allow users to integrate with any backend of their choice or other observability tools. +[00:11:30] **Austin:** Yeah, I mean, it was a process. Not that we really had to convince ourselves; I think there was a big upswell and a big push. Just from a software point of view, it's tough to bring in a dependency to a project because it's an unknown. At that point in time, we were pretty unfamiliar with the OpenTelemetry Swift SDK itself. We had to do our due diligence and some very tedious work to make sure that it fit how we expected it to, to ensure that any edge cases that we could think of we could test and verify that it would work how we expected it to, and then also make sure that the performance matched the scale we were expecting. That's just very tedious work, especially when you're compressing it into deadlines. That's not really OpenTelemetry specific; that's just software and dependencies in general. -## Challenges with OpenTelemetry +A challenge that was probably more specific to OpenTelemetry and observability is that our first instinct is when we instrument something, we want to just go instrument and get it done. You kind of have to catch yourself and say, "Oh, there's actually some prior art here," and search for these. OpenTelemetry has the specification and the shape of the data model and what it should be, and then on top of that, there are all these existing semantic conventions on how to use something and how to fit into a system—how to record a span for a network request, what attributes you should use for that network request, where the URL goes, what the name of that span is, and so on. Because we are all in, we want to make sure we adhere to those semantic conventions. It's tough when there's something very similar to what you're instrumenting, and you have to have discussions about whether this is something new, whether it's the same, or whether we should leverage some of that and massage it a little bit to fit our needs. That's just a tedious process to ensure that you're not stepping on any toes, but you're innovating and ultimately getting what you need done. -Elli: -When we first decided to implement OpenTelemetry in our SDKs, we faced several challenges. One was the unfamiliarity with the OpenTelemetry Swift SDK and the hesitation around adding a new dependency. We had to do due diligence to ensure it fit our expectations and could handle edge cases effectively. +**Elli:** When you did this, were you just updating your existing SDK, or was this a complete rewrite when you made this decision to use OpenTelemetry? -Austin: -A unique challenge in mobile observability is the chaotic environment. Mobile apps are affected by many factors beyond our control, such as battery life and network status. We need to ensure our SDKs can recover gracefully from these issues. +[00:14:50] **Austin:** For the iOS team, we took the opportunity to rewrite into Swift. We had an existing observability SDK that was mostly Objective-C. Being a larger shift, we were able to take this opportunity to really modernize it and present it to Apple developers in the year 2024. Our Android team— their SDK was a little younger, if that's a term you can use for SDK—but they were able to maintain that core codebase and then start shifting just the data model piece. I think they also took the opportunity to modernize a lot of the architecture because it changes a bunch when you're pulling in something like the OpenTelemetry SDK. There are a lot of concepts in there that are very useful that you either massage to fit or completely replace. It was different for each platform, but yeah. -## Contributions to OpenTelemetry +**Elli:** Some folks who are listening in or who will listen to this in the future might be wondering, you know, OpenTelemetry does have some SDKs tailored for mobile, so what's the advantage of using the Embrace SDKs in that case? -Austin: -I started attending the client-side SIG meetings a few months ago. I volunteered to represent iOS after noticing a lack of representation. The discussions have been great for sharing ideas and proposals, such as formalizing user session events and how to handle crash telemetry. +[00:16:30] **Austin:** It's really just to simplify. We use the SDKs as dependencies; they underlie what we do, and we're trying to add a layer on top of that to make it a little more accessible. We manage a lot of the setup process for the SDK and try to simplify and streamline that. We also want to minimize the mental overhead of, "Okay, when I'm creating a span, what do I need to do?" Hopefully that's one simple call into our SDK instead of getting your tracer provider, building a tracer from that, taking that, and creating a span builder. Once you have that span, there are a couple of steps that the SDKs themselves have; that's part of the spec and there's good separation for a reason. But we want to streamline that. If you're manually instrumenting things, we also want to add our own instrumentation that is mobile specific, which is the space that we're really in and are experts in. When you're on an iOS device, there are things that occur that are very specific to the iOS system or just users of that application that don't really apply to a backend system. We want to make sure that we can automate that instrumentation so that the users don't need to reimplement that or worry about it. There's definitely a push for us to get that instrumentation upstream because the goal is to benefit the community, and then it's off our maintenance burden. I mentioned there were two layers of the spec and those semantic conventions. The third on top of that is the Embrace semantic conventions that we're hopefully keeping as thin as possible and pushing down into those OpenTelemetry semantic conventions when we do create or think of new things. -## Audience Questions +[00:18:00] **Elli:** That's awesome! Basically, your SDKs serve as a wrapper, but then also any sort of things that you come up with that would benefit the community, you contribute them back upstream, which is very cool. That really speaks to the power of community because I think OpenTelemetry's success is due to the fact that we have so much support from various vendors who have all decided not to work on an island. I really want to emphasize that stuff like this is extremely beneficial to the community. -**Paige Cruz:** -What learning resources do you recommend for understanding OpenTelemetry? +Now, when you switched your SDKs to using OpenTelemetry, what kind of benefits did you start seeing? -Elli: -We have a book club around the "Learning OpenTelemetry" book by Ted Young and Austin Parker. This book is excellent for understanding observability concepts, especially as they relate to mobile. +**Austin:** For me, it was the common language. It became very easy to discuss with people what OpenTelemetry is, using terms like trace, span, and log, and what those glossary items are. One thing that really bugged me before is that pretty much every vendor models these concepts but calls them different things, and we were doing that too, so we're at fault as well. Being able to use just a common envelope and a common language was a massive benefit when we started switching to OpenTelemetry, even at a non-technical level. -Austin: -From a visual perspective, seeing a trace tree laid out can help make observability concepts more tangible. +**Elli:** Can I add one thing to that? I think from an organizational perspective, it also helped to break down some silos for us internally. Even across our SDKs, across the different platforms, prior to OpenTelemetry, our Swift SDK, our Android SDK, React Native, Unity, etc., were all built with specific implementation details for those platforms in mind and kind of lived a little bit in a silo. When you think about our customers, a customer of ours that has an iOS or an Android app builds it specific to those platforms, but in terms of performance, their mobile teams want to understand that a consumer is having a great experience regardless of whether they're on an iOS device or an Android device. Building the underlying SDKs in silos internally or I should say the other way around, when we moved to OpenTelemetry, being able to use those common signals helped us break down those silos such that we could really focus on the insights that we're giving to our customers, which is really the ultimate value that we aim to provide. -**Audience Member:** -What’s being developed in terms of crash events in OpenTelemetry? +**Elli:** That's awesome! That's a really great point. Now I want to switch gears a little bit because, Austin, you mentioned that you make some contributions upstream to the client-side telemetry. Can you tell us about your involvement with that SIG and how you started getting involved? -Austin: -We're currently working on a proposal to represent crashes as events, which is particularly challenging in the mobile space due to the variability in crash reports across different platforms. We aim to formalize this to make it easier for developers to send and parse crash data. +[00:21:30] **Austin:** I started attending the client-side SIG probably five or six months ago, right at the beginning of 2024. My coworker Hansen had been joining those before me, and I had been joining the OpenTelemetry Swift SIG meetings before that. Somebody just popped their head into a Swift SIG meeting and said, "Hey, we have this client-side SIG; there's not really any iOS representation. Would somebody please volunteer to come hang out with us?" I volunteered, and I've been part of that SIG since, joining every week. It's been great! Community ideas are shared; we talk about what's going on, what proposals are happening, or what problems people are trying to solve. When the time comes, it could just be unmuting yourself on this Zoom call and sharing an idea or even just sharing how it works for you. That could be going into the GitHub repos for some of the semantic conventions, adding a comment, and starting the discussion there or responding and continuing the discussion. Hansen has done a better job than I have, but he's led some semantic convention proposals for things that are mobile-specific or client-side specific. Some things like a user session are a little more formalized and standardized than they are now, and a big thing for mobile apps is when a mobile app crashes—how do we model that as telemetry? Now that the events data model has taken shape in the spec, there's a semantic convention proposal around modeling client-side crashes as events. That will be very useful because we deal with crash reporting as well at Embrace, and to formalize that and say, "Okay, here's how other people interpret crash reports," gives that flexibility to send that data anywhere, ridding ourselves of vendor lock-in because it's burdensome when you have it. -## Conclusion +**Elli:** That's so awesome! When you started attending those OpenTelemetry meetings, did you do that when the decision was made to start using OpenTelemetry internally, or when did that come about? -Elli: -Thank you, everyone, for joining us today! It was great discussing mobile observability and OpenTelemetry with you all. +**Austin:** I would say the decision was made when we started to switch, like, "Okay, we want to take this on; we want to be part of this community." I procrastinated a little bit just because I'm a person. I didn't actually start joining until the beginning of the year, but that's just the realistic approach, I guess. It's tough because you don't think that you have anything to contribute, and so there's an impostor syndrome that happens. It takes a bit to rid yourself of that, and I'm here to tell you to rid yourself of that, please join! We love hearing voices from all types of people using the tools provided. You can just come and hang out and chat, and if something comes up that you're comfortable with, then unmute yourself and join the chat verbally. -Austin: -We appreciate your time and interest. Looking forward to more discussions in the future! +**Elli:** That's great! Thank you for calling that out. I think it's so common. The impostor syndrome is very real; it's very scary. You are being so vulnerable there, especially when you join a group that seems to be very well-established and appears to be full of very intelligent people. You start thinking, "Oh my God, am I worthy of this?" and the answer, as you said, is yes, you are worthy of it. Everyone has a contribution to make, and it can be any kind of contribution. ---- +So often, I've used something from an open-source project and followed whatever in the documentation, and the docs are wrong. My first reaction is to be mad, like, "Oh my God, these idiots don't know what they're doing." But you should turn that frown upside down because if you notice an issue in the documentation, there's nothing preventing you from filing a pull request to fix that documentation and prevent other people from getting frustrated. That is the simplest thing you can do, and it makes such a huge impact. It doesn't have to necessarily be code; it can be documentation, and it can even be a minor correction. That's my little PSA there. -Thank you for attending this Q&A session on mobile observability and OpenTelemetry! +**Austin:** That was exactly our approach too! We're like, "Awesome! We want some first issues; let's go read the documentation and see if there's anything we can adjust or make more clear." + +**Elli:** That's so great! I love hearing these stories, and I feel like these are the types of things we just need to keep repeating in our community—telling people, "Yes, this is a great way to contribute." We talked about the benefits that you saw with integrating OpenTelemetry into your SDKs. What are some challenges you've seen in OpenTelemetry, especially in the mobile space, now that you've dipped your toes a little bit more into that community? + +**Austin:** I would say the biggest challenge that we faced is really—we have this layer above the OpenTelemetry SDK, and we're trying to streamline the process of getting into observability. It's mostly because mobile developers just aren't familiar with this territory. What’s interesting is that a lot of developers are, and there are a lot of existing tools out there that log events and things, but we're still trying to flatten the learning curve for observability. A lot of the challenge is just educating and figuring out what clicks for people, why this is valuable, and why it's useful. Then going from there, educating them on how to use our product specifically is probably one of the biggest non-technical challenges. + +The biggest technical challenge is that in the mobile space, the environment is just chaotic—it's something an app developer does not control. This is not a server in a box on some air-conditioned shelf; it is in somebody's pocket. It could be on a nightstand—you have no idea on the network status, how much battery is left on that device, or how angry the operating system is and whether or not it's just going to kill your app for who knows what reason. The variability is ever-present, and we need to be able to recover if anything goes wrong. We have to handle things like, "Okay, we're trying to write to disk locally to recover this data, but there's no disk space. What do we do?" or "The battery is at 2%. What do we do?" It's just a little more hectic than you would expect or hope for if you're an app developer. The worst thing is if you're an app developer that's worried about performance; these things are out of your control, but the user is going to blame you if the app is not performing well. If they are on 2% battery and they're rushing to call a car to get to wherever they need to be, you need to ensure that they can complete that operation before the system decides to sleep. + +**Elli:** Yeah, it's definitely a whole different ball game in the mobile space compared to the non-mobile space, where it feels like there are fewer unknowns. + +**Austin:** Right, and we're monitoring more than just the application's performance. We're also looking at how the user is interacting with this application and what their behavior is. You don't really have to deal with that if you're in a microservice architecture—there are very consistent entry points and exit points. In any client-side application, not just mobile-specific but even in a web browser, the user might be doing something on the page that you just don't know how they got there. They came in with state that you didn't expect, and you need to understand how to reproduce the issue. It works on my machine, but that's not an excuse. + +**Elli:** Absolutely! Now, switching back to contributing to OpenTelemetry, you mentioned getting started in contributing. Now that you've been doing it for several months, what has the contribution experience been like for you? + +**Austin:** It's good! I don't really have any bad or ugly experiences, which maybe is just me. When I joined the Swift SIG, it was like a week or two in—these are weekly meetings—so my second or third meeting, we were discussing an issue. Nacho just said, "Oh hey, Austin, can you take a stab at this?" I was like, "Okay, sure! You trust me to do that? That's awesome!" That was kind of a good sign of faith and confidence. I thought, "What can I do to help out?" Some of the things we're trying to do to help out are to push proposals upstream into the semantic conventions and make updates to the documentation to flatten that learning curve. You know, specific examples on how to instrument, like what this view controller is doing or making a network request with URL session—these are very common patterns for iOS developers. It just makes it very easy if there's a snippet out there that you can grab and tweak for your application. Those are the contributions we're trying to make. + +In terms of feedback for the SIGs, I don't know, be less friendly? I don't have much feedback; it's been great! + +**Elli:** That's awesome to hear! In terms of improvements, do you have anything you'd want to see tweaked? + +**Austin:** I don’t know if it’s an improvement, but finding that prior art is tough sometimes. A lot of these proposals are ongoing, and many things are just marked experimental, so you don't know how experimental they are. There's a process for every proposal or specification change, and making sure that those signals are up to date would be very useful and helpful as proposals go through the process. But that's just bookkeeping. If you're joining and talking about things, that's something that if you just ask a question, someone would help you out very quickly. + +**Elli:** That's actually a really great callout. + +**Austin:** I think it takes a bit longer than you would normally expect to get contributions accepted and move through that process, like Austin is talking about. But one thing to be aware of is that this is by design. When you're used to working at a startup, we try to make really quick decisions—implement them and see what happens. Working in a community like this takes more consensus. The goal is to create a unified solution that anyone can use, and that means decisions take a little longer than you might be used to when you're new. + +**Elli:** That's a really great point! Now we are finished with the main questions portion. Does anyone from our audience have questions they would like to ask? + +**Paige:** Hello! I am really curious what learning resources you would recommend—if there was a particular blog or metaphor or tutorial that helped you have that aha moment for you both. The follow-up is any helpful metaphors you use for teaching your customers and end users. + +**Elli:** Paige, real quick before we answer that, can you tell us who you are and what your goal is? + +**Paige:** Yes! I'm Paige Cruz. I work as a principal developer advocate over at Kronos. I used to be in SRE, mostly focused on those beautiful backend systems—the microservices running in beautifully air-conditioned data centers. I'm looking to learn more about mobile. I'm very interested in making sure observability is useful for everybody up and down the stack, and mobile is an area that I need to brush up on, so I was here to learn. + +**Elli:** Thank you! Just wanted to make sure I knew who I was talking to. Actually, it's funny that you mention blogs and books and resources. It is required reading at Embrace—the learning OpenTelemetry book by Ted Young and Austin Parker. We've started a book club around that at Embrace for everyone in the company. I think it addresses one of the other challenges we face that’s not technical. When you think about observability concepts in OpenTelemetry, it’s been primarily focused on backend to date. Taking those concepts and introducing them to front-end and mobile developers and mobile teams and why they're important in observability as a whole is a challenge. Before we can help them implement our solutions or understand why they're important, they need to understand the fundamental concepts. In our journey over the last nine months, it’s been various levels of education through our entire org, not just myself and Austin who are involved deeply in the SDKs but also our CSM teams, go-to-market teams, and SDRs, so we can all speak the same language and understand the benefits of OpenTelemetry—not just for backend observability but for client-side and mobile as well. Austin, I don't know if you have others. + +**Austin:** Oh, and we attend a bunch of conferences too. Just being in the same space as people and learning what they're going through has been super helpful. + +**Elli:** I can't remember when it clicked for me, but I'm a very visual person and a very hands-on learner. I think it was probably the first time I saw a trace as a trace tree visualized out in a timeline. I think that's when it was just like, "Oh, this is my code running right now, and I can see it and the differences." If I'm comparing two that should be identical but aren't because of stuff, that's when it was like, "Okay, this is interesting and useful." + +**Paige:** Does anyone have any other questions? + +**Audience Member:** I was curious, Austin. You mentioned the challenges with the mobile environment just being chaotic, and you mentioned the data model for crash events. I was just kind of curious what's going on with that. Are you developing anything, and what are the current topics in that area? + +**Austin:** I missed the client SIG this week because it was OpenTelemetry Day, but the latest I heard, as of last week, is that the proposal is up in the GitHub repo. The PR has been made—maybe it's just in a Google Doc, but there’s a document shared that describes a crash report as an event. The big thing is that events are built on top of logs, and the key difference is there's an event name attribute that you can key off of. The value of that attribute represents the schema of the log body. The other attributes in that log record are important. If you start calling an event with a name, like "client crash," then the body can take a shape that's a little more formal. The interesting part is that not all crash reports are the same in a mobile environment. On Android, you have the JVM, which is the Java virtual machine and that runtime. Then there are the native crashes below that—mostly C and C++ crash reports. On iOS, you have Swift crash reports; there are some C++ crash reports that are kind of weird. Now we have these hybrid apps, React Native and others, which are interesting because you might get two crash reports and two stack traces for a single crash. There's some weirdness there because if a React Native app crashes, it might crash at the bridge layer or in the JavaScript and throw an exception down to the native code. You want to capture all this. What's going into the proposal is figuring out how to represent this in a format that's consumable while understanding that it can be very different when an OpenTelemetry collector receives this data. I definitely recommend checking out the proposal as soon as it's up. Sorry, Hansen, for putting the pressure on you, but we're looking forward to it because it's one of the biggest things that when we came into OpenTelemetry, we asked ourselves, "How are we going to represent crashes?" We struggled with that, but we found a way—we just kind of shove everything in as a log record right now and said, "This isn't good enough; this could be better. Let's try to make this better." + +**Audience Member:** Interesting! Okay, I guess I didn't think about low battery as being something that would affect my apps, of course. Like you said, "Stupid app!" + +**Austin:** Low battery is one where the systems will start throttling the processor. Mobile devices have these high-efficiency processor cores and the AI crazy-fast processor cores. When the system sees that the battery is getting low, it starts throttling things and pushing things to those high-efficiency CPUs. That has a downstream effect on your logic, making it less performant because it's either bottlenecked and waiting behind something or just the processor is slower. It's just naturally not as quick as it would be if it was charging or had a full battery. + +**Audience Member:** Interesting! Thank you for sharing. I'm sure I'll have more questions once I've processed this more, but this is good. + +**Elli:** No problem! Well, as we prepare to wrap up, we will turn the tables and allow you to ask us questions. + +**Austin:** I'm curious. I think both of you are—I think Reese, you're a developer relations engineer, and Paige, you work as an SRE. To the other attendees as well, I'm not sure exactly your roles, but I'd be curious to what extent mobile observability is part of your current observability practice and how that fits in or how you marry that with your backend observability that you have in place. + +**Audience Member:** I don't personally monitor any mobile applications. I work with Adriana in the OpenTelemetry SIG. Anecdotally, I've heard a lot of end users being interested in this space. Like Paige, I want to learn more about it because I just don't really know much about mobile observability, and I'm really interested. + +**Audience Member:** Honestly, I think it was learning that Embrace existed. I'm like, "Oh, of course you'd want to do observability for your mobile applications as well!" That was when I made the mental connection of, "Duh! Why aren't we talking about it?" I see this as an opportunity to bring awareness to folks that, of course, you should be doing observability for your mobile apps. + +**Austin:** That triggers something you said earlier that I meant to note down for this question section. What tactics have you found work best in that regard? For instance, we have deep expertise in mobile. We've just started working in OpenTelemetry over the last six to nine months, and we want to bring that mobile expertise to the OpenTelemetry community. I think there's a thing around education from the mobile perspective that we also have to do that relates to a lot of the questions we talked about today. What tactics have you found to be particularly impactful in that regard, whether it's something you've done or you've seen done in terms of educating others on the nuances, whether for us it'll be mobile or it could be whatever platform? + +**Audience Member:** I would say talks, blogs—apply to talk at conferences to bring awareness for sure! Blog posts, YouTube videos, get on people's podcasts is another way to bring that awareness. + +**Audience Member:** For me, it was really eye-opening. At an org, we sent all of our application data to the same observability platform. I was mostly focused on the website and just for fun looked at the mobile side to see what was going on there. I saw, "Oh my God, like 90% of our requests came in through our iOS and Android apps!" I was like, "I should just honestly ignore the website because the impact would be so much greater if we made these optimizations." Business observability is not always talked about inside orgs, so really even just asking folks what the split is between entry points from mobile versus web can really be a thread to start pulling on. You also get your managers and other parts of the business, like PMs, interested and invited to the party. + +**Audience Member:** Last thing is at SREcon this year, I posted a link in the chat. Isabelle gave a great talk called "The Invisible Front Door: Reliability Gaps in the Front End," and touched a bit on mobile monitoring and how SREs really need to turn their eyes towards this space. + +**Elli:** That's awesome! Thank you for sharing that. + +**Austin:** Well, we are almost out of time! I thank you, Austin and Elli, for joining us today. This has been really great and educational. Thank you to everyone who joined us as well. See you for Otel in practice! ## Raw YouTube Transcript diff --git a/video-transcripts/transcripts/2024-10-04T15:54:54Z-otel-q-a-with-steven-swartz.md b/video-transcripts/transcripts/2024-10-04T15:54:54Z-otel-q-a-with-steven-swartz.md index 9536b26..990764c 100644 --- a/video-transcripts/transcripts/2024-10-04T15:54:54Z-otel-q-a-with-steven-swartz.md +++ b/video-transcripts/transcripts/2024-10-04T15:54:54Z-otel-q-a-with-steven-swartz.md @@ -10,260 +10,293 @@ URL: https://www.youtube.com/watch?v=3c9Bnldt128 ## Summary -In this YouTube Q&A session, Stephen Schwarz, a Form Engineering expert at a payment processing company, discusses his team's experience with observability using OpenTelemetry. The conversation, moderated by Dan, covers topics including the migration from a proprietary observability vendor to OpenTelemetry, the architecture and deployment strategies employed by Stephen's organization, and the challenges faced with scaling collectors. Stephen explains the use of the OpenTelemetry operator for automating instrumentation, the benefits of tail-based sampling, and the integration of Prometheus for monitoring metrics. He also shares insights on the learning curve for teams transitioning to OpenTelemetry and provides feedback for the OpenTelemetry maintainers regarding documentation and configuration examples. The session concludes with Stephen responding to audience questions, emphasizing the importance of community engagement in advancing observability practices. +In this YouTube Q&A session, Steven Schwarz, a Form Engineering expert from a payment processing company, discusses his experiences with observability, particularly focusing on OpenTelemetry (OTel). He shares insights into his team's migration from a proprietary telemetry solution to OTel, highlighting the challenges and advantages of using OTel's architecture, including Kubernetes and the OpenTelemetry operator. Steven explains their use of sidecar collectors, auto-instrumentation, and the management of resources, as well as the importance of sampling strategies and the integration of metrics monitoring. The discussion touches on the learning curve for teams transitioning to OTel, the significance of proper documentation, and the need for community support in navigating complex setups. The session concludes with audience questions addressing various technical aspects of using OTel, showcasing the engagement and interest in observability practices. ## Chapters -Here are the key moments from the livestream with their corresponding timestamps: +00:00:00 Welcome and introduction +00:01:00 Guest introduction: Stephen Schwarz +00:02:40 Architecture and deployment overview +00:03:50 OpenTelemetry operator usage +00:06:01 Migration from proprietary solution +00:08:10 Challenges in scaling collectors +00:09:36 Load balancing collectors +00:12:40 Experience with OpenTelemetry operator +00:14:50 Building custom collector distribution +00:17:00 Manual vs. auto instrumentation +00:20:00 Audience Q&A session -00:00:00 Introductions and format explanation -00:01:30 Stephen Schwarz introduces himself and his role -00:05:00 Discussion on the architecture and programming languages used -00:11:30 Overview of the OpenTelemetry setup and Kubernetes usage -00:17:00 Explanation of tail sampling challenges and solutions -00:22:00 Discussion about migrating from a proprietary observability vendor -00:26:00 Challenges in scaling collectors and resource allocation -00:31:00 Experience with the OpenTelemetry operator and contribution to the project -00:37:00 Insights on manual vs. auto instrumentation in OpenTelemetry -00:42:00 Audience Q&A session begins with various questions about implementation and best practices +**Speaker 1:** Well, thank you everyone for joining. We have quite a good audience for the OTEL Q&A. We have Steven Schwarz today with us. -# OpenTelemetry Q&A with Steven Schwarz +Just a quick note on the format, which Dan actually posted in the chat. I'll be asking Steph some questions, and then afterwards, time permitting, you'll have the opportunity to post some questions for Steven as well in the link that Dan provided. -**Moderator:** Well, um, thank you everyone for joining. We have quite a good audience for our OpenTelemetry Q&A. We have Steven Schwarz with us today. +All right, so first things first, Steven, do you want to introduce yourself? -**Format Note:** Just a quick note on the format which Dan actually posted in the chat. I’ll be asking Steven some questions, and then afterwards, time permitting, you’ll have the opportunity to post some questions for Steven as well using the link that Dan provided. +[00:01:00] **Steven:** Yeah, sure. My title at work is Form Engineering, and I work for a company that does Payment Processing. So you know, when you're working with payments, you're working with your money; you need to be kind of exact with your observability. ---- +What my team does is we manage all of the observability infrastructure. We've got about a hundred technical folks that we're managing it for, and we try to be sort of like the experts in observability and share some of the best practices and help them solve their problems. -**Moderator:** So, first things first, Steven, do you want to introduce yourself? +I recently joined this team about almost a year ago, and the project that I started on was actually migrating from another observability vendor, where we were using proprietary instrumentation, and we adopted OpenTelemetry and moved to a new observability vendor. -**Steven:** Yeah, sure! My title at work is Form Engineering, and I work for a company that does payment processing. So, when you're working with payments, you're dealing with money, and you need to be exact with your observability. My team manages all of the observability infrastructure for about a hundred technical folks. We try to be the experts in observability, sharing best practices and helping solve problems. +So hopefully, some of those learnings are useful to other folks going through that process. Also worth calling out is that I really enjoyed contributing to the OpenTelemetry Collector contrib repo. I had to do that to unblock myself with a few bugs that we faced, but it was also just a really good way to understand what the collectors are doing under the hood. I was completely new to Go, but the code is really approachable. So yeah, I'd recommend taking a look if you haven't. It made my job a lot easier for sure. -I joined this team almost a year ago, and the project I started on was migrating from another observability vendor that used proprietary instrumentation. We adopted OpenTelemetry and moved to a new observability vendor. Hopefully, some of those learnings are useful to others going through a similar process. +[00:02:40] **Speaker 1:** That's great! Thanks for the intro. Now, can you tell us a little bit about the architectures that you use in your organization? Like what programming languages are used and the deployment landscape, that kind of thing? -I also really enjoyed contributing to the OpenTelemetry Collector contrib repo. I did that to unblock myself from a few bugs we faced, but it was also a great way to understand what the collectors are doing under the hood. I was completely new to Go, but the code is really approachable. I’d recommend taking a look if you haven't; it made my job a lot easier for sure. +**Steven:** Yeah, so I have that architecture diagram I shared with you. Would it be a good time to share that? ---- +**Speaker 1:** Yeah, go for it. -**Moderator:** Thanks for the intro! Can you tell us a little bit about the architectures that you use in your organization? What programming languages are used and what’s the deployment landscape like? +**Steven:** Okay, all right. So I believe my screen is visible. Everyone can see a diagram? -**Steven:** Sure! I have an architecture diagram I can share. Is it a good time to do that? +**Speaker 1:** Yep, I can see it. Hopefully, everyone else can as well. -**Moderator:** Yeah, go for it! +**Steven:** Well, okay. So going kind of left to right, we can dive into specific parts that seem interesting. -**Steven:** Okay, I believe my screen is visible. Everyone can see the diagram? +So we are running on Kubernetes. We have multiple production clusters, and within each of those clusters, we are using the OpenTelemetry operator. That gives us a way to inject common configuration of the SDK into our teams' applications, into their containers. It also lets us automatically turn on auto instrumentation, so we get consistent spans collected from all the apps. -**Moderator:** Yep, I can see it. Hopefully, everyone else can as well. +[00:03:50] Another unique part is we are using OpenTelemetry. We're running OpenTelemetry as a sidecar, meaning that alongside each instance of the application, we have a mini collector running, and the app can send all their telemetry over localhost to that sidecar. -**Steven:** Great! Going from left to right, we can dive into specific parts that seem interesting. We are running on Kubernetes with multiple production clusters. Within each of those clusters, we are using the OpenTelemetry operator, which allows us to inject common SDK configuration into our teams' applications within their containers. It also lets us automatically turn on auto instrumentation, so we get consistent spans collected from all the apps. +So we are currently sending metrics and spans. Logs, it's a bit of a long story, but we aren't currently using logs through OpenTelemetry. For metrics, we use OpenTelemetry, but as sort of like a hop. We also use Prometheus to scrape the sidecar before it pushes to Grafana. -Another unique part of our setup is that we are running OpenTelemetry as a sidecar. This means that alongside each instance of the application, we have a mini collector running, and the app can send all their telemetry over localhost to that sidecar. +Another interesting challenge was getting tail sampling to work because we have spans coming from multiple Kubernetes clusters, and we only want to keep some of them. But we don't want to make the decision about which traces we keep until we have all the spans—maybe we wait to see if there's an error. -Currently, we are sending metrics and spans. For logs, it's a bit of a long story, but we aren't currently using logs through OpenTelemetry. For metrics, we use OpenTelemetry as sort of a hop; we also use Prometheus to scrape the sidecar before it pushes to Grafana. +So we have a separate cluster that all the spans go to, to make that decision. We load balance just to help scale that. We have these collectors in that common cluster; they just load balance by Trace ID, so we don't have a ton of spans piling up on one collector. Yeah, that's sort of like a high-level overview of our setup. -Another interesting challenge we faced was getting tail sampling to work. We have spans coming from multiple Kubernetes clusters but only want to keep some of them. We don’t want to make the decision about which traces we keep until we have all the spans; maybe we wait to see if there’s an error. So, we have a separate cluster that all the spans go to in order to make that decision. We load balance to help scale that; we have collectors in that common cluster that load balance by Trace ID, so we don’t have a ton of spans piling up on one collector. That’s sort of a high-level overview of our setup. +**Speaker 1:** That's awesome! It's really cool to see this kind of setup out in real life because I think that's, I do feel like that's kind of a burning question with a lot of our practitioners: how do you set up your collectors? So it's very nice to see. ---- +And cool that you're using the OpenTelemetry operator. I think you're one of the few people that I've spoken with. Not to say people don't use the operator, but you're definitely one of the few people I've spoken with recently who has been using the operator. -**Moderator:** That's awesome! It’s really cool to see this kind of setup in real life, as I think that’s a burning question for a lot of practitioners: how do you set up your collectors? It’s nice to see that you’re using the OpenTelemetry operator. You're one of the few people I’ve spoken with recently who has been using the operator. What aspects of the operator are you currently using? Do you leverage the auto instrumentation capabilities? +What aspects of the operator are you currently using? Do you leverage the auto instrumentation? Do you leverage the OpAmp capabilities? What's the other one? Do you use the Collector deployment capabilities? Do you use all three or a combination? -**Steven:** The main aspects we’re using are injecting the common SDK configuration. That saves us a ton of time because we don’t have to get teams to make code changes; we can manage that centrally and push it out to all the teams. +**Steven:** So the main ones we're using are just injecting the common SDK configuration. That saves us a ton of time. We don't have to get teams to make code changes; we can just manage that centrally and push it out to all the teams. -We also use the auto instrumentation, where they just add a line to their pod configuration to get the instrumentation. The other capabilities you mentioned, we aren’t using those, but we are using the sidecar, which is part of the OpenTelemetry operator. +Same with the auto instrumentation. They just add a line to their configuration of their pod, and then they get the instrumentation. The other ones you mentioned we aren't using, but we are using the sidecar, which is the OpenTelemetry operator inject. ---- +**Speaker 1:** Cool, cool. I have to put a plug for the operator. Auto instrumentation is actually so cool. You literally don't have to write code; you just put an annotation in the manifest. And the instrumentation CR, of course, that you have to figure, but it's pretty cool, I have to say. -**Moderator:** I have to put a plug in for the operator; auto instrumentation is so cool! It’s a minimal effort for teams. +**Steven:** Yeah, you know, the least effort you can make for teams, the better. Even getting them to add the annotation can sometimes take time, so I think it made the migration go much faster by being able to minimize the amount of work they have to do. -**Steven:** Absolutely! The less effort you can make for teams, the better. Even getting them to add the annotation can sometimes take time, so I think it made the migration go much faster by minimizing the work they had to do. +[00:06:01] **Speaker 1:** Absolutely. Now, I wanted to ask because you mentioned that you had migrated from a previous vendor using their own proprietary telemetry solution. What made you decide to do that migration? ---- +**Steven:** Yeah, so cost was a big one. It was hard to control with the previous vendor. I guess support wasn't good, so those are the main drivers. A lot of stakeholders were unhappy with it, and we wanted a bit more. The Collector gives us a lot of control over cost that this other vendor doesn't. -**Moderator:** I wanted to ask: you mentioned that you migrated from a previous vendor using their proprietary telemetry solution. What made you decide to do that migration? +[00:08:10] **Speaker 1:** Right, right. Yeah, that makes a lot of sense. Now, another thing that I wanted to ask in terms of scaling your collectors: have you experienced any challenges in scaling your collectors? Because you mentioned that you have a sidecar deployment pattern. How many different sidecars are you running at a particular point in time? Is it a lot that you're managing? -**Steven:** Cost was a big factor; it was hard to control costs with the previous vendor. Additionally, the support wasn’t good, and a lot of stakeholders were unhappy with it. We wanted more control over costs, which the collector gives us. +**Steven:** Yeah, so for the sidecars, we run one for each pod or instance of the application, so that kind of almost scales on its own, which is quite nice. That works. ---- +The tricky part with the sidecar is configuring how many resources to allocate, like how much memory to ask from Kubernetes, just because every application is slightly different in their volume. There currently isn't a way for a specific application to say that they want a specific amount of memory. We need to configure that in our centralized configuration, so it creates a lot of coordination overhead if we want to tune that. -**Moderator:** Have you experienced any challenges in scaling your collectors? You mentioned that you have a sidecar deployment pattern. +But we've only had to tune it for one application so far—one of our monoliths. So that has actually been quite easy to scale, the sidecar. -**Steven:** Yes, for the sidecars, we run one for each pod or instance of the application, so that scales on its own, which is quite nice. The tricky part with the sidecar is configuring how many resources to allocate, like how much memory to request from Kubernetes. Every application is slightly different in volume, and there’s currently no way for a specific application to request a specific amount of memory; we need to configure that in our centralized configuration. This creates a lot of coordination overhead if we want to tune that. We’ve only had to tune it for one application so far, so it's been quite easy to scale the sidecars. +[00:09:36] **Speaker 1:** Okay, that's great! Now, what about your load-balanced collectors? Did you encounter any challenges in setting that up? ---- +**Steven:** Yeah, well, I mean at the beginning, there was a regression that caused a memory leak in one of the components we're using—the span metrics one. So that was a problem. That was actually one of the things that I fixed as an open-source contribution. -**Moderator:** What about your load-balanced collectors? Did you encounter any challenges in setting that up? +I guess our setup is quite memory-intensive because we have to cache our metrics in memory for Prometheus to scrape them. So yeah, it's really been a matter of keeping memory at a reasonable level. Right now, we use the Kubernetes horizontal pod autoscaler that watches the memory of each of the instances of the collector and makes scaling decisions based on that. -**Steven:** Yes, there was a regression at the beginning that caused a memory leak in one of the components we were using for handling spans and metrics. That was an issue I fixed as part of my contribution to the open-source community. +It works pretty well; it took some tinkering. One thing I've noticed, and maybe other people have a good solution for this, is when our vendor has an outage, I find the telemetry starts to pile up, waiting for service to be restored, and we see this big spike in CPU and memory. So it's quite hard to... I guess if the outage were to go for quite a while, probably the memory usage would just start to kill the collectors or be very high. So that's a bit of a problem. -Our setup is quite memory-intensive because we have to cache our metrics in memory for Prometheus to scrape them. So, it’s been about keeping memory at a reasonable level. We use the Kubernetes Horizontal Pod Autoscaler to watch the memory of each instance of the collector and make scaling decisions based on that. +**Speaker 1:** Right, right. Now, another question that I had for you, just going back to the OTEL operator: what made you decide to use the operator in the first place? Is it something that you had been made aware of, or did you hear about it from folks in the community? -One thing I’ve noticed is that when our vendor has an outage, telemetry starts to pile up, causing a spike in CPU and memory. If the outage lasts a long time, the memory usage could become a problem for the collectors. +**Steven:** I heard about it actually from a colleague. It was their recommendation, but it fit with the kind of platform ethos of trying to get out of the way of the developers and have a way to do things without the developers having to get involved. ---- +**Speaker 1:** Right, right. Yeah, fair enough. That is one thing I like about the operator. Now, what was your experience in running it? Was it something that you felt was straightforward? -**Moderator:** Going back to the OpenTelemetry operator, what made you decide to use it in the first place? +Because one of the things that we really value about these sessions is not only learning how you're using these things in real life but also learning your user experience with using components of OpenTelemetry. Was it something you felt was documented well enough? Did you have to do a little bit of digging, poke around, ask some questions in the operator Slack? -**Steven:** I heard about it from a colleague, who recommended it. It fit with the platform ethos of trying to get out of the way of developers and allowing them to do things without needing to be heavily involved. +[00:12:40] **Steven:** I'd say generally, compared to the collector itself, it was very smooth. I'm not sure why that is, but yeah, it worked quite well. The maintainers were very responsive in the Slack channels when we did have minor issues. ---- +**Speaker 1:** Awesome! That's good to hear. I've always had a really good experience with the OTEL operator maintainers, so that's nice to hear as well. -**Moderator:** What was your experience running it? Did you find it straightforward? +Great! Now, the other thing that I wanted to ask is, when you're using your collectors, are you building your own collector distribution? -**Steven:** Generally, it was very smooth compared to the collector itself. When we had minor issues, the maintainers were very responsive in the Slack channels. +**Steven:** Yeah, we are. That was because when we faced some of those bugs, we couldn't wait until it was pushed to the main distribution, so we forked it just to fix it earlier. Once that happened, we started to make other changes, so it's a bit hard to get off the fork now, but it's something we do want to do. ---- +**Speaker 1:** Now, does that involve using the OTEL Collector Builder tool when you're deploying your collectors? -**Moderator:** Are you building your own collector distribution? +**Steven:** Yeah, it does. I looked at the Dockerfile, how they build the container, and just kind of copied the commands from there. -**Steven:** Yes, we are. When we faced some bugs, we couldn’t wait for them to be pushed to the main distribution, so we forked it to fix issues earlier. Once that happened, we started making other changes, so it’s been a bit hard to get off the fork, but it’s something we want to do. +**Speaker 1:** Oh, okay, okay. Have you used the actual tool where you specify the components, like which receivers, which processors, etc., that you want to include? Have you tried using that tool? ---- +**Steven:** Yeah, that's what we're using to build it. We copied the list of components. -**Moderator:** Does that involve using the OpenTelemetry Collector Builder tool when you deploy your collectors? +[00:14:50] **Speaker 1:** Oh, okay, okay. Got it, got it. Sorry, I misunderstood you. Awesome! You mentioned you did your own customizations, and that kind of forced you to contribute back to the collector. How was the experience of contributing back to the collector overall? -**Steven:** Yes, we are using that tool. We copied the list of components from the Dockerfile to build it. +**Steven:** It was a good experience. People were helpful during the code review. It took a little while to get the code merged. I noticed you'd have a separate person for approving it, and then it would sit unmerged for a few weeks, and then you get all these merge conflicts because the versions would be upgraded. So I had to go back and update the Go mod files a bunch of times. ---- +That was a bit of a challenge. I think there's room to merge it a bit faster once someone approves it, but I don't know the rationale behind that. -**Moderator:** You mentioned your own customizations, which led you to contribute back to the collector. How was that experience? +[00:17:00] **Speaker 1:** Fair enough, fair enough. The other thing I wanted to ask, you mentioned that you use auto instrumentation. Have you done any manual instrumentation as well? -**Steven:** It was a good experience. People were helpful during the code review. It took a while to get the code merged, though, and I had to deal with merge conflicts due to version upgrades. I think there’s room to merge things faster after approval to avoid those conflicts. +**Steven:** Yeah, we had to show teams because we had manual instrumentation using that proprietary format. We had to show teams how to convert it over to the OTEL format and also educate them on how to use OTEL manual instrumentation. ---- +**Speaker 1:** And how did that go? Did you find that teams were receptive to that? -**Moderator:** Have you done any manual instrumentation as well? +**Steven:** I think they're slowly doing it. I think tracing isn't maybe as obvious, so I probably just need more examples of why it's useful to give them the motivation to add more metadata to their spans, for example. -**Steven:** Yes, we had to show teams how to convert their manual instrumentation from the proprietary format to the OpenTelemetry format. It’s still a slow process; I think the benefits of tracing aren’t obvious to everyone, so I need to provide more examples of why it’s useful to motivate teams to add more metadata to their spans. +**Speaker 1:** Right, right. Now, does your team ever get asked to instrument stuff for other teams? ---- +**Steven:** Not specifically for their applications. -**Moderator:** Does your team ever get asked to instrument stuff for other teams? +**Speaker 1:** Yeah, I'm very glad to hear that. One of my observability pet peeves! Well, that's good to hear. -**Steven:** Not specifically for their applications, and I’m glad to hear that. +Now, in terms of the instrumentation, it looks from your diagram that it's primarily a Java app, is that correct? ---- +**Steven:** Yeah, mostly Java. We do have some Go as well. -**Moderator:** I noticed from your diagram that it looks like you're primarily using Java apps. Is that correct? +**Speaker 1:** Okay. Now, how did you find it in terms of the learning curve for instrumenting stuff in OpenTelemetry? Since you're in a position to teach folks how to instrument their code with OpenTelemetry, was it a big learning curve for your team as far as understanding how the manual instrumentation works for both Java and Go? Did you notice anything that was maybe easier in one language compared to the other, or where you had to go back to the SIGs to ask for clarification? -**Steven:** Yes, mostly Java. We do have some Go as well. +**Steven:** So Java, they've done a really good job with it. I think the language just makes it easier. Because we use Micrometer, which is a Java library for emitting metrics, they didn't actually have to change anything to get the metrics to work. ---- +The span API is pretty intuitive as well. For Go, I mean, it's not my personal strength, but I found it's a bit more manual. It was definitely harder to use than the Java one. I'm curious if anyone's used the auto instrumentation; I haven't had a chance to play around with that, but it would be nice if we could get that working. -**Moderator:** How did you find the learning curve for instrumenting stuff in OpenTelemetry? +**Speaker 1:** For folks who are listening, does anyone have experience with that that would like to share? Feel free to raise your hand if you do have any experience. -**Steven:** Java has a good API, and the instrumentation process is pretty intuitive. We use Micrometer, a Java library for metrics, so the teams didn't have to change anything to get the metrics to work. With Go, it’s a bit more manual and harder to use than the Java one. I’m curious if anyone has used the auto instrumentation for Go; we aren't currently using it, but it would be nice to get that working. +No takers? Alright. ---- +[00:20:00] Okay, I guess I'm just about done asking the questions. The only other thing I was going to ask is do you have any general feedback that you wanted to provide to the OpenTelemetry maintainers around your OpenTelemetry experience, as far as using components? You mentioned the contribution experience; you said it was a little bit challenging with approving the PRs. Anything else that you wanted to add to that? -**Moderator:** Do you have any general feedback for the OpenTelemetry maintainers about your experience? +**Steven:** I think it felt like we were kind of figuring out a lot of stuff for ourselves. It took us quite a few iterations to get to this point. It might be hard to have a recipe for every combination of vendor and backend, and all that, so that could be part of the problem. -**Steven:** I think we figured out a lot by ourselves, and it took several iterations to get to this point. More example configurations would be helpful for common problems, like tail sampling. Monitoring collectors is another area where I had to dig through the codebase to find what metrics are exposed, so documentation on useful metrics to monitor would be great. +I guess more example configuration would be helpful. For example, the load balancing—originally, we actually had another set of collectors that all they did was load balancing, but we realized we can actually load balance this deployment of collectors to itself, and that just is way easier to maintain. ---- +Yeah, and I guess more example configuration for common problems, like the tail sampling that we're doing here. -**Moderator:** Thank you for the feedback, Steven. Now, let’s check the questions from the audience. +**Speaker 1:** That's really good feedback. I have heard from other folks as well that we have the OpenTelemetry demo, which is a great showcase. I think it's really focused on application instrumentation, which is awesome, but I think there is definitely a desire from the community for more collector-specific use cases, and specifically stuff that's a little bit more complex than what you would necessarily see in the demo. ---- +**Steven:** For sure. Another one is monitoring your collectors. I found I had to dig through the codebase to find what metrics are exposed and what they are doing. I feel like there are probably some common alerts or things that for most setups would be useful. -**Dan (Moderator):** We have many questions from the audience, which is great! Feel free to vote for the ones you find most interesting. I have marked one of them as answered already. +Maybe someone already has this, but like, you know, some documentation like, "Hey, these are useful things to monitor." The dashboard for Grafana they have was pretty useful, but I think on the alert side, I didn’t find anything there. -Starting with the question about tail sampling: Do you only use tail-based sampling, or do you also use head-based sampling? +**Speaker 1:** Right, right. Cool! Well, thank you for the feedback on that. Now, I guess we can do one of two things. We've got the Lean Coffee board where folks can post questions for Steven. I'm actually going to check the board. -**Steven:** We only use tail sampling. It’s more work to set up, but having the full trace to make sampling decisions allows us to make the best choices. +Oh, we have some questions. Yeah, let's go through that, and then if we have some time afterwards, then Steven, you can ask some questions to our viewers as well. ---- +Dan, do you want to take it on from here on the questions from the board? -**Dan:** How do you handle container sizing on the collector sidecars? +**Dan:** Sure thing! We've got many questions from the audience, which is great. Also, feel free to go and vote for the ones that you think are the most interesting. -**Steven:** We don’t have one size that fits all. It would be nice if pods could add an annotation for the resources they need. Currently, we have a centralized configuration that creates sidecar configurations for different resource combinations, which requires coordination. +I have marked one of them as answered already because we talked about the collector builder and how you build your own dist for the collector on your sidecar. ---- +Starting from the one that we talked about tail sampling, a little bit about how you use the tail sampling, but I wanted to ask this question here: do you only use tail-based sampling, or do you also use head-based, and any tips or thoughts on mixing these approaches? -**Dan:** Do you use serverless? If so, do you have any tips for instrumenting serverless workloads with OpenTelemetry? +**Steven:** Yeah, so we only use tail sampling. I guess it's more work to get it set up, but when you have the full trace to make your sampling decision, I find that you can make the best decision. That's why we lean on tail sampling completely. I guess with head-based sampling, there's always a risk you're going to miss something important, right? Because it's completely probabilistic. -**Steven:** We don’t use serverless, so I don’t have any advice there. +**Dan:** Good point! Moving on to the next one: how do you handle container sizing on the collector sidecars? You mentioned this already, that you had to customize or tune one of the collector sizes. Do you have one size that fits all, or anything you would like to improve in that setup, or resource allocation? ---- +**Steven:** I think what would be nice is if pods could add an annotation saying this is the resources I need. Instead, we have to do something kind of hacky. For each sidecar, it's a separate Kubernetes resource or configuration file. -**Dan:** You mentioned using FluentD for logs. Are you considering moving to the OpenTelemetry log file receiver? +So what we do is we have a for loop that goes through and creates those configurations for different resource combinations. It's something we have to coordinate, like, "Hey, you have to update the sidecar configuration that your pod wants to use to this OTEL sidecar, 500 megabytes, one CPU." It's a bit of extra coordination work there. -**Steven:** It's something I’d like to explore. We had issues with logs initially, so we pivoted to using FluentD, but I’m curious to see if others have found good solutions for durability in logs. +**Dan:** Cool! The next one is related to serverless. Do you use serverless, and any tips or go-to's to look out for in instrumenting serverless workloads with OpenTelemetry? ---- +**Steven:** We don't use serverless, so I don't have any advice there. -**Dan:** Regarding monitoring your collectors, do you use OpenTelemetry to monitor your OpenTelemetry collectors? +**Dan:** That's fine! I wanted to ask that in case you had some experience with that. Also, you mentioned you use FluentD for logs reading from standard out. Are you considering moving to the OTEL log file receiver? -**Steven:** Yes, we use Prometheus to get metrics from the collectors and FluentD to collect logs, and that’s worked pretty well. +**Steven:** Yeah, it's something I'd like to explore. We sort of had a bit of an issue when we first tried using logs. I'm curious if anyone has found a good solution for this. What we wanted to do is store our logs on disk so if there's an outage, we don't lose them. We ran into a bug with the durable storage extension in the collector, and that kind of spooked us. We already had FluentD running, so that's why we kind of just pivoted to that. But I don't know; that was a while ago. Maybe people have had good success with getting that durability for their logs. ---- +**Dan:** Just to add on this, with FluentD, are you decorating your logs with some of the semantic conventions or trace ID, span ID, so you can then correlate with your traces? -**Dan:** You have a dedicated platform team managing the collectors. Do you have thoughts on not having a collector gateway and sending telemetry data directly to your observability provider? +**Steven:** Yeah, so we definitely have the trace IDs and the span IDs, and we're actually adopting a specification for the event—they call it log events. It's sort of a structured format for logs in the OpenTelemetry specification, so we're trying that out as well. -**Steven:** We decided to use collectors to control costs and provide durability if the vendor goes down. While it simplifies things, there are trade-offs. +**Dan:** Cool! Do you use OTEL to monitor OTEL collectors? ---- +**Steven:** Yeah, it's like a big—it's always called monitoring your monitoring. We use Prometheus to get the metrics from the collectors and FluentD to collect the logs from the collectors, and that’s worked pretty well. -**Dan:** Do you find that developers are aware of auto instrumentation, or do they tend to do manual instrumentation unnecessarily? +**Dan:** Okay, moving on to a bit more of your team. Sounds like you folks have a dedicated platform team that manages the OTEL collectors in your organization. Do you have any thoughts on the opposite of this, where you have no collector gateway or no collector endpoint, basically 100% of your telemetry data is then sent to your observability provider? -**Steven:** Yes, we have seen some redundant data from applications that instrument on their own. It comes down to education and tracking cardinality to help teams understand when they can disable unnecessary instrumentation. +**Steven:** I guess in our case, we decided to use the collectors so that we could do sampling to control costs of our traces. We also try to have some durability; if the vendor goes down, we can move that retry logic off to the collector. ---- +I'm curious if anyone has done that in production. It would definitely simplify things, but I guess there are some tradeoffs as well. -**Dan:** How do you manage sampling policies? Can teams dial that configuration themselves? +**Dan:** Right, there are tradeoffs as well of authentication to a provider. It does sell in one place as well, some of the benefits, pros and cons I guess. -**Steven:** We have a simple setup where we sample based on errors and trace lengths. Teams can force sampling by adding an attribute to their span, but we monitor its use to prevent issues. +The next one is related to auto instrumentation and the OTEL operator. You inject those instrumentation packages, and we also talked about the fact that you are also using manual instrumentation. I guess the question is, how do you find if your teams found that auto instrumentation is sufficient, or is there a lot of custom instrumentation that's needed? ---- +**Steven:** One thing I haven't talked about that has been great for us is the span metrics component. We generate metrics—frequency, latency, error counts—for every single span. -**Dan:** Do you currently use any configuration that may not be supported through environment variables, like metric views? +Because we're using the auto instrumentation, every application emits the spans in the same format. Out of the box, without adding any instrumentation, we get some what folks call APM or RED metrics—like all your API calls, all your database queries. We have metrics on that automatically. -**Steven:** We haven’t had to do that much. When we do, we usually show teams a merge request to copy paste from our configurations. +That goes quite a long way, and we can build common dashboards and common alerts. All these apps just get out of the box because of that standardization, so it's a really good starting point. If teams want to add metadata, they can. I would definitely recommend span metrics if that's an option for people. ---- +**Dan:** I'm going to add a bit to this one as well. Do you find that if you're moving from a world where you had to instrument everything, and now there is a lot of auto instrumentation out there, do you find that engineers normally tend to not know about that auto instrumentation, and they do it themselves? How do you manage that sort of education piece, saying that you may not need to instrument things? Has that been a challenge within your organization, or has everyone just...? -**Dan:** How do you handle supply chain security for the OpenTelemetry SDKs? +**Steven:** I'd say it is. We have some redundant, like for example our API metrics—Spring Boot and Java automatically emit some metrics, so a lot of applications were getting some redundant data. So yeah, that has been a problem, but I guess it just comes down to education. -**Steven:** I created a wrapper dependency to manage OpenTelemetry library versions that work together, but it can cause dependency conflicts in some cases. +Then having a way of tracking the cardinality, when we've seen, "Oh, you're sending a ton of API metrics, and we already captured the span metrics," we can reach out to that team and say, "Oh, you can probably disable this. You don't really need this. We don't even pay for it." ---- +**Dan:** Cool! Regarding sampling, I'm assuming this is related to sampling policies. Can you advise on whether that’s the collector where you manage the policies for all traces, or can teams dial that config up and down themselves? -**Dan:** Is the OpenTelemetry operator your preferred solution, or can it be used in parallel with SDKs? +**Steven:** Ours is pretty simple. We sample errors; we keep traces that are longer than something like five seconds. We do have an escape hatch if they add an attribute to their span to force sampling. So far, no one has caused any problems with that, but it might in the future. Teams may overuse it, but that works so far. -**Steven:** So far, we haven’t had issues with teams deviating from the injected configurations, but I’m curious how that will evolve over time. +**Dan:** Related to some of that SDK config that you currently inject through environment variables, do you currently use any type of other config that may not be supported through environment variables, like metric views, for example? Do you have a way to apply defaults across the board for that type of config? ---- +**Steven:** We haven't had to do that too much, and when we have, we do have—our company is lucky in that we have a tool that can use to generate applications from scratch. There’s a way to sort of regenerate it. -**Dan:** Are developers using the LGTM Docker container from Grafana for local testing and troubleshooting OpenTelemetry implementations? +The regeneration doesn't work too well, but we know that most teams have a pretty similar format to their application, so when we do need to make a change that we can't do with the operator, it's usually just boilerplate copy-paste. We can show them a merge request, and they can copy it in. But it does take a couple of months for every team to adopt that. -**Steven:** Not yet; they can only test by deploying to our clusters. We need a better local testing solution. +**Dan:** Cool! Okay, I guess one related to supply chain security with the OTEL SDKs. How do you handle supply chain security so that the OTEL SDKs are safe to include them in application code, or do you build them from source yourself and then run your own security tests? ---- +**Steven:** Are you thinking this would be, you know, if there's a CVE in one of the SDK versions, how would we sort of manage patching that for everyone or downstream or upstream dependencies from OTEL packages? -**Dan:** If you had a magic wand, what would you love to add, remove, or change in OpenTelemetry and why? +Yeah, for different reasons, I kind of created a wrapper dependency that wraps all the OTEL library versions we use and we know they work together just because we found some versions were kind of incompatible. -**Steven:** I would get rid of Prometheus in our architecture because we rely on it for aggregating data for cost savings. The collector doesn’t currently offer that functionality, so it would simplify our architecture. +From the app team's perspective, they depend on one dependency, and then they get all the OTEL dependencies as transitive dependencies. If we need to upgrade it very quickly, I guess it would just be one thing they need to update. ---- +Although that works in theory, it has also caused dependency conflicts just because of how Maven works in Java, so I don't know if that's actually the best thing to do—it might be more of a headache than it's worth. -**Dan:** Are you using multiple OpenTelemetry backends for processing, or does everything go to one vendor? +**Dan:** Thank you! Okay, moving on to the next one. Related to the OTEL operator and automatic instrumentation: is that your preferred solution, or can the OTEL operator be used in parallel with SDKs, for example, in the case of .NET? Do you provide some balance of scalability and configurability? Is there a recommendation here? -**Steven:** We have one vendor that we’re pushing to; we don’t push anything internally to anywhere else. +**Steven:** My immediate concern is that the manual instrumentation inside the application could make it difficult to configure and manage at scale. So far, we do—you can't override the environment variables that are injected into the application automatically by the platform team. ---- +So far, I guess it hasn't been too much of a problem. Teams haven't really wanted to deviate too much from the configuration we've given them, but I’d be curious long term if that does cause problems. -**Dan:** Given the potential for bottlenecks in collectors, do you see disadvantages in having the SDK export data directly to the back end? +**Dan:** Thank you! This one is interesting: are your devs using the LGTM Docker container from Grafana to develop OTEL implementation and then do local troubleshooting? -**Steven:** It could be easier to handle smaller amounts of data directly from the pods, but it shifts the back pressure to the application, which could be impactful. +**Steven:** Not yet. Right now, they can only really test it by deploying it to our clusters, so we do need a better solution there so they can test locally. ---- +**Dan:** I personally love that there are more and more solutions coming up that are open-source that allow you to see that telemetry in your own computer for testing. So yeah, a lot of really cool solutions out there at the moment. -**Moderator:** Thank you, Steven, for joining us today and for answering all these questions. It was great to see such engagement from the audience! +If you had a magic wand, what's the thing that you would love to add, remove, or change in OTEL and why? -**Steven:** Thank you, Dan, for hosting this and thanks to everyone for the great questions. It’s a topic that really interests people, and I appreciate the engagement. +**Steven:** That is a logic question! I think this is maybe short-sighted, but if we could get rid of Prometheus in our architecture, the reason we can't right now is because we rely on Prometheus to aggregate our data together for cost savings. -If anyone out there would like to participate in OpenTelemetry Q&A or OpenTelemetry in practice, we’re always looking for folks to share their use cases. You can DM either Dan Reyes or me or message us in the user channel in the CNCF Slack. +For span metrics, we generate like 50 histogram buckets for every span, so it gets really expensive. If each pod has its own set of 50 buckets, our cost scales with the number of pods. But we use Prometheus to aggregate all the counts from all the pods, and that cuts our cost significantly. -We’ll post the recording of this session on YouTube, so let anyone know who may have missed it. Thank you again, everyone! +I don't think the collector can do that today, so yeah, that would make our architecture simpler. + +**Dan:** Makes sense! Are you using multiple OTEL backends to process some of the things internally compared to your vendor, or is everything going to the vendor? + +**Steven:** We have one vendor if that was the question that we're pushing to. + +**Dan:** So I guess you're not pushing anything internally to anywhere else; just everything goes to... + +**Steven:** Oh right, yeah, we don't. + +**Dan:** Okay! I think we've got time for one last one, and that is: given that the collector can quickly become a bottleneck due to back pressure, do you see any disadvantages to having the SDK export that data directly to the backend, except for the cost? + +**Steven:** Yeah, I think that was similar to the other question about why not just push directly to your vendor from your pod. It does make it easier to kind of scale; you don't have to worry. Well, yeah, that's a good question. I guess you're shifting the back pressure then to your app. + +It might be easier to handle if it's a smaller amount of data, but yeah, that one I'm not too sure about. + +**Dan:** That would be interesting to discuss. It might be easier to handle but also more impactful for the application. I guess there are pros and cons there as well, right? + +So if you have all your telemetry shipped from your pod as soon as possible, there are benefits for that as well in terms of impacts. + +**Steven:** Yeah, exactly! + +**Dan:** Okay, I think that's all we have time for. Thank you very much, Steven, for joining us. We had quite a lot of questions we went through from the audience. Thanks for everyone that joined and asked questions as well. It's super nice to see engagement in a session Q&A. + +Do you want to say a few words to say goodbye? + +**Steven:** Yeah, thanks, Dan, and thanks for doing the questions. It's really nice to see the engagement from folks in the Q&A. I think this is a topic that really interests people, so we love it when you engage. Also, if anyone out there would like to participate in OTEL Q&A or OTEL in practice, we are always looking for folks to share their use cases with us. + +You can DM either Dan Reyes or me, or if you want, you can also message us in the user channel in the CNCF Slack. We would be happy to hear your use cases and have conversations. Also, as I mentioned, the recording for this will be up on YouTube, so anyone that you know that wishes they had attended but could not, let them know. + +We usually post on the OTEL socials and also in the channel once the video is up. Thank you, Steven, for joining us and answering all the various questions. We really appreciate your time. + +**Steven:** Yeah, that was a lot of fun! Thank you for hosting this. ## Raw YouTube Transcript diff --git a/video-transcripts/transcripts/2024-11-04T21:03:07Z-otel-q-a-with-dan-ravenstone.md b/video-transcripts/transcripts/2024-11-04T21:03:07Z-otel-q-a-with-dan-ravenstone.md index 6af26fa..4eb3a83 100644 --- a/video-transcripts/transcripts/2024-11-04T21:03:07Z-otel-q-a-with-dan-ravenstone.md +++ b/video-transcripts/transcripts/2024-11-04T21:03:07Z-otel-q-a-with-dan-ravenstone.md @@ -10,100 +10,358 @@ URL: https://www.youtube.com/watch?v=BMkckCQN8eg ## Summary -In this YouTube video, Dan Ravenstone, a staff engineer at Top Hat, discusses the evolution of monitoring to observability, particularly in the context of using OpenTelemetry. The conversation includes insights about the challenges organizations face when adopting observability, the importance of user experience in guiding instrumentation choices, and the cultural shifts necessary for developers to embrace new practices. Dan reflects on his experiences transitioning Top Hat from a vendor-specific monitoring tool to OpenTelemetry, emphasizing the need for effective communication and patience while educating teams. He highlights the role of structured logging, the significance of service level objectives (SLOs), and the necessity of aligning tools with organizational needs. The discussion culminates with audience questions addressing telemetry quality, transitioning from logs to tracing, and best practices for ensuring user satisfaction. +In this YouTube video, Dan Ravenstone, a staff engineer at Top Hat, discusses his extensive experience in the fields of monitoring and observability, particularly focusing on the transition to open telemetry. The conversation is led by Adrian, who shares insights on the upcoming Cucon event in Salt Lake City. Dan introduces his background, highlighting his journey from traditional monitoring tools to modern observability practices, and emphasizes the importance of understanding user experience as a key driver for effective observability. The discussion also tackles challenges organizations face in adopting observability, including cultural resistance, the importance of communication, and the need for developers to take ownership of instrumentation. Dan shares anecdotes from his experience at Top Hat, where they are migrating to open telemetry and highlights the necessity of aligning observability practices with user needs to drive meaningful outcomes. Throughout the Q&A, they address various audience questions related to telemetry quality, instrumentation strategies, and the impact of user experience on organizational priorities. ## Chapters -Sure! Here are the key moments from the livestream along with their timestamps: +00:00:00 Welcome and intro +00:01:20 Dan Ravenstone introduction +00:03:40 Observability evolution +00:04:00 Challenges in alerting +00:07:00 User experience and observability +00:09:30 Observability adoption challenges +00:10:40 Vendor-specific libraries issues +00:12:50 Cultural shifts in instrumentation +00:20:00 Communication challenges +00:30:10 Instrumentation culture at Top Hat +00:34:22 Collector landscape discussion +00:40:11 Questions from the audience -00:00:00 Introductions and event overview -00:02:30 Dan Ravenstone introduces himself and shares his background -00:05:45 Discussion on the evolution from monitoring to observability -00:10:00 Key changes in observability and the challenges with alerting -00:15:15 The importance of understanding user experience in observability -00:20:00 Challenges organizations face in adopting observability -00:25:30 The significance of using open Telemetry versus vendor-specific libraries -00:30:45 Cultural and technical shifts required for observability adoption -00:35:00 The role of instrumentation in development and its impact on user experience -00:40:15 The current state of observability at Top Hat and future plans -00:45:00 Q&A session begins with audience questions +**Host:** Welcome everyone to the Otel Q&A, and thanks Dan for joining us for Otel Q&A, especially on such short notice. I'm glad we're able to squeeze you in for October, especially with KubeCon coming up in November. So for folks who are going, a few of us will be there. Dan and I will be there from the Otel team and user Sig, so it'll be busy. -Let me know if you need any further assistance! +**Dan:** Yeah, it's going to be busy. When is that happening, by the way? -# Otel Q&A with Dan Ravenstone +**Host:** It's in Salt Lake City. -Welcome everyone to the Otel Q&A! Thank you, Dan, for joining us on such short notice. I’m glad we were able to squeeze you in for October, especially with KubeCon coming up in November. For those who are going, a few of us will be there—Dan and I will represent from the team. +**Dan:** Salt Lake City. Nice! I've never been there myself personally. -### Introduction +**Host:** Yeah, me neither. -**Dan:** -Hello! I’m Dan Ravenstone, a staff engineer at Top Hat. I’ve been here for a bit over two and a half years, almost three by May. My background is mainly in monitoring and observability. I got my start at a company called Affiliat in Toronto, which managed the .info domain registry. +**Dan:** Oh, so this will be an experience for you too. -I started in tech support but quickly realized that I was tired of customers calling us to report downtime. So, I took the initiative to learn monitoring tools. Back in those days, we had tools like Big Brother, Nagios, and Cacti. I loved monitoring because it was like being on the front line, finding problems before they impacted customers. +**Host:** Yeah, looking exciting. -As technology progressed, I fell in love with observability and how it provides a different perspective. Being proactive is far superior to reactive strategies, which many people still struggle to grasp. +**Dan:** Um, okay, let's start asking you questions. I realize I'm the one here who is supposed to be with the breaks. Folks, it's going to be a fun one. -### Changes in the Industry +**Host:** Oh yeah, not sure about informative, but it will be fun. -**Interviewer:** -You’ve been in this field for a while. What major changes have you seen in monitoring and observability? +**Dan:** Well, here we go. Okay, let's start. Why don't you introduce yourself and tell us what you do? -**Dan:** -One thing that hasn’t changed is alerting; it remains a problem. However, in observability and especially with OpenTelemetry, we’ve learned to look at our services differently. In the past, monitoring was reactive—we only had symptoms to deal with. For example, Nagios would alert us to issues based on CPU, memory, or disk space, but that’s not always indicative of a problem anymore. +[00:01:20] **Dan:** Hello, I am Dan Ravenstone. I am a staff engineer at Top Hat. I've been there for, I guess, two and a half years maybe? No, a little bit longer than that. Come May, I think it'll be three. A little bit of my background: I have been doing monitoring and observability for the bulk of my technical career. I started back in, um, where I actually refer to as like where the bug bit, the monitoring bug bit me, was when I was at a company called Affiliat, which was at Young and 401 area in Toronto. Affiliat managed the info.org domain registry in the back end of it, for back in the day. I started there in tech support, but then I one day opened up my big mouth and said, "Hey listen, I'm tired of our customers calling us up and telling us our stuff is down. Can we fix this?" And so they said, "Well then you ask for it, you get it." So I went out and that's how I started learning about monitoring tools. -With the rise of Kubernetes and containers, we’ve had to adapt. We started aggregating logs, but even that posed challenges. Despite best practices, people would often log inconsistently. Observability has taught us to focus on overall user experience rather than getting lost in the minutiae. When in the middle of an incident, it’s crucial to decipher which data points truly matter. +**Dan:** And that's back in the day of Big Brother, Nagios, Cacti, all those fun things. And I just loved it. This kind of like, I think one of the things I liked about monitoring in those days was that I like looking for patterns. I like detective novels; I like all the fun things. And what this does is it kind of allows for somebody to be on the front lines in operations and dive in and find problems and report them back. -### Challenges in Adoption +**Dan:** The better your monitoring tools were, the more likely you were able to follow and find the problems a lot sooner before they actually become customer impact. And that has always been my kind of push when I did this. And of course, as things progressed and modernized and changed, we got into the wonderful world of observability and a whole different way of looking at it. I fell in love with that obviously as well and tried to grow with the times and see how being proactive is a lot better than being reactive, which is kind of like, you would think is an obvious thing, but it takes a lot of people time to kind of get their heads wrapped around. -**Interviewer:** -What are some challenges organizations face today regarding observability? +**Dan:** So yeah, that is kind of how I got to this part of the world and a little bit about me. There's tons of other things, but we don't want to go into those because then we'll get lost in the weeds and we'll never get out and we'll be lost forever. -**Dan:** -Communication is a big issue, often stemming from cultural barriers. There’s no technical reason for organizations not to adopt observability other than resource constraints. For instance, our team is moving away from a vendor-specific library to OpenTelemetry. Transitioning takes time and can be a tough sell to executives, especially when discussing costs. +[00:03:40] **Host:** Sounds good! Well, thanks. And, you know, you've been in this for a while. Like, when it started out with monitoring, evolved into observability. What kinds of big changes have you seen in the course of that time? -Additionally, once organizations make the shift, they often struggle with implementation and proper usage. Many teams try to adopt OpenTelemetry but end up overloading their systems with data without understanding how to manage it effectively, leading to rising costs. +[00:04:00] **Dan:** Let me start off with the things I haven't seen change, which is still a problem: alerting. Okay, alerting still has not changed, surprisingly. But what has changed, and I think this is one of the things that we're trying to get to—I focus on alerting because I was giving myself time to think—what has changed though in observability and in OpenTelemetry especially, too, is it allowed for us to actually look at our services a lot differently where monitoring was kind of a very largely reactive thing and we only had the symptoms of what was going on. -### Tools and Cultural Shift +**Dan:** So if everybody has ever used Nagios, they're well aware of the sort of out-of-the-box HTTP monitoring sort of plugin you usually include, which is like your CPU, your memory, your disk space, and these things have been carried throughout time to be indicators of a problem. But that's usually not quite the case anymore, is it? Especially when we have Kubernetes and containers and other tools now that these things shouldn't really play a role for us. So it's more like, what else is impacting the service? We never really had the ability to look at it until we started aggregating our logs. And even when we aggregated our logs, it was still strangely problematic because, you know, we had guidelines on how to do structured logging and that kind of thing, but still people would do their own thing. -**Interviewer:** -You mentioned that the adoption of observability tools is as much about culture as about technology. How do you approach changing the culture around instrumentation? +**Dan:** And things were never, and so even with all the good documentation and sort of best practices around it, logs were still problematic. And again, you still have to collect a bunch of data before you can actually understand, oh wait a minute, this is a problem before actually kind of knowing beforehand what observability has taught us is how to sort of like set things up in more of a way that we're looking at it as an overall user experience. We don't get sort of sucked into minutia sometimes, and there's because there's a lot of red herrings in operations, especially when you're in the middle of an incident. You're looking at all this data and you're like, what is the actual problem? We have high CPU over here, and we have slow load times over here, we have latency here, we have some errors here, but which one of these is the smoking— I would not want to use a gun, but for lack of a better word, smoking gun, right? -**Dan:** -It's essential to connect instrumentation to the user journey. When developers understand how their work affects the user experience, it becomes easier to convince them of the importance of proper instrumentation. We encourage teams to document their user interactions and understand the dependencies and prerequisites. +**Dan:** And I see that with observability has changed how we look at that. However, that's for this group; a lot of people are familiar with it. There's a whole world of engineers out there who are still trying to get their heads wrapped around this concept. And this is why I'm glad we have these conversations. I try to get those people, this is targeted towards them, of why this is so important, why you need to start looking at this. -Ultimately, the goal is to make instrumentation part of the development culture, not just a checkbox. We must support our developers in learning and adapting to new tools without overwhelming them. +[00:07:00] **Dan:** Because there's a number of good things. One, you know what the user is experiencing, and that's all that matters. If you know what the user is experiencing, that means you can translate that to how much you're making. It's a cost thing and it's a money thing. If you know if your users are happy, that means you're making money because they're spending money because they're going to keep using your service. If they're not happy, they're not going to be using your service, and they're going to walk away and use something else or just not use it that often and try to avoid it as much as possible. That means you're not going to be making the money anyway. -### Observability in Practice +**Dan:** And I'm going to get in the weeds, so you're going to hear me say that a lot because I just do. So, Adrian, it's all up to you to rope me back in. -**Interviewer:** -What is the instrumentation culture like at Top Hat? +**Host:** Don't worry, don't worry. But I do want to— you said something really interesting in terms of translating it in terms of money because, you know, like observability adoption— and maybe I'm getting ahead of myself here— but I like the way you said in terms of observability adoption, like it's as much a top-down thing as a bottom-up thing. And one way to speak to the benefits of observability is in dollars and cents, and executives speak in dollars and cents. -**Dan:** -Currently, it’s still a work in progress. We have some auto instrumentation in place, but many developers are still getting used to it. There’s a tendency to view instrumentation as just a task to complete rather than a valuable part of their work. Our goal is to make it as seamless as possible and to show the value in real-time data. +**Host:** So proving the worth of having an observable system, I think, becomes really really important. And being able to use that language with executives to be able to convince them, I think, is very valuable. -### Final Thoughts +**Dan:** And that's what I've been trying to focus more on. There are quantifiable reasons why to look at this. We can do that. Why observability, excuse me, provides value toward us at the end of the day. But I think we're still trying to get that figured out. I feel like that, um— anyway, you know what? Before we jump in, let's keep moving forward because I could go into a whole other area and I don't want to sort of suck a lot of time in there. -**Interviewer:** -What advice do you have for teams adopting observability practices? +**Dan:** But it is one of those things I think that is worth having conversations, not just with those in the community, but also talk. Like, I mean, if you're out there, find out what the observability sort of platform or the concepts are where you work. What are the drivers? Why aren't you doing it? Why aren't you using these things? And ask those questions. And then maybe we could help feed that back to the community because that could help us build out the reasons why you should be doing this because it does make better sense eventually. -**Dan:** -Always focus on the user. If users are unhappy, it doesn’t matter how much data you collect; you need to know what frustrates them. Having clear metrics and objectives based on user experience is vital. The right tools can help, but they should be the last consideration. +[00:09:30] **Host:** I completely agree, and I think this takes us to our next question, which is, you know, like you've done this for a while now— first the monitoring space, now moving into the observability space. What do you think are some of the main challenges that most organizations are facing these days when it comes to observability? -### Questions from the Audience +**Dan:** There's a lot of challenges and I've been trying to get my head wrapped around some of this stuff. One is communication, I think. Some of it I think is mostly cultural, really, if you think about it. I mean, there's not a technical reason why people shouldn't go to this other than resourcing and time. -1. **What is your opinion on telemetry quality as a way to reduce spending? Is more data always better for observability?** - - Quality is key. More data doesn’t necessarily mean better observability. It’s essential to capture data that reflects user experience. +[00:10:40] **Dan:** And I mean, this might be jumping ahead a little bit, but where I am right now, we are using a current vendor, which will remain nameless, but we use a current vendor right now for our monitoring and observability purposes. But we're getting away from them because the library we're using is vendor specific, and that means that we're basically—when it comes to doing any kind of telemetry, it has to use that particular library. And theirs is only getting— and that's one challenge because if you think about it, like, okay, so you're using this one vendor, it's costing you X amount per month or per year. It's exorbitant. -2. **How do you approach the "what to instrument" question with your teams?** - - We start with understanding the user journey. Documenting user interactions helps pinpoint what needs to be instrumented. +**Dan:** However, to get off of that one and go into another one where you have more options, like with OpenTelemetry, takes time. It's not like, anybody who's ever gone on any kind of migration from one tool to another knows how long it takes. And you have to have a few things in place to actually say why you should do this. So one is, of course, obviously the cost, right? But you know, it's hard for some executives to say, well, we're spending this, but it'll cost us this much to get over here and then we'll see the cost value. -3. **In an organization that has adopted distributed tracing, how do you make teams leave behind outdated practices based on logs?** - - You need to understand their reluctance and restructure discussions around their viewpoints. Help them see the value of distributed tracing in understanding service interactions. +**Dan:** So there's this huge challenge of just trying to shift things over, spend the time to do it, and then do it properly. And that's a whole other challenge is to say, okay, yeah, doing OpenTelemetry is one thing, but doing it properly, like every other software or any other tool that you have out there, you have to still use it for the right reasons and do it properly. If you don't, you'll still run into issues, and whatever you're trying to sort of get away from over here will still might come up in a different way over here. -4. **What questions do you ask to understand what is most important to someone's organization?** - - Always focus on the user experience. If you understand their needs and frustrations, you can tailor your approach effectively. +**Dan:** So that's one of the challenges. Just doing the whole shift over, I think that's where a lot of people— and they don't know how to do it. And so what happens too is then they do a shift, they try it out, and then they get burned because, you know, they don't know how to do sampling or something like that. And then they feed it off to some tool like, let's say, a tool that you get charged by event volume. They're going to— even though they're using OpenTelemetry, they still became like, oh, I saw one company who did that. They started shifting to using OpenTelemetry, but they started sending all this volume, all this data to it. They didn't do it right; they were having like a hundred thousand spans per trace, which is a huge amount, and they weren't really getting a lot of value. -Thank you, Dan, for sharing your insights today! It was a pleasure having you on the Otel Q&A. +**Dan:** And their costs were soaring. And so it's like, well, you know, then they kind of just dropped and said, well, we'll figure it out another way. We'll just use some other tools to figure out when there's problems with our service, which is sort of like a defeatist kind of attitude. But it does happen because they don't, you know, because there's a lot of work involved. Doing this is more than just a simple lift and shift; it's a lot more involved. + +[00:12:50] **Dan:** I'm trying to think of other challenges— those to me are some of the bigger challenges. And I think a lot of people could probably empathize with that one as well. There are others, obviously, but I mean, we don't have a whole lot of time. We could probably go on just you and I alone. I bet you we both have experiences on some of those other challenges we have to experience. But I mean, I like to see like, how do I get past that is kind of where I want to target. How do we get past these challenges and move forward, which I don't have a proper answer to, but I do want to talk about it. + +**Host:** You touched upon something that I think is worth mentioning as well, which is moving away from vendor libraries to OpenTelemetry. Because, you know, even though I think one of the wonderful things about OpenTelemetry is that it's vendor-neutral, and so it makes it, once you've moved all of your instrumentation to that, super easy to switch vendors. I would say relatively easy, right? Because they're still— like from the instrumentation standpoint, it makes it super easy. + +**Host:** But the challenge though is convincing folks to move away from those vendor libraries and I think that becomes a bit of a sticking point because you need to move away from them because maybe that vendor is too expensive and you don't want to go with them, and you want to try something fresh, but there's so much time and effort and learning curve and all that stuff associated with reinstrumentation. What are your thoughts around that? + +**Dan:** Well, and that's a very powerful point you’re making because first off, with vendor libraries, you get in the habit of doing things a certain way, right? So again, I do not want to—there's no—like, but the thing is, so some vendors may do things one way, but it's not part of the industry standard or the best practice we see across the industry. So they may like name things a certain way or do things a certain way, and people get used to that. + +**Dan:** So not only is it just a matter of shifting a library over—if it was just a matter of shifting a library to one or the other, that's in itself a challenge. But there's also all the behavior that gets attributed to using that older library. For some things, you got to go, well listen, you don't need to look at it like that anymore. You need to look at it like this now. And so it's not just a technical shift; it's also a cultural shift and how they even envision how their service is working with the new library, with using OpenTelemetry. + +**Dan:** Because OpenTelemetry is current; it's adding new things as we speak every day. We're seeing a lot more love now for mobile, which is wonderful. I'm so excited to see some of that because I feel that's largely been ignored for a long time. But we're seeing these things getting pulled together, and we're having this— and it's kind of like keep following these sort of patterns and following these standards. + +**Dan:** And so it forces the developers to think a certain way. A lot of them are not used to that yet. A lot of them still want to develop—and I realize one of the things I noticed too was from a developer's perspective— I don't know if Adrian, you do—you're a coder by nature, right? Like that's kind of—yeah, that's your back, so it's not mine. But I know that when I did do some coding though, I would always print a line at certain parts in my code to say, okay, where am I at? + +**Dan:** So even logging was part of your local development experience, right? Now we're telling them, don't worry so much about logging; use instrumentation. Well, you still have to have a mechanism for them to actually print that to the screen. And so it's just changing how they even develop in the beginning. And that, I think, is where we're facing some challenges. It's not just a technical shift; it's cultural. It's about even how what you've been taught in the universities and how you've been taught how to develop is changing a little bit. + +**Dan:** And not everybody's quick to embrace change. And you know, especially when you've got your CEO or your features team or product team saying, we need this out now. Well, you know, you're not going to worry about trying to learn a new way of doing things. And you’re saying, oh, well now how is my service working from an OpenTelemetry perspective? It's like, well I just need to get this out, so I just slap these log lines in, boom, gone. It passed, you know, and we're going now; it's in production, and we move on, right? + +**Dan:** So there's that kind of like also encouraging and supporting our developers to have the time to learn this and work with it so they can actually use it to their benefit and perhaps be able to do better code before it even hits staging or development. And then once it's in production, you know how it should be based on all your stuff. But that's—it's getting that linking those people together, getting them to work together in that thinking. That's a huge challenge. + +**Host:** Yeah, absolutely. And getting them used to, as you said, the nuances of, you know, instrumenting with OpenTelemetry, which OpenTelemetry wants to standardize how you're doing your main signals in a certain way, right? Which, as you said, may be different from how you've been used to doing it, whether it's like through your print statements or vendor libraries or whatever. + +**Host:** It's kind of like—so back in the day, this might be pre a lot of folks, I don't know, but I used to work a lot with SNMP. And what I liked about SNMP was it was basically a protocol that was agreed upon by the community, and it was very basic. But even then, I saw this with logging too, but even with SNMP, I would still come across MIBs. I would still come across libraries that were not properly done, so it required me even—because I knew it, I would have to go in and kind of redo maybe a MIB to translate the OID properly, so when it goes into our— or create a MIB because it wasn't available. It just had these OIDs that were available from an interface or something, and you just had to guess what they were. + +**Host:** Even with standards in place like that, something that has been in our sort of part of our industry for so long, no longer really being used anymore—well, actually, it still is by network devices, but that's not—hopefully we don't have to worry about that right now. + +**Host:** It’s that, you know, even with those things in place, it still takes time for people to even—there are still mistakes and things that happen. So with a new thing like OpenTelemetry and being setting standards, it's taking a long time for people to understand how to use it to get the most value out of that instrumentation, the most value out of using that from the beginning of the development cycle. + +**Dan:** Yeah, absolutely. I do want to switch gears a little bit and talk a little bit about specifically the challenges that you're facing currently in your role when it comes to observability. + +[00:20:00] **Dan:** Oh, I'm really not my own mental health problems. Okay, that's different. Yeah, so my challenges have pretty much always been—I feel like it's always been the same—it's communication. And not because I'm—my challenge for me, so I've been in this business for a while, so what happens sometimes is I make the mistake or the assumption that folks know what I'm talking about. + +**Dan:** I have to be careful sometimes. I can have these conversations with many people in the same community, like within the monitoring and observability community, and we could go on for hours about little nuances of something, whatever it may be—logging, or tracing, anything like that. And we can go on and talk about that ad nauseum. However, when you're talking to other people and you talk about basic things like the rate of errors, rate of requests, duration of requests, which is a very basic kind of concept in the monitoring world, which is helped—kind of like we see a lot in observability too to a certain degree—people don't understand that. + +**Dan:** You make the assumption that they actually kind of look at things in that light, but they don't. They still look at it from the old school and then—like honestly, at some places I've been, it's like they still see CPU as an indicator of a performance problem. But it shouldn't be, in my opinion. It should be because CPU can be cheap; you can get more. So that should be a scaling problem. Make sure you're scaling properly; you should be good. + +**Dan:** However, I make the assumption people know what I'm talking about, and that's one thing is I got to—so I have to constantly be patient and communicate the same message over and over again to make sure they understand. Because even if I tell them two or three times, they still don't clue in. I had one conversation with one engineer for about a month, and it took a month for them to actually understand what I was driving at. + +**Dan:** And this is like because they were in different time zones, so that's why it took a little bit longer. But it took a while before they actually, oh, that's what you meant. And as soon as they were able to make that change, everything was working better. And like, oh, now I see where you're driving at. I now see the point you're trying to make. And it's just you have to be diligent; you have to be patient; you have to communicate. + +**Dan:** And you have to change how you communicate sometimes too. Like you just can't say the same words over—you have to rethink, okay, you're not getting what I'm saying. What will help you understand? And then try to reword it so people understand. + +**Dan:** Adrian, you met my friend Damian, who I work with. He's actually at the same place as I am. But Damian, who also works with me on a lot of stuff, he has a different background, but he sometimes steps in and sort of translates what I'm saying because sometimes it helps to have a different—because he puts a different spin on it. And oh, now I get it. + +**Dan:** So those are some of the biggest things I find today are still the biggest sort of people get it. It's just a matter of once—just getting them—just lead the horse to water type thing, right? But I am—it's just leading them there. Once they're there, they eventually do start to drink because they kind of get, oh, I see this is good for me; I will do it now. + +**Host:** For sure. But getting there, sometimes you gotta kind of be patient. You gotta pause for a while and say, okay, yeah, you want to look at the pretty flowers for a moment. That's cool. You watch the grass grow. Great. + +**Dan:** Okay, finally, I'm equating developers and engineers to horses, so hopefully nobody will take offense to that, but in the nicest possible way. Honestly, I mean, you make such a good point though with experimenting with different ways of doing the messaging because, as you said, you can't just keep repeating the same thing and then hoping that people are going to get it. + +**Dan:** You have to come up with different ways of saying it. And another thing that you and I have discussed before also is you can't tell them that they have an ugly baby. You can't say, your thing sucks; my way is the better deal because people will get very offended, rightfully so. So you have to be very tactful in your messaging. + +**Dan:** Exactly. No, you're— and that's right because they'll just take offense, and then everything you say afterward falls on deaf ears. So you don't want to attack them; you've got to draw them out. We wear a lot of different hats in this role when it comes to observability, I feel. It's not just about knowing how to do it technically; it's about assisting folks to see how they can learn from it and grow with it too. And that is a huge challenge. + +**Dan:** That's why I've done a lot of my videos, not too many of them have been exposed publicly; I've done them internally at work. But I will put on different characters. Like I would honestly do different characters to help people understand just so they have something. So it’s not just another boring—oh, here we go, getting told off about monitoring observability. Yes, we know Dan, logs are not monitoring. Okay, but I try to build it up so they resonate a little bit because they're more engaged. + +**Dan:** Because, oh, Dan's put on a funny wig and he's making a funny voice, but that point actually does make sense. So then, you know, it starts to click eventually, I hope. Maybe I'm wrong; maybe they just look at me and laugh and just walk away. + +**Host:** Nothing sticks. + +**Dan:** Nothing sticks! I like that though because you're also putting on different personas, right? And I think people also resonate with different personas. They have to, at the end of the day, it's got to be how does this benefit me, right? Because otherwise, if they don't see where they fit into all this, then it's falling on deaf ears. + +**Dan:** Exactly, how does it benefit me? That's a good valid point. And it's getting them to see that there is a benefit to them. It's just like, alright, well, you just have to sort of showcase this several different times. + +**Dan:** I know somewhere along the way I heard somebody say that when you're creating a message for your organization, you've got to do it like seven times in different formats to get it so people all kind of get it and think about it and understand it. And I think that's got some truth to it, to be honest, because even to this day, I'm still telling people, by the way, this is the reason why we're doing this. And you kind of go through that process. + +**Host:** Yeah, no, it's totally true. + +**Dan:** Yeah, and I think there's also the seeing is believing sort of thing to it too, which is like, yeah, okay, understand how OpenTelemetry works in theory, right? And, but it's not until you put it in practice and instrument your code for the first time, and even if it's like, you know, I added a trace, I added a log, whatever—then you see it go through the collector and into your observability backend. And you're like, what? My life is transformed just even—and you go like, now what I do, yeah, right? + +**Dan:** It's like you crave more, right? Give them a taste; the proof is in the pudding, as the saying goes, is very, very true too because I have seen a lot of success once you get the ball rolling. They start to implement something, and I’ve seen it over the years in different tooling. I think only once have I seen it not work, but that's only because that was more my arrogance than anything else. So I've dropped arrogance at the door after that and never did it again because I wanted to use a specific tool for the job and realized that that's not the way to approach it. + +**Host:** And that's the other thing too; as engineers, as leaders, technical leaders, we should never push a tool, in my opinion. I think we should always look at what the organization needs, what's the most important thing, what's the most value to the organization, and then make sure we have the right practices in place, the right standards in place, and that will eventually then we can figure out what tooling we'll use after that. Tooling should be always the last thing you should decide on how you want to visualize or work with your data. That's just my two cents. I don't know how the people that might cause problems, but who knows? + +**Dan:** Now here's a question for you along the lines of tooling because we see this a lot in tech where you've got a team, they start using a tool, but then the corporate standard for tooling comes out. How do you reconcile the teams that are off doing their own thing with the corporate standard? Because sometimes it's really hard to get folks unstuck from that. + +**Dan:** You can, you know, try to force them, but some folks are very adamant and passionate about the tooling that they use, right? + +**Host:** They are, and you know what? You've got to—I honestly personally, when it comes to that thing, is like, okay, try to get them to—as long as they're following best practices, following industry standards, then you can only push so far. You know, like you could say, hey listen, you know, stop using this particular tool because it's archaic; it doesn't give you the same value for what you have. But, you know, and then you—you’re going to have pushback. I mean, it doesn't matter where you are. Even when everybody's on board, you're going to get pushback because people want to do things their own way, right? + +**Host:** There's always that kind of like I know better; I know what I'm doing. + +**Dan:** Yeah, and you have to be delicate, and you have to be patient, and you have to be tactful. So it's better to just—I would say just try to encourage them to use best practices and standards, but don't try to push them off the tool if that's—I mean, if somebody higher up wants to make that decision and call them out on it, let them do it. + +**Dan:** But you want to—I think it's more important to have the standards in place because if you can tie everything together and you have a proper process flow and everybody can still get to the root cause of the issue, then that's better than just saying you need to use X for your stuff moving forward. Tough luck, because that kind of thing doesn't work either, in my opinion. + +**Host:** Yeah, that makes a lot of sense, especially like, you know, I'd almost prefer to like, okay, you use whatever observability backend you want as long as we're all instrumenting with OpenTelemetry. I feel like that is the main thing. + +**Dan:** Yeah, that is the bar. + +**Host:** Now when you joined your organization, was Top Hat doing observability at the time? What was the landscape like at the time? + +[00:30:10] **Dan:** So at that time, we were not, but we were. So this is one of those things where I kind of had to be very patient. But they were like—so when I first joined, we were using a vendor. Everything was there; there was instrumentation in place. There was logging in place. Well, I should say there were traces in place and there was logging in place. They were linked, but they weren't—it was done—initially it was rolled out about five years ago, but no real work was done; it was just kind of building on what they already knew moving forward. + +**Dan:** I was asked to come on and actually bring—take the company off the vendor and migrate to OpenTelemetry, so that was two and a half years ago. We are almost there. I think we have one more service to migrate over to OpenTelemetry this month, and then we're done. So it's taken us almost about two and a half years to get to this point. + +**Host:** Amazing! + +**Dan:** Yeah, it was hard, obviously, but it was also—it was challenging. It had to be a lot of patience involved. And what I did though in the beginning was actually kind of go back to—remember when we were at Monorama together? I did a talk on alerting, and I did that talk because I kind of was—that was a precursor to where I really wanted to go, which is actually getting into SLOs, right? + +**Dan:** But that was sort of like, here’s how you can first sort of understand your landscape when it comes to alerts and why you’ve got to categorize them in a certain way and realize a bulk of them were a lot of noise and didn’t have any value. Help people start seeing those things in that kind of light and then start showing the value of using service level objectives and how that can actually help with end-user journeys, help actually bring the user experience home to the developer. + +**Dan:** So they understand, okay, I don’t need to alert on, you know, when the load balancer is hitting 5xx errors when I should be alerting on whether or not the service is being impacted by that and how it’s being impacted. And then, yeah, I mean those 5xx errors are important, but where are they important? How is that impacting the user experience? If it's for some feature that we don’t really care about, we don’t care about the 5xx because we don’t need or we don’t need to be alerted about that in the middle of the night. + +**Dan:** But that was how we were trying to get that process going. And then they start doing the migration. Okay, now that we know how our service should look, now we can do our instrumentation on that moving forward. So we've started off a lot of instrumentation, but now we have those user journeys to reflect upon and go, okay, we’re missing this function in our auto instrumentation. Let’s make sure that’s part of that because that impacts the user’s experience. + +**Dan:** So yeah, it’s been a while, but we are getting closer and closer to getting over to OpenTelemetry. We've set a lot of service level objectives—two of them have already helped out, like alerted us like two days before an incident actually happened. So we're already seeing some value in this. So it does, again, proof is in the pudding, right? And we're already seeing some of that, and we're, you know, it's great. A lot of churn, but a lot of like turmoil. A lot of folks were like, oh no, we’re losing this, but they’re seeing that they’re getting more value out of how we’re doing things moving forward, and they’re able to query things better because they’re able to have more richer data available at their fingertips. + +**Host:** So in terms of vendor, did you end up switching vendors from the one that you were using when you first came in? + +**Dan:** So we had the one vendor we were using; we're still on that same vendor. But as of the end of this month, we're no longer going to be using that vendor. We'll be on another vendor moving forward because we're pulling all their vendor-specific SDKs out, and we're just putting in OpenTelemetry, and we're doing that replacement. So it's kind of like now we're getting to the point where the last of those OpenTelemetry SDKs are pointing to a different vendor for our visualization of data. + +[00:34:22] **Host:** Cool, awesome. Alright, moving on. The next question I had is, you know, now that you've gone through the exercise of doing the instrumentation, what's the instrumentation culture like at Top Hat? + +**Dan:** Um, not as of yet. And there's a reason why, and this is nothing against anybody, so I hope anybody who's at Top Hat listening doesn't take offense. I love you all! + +**Dan:** No, but it's because we were kind of like, once we started getting going on this whole transition, we had a very short period of time. So there have been a few instances where staff engineers and principal engineers have stepped in and done some of the work, laid the groundwork for the other team, and they kind of just flipped over things and did things. The most effort they put in was actually in the user journeys themselves. + +**Dan:** So the auto instrumentation has been more largely kind of like, we moved this, put this in, and they’ve just looked at it and done it, so they haven't really— it hasn't been ingrained in part of their cultural thinking yet. It's still largely kind of like a checkbox for them to sort of, okay, we've gotten, we've moved, migrated to OpenTelemetry, job done, move on to something else. + +**Dan:** And little do they know that there's a lot more work involved. But don't scare them off yet because it will help them. But it is something that has to be—I think we've tried to do it so as easy as possible, less not as impacting as far as their day-to-day life, but try to slowly get them to start looking at things and so going over things and saying— and using the tooling that we have available saying, here, we know you did it in this particular tool; this is how you should do it in this tool, and providing that data and then doing training on that too. + +**Dan:** So trying—so it's slowly becoming part, but I would not say it’s 100%—it’s far from 100% yet. There’s still a lot of work to be done there just so that they start to understand why it’s part of the value, right? + +**Host:** In terms of instrumentation, like what languages are part of your application landscape? + +**Dan:** Mostly Python. + +**Host:** Okay, okay. So mostly—so you alluded to some auto instrumentation there. + +**Dan:** Yeah, so we did auto instrumentation in our Python code, and we use a couple different frameworks, but I think we're just trying to work towards getting to one framework. And then we have—unfortunately, there’s not—so we can almost kind of get away with doing a wrapper sort of library for people and have them just use that. + +**Dan:** And that’s kind of what we did. We have this little wrapper library set up that uses OpenTelemetry with all the different things that they need to pull in there, and people just kind of just dump it in and do auto instrumentation and away they go. So it’s kind of like all done for them, and they just kind of include this and away they go. + +**Dan:** But I think as we progress, it’s going to be like, okay, let’s start targeting—this is my next stage—talking to first off the PMs, talking to them: okay, this is what we have so far, these are the gaps we have in our telemetry data, these are the SLOs we have set so far; let’s decide on how that should look because your team needs to own this. + +**Dan:** Your team needs to act on this. So when your error budget gets exhausted, your team needs to take that on in the next sprint. And that’s one of—again, the technical side is not that difficult; it's the cultural change process change requires a lot of effort, right? Because now you have to get people to say, okay, well, instead of when we burn through our error budget, we need to deal with this next sprint—well, that's usually, well, that feature that we need to push out. + +**Dan:** Which one do we do? And so—but that’s what I said when I first started this is that you have to agree upon this process. You have to think about you are making a contract with the company to say, yeah, we will act on these SLOs because they will impact the user journey. They will impact our users, and that's what's most important to us today. + +**Host:** For those who are not aware, Top Hat is an educational platform, so our users are teachers or professors and students. So imagine, you know, if they're taking an exam and then everything goes sideways, that's going to be really— I mean, anybody who has kids or has been to university knows that that could be a huge frustrating factor when you're not—you know, if that's not working properly. + +**Dan:** Yeah, so it's important that—oh, I did get a message. Somebody did tell me, we all love it. Thanks, Mark! + +**Dan:** Yeah, so now I just completely thrown off there. What was I talking about? + +**Host:** ADHD! + +**Dan:** You were talking about the SLOs and how frustrating it is if the consumers of your platform if it's not working as they expected. + +**Dan:** Yeah. So, and that’s this kind of fundamental shift is going to SLOs and helping them see that and get through that. + +**Host:** Yeah, because I think—now I just went off the rails. + +**Dan:** I can relate! So, but that’s all good. + +[00:40:11] **Host:** Cool, I have one more question that I wanted to ask, and then we actually do have some questions from folks. See that our on our board, which I’m very excited about. Final, final question. I guess there are two questions, I guess. Are you using Kubernetes, and if so, are you using the OpenTelemetry operator to manage your collectors and to manage your auto instrumentation? + +**Dan:** No, no. + +**Host:** Yeah, so we—I'm glad we're not just because—and this is not because I think that Kubernetes has its place and all tools have their place. And I think for what we're doing today, it would not make sense. It would add a layer of complexity that would completely destroy our service. So glad we're not using Kubernetes; I don't think it—not saying that there’s anything wrong with Kubernetes; I just think that if you’re going to run a Kubernetes environment, you need to know it. + +**Dan:** Yeah, absolutely. + +**Host:** And if you don't know what you're doing with Kubernetes, you can really mess up your service. And I’ve been at places where we only had one or two people who knew Kubernetes and we had a number of problems always with our clusters. So I’m—when we’re ready though, I think we’ll probably make that push. But right now, we’re just in containers, so it’s all just through—yeah, just containers, but not Kubernetes. + +**Host:** Gotcha, gotcha. So you're using like another container management service, like a cloud-provided one kind of thing? + +**Dan:** A cloud provider container service, yes. + +**Host:** Gotcha, gotcha. You're trying to avoid all names of vendors right now. + +**Dan:** Yeah, yeah. Let's be agnostic. + +**Host:** And final question before we get to the questions. Collectors, what's your collector landscape like? + +**Dan:** So we're using a combination of OpenTelemetry Collector as a SAR, and then a primary one that aggregates it. And then for our traces, we use one company that we're working with. They have their own collector that we send our traces to that we can do additional sampling rules on and send off. + +**Dan:** Yeah, so we—right now because, and I really, really love the fact that you could choose. You could send from code itself, but I love the OpenTelemetry Collector simply because you have these processors, you have these exporters, and you can send to different places as you see fit. I've had conversations with our data team saying, hey listen, if you want stuff, OpenTelemetry is collecting a lot of this, right? + +**Dan:** And we don’t have to send all of it to our— for production, as far as what we're looking at, as far as like would give us an idea of the user experience. But there may be things you are collecting from OpenTelemetry that we can send to an S3 bucket that you can pull in later for whatever you need. + +**Dan:** However, you know, we still haven't gotten to that point, but that's the beauty of the OpenTelemetry collectors, that you can kind of like just point to different things. And I love that because then there's no code change. So if we decide we no longer want to use this or we want to add logging or we want to add—let's say we start using Prometheus as an open-source tool, as an example, though, you would—we can easily just start getting that information right off the mark. + +**Dan:** So I just—if anything, try not to use the sampling and the old push from this code itself. Use a collector because I love the fact that it makes life so much easier. Then you can just—if you have to do little tiny changes or tweaks to your sampling or whatever, you don’t have to go push it out to like the production environment again. It’s just a small change on your collectors themselves, which makes life so much easier. That's just how I think of it. + +**Host:** Absolutely. And you're using all three major signals: traces, logs, metrics right now for OpenTelemetry, or mostly traces right now? + +**Dan:** So there was a decision to not do—so metrics we still are capturing the infrastructure metrics from the cloud, and the logs are primarily staying in our cloud provider. So all of our services are just logging by default from container to the logging in our cloud provider, and they are staying there for the time being because that was another thing to start tackling. + +**Dan:** And I didn't—I just—I felt like that’d be too much for folks as they're transitioning from the one vendor to OpenTelemetry for them to like get their heads wrapped around would be like, oh, by the way, you also need to have structured logs too, and this is how you should do it. No, this is just going to—it’ll make more confusion. + +**Dan:** And so I figured just focusing on one and then adding things as they go along, as they prove useful, so yeah, that's how we kind of approach that whole process. + +**Host:** Makes sense, baby steps. + +**Dan:** Exactly, cool. Okay, so we're going to turn to the board. And if you want, you can take a peek at the questions; they're on the chat window. There's a link to the board. + +**Host:** Take a look. So let's have a look at the questions. + +**Host:** So the first one is, what is your opinion on telemetry quality as a way to reduce spend? Is more data always better observability? + +**Dan:** Great question! Awesome question, and that's a tough one. I don't think I can—I hope I can answer that. In my opinion, it's—quality is—you need to actually—because really with the end of the—because you can have a bunch of things going through, but if you're sort of—so let's just take a standard trace, right? If you have a bunch of spans in there that are doing some things that you really know it's just back and forth and you really—like there's—I’ve noticed some traces where there— I forget what it's called, and I wish I had a sample of the code before. It's been like two weeks ago; one of our engineers cleaned this up, and what he did though is he looked and realized we’re making these calls in the code that we’re adding span events to, and we don’t need to because—so you're looking at the quality there, and then you're not—you're seeing what's invaluable and what's not. + +**Dan:** So more data doesn't always mean better telemetry; quality data means better telemetry. So in my opinion, you've got to look at what you're getting, making sure—does that reflect the user experience? If it doesn't reflect the user experience or is not exposing things where you would say, oh, you know, like that's kind of like the whole concept of the unknowns, right? Like you're exposing those unknowns. You're not able to see those things, and then you're not—the quality doesn't matter how much you have if you're not seeing that stuff, then it doesn't mean you're—it's the quality of the data that's most important, and that will help you reduce spend because then you know what's good, you can sample that partially out, and you're going to make sure that those error-prone things bubble up and become more apparent when you're looking at your data. Hopefully, that answered the question. + +**Host:** Yeah, that was a really great answer. + +**Dan:** Yeah, and I agree with you. It's really like what's the point of capturing telemetry for your system that's useless to you? Like that's just—and I think that's a pitfall that a lot of people run into also with auto instrumentation initially, right? Because there's a lot of stuff coming in. And fortunately now, I think there's a way to actually switch off auto instrumentation for certain libraries so that you can really zero in on the stuff that's super meaningful to you. + +**Host:** So, yay! + +**Dan:** Yay! + +**Host:** Cool. Okay, next question. How do you approach the what to instrument and how much to instrument questions with your teams? + +**Dan:** So that kind of is linked to the previous question too, in my opinion, because what to instrument is again—it goes back to this. So I went through the whole process of working with the development teams and the product teams to understand what the user journey looks like. So I went through and said, okay, what is the—so we did the user journey. We looked at their servers, and I asked—some did more than others—but more or less it was like I asked them what are the typical interactions a user would have with your service? And write it down. + +**Dan:** And then write in plain language, and then in plain language describe the tech, what's happening in the background, and then we also talk about the prerequisites of what people expect when they interact with that system. So they have to be logged in, they have to have this tech—maybe this cookie or this cache or whatever—and then what are the dependencies? And then we just kind of work through and go, okay, what are the things that where a user will potentially see problems based on this information? + +**Dan:** And we went through that. So that's how I kind of approach what to instrument because then once you kind of know, okay, so we know that user logs in, so they know they have to do that. They have to interact with maybe—let's hypothetically say they interact with a database, they interact with an authorization service, maybe they interact with something in the back end for something else—another database—and so all these queries are checked in there. + +**Dan:** Now that all those things—all those functions are like talking to these different things. If they don't work, it means they can't log in. Therefore, we have key spots what they should be mostly looking at to make sure. There are always going to be times where people miss things. This is why we have the SLOs. This is why we have to have this constant part of our cycle to go back and make sure that those things are identified. So you know, we’re having problems with our servers, but we don’t know why that means maybe you’re not instrumenting. + +**Dan:** So what that means you have to go revisit some of the things you did because maybe you did some changes that no longer use this kind of database. Maybe it uses a cache service or uses a Redis cluster instead of these things. And so does your code or your user journey reflect that change? And does your instrumentation reflect that change? And if not, let’s make sure they do. + +**Dan:** So, but it always boils down to the user journey and what the user is experiencing. If as long as we're capturing that, you're like 80% of the way—in my opinion, 80% of the way there, because you at least have a general idea. But you have to know—you have to have that conversation. + +**Host:** Awesome, thank you. + +**Host:** Okay, next question. In an organization that has adopted distributed tracing, how do you make individual teams leave behind their outdated practices based on logs? How do you show value? + +**Dan:** That was a spicy question! That's a spicy one. You slap them around and tell them what—no, you do not. That's not the way to do things. You have to be very—exactly, that’s number one: do not call their baby ugly. + +**Dan:** Um, that's a tough one because you really, really have to be patient. And sometimes you do have to say, okay, listen, this has been dictated by the upper management that we're doing this. And you try to offer them as much help as possible, but you have to find out too why they’re reticent about switching over. + +**Dan:** Having these conversations is important because then you understand how they're looking at things, and then you can restructure your discussions around that. So, oh, I like using logging because, you know, I get to see immediately when I'm doing my development. I can see it point out on the screen; I know where things are broken, so I can go back and fix it and blum blum blum. And that's why they love logging, as an example. + +**Dan:** So how you have to then help them bridge that gap and say, okay, well this is how you would do it with tracing, and this is why you would probably want to do it because then with distributed tracing, you're not just getting the value out of your service, but you're seeing how service A interacted with your service, service B, and how service B is going to interact with service C and how all that kind of works together. + +**Dan:** So this is how—why you want to have these pieces in the puzzle so you can get that full picture. But it takes time; it takes patience. It takes—and again, going back to which one of the things we talked about before, restructuring how you approach their challenge of switching over. It's not just repeating yourself with the same words over and over again because eventually, people are just going to tune you out. + +**Dan:** Like at my work, I think they tune me out half the time, okay? + +**Host:** Final question; we can squeeze it in hopefully in two minutes. + +**Host:** I really love what you said about the tooling. Is the last thing you should think about, and that you should lead with the question, what is most important to your organization? What are some of the questions you ask to understand what is most important to someone's organization? + +**Dan:** It's always about the user, in my opinion. It's always about the user because first—because you're looking at—if the user is not happy and you can't measure that or you don't know that, then you're in trouble, right? + +**Dan:** And I always use streaming services, like video streaming services, as an example because everybody has tuned into one of many of these streaming services we have now today. So if you pull up a movie, you push play, and it gets all pixelated for the first three seconds, and then it clears up after that, do you care? No. + +**Dan:** So if a person comes by—do the streaming service know that? They probably are measuring that because they want to make sure that that doesn't become a bigger issue because they know that runway of where you're okay to where you're not okay and you're going to get frustrated and go, I'm going to go switch to this now because they have better quality. + +**Dan:** And so it's not—it's always about what the user—how we are—we measuring the user experience. Do we know what the user experience is like? Do we know what frustrates them? And if we don't, how do we get there? And that's how we get—so that is usually with the tooling that we have available today, with the SDKs and other things and the philosophies we have. + +**Dan:** Eventually, then you can go—because you know what? The tools come out all the time. There are so many tools out there that the landscape has gotten bigger and bigger as people become more and more involved, but it doesn't mean they're all good. And you know, but you have to find the one that kind of works for you at the time. + +**Dan:** And this is why we like OpenTelemetry is because you can switch from one to the other to the other without with very minimal change. And that’s why I like about getting people on OpenTelemetry. But when it comes to the tooling, that should be your last thing to think about. Don't build your solution around this tool in your spot. Build your solution around what your users are thinking, and that's how I think a lot of people—that resonates when you start talking in that light, especially with the higher-ups. + +**Host:** Anyway, I think we got through them all. + +**Dan:** Yeah, absolutely. + +**Host:** Well, we got through all the questions! Yay! We filled up this hour quite nicely. So thank you so much, Dan, for joining us for Otel Q&A. Stay tuned for our next Otel Q&A and/or Otel in Practice. And again, thank you, Dan, for joining today. I really appreciate it. + +**Dan:** Oh yeah, you know, it’s my pleasure. Thank you for having me. You know how much I love talking about these things. So, you know, I think I actually did not go down a rabbit hole today. I think I was pretty good. + +**Host:** Perfect! It was great. Thank you again! + +**Dan:** Alright, thank you everyone! Thank you for showing up. See you! ## Raw YouTube Transcript diff --git a/video-transcripts/transcripts/2024-11-15T06:30:37Z-humans-of-otel-live-from-kubecon-na-2024.md b/video-transcripts/transcripts/2024-11-15T06:30:37Z-humans-of-otel-live-from-kubecon-na-2024.md index a224ace..ea5ccf8 100644 --- a/video-transcripts/transcripts/2024-11-15T06:30:37Z-humans-of-otel-live-from-kubecon-na-2024.md +++ b/video-transcripts/transcripts/2024-11-15T06:30:37Z-humans-of-otel-live-from-kubecon-na-2024.md @@ -10,154 +10,335 @@ URL: https://www.youtube.com/watch?v=LjD454EPMgQ ## Summary -The video features a live discussion from CubeCon North America 2024, hosted by Rees, and includes special guests Hazel and Ted, both prominent figures in the OpenTelemetry community. The conversation delves into Hazel's journey into observability and OpenTelemetry, highlighting her frustrations with existing tools and her desire to improve the process of teaching others how to navigate these systems effectively. They discuss the importance of observability in platform engineering and the need for better integration of technology with business processes, coining the concept of "observability 3.0," which focuses on embedding tech understanding into non-tech stakeholders. Ted shares insights on his role as a co-founder of OpenTelemetry, the project's evolution from merging OpenTracing and OpenCensus, and the importance of community trust and involvement in maintaining a vendor-neutral approach. They emphasize the need for user feedback and engagement to refine the tools and processes within OpenTelemetry. The video concludes with a call to action for viewers to participate in surveys and community discussions to enhance the project. +In this live segment from KubeCon North America 2024, host Rees is joined by special guests Hazel and Ted, who are prominent figures in the OpenTelemetry community. They discuss Hazel's journey into observability, the significance of OpenTelemetry, and the challenges of integrating observability into platform engineering. Hazel emphasizes the importance of teaching others about observability tools and her commitment to empowering users, while Ted shares insights on the OpenTelemetry project's governance and community dynamics. They highlight the project's goals of unifying tracing, metrics, and logs to create a coherent view of systems, as well as the need for collaboration across various tech and business sectors to enhance observability practices. The conversation underscores the project's evolution, future aspirations, and the ongoing need for community engagement and feedback. ## Chapters -Sure! Here are the key moments from the YouTube livestream along with their timestamps: +00:00:00 Welcome and intro +00:01:20 Guest introductions +00:02:30 Hazel's observability journey +00:04:50 OpenTelemetry discussion +00:06:00 Platform engineering insights +00:08:20 Observability challenges +00:10:00 OpenTelemetry project overview +00:12:10 Observability 3.0 concept +00:22:00 NLY Foundation background +00:29:10 Transition to next guest +00:30:00 Ted Young introduction +00:31:16 OpenTelemetry governance and community +00:36:30 Community trust and collaboration +00:40:00 Project structure and incentives +00:44:35 Updates from project +00:55:00 Future of OpenTelemetry +00:58:00 Closing remarks and survey reminder -00:00:00 Introductions and welcome to the livestream from CubeCon North America 2024 -00:01:30 Guest introductions: Hazel and Adriana share their backgrounds -00:03:15 Hazel discusses how she got involved in observability -00:05:00 Transition to OpenTelemetry: Hazel's journey into the project -00:07:45 The importance of observability in platform engineering -00:10:30 Hazel shares her thoughts on the fragmentation within observability tools -00:12:15 Discussion on the challenges of data collection and operational context -00:15:00 Introduction to observability 3.0 and its significance -00:18:00 Overview of the OpenTelemetry Foundation and its origins -00:20:30 Ted Young joins the stream and introduces himself and his role -00:23:00 Ted discusses the merging of OpenTracing and OpenCensus into OpenTelemetry -00:28:00 Insights on maintaining community trust within the OpenTelemetry project -00:30:45 Ted shares updates on the project's current state and future direction -00:35:00 Discussion on the importance of user feedback for the SDK and instrumentation APIs -00:37:30 Ted talks about the vision for OpenTelemetry in the next five years -00:40:00 Closing remarks and encouraging audience participation in the project +**Reys:** Well, he hasn't done the countdown, so YouTube... Oh, is this on the channel? Why should I just shut up, right? So, I launch the counter of 10 minutes. That's too much, probably drop it. No, we still have this. Oh, it's OpenTelemetry official. Oh, he's checking the OpenTelemetry channel to make sure that it’s queuing up on their live. Okay, cool, perfect. Are you ready, M? -Feel free to reach out if you need more details or information! +**M:** Yeah, and I will in... -# CubeCon North America 2024 Live Stream Transcript +[00:01:20] **Reys:** Hello, live from Salt Lake City, I'm Reys, and I'm here with some special guests. We are live at KubeCon North America 2024, and we're here to talk with a couple of really cool people in the OpenTelemetry community about how they got involved and kind of learn a little bit more about them. Let's go! -**Reys:** Well, he hasn't done the countdown, so YouTube... Is this on the channel? Should I just shut up? Right, so I launched the counter for 10 minutes. That's probably too much. Let's drop it. No, we still have this. Oh, it's OpenTelemetry official. He's checking the OpenTelemetry channel to make sure that it’s queuing up on their live. Okay, cool. Perfect! Are you ready, M? +**Adriana:** All right, good morning, good morning, good morning! I'm really glad to be here. So, for those of you who don't know, I am Adriana Vela, and I work with Reys on the OpenTelemetry and User SIG. We're excited to be talking to our awesome OpenTelemetry guests today. -**Adriana:** Yeah, I will be in. +**Reys:** So, Hazel, how did you get involved in OpenTelemetry? What is your story? How did you get involved in observability? -**Reys:** Hello, live from Salt Lake City! I'm Reys, and I'm here with some special guests. We are live at CubeCon North America 2024, and we're here to talk with a couple of really cool people in the OpenTelemetry community about how they got involved and to learn a little bit more about them. Let's go! +[00:02:30] **Hazel:** So, I think I got involved with observability in a very similar way that a lot of people do, which is to say I didn't. What happened was I got really, really frustrated at things happening, and I needed to figure out how to actually ask better questions about my systems and figure out what was going on and dig into them. The tools that I had weren't very good, and so I dug around looking for better tools, better ways to do things, and more importantly, better ways to teach other people what we're doing and how it works, because one of my superpowers is being able to take the entire system and hold it into my head and be able to sort of intuitively think about it. Like, when I was in an operating systems class in college, the point of the class was to kind of almost force you to learn how to debug. But I never did, and I still don't know how to debug, because I just looked at the code and mentally traced the code in my head and debugged panic flaws in the sheer by just like staring at it. But I can't teach that to people, and I can't show them how to think about that. So I've always said I want to figure this out with just, you know, print F debugging or whatever I want to do, but how do I actually take the vibes and turn them into a tool and an explanation and a technique and a way to teach people that you have this? You already know the answers. You know how to find things. You're already like 80 or 90% in the way there. It's not this magical new toolkit; it's this asking questions that are meaningful to you, getting those answers that are useful, and then doing something useful with that, like learning. After effectively on that, and you don't do that, you already know how to do that, and just taking that and looking for really, really nice things ended up getting me into that observability space where I met a lot of people that were also trying to do that. ---- +**Reys:** And how did you get into OpenTelemetry itself? -**Reys:** Good morning, good morning, good morning! I'm really glad to be here. For those of you who don't know, I am Hazel Ha, and I have thoughts—lots of thoughts. They never stop thinking, and I am super glad to be here to talk with you all and hopefully make you laugh so hard you cry a little bit. Thank you! +[00:04:50] **Hazel:** So, OpenTelemetry... For me, it was actually I looked at the shape of all the different vendor-specific things or the different instrumentation of like logging or metrics, or we didn't really have traces when I first started. It was actually really frustrating because I really, really hate doing things that are undifferentiated in a different way. I would much, much rather do a bit of work to figure out how can I slice it down? How can I think about this concept of optionality and putting everything together? Because one of the best things about Linux is it turns out you don't need to write a kernel in order to get something working; you can use one. One of the best things about Kubernetes is sure, you write, you know, your own custom flavor on top, but it's the thin veneer over this core. So for observability, I started looking for that core, the essence of it that you could build that thin veneer, that layer on top, and I found that in OpenTelemetry. -**Adriana:** And I'm Adriana Vela. I work with Reys on the OpenTelemetry and User SIG, so we're excited to be talking to our awesome OpenTelemetry guests today. Hazel, how did you get involved in OpenTelemetry? What is your story? +**Reys:** That's amazing! So how long have you been using the project or contributing to the project? -**Hazel:** I think I got involved with observability in a very similar way that a lot of people do, which is to say I didn’t. What happened was I got really, really frustrated with things happening, and I needed to figure out how to ask better questions about my systems and understand what was going on. The tools I had weren't very good, so I started looking for better tools, better ways to do things, and more importantly, better ways to teach other people what we were doing and how it works. One of my superpowers is being able to take the entire system and hold it in my head, thinking about it intuitively. +[00:06:00] **Hazel:** So I've been using the project actually primarily, usually not directly, because I'm low enough on that platform side. What I'm almost always looking to do is enable people, and so I have looked at the OpenTelemetry collector a lot, being able to say how can I empower people? How can I enable them? How can I take them from these different places where maybe they're using one vendor here and one vendor on another team and one vendor on another team or a third-party integration or the OpenTelemetry code? So as far as being able to enable people and think about that, I started, I want to say, really digging deep in building those platforms for other people around in 2019. -When I was in an operating systems class in college, the point of the class was to almost force you to learn how to debug, but I never did. I still don’t know how to debug because I just looked at the code and mentally traced it in my head. I can't teach that to people, though, so I always wanted to figure out how to take the vibes and turn them into a tool, an explanation, a technique, and a way to teach people that they already know the answers. They know how to find things; they’re already 80 or 90% of the way there. It’s not about a magical toolkit; it’s about asking meaningful questions, getting useful answers, and then doing something with them. +**Reys:** Okay, wow! So basically from the beginning, really? -After realizing this, I ended up getting into the observability space, where I met a lot of people who were also trying to do that. +**Hazel:** Yeah, yeah, that's great! Awesome! Which is so weird because like 2019 was not that long ago; it's like two weeks ago. ---- +**Reys:** I know, right? It does feel like it, absolutely. Yeah, I was going to say, you're also heavily involved in the platform engineering space, and I feel like observability is one of those things. Like, we often talk about it as a separate thing, but really, like, you can't have effective platform engineering without observability. You can't have effective SRE without observability. And I know you have many thoughts. -**Adriana:** And how did you get into OpenTelemetry itself? +[00:08:20] **Hazel:** Yeah, so it's actually something that irks me a little bit a lot of the time because I get why this happens. You have a massively wide platform, and you have a massively wide tool chain, and people keep adding more and more context into something, and the context goes deep, and it goes wide, and it goes up, and it goes in all the different places, and you can't possibly hold it all in your head unless, you know, you're weird like me and you can hold way too many things in your head. But you do end up in this situation where, yeah, OpenTelemetry, you can dive so deep into it; it can become your whole thing; it can become like the only thing you really think about. When you see a lot of companies with the platform teams, they start off with an IT team that's really just rebranded Ops. Then they take the rebranded Ops team, they split it into two or three teams, and then one of those teams ends up being in charge of, you know, the Kubernetes part, and that gets labeled the platform team for some reason. Then the platform team gets turned into two or three different teams, and then you keep going on, and you know someone gets stuck with the observability thing, and it ends up being really complex because there's a lot of moving parts. So I see how things get split out and sort of fragmented, you know, much in the same way as the CNCF landscape is so massive. Everything has its own kind of corner; nobody really talks to each other. But I wish people would do that because they often end up re-implementing the same thing over and over. -**Hazel:** For me, it was actually looking at the shape of all the different vendor-specific things or the different instrumentation of logging, metrics... We didn’t really have traces when I first stepped into it. It was frustrating because I really hate doing things that are undifferentiated in different ways. I would much rather do the work to figure out how to think about this concept of optionality and put everything together. One of the best things about Linux is it turns out you don’t need to write a kernel to get something working; you can use one. One of the best things about Kubernetes is that you can write your own custom flavor on top of a core. +**Reys:** Right, exactly. -So for observability, I started looking for that core essence, and I found that in OpenTelemetry. +[00:10:00] **Hazel:** So like in the profiling side of things, we have this Open Profiling aspect, and they're looking at a lot of the OpenTelemetry stuff, and they're reinventing a lot of the same discussions that they need to have. A lot of saying, do you need standards? Do you need naming? Do you need plugins? How does the collector work? And I'm like we solved that five years ago! Like, look at the pre-existing stuff, tweak it a bit, and then you think about it. Or you have the event streaming architecture people where you have like database streaming, you have lake house people, and you have business analytics and business intelligence and all these different data science types of things. It turns out that if you are trying to take a metric load of data and derive useful actionable insights from it and then share that with people who are doing data science, that's kind of what we decided to call it, you know, 20 years ago. Then we forgot about that, and we reinvented OpenTelemetry instead of making a data pipeline. And then we reinvented profiling instead of making a data pipeline, and now we're kind of looking at all these things, and then we're going, "Oh, we should do this thing," and then the, you know, the Power BI or the business analytics people in the corner are just crying a little bit because they've been doing this since the '80s. ---- +**Reys:** Right. -**Adriana:** How long have you been using or contributing to the project? +**Hazel:** And then way over here in the financial side, did you know this is one of my favorite things actually? So KDB is a financial analytic state database, and it is a column database. If you look at high-frequency trading or, you know, data analytics people that specialize in the financial world, they actually predate the usage of all of these, like, oh no, queries that rely on indices. You can just grab the data, and they've been doing that for 40 years. They had to, and nobody else ever really thought to look it up. It's fun to see all the communities reinvent things over and over, bring their own flavor and context into it, and then hopefully we can, with platform engineering, start stirring all the people together, talking to them, and getting them to actually look outside the little window and go, "Oh, we solved the same problem!" Well, that's cool. How can I learn from how you approached it, and how can you learn from how I approached it, and can we build kind of a more common thing that's really awesome? -**Hazel:** I’ve been using the project for quite a while. I usually don’t get involved directly because I’m low enough on that platform side. What I’m almost always looking to do is enable people. I’ve looked at the OpenTelemetry Collector a lot, wanting to empower people and help them move from using one vendor here and another vendor on another team or a third-party integration to the OpenTelemetry Collector. +**Reys:** Given that there's so many interesting points that you just brought up and like the way your brain works and how you can look at something and kind of intuitively understand what's happening, what pieces of the open projects are intriguing you? -I want to say I really started digging deep into building those platforms for other people around 2019. +**Hazel:** So one of the things that's really intriguing to me about the OpenTelemetry project is you have this dichotomy of OpenTelemetry as the end-user kind of part where you have this SDK and this API and there's how you use it, and you have this mental model. Then you have the operational side of how you collect the data, how you store it, how you enrich it, how you sample, and how you do all these things, and they're all deeply disconnected. They really don't need to be, but the reason they're often deeply disconnected is because they're all non-intentionally or maliciously horrifically lying to each other. So the mental model, for example, that you have with traces is, oh, there's like this little box and it's this start of it and the end of it, and in the box, I put all my information stuff, and I draw some information on the outside of it. Then in the information stuff, I have another box, then I have another box, then I have another box. I have so many boxes! When you actually use the SDK, when you actually send things, that's a complete lie. You're not doing a box; you are reinventing the concept of distributed transactions really badly and without transactional semantics because you're just firing stuff off like a stream of events. But you don't have like this read-ahead lock or any of the other stuff that the database people invented in the '60s. So there's that kind of lie of it. Then when you get into the operational side, increasingly, it turns out sending everything is expensive, you know, B, really, really time-consuming, C, actually a waste of effort and time and resources, and you shouldn't do that. So you need to take that, enrich it, correlate other data, look at better things, figure out associations, sample it, and sort of figure out and finesse how you understand the data in this very operational context, which we often bundle that into the OpenTelemetry collector. But we never ever talk about the OpenTelemetry collector as like a required thing or even, you know, a thing, and then so it hampers a lot of the potential of the SDK make, because for example, you can't update a span because a span is treated as an append-only immutable log. Except it's a box, but no, it's a dependently immutable log; it's a stream of events. But at the same time, it turns out if you break this at any point because not all are reliable, you completely destroy the entire concept of what you're doing and embreeze all your ingestion pipelines and embreeze your UI of everything building everything, and it's super annoying. ---- +**Reys:** Right. -**Adriana:** Wow, so basically from the beginning, really? +**Hazel:** So I would love for people to think about what it's like if we made the concept of a collector or this concept of enriching, rewriting the tree, flattening the tree, doing all these weird sorts of correlation concepts or flattening or changing the shape of stuff to make it more malleable for you. What if that was a more integral part of how we designed the SDK? And if we designed the SDK with more use cases in mind, how do we do so in a way that gives people a simple mental model but doesn't lie to them about what they can expect out in the platform? So like there are client SDKs, for example, that don't send the data at all ever until they have received on the client the ending span, which means that if your span is 10 seconds long and on second 9 and 2, the client process gets shut down and does not have time to send that span, you lose all of that data. But if you send it all to the collector immediately, you have so much traffic you can't really afford to do that. But if you were able to send like a snapshot thing of a read-ahead log of this isn't done yet, but I'm going to window it like databases do in the '70s, then okay, you can work with it. You can work with it, and then you can patch it up. Some vendors have actually started to work cleverly around OpenTelemetry and work cleverly around the specification in the middle part to allow that capability in places where they need it, like mobile clients, and then patch it up and make it, you know, OpenTelemetry compliant by the time you send it to your backend. So if we open up the Pandora's box a little bit of, okay, maybe this is kind of necessary, maybe we need to think about this in a weird way of we have to have all the things talk to each other, and we need to make it more possible to do these things that make the mental model a bit more coherent, when can we do that? -**Hazel:** Yeah, that’s great! It's so weird because 2019 was not that long ago—like two weeks ago! +**Reys:** That's a lot! You got thoughts! -**Adriana:** I know, right? It does feel like it! +**Hazel:** Um, I wanted to just switch gears a little bit because before we started the stream, we were talking about, you know, you have many thoughts on many things, and one of them was Observability 3.0. Now we've heard Observability 2.0 has kind of come into our vernacular lately, and you were talking now about Observability 3.0. So can you tell us what you mean by that? ---- +**Hazel:** So, what I mean by that is a really, really fun thing where it gets into the heart of what I like about observability, which is essentially the same thing that everybody ignores, and I would like them to not ignore that. And so if I brand it a little bit with like a cute little sticker, maybe people will care. So Observability 1.0 and 2.0 is sort of a reframing of the natural progression that has happened in observability and the vendors and the capabilities and the needs of the platform and what we need to do in order to ask the right questions. Before, it was sort of this is observability, and this is observability, and then we're like, well, no, it is all observability. It's all about asking questions. It's more what's the fidelity of the questions? What type of questions can you ask? How much information is there? How rich is it? What are the properties of asking those types of questions? So if we think of Observability 2.0 as sort of being the ultimate insight in the tech context of a company or a program or a platform, can we ask essentially a question? That's kind of the ultimate goal of observability 2.0 to me. -**Hazel:** I feel like observability is one of those things we often talk about as a separate thing, but you can’t have effective platform engineering without observability. You can't have effective SRE without observability. +So for me, Observability 3.0 is defined by the idea that you need to take non-tech people and non-tech problems and the larger context of the business and embed it into your ability to reason about your system. You need to take the system and embed that into the rest of the company so the company can reason about that. So Observability 3.0 has a stark difference in that it's not about tooling; it's not about capabilities. It's not about all the sort of the technical side; it's very much now we need to take the tech and we need to take the people and the processes and this, you know, massive budget that the IT industry gets and stop spending it without accountability and stop spending it as this massive black hole where money goes in, magic stuff comes out, and nobody can explain why. So can your business analysts, can your marketing people, can your salespeople, can your product people, can your everybody else, your customer success team, can they all sit there and understand how to better serve the customer and how to better interoperate with the technology people without the technology people having screen feeding to them? Can you give them the capability to do a better job, and can you take what they know and take all this cool stuff that they do and put it into context that lets platform engineers and product engineers and backend and all these people do a really, really good thing of being able to present technological options and be able to actually interoperate with what the company needs, as opposed to what the company has asked for, you know, in a guessing manner? To me, Observability 3.0, to kind of sum it up, is the business context comes in tech, the tech context goes into the business, and they become one harmonious concept. -**Hazel:** It actually frustrates me a lot of the time because I understand why this happens. You have a massively wide platform and toolchain, and people keep adding more and more context into something. The context goes deep and wide, and you can’t possibly hold it all in your head unless you're weird like me. But you end up in a situation where OpenTelemetry can dive so deep into it that it becomes your whole thing. +**Reys:** That's amazing, and I'm sure this is going to spark a lot of conversations! I'm really excited to see other people's thoughts and kind of hopefully, you know, see your vision. -A lot of companies with platform teams start off with an Ops team that gets rebranded, and then they split into two or three teams. One of those teams ends up being in charge of the Kubernetes part, labeled the platform team for some reason. This leads to complexity and fragmentation, much like the CNCF landscape, which is so massive that everything has its own corner, and nobody really talks to each other. +**Adriana:** I also wanted to chat with you; you know, OpenTelemetry is an open-source project. Obviously, we at KubeCon, and you work at the NLY Foundation, and I would love for the audience to learn more about kind of the story behind the foundation because, yeah, you shared it earlier, and I would love for you to share that. -I wish people would do that because they often end up re-implementing the same thing over and over. In profiling, for example, we have this Open Profiling aspect looking at a lot of OpenTelemetry stuff, and they’re reinventing the same discussions they need to have. +[00:12:10] **Hazel:** The NLY Foundation was started by Chris Noda, and it was one of those last things that she did before she passed away. The NLY Foundation actually kind of started as an idea that germinated from Hacker, the Mastodon instance that we all built together, made into this massive thing, and then did a huge migration. We live-streamed it, and everybody learned from it; it was awesome. The reason in Hacker that we chose the technology that we did is because you had a bunch of brains; you had a bunch of people that are kind of the world’s support to doing a lot of things, but we didn't want to rely on that because we wanted to experiment with the idea of what does it mean for a community to deeply own their own community, their own concept, their own architecture, their everything? This was right around the time when the community, especially the tech community, was first, you know, grappling with the idea of we built this little thing, and now it's kind of being taken away, and we don't have control over our own, you know, engagement platform, our own sort of communication. How do we never lose it again? Then it turns out that as the massive migration initially happened, we almost closed registrations on Hacker, not due to operational concerns, but due to a massive legal liability. The course of how federation works on Mastodon ends up being a very, very abusable vector for putting illegal content somewhere because that content has to get syndicated onto your server. If you run a Mastodon instance, you are liable for anything that touches your computer, but you don't really necessarily control what that is, and so that is a very difficult concept. Nova kind of ran into this with her legal background and her legal brain. She was like, "Oh, absolutely no, no, no, no, no, no, no." She ended up immediately finding lawyers, sitting down, and building like an LLC. We wrapped Hacker around NLY and very quietly spread this kind of information to a lot of other larger instances of, "You need to now care about this; you need to immediately start caring about this because you are at risk of a huge vector." There's a huge surge of potential heat coming, but it made us realize as we sat down and we kind of solved this problem, when you go from a fun toy project and you take this toy project, and it becomes a community, there's this invisible cliff of I have like my vision, I have my dream, my people, and now there's everything. I can't half-ass my licensing anymore; I can't half-ass my contributor CLA agreement. How do I get people from large companies or from regulated industries to be able to contribute? It turns out you can't just make their repository open source; you have to actually make it so that their paperwork is okay with it. ---- +**Reys:** Right. -**Adriana:** That’s really fascinating, and I appreciate your perspective. +**Hazel:** And then how about international people? How about people with disabilities? How about people with different needs? How about people with different backgrounds? Okay, you know, we haven't even begun to solve the how you fund open source. How do you find open source? No, nobody knows the answer; nobody's even gone close. People are trying, like Eva Black, as she says, doing a fantastic job trying to create this sort of top-down way and path and avenue for the ability for companies to pay open source and for governments to pay and for all these things to happen. It's going to take time, but as a community, we need to bottom-up sort of figure out how do we pass this cliff? It's a massive cliff, and nobody really knows it's coming, and you don't have time to prepare for it because you don't get to choose when you become adopted and beloved by the community. Nobody sits down and says, "We're a community now." You just look around one day, and you go, "These are all my friends, and I love them, and this is great!" and then, oh no! It turns out that's only step two because it turns out that you can't really call the CNCF community anymore; it's an ecosystem. That's kind of that third massive cliff that very few people talk about, where you have this one project and this community, and that's cool. The community maybe grows other things when it reaches this stage of becoming a generation and incubation hub of innovation and experimentation, and it starts generating a fractal of different communities that all come together in this ecosystem of this massive idea exchange of community knowledge sharing and community growth when it becomes this messy, inarticulate sort of ill-defined but beautifully growing organic thing that takes on its own life. That's an ecosystem, and we don't even really know how to build those. We definitely don't know how to fund them; we definitely don't even know the differences between all the legal and the compliance. How do you facilitate that? How do you get there? So we looked at this problem, Nova and I, and a bunch of other people, and we thought someone needs to start thinking about this. It's going to take a while; it’s going to take several years, maybe even a decade or two, to really deeply help the world understand how to pass those cliffs and turn them into this growing ramp of taking ideas and sharing with the world in a way that generates further ideas in a way that becomes this ever-growing thing. The NLY Foundation, which was originally sort of kind of starting as like a legal cover and legal ability to do this, always was intended to be a vehicle that helps you understand that problem and helps figure out how do we make this beautiful growth and knowledge sharing possible. -**Hazel:** One of the things that’s intriguing about the OpenTelemetry project is that there’s this dichotomy between the end-user part and the operational side. The mental model for traces is often misleading—like there’s a little box that represents the start and end, but in reality, it's more complex. When you start sending data, it’s not a box; you’re reinventing distributed transactions. +**Reys:** That's so great! I think are we coming up on time? -Sending everything is expensive, time-consuming, and often a waste of resources. You need to enrich it, correlate it, and finesse your understanding of the data in an operational context. This is often bundled into the OpenTelemetry Collector, but we don’t talk about the collector as a requirement, which hampers the SDK’s potential. +[00:29:10] **Adriana:** So we are coming up on time. We are going to have our next guest on. Thank you so much, Hazel. It was lovely to talk with her, and we will share socials, like how you can get in touch with her at the end, and as well as like how you can get involved in the future Humans of OpenTelemetry segment as well. I'm really excited to see what kind of interactions come up from these conversations that we just had. And yeah, whatever questions, comments, thoughts you have, definitely please let us know. We would love to hear them and help connect you with Hazel so we can learn even more and move forward on kind of her vision. Also, secretly jealous of her mental superpowers. I'm going to have to see if I can pick up any tips, but now we have our next guest on, and I'm so excited! Hopefully, you can see his cat pants because they're amazing. ---- +**Ted:** Hello! -**Adriana:** Switching gears a bit, we talked about Observability 3.0. Can you elaborate on that? +**Reys:** Hey! Hello, how's it going? -**Hazel:** Sure! Observability 1.0 and 2.0 represent a natural progression in observability, focusing more on asking the right questions. Observability 3.0 is about embedding non-tech people and non-tech problems into your ability to reason about your system. It’s about making the tech and the people work together harmoniously. +**Ted:** Going great! How are you all doing today? -I’d love for businesses to understand how to better connect their operations with technology without it being a black hole where money goes in and nobody knows why. Can your business analysts, marketing teams, and customer success teams understand how to better serve the customer without tech people feeding them information? +**Reys:** Not bad! It's KubeCon! Awesome times! It's like a big friend reunion! Day two of the main KubeCon events, which, you know, everyone is on fumes at this point. ---- +**Ted:** Yeah, we had a busy start because we had Observability Day, which just started with Rejects, and then, yep, Observability Day and KubeCon Day one yesterday. -**Adriana:** That’s a fantastic vision. Finally, can you share the story behind the OpenTelemetry Foundation? +**Reys:** I know! It just feels like we've been here forever, so we're so excited to have you! Can you tell us a little bit about your role in OpenTelemetry? I'm sure a lot of our viewers who are more familiar with the project are already aware of who you are, but we'd love an introduction. -**Hazel:** The OpenTelemetry Foundation was started by Chris Nova. It germinated from our experience with a project that became a massive thing, and we wanted to ensure that the community owned its architecture and engagement platform. We faced many challenges, particularly around legal liabilities and how to ensure that the community truly owned their project and didn’t lose control of it. +[00:31:16] **Ted:** Yeah, sure! My name’s Ted Young, and I'm one of the co-founders of the project, coming from the OpenTracing side of the family. For people who don't know, OpenTelemetry is actually a merging of two prior projects: OpenTracing and OpenCensus. OpenCensus was mostly Google people and some Microsoft people, and OpenTracing was everybody else. Then we merged to form OpenTelemetry, and I continue to work on the project as a member of the governance committee, mostly just focusing on the various spec SIGs, implementation SIGs that need an extra helping hand when it comes to finding consensus and figuring out what design is actually going to work. -The foundation's goal is to bridge the gap between a fun toy project and a community-driven initiative. We want to figure out how to navigate the cliffs of open source and ensure beautiful growth and knowledge sharing. +**Reys:** Right. ---- +**Ted:** There are some areas in OpenTelemetry where it's technically challenging enough that ironically it's easy to come to a design, or at any rate, it's easy to determine which design is the correct one, right? Because it's so technically challenging that the requirements are very rigorous. But then there's other parts of OpenTelemetry where it's like a little more squishy, where there's like one way that would be a really, really good way, and then there's some other ways that would like sort of work, right? You can't say they definitely would not work, but they're not like great. But when you have that situation, it's so, so, so much harder to find consensus, right? Because people will lock in to a particular idea, and then if it's possible to find some way to make it work, because they're engineers, they will continue to promote, but "But there's this way it could work!" and trying to like find consensus there. -**Reys:** Thank you so much, Hazel. It was lovely talking with you. +**Reys:** Right. -**Adriana:** We will share social links at the end and how you can get involved in future Humans of OpenTelemetry segments. I’m really excited to see the interactions that come from these conversations! +**Ted:** Yeah, of being like, "Yes, okay, that would work, but this would work better," and getting everyone to be like, "Well, even though you prefer that one, would you agree that you're not going to convince everyone at this point to go that way?" Like, "Yes!" And like, "Would you be able to get everything done with this other one?" Like, "Yes!" And like, "Okay, well then would you be willing to come on board with this and help us move this forward because we all actually need it to get done?" So that's like a bit of just more like community organizing that requires maybe kind of like an engineering design background. ---- +**Reys:** Yeah, and I feel like that's where I provide the most value these days to the project. -**Reys:** Now, let’s bring in our next guest! +**Ted:** That's so awesome! Cool! I feel like there's also, you have to like have some good diplomatic skills as well. -**Ted:** Hello! +**Reys:** Right. -**Reys:** How's it going? +**Ted:** Building trust in the community is important, just absolutely having people know that you don't have an agenda. -**Ted:** Going great! It’s CubeCon—a big friend reunion. +**Reys:** Yeah, yeah. Right. And that you really are just there to listen to everyone and help everyone really clarify the requirements and help everyone really determine what the best design solution would be for those requirements. -**Reys:** Can you tell us about your role in OpenTelemetry? +**Ted:** Yeah, yeah, yeah. And you know, on a similar vein, it always brings me back to like the whole thing with OpenTelemetry being such a lovely community and that it really takes the whole thing of being vendor-neutral very seriously. I mean, we're all at three different places, and we all get along. Like, we're all competitors, but like not really, because we're all moving towards the same goal. I was wondering if you could comment on like what has to go on to really continue to foster that sense of community and not in us versus them. -**Ted:** Sure! I’m one of the co-founders of the project, coming from the OpenTracing side of things. OpenTelemetry is a merging of two prior projects: OpenTracing and OpenCensus. I work on the governance committee, focusing on various SIGs that require consensus and design solutions. +**Ted:** Yeah, that is great! So I really think there's two parts. The simple part is that like humans, generally speaking, are good people. And also, even though we might all work at different companies now, the reality is for the most part, there are just like many tech domains. There's like an observability scene, right? There's like the engineering and product people or whatever who's just like, that's their bag, and they're good at it, and they just tend to kind of like circle through the different orgs and companies that happen to be paying people to work on that stuff in that time. So I think that's the thing that makes it easier, right? Like it's not like we're from different planets that are in some intergalactic war, and we have like nothing in common or something. We've got all this stuff to overcome. It's just like we all know each other. ---- +**Reys:** Right. -**Reys:** That’s fantastic! What do you think helps foster the sense of community in OpenTelemetry? +**Ted:** And when you're around long enough, it's like everyone cycles two or three jobs, so you stop really looking at people as like representatives of a particular company. -**Ted:** Humans are generally good people. We all share a common interest in observability. When you’re involved long enough, you see people cycle through jobs, which makes it easier to trust one another. +**Reys:** It's so true! -The second part is structuring the project in a way that discourages bad incentives. We made it clear how people are going to make money and how OpenTelemetry provides value for end-users and vendors, which has helped build a healthy community. +**Ted:** Yeah. I think one of the things too, just from like my, you know, very short experience, is even when they are moving companies, people want to stay involved in the project. They're actively seeking out roles where they can continue contributing to the project in their role, so I think that speaks volumes. ---- +**Ted:** The fact that you can look at our maintainers and community members and there's like quite a long lifespan at this point. I actually haven't run the math, but I would not be shocked if like the average age of a maintainer in OpenTelemetry was measured in years at this point. Like, on average, it's like two or three years. I wouldn't be shocked if it's that high. Like people really do stick around. -**Adriana:** That’s insightful! What’s on the horizon for OpenTelemetry? +**Reys:** It's so true! -**Ted:** We’re at an interesting point where we’ve stabilized tracing, logs, and metrics. We’re looking at how to bring in profiling and other data sources. I think we’ll continue to see amazing new analysis tools that leverage all the data we’ve captured. +[00:36:30] **Ted:** And the second part of that is structuring the project because companies, there are very few like companies, but there are a lot of bad incentives. So when we started the project, coming from other open-source projects where I had borne witness and had to deal with all the fallout of these bad incentives or muddy incentives causing various companies to like test the waters and like get into stuff with each other. So when we started this project, we had a goal based off of our experiences of like how do we actually structure this project in a way that makes it obvious that these bad incentives are not present? Where people can look at that and be like, "Oh yeah, that won't work, so we're not going to bother to try to see if we can just buy the whole TC, right, and take over the project." ---- +**Reys:** Right. -**Reys:** Thank you, Ted! Before we wrap up, any last thoughts? +**Ted:** By like, "What if just everyone who works on who's in a leadership position works for Splunk or something?" Right? Like you can't do that because once you get more than two people on the TC who work at one company, and you want a third person, well one of those first two people has to drop out or remove companies or something. -**Ted:** I encourage end users to get involved in the Developer Experience SIG and provide feedback. We’re looking to clean up the SDK and instrumentation APIs. +**Reys:** Right. ---- +**Ted:** Yeah, and then when it comes to the actual shape of the project, prior projects, even these big industry things like Kubernetes and whatever, they tend to have this veneer of utopia like, "Oh, we're all just getting together for this altruistic reason of making container scheduling work!" And it is like as an individual, it's somewhat altruistic in that like I'm really interested in it. I would love to like push this domain forward. I want a real distributed operating system. I don't even want Kubernetes; I want the next thing, but sorry, Kubernetes! -**Adriana:** We also do monthly events and end-user interviews to gather feedback. Don’t forget to take the doc survey that’s open until the end of the week. +**Reys:** Right. -**Reys:** Thank you, everyone, for tuning in! We’re looking forward to more engaging discussions ahead. +**Ted:** But I also want the next OpenTelemetry; I want to keep pushing this stuff. + +**Reys:** Exactly! + +**Ted:** So I'm motivated, but like these are like billion-dollar projects almost, right? The amount of engineering effort that goes into Kubernetes... If you calculate the salaries that I'm sure that's at this point, that's probably a billion dollars. Maybe that's crazy; maybe it's like a hundred million. + +**Reys:** It's a chunk of change. + +**Ted:** It's a chunk of change! A lot of money when you think about developer salaries and how many people work on these projects. OpenTelemetry, on average, like monthly average number of contributors is like a couple hundred engineers, I think. + +**Reys:** Wow! + +**Ted:** On average, it's a couple hundred engineers pushing some kind of PR or something at any given moment on OpenTelemetry. It's a lot of people; there’s a lot of money. The only reason people are getting paid to do that is because there are some incentives. + +**Reys:** Yeah. + +[00:40:00] **Ted:** And the problem Kubernetes had when it first started and other projects that I worked on is there was all this interest and all this willingness to pay people to work on bootstrapping Kubernetes, but why? That question was actually nebulous for maybe some vendors or cloud providers. It was clear how they were going to make money and thus why they should be paying people to work on this, but for a lot of companies, a lot of groups getting started, especially the startups, it was not clear how they were going to make money off of Kubernetes. + +**Reys:** Right. + +**Ted:** So it was like step one: spend all this money to bootstrap this thing; step two: figure out how to exploit it. And just leaving that door open, right? Just the fact that it wasn't really clearly defined how Kubernetes was to be organized, how everyone was expected to make money, what part of this is going to be some shared open stack versus what part are we going to like sell things or whatever? That was all just like a big question mark in the early days and should all the code live in a Kubernetes GitHub? Or should various startups and companies be allowed to have complete ownership of some component that lives within Kubernetes and thus it lives within that company's repo? + +**Reys:** Right. + +**Ted:** All things like this led to just like a lot of problematic stuff, right? Like what if one company kind of owns a component of OpenTelemetry? They're going to build a lot of like startup brand around that component naturally, right? That would be the natural play, would be to talk that component up and how it's kind of yours. + +**Reys:** Right. + +**Ted:** And then OpenTelemetry, a project, was like, "That was a cool component, but now we're going in a different direction, so we should deprecate that one and get a new thing." Well, that would be like the kiss of death to a startup. + +**Reys:** For sure. + +**Ted:** So you can start to see how like just leaving the door open starts to like create a situation where things like people's incentives can start to get at loggerheads. + +**Reys:** Right. + +**Ted:** And working on containerization stuff that led to these situations where you'd be in spec meetings, right? People are making some technical proposals, and everyone else is like, "I can't tell if this is a good idea or not. This might be a good idea, or this might be the beginning of some sneaky play to like insert some nasty thing so that this company can pull some crap on the community later." + +**Reys:** Right. + +**Ted:** And why are they doing that? They're evil? No, but because they need to make money off of all this money that they're spending. So when we started OpenTelemetry, we felt it needed to be very, very, very clear how people were going to make money and how this project was going to provide value for end users, for vendors, for cloud providers, for all the people involved that need to be crystal clear. Because otherwise, we were going to end up in that situation. + +**Reys:** Right. + +**Ted:** But because we learned our lessons, we did make it crystal clear. + +**Reys:** Good! + +**Ted:** Yeah, right! All the code lives in OpenTelemetry. Do you want to donate something to OpenTelemetry? That's fabulous! You need to move it into OpenTelemetry; you need to change the copyright and everything to be the OpenTelemetry authors. You need to completely relinquish any individual ownership of this; it's now collectively owned by OpenTelemetry and is now part of the CNCF. It's no longer part of your startup. + +**Reys:** Right. + +**Ted:** And also, like, what's the boundary of the project? OpenTelemetry, we're going to standardize the data being emitted by all of these systems, but storing and analyzing that data, that's never going to be part of OpenTelemetry because you wouldn't want to standardize that part. That's like green field; that's the part where we're trying to figure out the futures. Like, what can we do with this information? And we want everyone to compete on that, and that's also where everyone's going to make money and making that very, very clear really did a lot in the early days as far as bringing in the second and third wave of contributing companies. + +**Reys:** Right. + +**Ted:** Like you have this first wave of people who are like, "We're just making this major bet because it's super clear this would be good for us." Then this second wave of companies that are like, "Well, okay, if those guys are all in, that kind of changes the calculus for me; I guess we should get involved." + +**Reys:** Right. + +**Ted:** And then once those groups get involved, there's like a third group that's like, "Well, now that all of those people are involved, I guess that changes it for us." And the fact that it was super clear what you would get out of this if you put something into it made that happen faster. + +**Reys:** So you mentioned, you know, OpenTelemetry is the merging of like OpenTracing and OpenCensus, thank you. I was like OpenTraces! Okay, day three is okay, sorry. + +**Ted:** No worries! + +[00:44:35] **Reys:** And you know, obviously, you guys did the project update yesterday; we heard a lot of new cool stuff coming on the pipeline. Was this part of your vision like at the beginning of the foundation of OpenTelemetry? + +**Ted:** Or is this kind of... you mean like where the project is today? + +**Reys:** Yeah! + +**Ted:** Yeah! I mean, we've really like our original mandate. We're at kind of this interesting spot in OpenTelemetry where we had this original mandate, which was to unify tracing, metrics, and logs. Those are the primary signals that we have available. But the problem in the past was these were siloed systems, and what you're trying to do is get a completely connected coherent view of the system. + +**Reys:** Right. + +[00:30:00] **Ted:** And those were the three major signals that we wanted to tackle first. So that was like our original pitch: we're going to merge these three signals into one graph of data that can then be walked by a computer system, and we can dump all of this analysis off onto the computers instead of doing it in our brains when it comes to finding correlations across all of this information. That was a great pitch; it's been part of actually transforming this whole industry, right? As the data completely changes, the products, of course, have to completely change. If you're going from a bunch of siloed systems to like a unified platform, you can see how that's just going to create a complete sea change within the industry because, well, if that's the future, then what's going to happen? Well, all these companies are going to consolidate. Right? One company that was doing logging or whatever is going to acquire some companies that did the other signals, and then they're going to try to merge all of that into a platform. Or if they're a new company, they're right out of the gate, they're going to be like, "Are we a platform? Or how do we fit into this new platform world where all the data is going into one spot?" + +**Reys:** Right. + +**Ted:** So that's like a complete industry shift that probably would have happened anyways without OpenTelemetry, but OpenTelemetry is like this massive accelerant. So you incentivize basically the unification of the signals—not just supporting the three signals but actually intertwining them so that you can really get the most out of observability, really. + +**Reys:** Right! + +**Ted:** Yeah, and I call, you know, the old version of it, I call the three browser tabs of observability because calling it three pillars gives it too much credit. Like, there was some intentional structure there; there was that was like never an intentional design; it's like a terrible design. What should we do? We should silo all of the data and use our brains to try to figure out how what connects to what. Like that's obviously dumb. So obviously, we didn't design that; it just kind of accrued over time. + +**Reys:** Right. + +**Ted:** And going from that to what I call the braid of data. So it is all this data, but it is being braided and connected in various ways. + +**Reys:** Right. + +**Ted:** And you can think of the value that comes out of a braid, right? You can think of like the individual strands in a rope, and then you can think about a rope and how much more you can do with a rope than just pieces of straw. + +**Reys:** Yeah. + +**Ted:** It's just so much stronger, right? + +**Reys:** Exactly! + +**Ted:** And can you give folks just a high-level overview of what some of the updates that came out of the project update yesterday? + +**Ted:** Yeah! I would say the super high level—and we didn't really cover this in the update, but—getting back to this industry sea change, right? Being at this moment where we're finally stabilizing tracing, logs, and metrics, right? We had the kind of last piece of that. There's a little bit of work that needs to be done related to resources, so if people don't know, resources are in a way like the fourth signal, right? Like when we, there's the data, there's the system, and there's like the structure of the system. The traces are kind of like the energy, right? That's the transactions; that's the stuff moving through the system. But then there's the actual stuff that it's moving through, and those are the resources—the Kubernetes clusters, the virtual machines, the application binaries, all of that stuff. When we started the project, there was this idea that came from OpenCensus that these resources were immutable compared to the lifespan of a service. When a service started, it was associated with some resources, and those resources did not change. + +**Reys:** Right. + +**Ted:** And this kind of made sense, yeah, because generally speaking, the resources you care about with server-side computing, they don't change. Now they do sometimes, but this is just like edge-casy enough that at Google, they kind of were just like, "Eh, we're just going to ignore that." + +**Reys:** Right. + +**Ted:** Like, you can freeze a virtual machine, move it into a different data center, and then like turn it back on, right? And now all of its resources changed! It's running; it's the same computer program, and now it's running somewhere completely different! But in practice, we don't do that, right? So maybe you could just ignore this requirement, and this was very convenient because you have this obnoxious thing with resources where you want to figure all of them out before you start sending data. You don't want to start sending partially indexed data, which is what would happen if you started emitting telemetry before you figured out your Kubernetes pod ID or whatever. But figuring out some of these resources can take time. + +**Reys:** Right. + +**Ted:** You have to go query some API and like wait for the answer, and you don't want to delay the booting up of your app necessarily, so do you move forward? Or is this gating? It turns out there's a lot of pernicious problems around actually gathering all of these resources and associating them. And the biggest problem is devs, because it's annoying at startup to be like, "Well, whatever. We'll do the dumb thing of, we'll just tack these resources on as they come back from their various APIs." + +**Reys:** Right. + +**Ted:** And then you end up with this situation where you have some partial data at the beginning. How do you prevent devs from making this mistake? Well, what if you made the resource API immutable, where you could only set it once? Then they would try to do this, right? They would try to come back with some late binding resource, and then the API would be like, "Oh, I can't! Oh, this is annoying! I guess I'm forced to do it the right way!" So it's not that resources are immutable by some intrinsic nature of what resources are; resources are mutable because that was a convenient way to force the Google devs to not make this like stupid mistake. + +**Reys:** Right. + +**Ted:** And to force them to do the extra effort figuring out how to resolve all of these resources at the beginning. So to protect them from themselves to a certain extent, this is my claim. + +**Reys:** Right. + +**Ted:** So, but then you get to clients, right? And all of this was fine. This was just like fine; all swans are black. And then you get to client development, which we've been a little slow to get to in part because almost all the engineers came from kind of a server-side background. It was just a little bit late before the client-side people started showing up, but when they did, client-side computing is very different, right? When the application starts and stops is actually like pretty arbitrary. Like it's pretty arbitrary when you hit quit on your web browser or whatever; it doesn't really signify anything. When program boots or ends, but it goes through all these different states, right? Like it might be foregrounded or backgrounded, right? It might be on Wi-Fi or a cellular network, right? It might get put to sleep and then wake up in Hong Kong, you know? + +**Reys:** Right. + +**Ted:** And so all of these resources are changing over the lifespan of this application. Oops! Oops, indeed! Damn it! So that's kind of where we're at with the project, where we finished our original mandate, but there's like a couple of squiggles around resources that are kind of like the last thing where it's like, "All right, we have to do this really annoying work." This is going to be really annoying for me personally because this is the exact kind of thing where it's like we need to fix this to make it work for the clients, but we already told people it was immutable, so can we like take that back and say like some of these are immutable and some of these are not? Like, would that screw anybody up or not? Like, this is going to be very tricky. + +**Reys:** Right. + +**Ted:** And then we're going to have to come up with a solution, and there's going to be a bunch of community people be like, "But we said it was immutable!" And then we're like, "But clients need to be like, 'But I don't care because I'm a Go developer!'" You know? + +**Reys:** Right. + +**Ted:** Like, yeah, so this is what I do, okay? Is try to bring everyone together so we can move forward. But if we can get past that, now we are in the green field of the future where we are looking at all of the data that we haven't captured yet, and we want to figure out how to start bringing that data in. How do we bring profiling in? How do we bring source code commits, versioning, all of that in? Yeah, config files! That's the future! + +**Reys:** That's exciting! + +**Ted:** Yeah! So we're, you know, at the five-year, five, six-year mark of the foundation. What do you think it's going to look like in another five years? + +**Ted:** In another five years? Pretty much the same, but with more stuff tacked onto it. Sorry, I'm going to be very boring with that answer because I think we move slow, and I don't believe in 2.0's. No, it is just 1.0 forever. + +**Reys:** Right. + +[00:55:00] **Ted:** And so I just think it'll be, but there will be like a sea change that happens once you get enough of this data actually connected. There are certain it unlocks certain things, so I don't think you're going to see OpenTelemetry radically change in the way it looks, but five years out from now is enough time for the analysis tools to start to catch up with where the data is. So where I think you're going to see some amazing things is some amazing new analysis tools that are actually looking at all kinds of data that we can't look at today and are condensing that into like a helpful co-pilot for an operator trying to figure out their system. + +**Reys:** That's exciting! + +**Ted:** Right on! And before we log off and get back into the real world, I have to learn about your pants. What is the story with these cat pants? Because they're fabulous! + +**Ted:** They have cats on them! You know these cats? + +**Reys:** Do I know them? Are they your cats? + +**Ted:** They are not my cats! I wish! I wish I had personally fabricated these pants, but I did not. The internet provided these pants! But I did know that these pants existed before I looked at them. I was like, "I want pants that have kittens and space on them!" And I knew the internet would give this to me. I had complete faith, and it took about 30 seconds. + +**Reys:** Wow! The internet came through! + +**Ted:** It came through! Yeah! You know, they can make me get out of bed; they can't make me get out of my pajamas. That's kind of my attitude. + +**Reys:** I love it! All right! What else would you like to share before we head back into the actual event that we're all here for? + +**Ted:** You know, I would just encourage anyone watching to, if you're an end user, please get involved in the developer experience SIG and give us feedback on your experience as an end user, in particular around installing the SDK and using the instrumentation APIs because we want to clean all that up now that things are stabilizing. So yeah, please get involved if that's what you're interested in. + +**Adriana:** And speaking of end users, Reys and myself work in the end user SIG. We do monthly events like OpenTelemetry in practice. We do end user interviews to learn more about your adoption and implementation process, you know, and get some feedback that we can share back with the SIGs. We'll have a QR code at the end that you can scan to get in touch with us on the OpenTelemetry SIG user channel on CNCF Slack. + +**Ted:** And don't forget the survey that's open until the end of the week! + +**Adriana:** Oh, the docs survey! The docs survey! It ends Friday! Yes! If you've used OpenTelemetry in any way, shape, or form, and you have something to say about it, whether good or bad or just comments or questions, definitely please take the survey! We will link that in the show notes, I believe we can do that. + +**Ted:** That is the thing we can do! + +[00:58:00] **Adriana:** Yeah, and we should also, if you go to the OpenTelemetry socials on I think every social platform, we're on BlueSky, X, LinkedIn, and Mastodon, there should be some tweets also with links to the survey. + +**Ted:** Yes, and we are available on Slack at any time, and yeah, reach out to us; we'd love to hear from you. + +**Reys:** And I think we're good! ## Raw YouTube Transcript diff --git a/video-transcripts/transcripts/2024-12-17T22:14:29Z-humans-of-otel-kubecon-na-2024.md b/video-transcripts/transcripts/2024-12-17T22:14:29Z-humans-of-otel-kubecon-na-2024.md index 9100d98..0a8e55a 100644 --- a/video-transcripts/transcripts/2024-12-17T22:14:29Z-humans-of-otel-kubecon-na-2024.md +++ b/video-transcripts/transcripts/2024-12-17T22:14:29Z-humans-of-otel-kubecon-na-2024.md @@ -10,106 +10,144 @@ URL: https://www.youtube.com/watch?v=TIMgKXCeiyQ ## Summary -In this YouTube video, several professionals in the software development and observability space share their experiences and insights regarding OpenTelemetry, an open-source observability framework. Participants include Hazel Weakly, Eromosele Akhigbe, Budha, Miguel Luna, Adriana Villela, David, Endre Sara, Braydon Kains, Christos, and Reese Lee. They discuss how they became involved with OpenTelemetry, emphasizing its importance in standardizing observability practices across various organizations and improving the efficiency of troubleshooting and monitoring software. Topics explored include the significance of observability in understanding system behavior, the role of community collaboration in the OpenTelemetry ecosystem, and the various telemetry signals such as traces, metrics, and logs that aid in effective software monitoring. The conversation highlights the transformative impact of OpenTelemetry on both individual careers and organizational practices, reinforcing its potential as a future standard for observability in software engineering. +In this video, a diverse group of tech professionals, including Hazel Weakly, Eromosele Akhigbe, Budha, Miguel Luna, Adriana Villela, David, Endre Sara, Braydon Kains, Christos, Reese Lee, and others, share their experiences and insights about OpenTelemetry, an open-source observability framework. They discuss how they became involved in the OpenTelemetry community, emphasizing its role in standardizing observability practices across various platforms and improving collaboration among engineers. Key themes include the importance of observability in software engineering, the benefits of using OpenTelemetry for troubleshooting, and the collective effort to establish common semantic conventions. The participants also reflect on their favorite OpenTelemetry signals, highlighting the significance of traces, metrics, and logs in understanding system behavior and enhancing user experience. ## Chapters -Here are 10 key moments from the livestream along with their timestamps: +00:00:00 Welcome and introductions +00:02:00 OpenTelemetry involvement stories +00:04:50 Contributions to OpenTelemetry +00:07:42 OpenTelemetry community impact +00:10:30 Observability definitions +00:14:00 Importance of user experience +00:17:30 Observability as understanding +00:20:00 Future of OpenTelemetry +00:23:00 OpenTelemetry as collaboration +00:25:22 Favorite OpenTelemetry signals -00:00:00 Introductions of speakers and their roles -00:05:30 Discussion on how individuals got involved with OpenTelemetry -00:10:15 Insights on the importance of open standards in technology -00:15:50 Explanation of the different signals in observability (traces, metrics, logs) -00:20:10 The role of OpenTelemetry in improving API management and troubleshooting -00:25:30 Personal experiences with OpenTelemetry and its impact on software engineering -00:30:45 Observability's definition and its significance in understanding system behavior -00:35:00 The collaborative nature of the OpenTelemetry community -00:40:20 Discussion on common standards and semantic conventions in OpenTelemetry -00:45:00 Final thoughts on the future of OpenTelemetry and its role in the industry +**Hazel:** Hey there. My name is Hazel Weakly and I have thoughts, lots of thoughts. They never stop thinking. And they never stop thinking. -# OpenTelemetry Community Insights +**Eromosele:** My name is Eromosele Akhigbe and I'm currently a software engineer at Sematext. -## Introductions +**Budha:** Hello everyone. I am Budha. I'm a developer advocate at Tyk. Apart from that I've got a very deep relationship with open standards because I'm also the board chair for the OpenAPI Initiative and a board member for the GraphQL Foundation. -Hello everyone! My name is **Hazel Weakly** and I have thoughts—lots of thoughts. They never stop thinking. +**Miguel:** My name is Miguel Luna and I'm a product manager at Elastic where I'm the product lead for the OpenTelemetry efforts across the company and what we contribute to the community. -I’m **Eromosele Akhigbe**, currently a software engineer at Sematext. +**Adriana:** My name is Adriana Villela and I'm a Principal Developer Advocate at Dynatrace. -Hi, I’m **Budha**, a developer advocate at Tyk. I have a deep relationship with open standards as I serve as the board chair for the OpenAPI Initiative and a board member for the GraphQL Foundation. +**David:** My name is David and I work at Monday.com. I'm a software engineer and I work there on the CRM product. -My name is **Miguel Luna** and I’m a product manager at Elastic, where I lead the OpenTelemetry efforts across our company and contribute to the community. +**Endre:** My name is Endre Sara, I'm the co-founder of a company called Causely. I started Causely two years ago. -I’m **Adriana Villela**, a Principal Developer Advocate at Dynatrace. +**Braydon:** My name is Braydon Kains. I'm a software developer at Google in the Google Cloud Org. I work for the Cloud Observability service and I mainly work on agents that customers install in their environments to collect telemetry signals and send them to Google Cloud. -My name is **David** and I work at Monday.com as a software engineer on the CRM product. +**Christos:** My name is Christos. I'm a software engineer at Elastic. I have been working mainly in observability over the past five years now and since last year I have been contributing mostly to the OpenTelemetry ecosystem. -I’m **Endre Sara**, co-founder of Causely, which I started two years ago. +[00:02:00] **Reese:** Hi, my name is Reese Lee and I am a Senior Developer Relations Engineer at New Relic. OpenTelemetry. I got into the project sort of almost accidentally, although I think at this point that's an answer that I give for everything. When I mean accidentally, it was I was looking for answers to questions that I had and more importantly, how do I teach other people to find answers to questions better and how do I continue to level up the teams that I worked with, the organizations that I worked with and in figuring out how to get people better at asking questions, getting answers, and learning from that? I finally stumbled onto OpenTelemetry. -My name is **Braydon Kains**. I’m a software developer at Google within the Google Cloud Org, working on the Cloud Observability service. +In March, I entered an internship called Outreachy and in Outreachy I was privileged to work on OpenTelemetry and I worked on building a logging bridge in Golang, and by the end of the internship I was able to build a logging bridge using OTel zerolog. -I’m **Christos**, a software engineer at Elastic. I’ve focused on observability for the past five years, contributing mostly to the OpenTelemetry ecosystem. +How did I get involved with OpenTelemetry? This is a multi parter question or answer, I think in this case, because there were a couple of reasons why it caught my attention. Starting off with actually advocacy from our new group product manager who had recently joined and she was a big proponent of observability and OpenTelemetry specifically. I kind of had played around with OpenTracing and OpenCensus for a little while, but I hadn't really looked into OpenTelemetry. But once she came in I was a huge advocate for it and that got my attention. That was trigger number one. -Hi, I’m **Reese Lee**, a Senior Developer Relations Engineer at New Relic. +Trigger number two was the fact that it was this open standard. So I think anything open standards to me is a no brainer. I've got a lot of time to invest in any sort of open standard that makes life easy. I think from a flexibility standpoint that's the way to go. So that was trigger number two. -## Getting Involved with OpenTelemetry +Trigger number three was actually when we started using OpenTelemetry. So we are an API management platform at Tyk. For us, OpenTelemetry was being used internally as well as externally. So internally we could already start seeing results in terms of how quickly and efficiently we were getting to troubleshoot problems and getting to the heart of issues. And not just limited to REST APIs but actually with GraphQL APIs as well, which you wouldn't have considered as a possible use case. But we were able to remediate some of those issues that we were facing with that. So that was sort of trigger number three. -I got into the OpenTelemetry project almost accidentally. I was looking for answers to questions I had and how to teach others to find answers better. This journey led me to OpenTelemetry. +And all of that collectively came together to say, hey, OpenTelemetry deserves attention. Initially I started as a product manager. It was a very interesting role because I started in a role where it was more about coordination rather than contributing directly. But I've been recently involved in the localization of the documentation. So that means translating the documentation, more specifically in my case among Spanish speakers. So, la traducción de la telemetria abierta. So, the traduction...the translation into the Spanish of the OpenTelemetry documentation. -In March, I participated in an internship called Outreachy, where I worked on building a logging bridge in Golang using OTel zerolog by the end of my internship. +[00:04:50] At my previous role, my manager at the time, as part of it, he encouraged me to actually join the OpenTelemetry community. And it was actually my first time ever contributing to open source and I never contributed to open source. I've been in tech for like over 20 years and my manager basically said, yeah, just attend a couple meetings. And my first meeting was for the OTel comms. And so that was kind of my gateway into OpenTelemetry. -My involvement with OpenTelemetry came from the advocacy of our new group product manager who was a big proponent of observability and OpenTelemetry. Prior to that, I had experimented with OpenTracing and OpenCensus but hadn’t delved into OpenTelemetry. Her advocacy piqued my interest. +I started my career in embedded applications and I was doing eBPF tracing before that was even a thing. I then moved into Dropbox where all our telemetry was in-house before OpenTelemetry was mainstream and now on Monday I continue doing trace-based testing. I started to learn about OpenTelemetry. I realized that this is such an opportunity for the whole industry to actually commoditize and standardize how instrumentation is being done and to be able to use common semantic conventions so people can understand what's going on. So I got instantly excited and I started to work on it. -OpenTelemetry caught my attention primarily because of its status as an open standard. I believe investing time in open standards makes life easier and offers flexibility. +First it was just a few test applications, then I played with and demoed to people on how to do instrumentation. But as we started our current company, from day one I said we have to make sure that we are properly instrumenting our software so that we can actually operate this as we get more customers for logs, metrics and disability testing, it has been helping us a lot. -At Tyk, we started using OpenTelemetry both internally and externally, which significantly improved our troubleshooting efficiency across REST and GraphQL APIs. +I got involved in OpenTelemetry because our team uses OpenTelemetry, namely the OpenTelemetry Collector, to support our customers collecting data off of their environments. When we had bugs and issues with OpenTelemetry in the past, there would be some light involvement from the team, but largely we would open an issue and sort of wait for it to get addressed. And I really wanted to change that within the group. And I wasn't the only one on our team who wanted to change that. So, you know, we all sort of started to make a more genuine effort to open issues that came with PRs. And that has generally moved our whole team forward into being more involved in OTel. -Initially, I began as a product manager, focusing on coordination rather than direct contributions. Recently, I’ve been involved in localizing documentation for Spanish speakers, translating the OpenTelemetry documentation into Spanish. +And I've ended up being much more involved in OTel to the point where now I'm a code owner on the Host Metrics Receiver, which is an important receiver to us, but I get to dedicate more time to making sure it's good for everyone and not just fixing our own problem. -My first experience with OpenTelemetry was at my previous role, where my manager encouraged me to attend community meetings. This sparked my first contribution to open source after over 20 years in tech. +I was originally asked to contribute to the OpenTelemetry by helping with the Elastic Common Schema donation to the OpenTelemetry, specifically to the specification and the semantic conventions. And since then I have been more and more involved in other projects like the Collector. And right now I'm mainly focusing on the semantic conventions and the OpenTelemetry Collector, specifically the Collector contrib project. -I started my career in embedded applications and was doing eBPF tracing before it was mainstream. After moving to Dropbox, where we handled all telemetry in-house, I continued to work with trace-based testing at Monday.com. +[00:07:42] The way I got involved in OpenTelemetry was at New Relic. And at first my first experience with it was through some support tickets that we started to get around some of our customers who had adopted OpenTelemetry. And then I had a great opportunity to join our dedicated OpenTelemetry team at the time as a developer relations engineer. And this was back in November 2021. And I was able to integrate within the OpenTelemetry community pretty soon after that. And actually my previous manager, Sharr Creeden, she kind of spearheaded the work to build the End User Working Group at the time, and now we are the End User SIG. -My team utilizes OpenTelemetry, specifically the OpenTelemetry Collector, to support customers in collecting data from their environments. Initially, we would report bugs and wait for fixes, but we aimed to change that approach by actively opening issues and contributing PRs. +OpenTelemetry has been really useful at my organizations that I've worked on because it's become something that you can tie into different vendors, tie into different tools, and into other intermediary ways. And the huge benefit of it for me is that I can take all these different bits of knowledge, not necessarily signals, but different bits of context from the company, tie it all together in a way that I can show people these answers to their questions, regardless of whether or not they're in engineering. And that is a new capability because previously engineering was kind of in its own bubble and increasingly it really can't continue to do that. -I was asked to contribute to OpenTelemetry by assisting with the Elastic Common Schema donation and focusing on the Collector and semantic conventions. +And so OpenTelemetry has been super impactful for me for bringing our knowledge outside of engineering and bringing the outside knowledge into engineering. Things have become a lot more efficient internally. When I talk to our SRE teams, our DevOps teams, they're a lot happier when they're interacting or working with different elements of our platform stack. It's a lot easier to manage and handle it. Now when I talk about the end users, they can truly talk about the value of it. -At New Relic, I began working with OpenTelemetry through support tickets from customers adopting it. I joined the dedicated OpenTelemetry team in November 2021 and integrated into the OpenTelemetry community quickly. +And personally, I think just the advocacy side of things, I think has been really, really enriching for me to learn more about it. Being involved with the community in different ways. Earlier this year I had the privilege of putting together a mini conference called LEAP, which was the API Observability Conference, where a lot of the folks from the community were able to scroll, speak to the different areas and elements of OpenTelemetry, not just limited to, again, the engineering side of things, but also how decision makers could perceive the value of adopting something like OpenTelemetry within their organization. -OpenTelemetry has proven invaluable across organizations by providing a way to connect different vendors and tools. It allows for greater collaboration and efficiency and enhances internal communication among teams. +[00:10:30] It all started when Elastic, we decided to donate the Elastic Common Schema, which was a natural fit to the goals of OpenTelemetry, or standardizing observability and driving efficiency across getting telemetry data converged into a single standard. When I just started my career, there was no OpenTelemetry, so I had to figure out how to do traces and how to correlate them with metrics and how to do logs. But now all this effort has already been standardized by the community, so new engineers that are onboarding into OpenTelemetry have a much easier time than I have. -Earlier this year, I organized a mini-conference called LEAP, which focused on API observability, gathering insights from various community members, including decision-makers. +In general, I think that the ability to be able to take signals from your application and to be able to use them to operate the environment to understand the behavior of the system is significantly easier with OpenTelemetry than it was with other proprietary instrumentations in the past. What I think is more interesting is what do you do with this data? Most of this information is being exposed to people in dashboards which are amazingly nicely presented, contextualized based on semantic conventions. -We at Elastic decided to donate the Elastic Common Schema to OpenTelemetry, aligning it with the goal of standardizing observability and improving efficiency in telemetry data collection. +But I think that the biggest advantage is to be able to use software to reasonably data. The main way OpenTelemetry has helped me personally is really learning how to interact with a large community. I already had some experience with open source communities and there is this sort of general culture of like, you know, you do the work, you get a say in the project. That's pretty common in the open source world and I think that's fine. -## Observability Insights +But OpenTelemetry has a very large... I really like working with the OpenTelemetry ecosystem in general because I believe that working with people from other companies, other teams, helps me personally as an engineer a lot because I see how other people do observability out there. So I keep learning a lot. So that's something that I really like and I believe in general my team is also really helped by this, by this fact and also for my job. I mean it's amazing because I really love open source, I really love working with open source projects. And yeah, I think that on a personal level it's really helpful. -Observability is essential for understanding the user experience in increasingly complex software environments. OpenTelemetry provides the tools to answer challenging questions about user interactions with our products. +OpenTelemetry has helped me personally in honestly really big way, in the sense that working in developer relations with OpenTelemetry, I've gotten to meet a lot of wonderful people which I talked about earlier. But as part of my role I get to submit topics to different events and part of that is being able to learn about all these different topics myself and being able to talk to people who are using it in production or trying it on themselves has been a really wonderful experience. -The primary purpose of observability is to understand and reason about system behavior. Merely collecting data is insufficient; the insights derived must drive actions and improvements in reliability and performance. +[00:14:00] My definition of observability, it is the process through which one develops the ability to ask meaningful questions, get useful answers and then act effectively on when you learn. So what I mean by that is it's not enough to be able to kind of figure out the answer. There's this process where you have to actually work on it over and over and over and you're developing a skill not just on a personal level, but on an organization level, on a group level, and in even broader an industry level. So as you continue to do that, continue to get those really, really useful answers and really, really meaningful questions that you can ask. You start to have this whole process of group learning that transcends the boundaries that we draw for ourselves and lets those boundaries become empowering rather than limiting. -To me, observability means having a clear understanding of your platform's pulse and recognizing what functions well and what does not. It’s about making informed decisions based on this understanding. +Observability Engineers are like the doctors of your system. So if something is going wrong in your system, you need us to be able to pinpoint where exactly or what exactly is wrong and how to solve whatever is wrong. So that's what observability means to me. -Monitoring answers known questions, while observability identifies unknown questions. Observability means gaining insights into your systems, transforming debugging processes into actionable insights. +What does observability mean to me? There is a technical answer to this, where it goes into the realm of perhaps monitoring, perhaps logging, and, you know, getting to the troubleshooting of all things. To me, it's all about understanding. It is essentially understanding the pulse of your platform that you have created. I work with APIs quite a lot, so everything underlying is all to do with API platforms. -Observability engineers are akin to doctors for your systems. When something goes wrong, we pinpoint the issue to resolve it effectively. +So understanding the pulse of your API platform, the different components coming together and knowing exactly what's functioning, not functioning, the good, bad and ugly of it all, that, to me, is what observability is all about. So to be able to get to that part of the problem, to be able to know what's working, what's not working, and making decisions more effectively. -OpenTelemetry represents the future of observability. It fosters a collaborative community where vendors work together to innovate without competing over signal collection. +Monitoring means knowing answers to questions that you know you needed to ask. Observability means knowing questions to answers that you didn't know that you need to ask. To me, observability means the ability to get insights into your system. And for me, like, this was extremely transformative, because, like, there's so many times in my life where, you know, I was debugging code, whether it was like my own code, like, as a developer or code in production, and not understanding, like, just looking through logs and not understanding, like, okay, but how does this relate to the bigger picture? -## OpenTelemetry's Community and Future +Like, I have so many memories of troubleshooting production issues, and it's like, oh, the system is slow. So you ask the person who's responsible for administering the app server, hey, can you check the logs? No, not my problem. You ask the DBA, no, no, it's not my problem. And then you ask whoever else, and you go down the whole line and, like, it's nobody's problem. And yet you're still seeing latency. And I feel like observability kind of like it. It uncloaks the whole thing because all of a sudden it exposes. Like, it exposes where the actual root cause is. And I think that's the magic and power of observability. -To me, OpenTelemetry is a home—a community that emphasizes collaboration among various vendors working towards the same goal of improving observability. +The most important thing in software engineering today is the user experience. And because our software is getting much more complex, it's getting harder to answer the question, how are my users experiencing my product? And OpenTelemetry allows us to answer these difficult questions and provide us with visibility into our software. -OpenTelemetry serves as a means to standardize efforts among engineers striving to tackle complex questions in software development. It enables collaboration, allowing companies to focus on adding value rather than competing over instrumentation. +[00:17:30] I think that probably the most obvious answer is to be able to collect signals. But I think that the real point of observability is to understand and reason about the behavior of the systems. Simply collecting data doesn't actually accomplish much. I think also with the becomes meaningful and valuable, and people are able to use this to drive actions, to drive decisions. Where do I need to improve reliability? Where do I need to improve the performance of my application? Where do I need to make architectural changes? I think observability is really serving that. Otherwise it's just a lake of data. -In my experience, OpenTelemetry is a fantastic place to learn and collaborate with passionate individuals in the observability space. +Observability means to me that you can tell what's going on. Computers are black boxes that understand what ones and zeros do. And being able as a human to understand what ones and zeros are doing at any given time, when a computer is blazing so fast, how would you ever be able to figure out what that means? So observability to me is the human version of understanding what a computer is doing. -## Favorite OpenTelemetry Signals +So for me, observability is something that I have been working on since university, and it's a really important area because I think that what really matters when we are running systems, it's the way that you can observe your systems, you can know if your systems are doing good or not. And specifically, I'm coming from an infrastructure background, as I mentioned before. So for infrastructure specifically, it's really, really important when it comes to cost reduction. And this sort of stuff or how the whole system is working is an important piece that you cannot miss. -- **Traces**: They provide a comprehensive overview of system interactions, helping to pinpoint errors effectively. -- **Logs**: They offer deep contextual insights and are foundational for understanding system behavior. -- **Metrics**: Essential for identifying problems and understanding performance trends. +Observability to me means that I, as an end user of various applications and software programs, get to have a better experience because the companies that build these products, you know, assuming that they're using observability and being able to stay on top of issues that are happening in their code, it means I get to have a better experience as an end user. -In conclusion, OpenTelemetry is not just a technical standard but a vibrant community that empowers engineers and organizations to enhance their observability practices and drive meaningful outcomes. +[00:20:00] OpenTelemetry to me is one really interesting approach towards building something that takes a very sort of capitalistic notion of companies need to be profitable, companies wanting to innovate, people wanting to compete, and people want to develop different solutions to things. And how do you wrap all that together in a project that's flexible enough to allow that competition, to allow those ideas to happen, and to allow this innovation to continue without limiting what's possible and without burdening the industry with the intermediate details of the evolution of that complex, the evolution of pursuing excellence. + +OpenTelemetry is, I believe, the future of observability. In March, when I started doing research on OpenTelemetry, I discovered how big this can be and I decided that I was going to go in fully into OpenTelemetry. So I believe that it's the future of observability and everyone should take it. + +What does OpenTelemetry mean? To me that's an extension of that understanding. In a way it's the... Well, again, the technical answer to this would be, is the open standard that essentially powers distributed tracing. That's all fine. To me it's the extension of that understanding by creating a common language or framework, however you want to put it, that the different components and elements of your platform stack can unite together to speak to the health of your overall platform. + +And that could go from the engineering standpoint all the way to the business standpoint. There are repercussions to both of those. So to me that's what OpenTelemetry as a technology brings, both from a business and a technical standpoint. But it's also about the community as well. It's sort of again the industry coming together and agreeing on a standard so that the life of SRE, DevOps organizations, tooling providers, end users, all of their lives are made a lot easier because by virtue of having an open standard, it means that your platform is a lot more flexible, you have a lot more freedom to evolve, to mature and actually be a bit more future ready. + +So that to me is the promise of flexibility and freedom that OpenTelemetry brings. For me it means a common language. So it's a place where we all, where users can made and at least understand that everything that we are going to collect is going to be collected in a similar way, with a similar mechanism. Also what we call things. So the semantic conventions we agree on common standards of what we are going to call things. So the telemetry is the same and it can be reusable. + +So yeah, so that's OpenTelemetry for me. What does OpenTelemetry mean to me? To me, you know, it feels like home actually, because it's been like my home for the last like two and a half years. So it's been like really transformative in my life because it's like I said, was my gateway into like open source, into the CNCF community. And so it takes on like a very personal flavor for me, just beyond like the, you know, the typical definition of OpenTelemetry, which is like this open standard for instrumenting your code. For me, it's just so much more than that. + +[00:23:00] It really is like this lovely community where we're working with different vendors from across the board and we're not enemies, we're all friends because we're working towards the same goal. OpenTelemetry is basically a way to standardize all the efforts, all the engineers that want to ask all the difficult questions about software. OpenTelemetry gives a way for multiple vendors to work together, to collaborate together and take the instrumentation as a given that is not a function of competition and really focus on adding value on how this information turns into an actionable insight. + +And I think that that is really where people, vendors, end users are expecting to innovate in. So OpenTelemetry is basically the enabler for vendors to focus on where the real value is. OpenTelemetry to me is a representation of the industry coming together and understanding, you know, what are we competing about really? Like what are, where are we really as different companies trying to fit in the market? + +And I think we all sort of collectively understand that the signals themselves really aren't worth differentiating on. It's generally a net negative for everyone for us to not agree on this stuff. And if we can agree on the signal part, it just leaves everyone, all companies, more time to differentiate in the ways that are actually tangible in terms of how the data works. + +Having been involved in OpenTelemetry over the past year, I think that OpenTelemetry is a great place to learn things and meet other people that are really passionate about the whole observability area. And I think that consists of people that really like what they are doing and they are really good at this. So it's a great place for engineers to come together and work and share the observability space to evolve. + +OpenTelemetry to me means a lot of things, you know, beyond the project and kind of the way it's helped different organizations, you know, move into and adopt open source throughout their stack. It's also such a huge, wonderful community. I really enjoy meeting the maintainers and getting to know the end users. I have really good relationships with a lot of the OpenTelemetry community people and that's what it means to me. + +[00:25:22] My favorite OpenTelemetry signal. I'm going to cheat a little bit here and I'm going to say my favorite OpenTelemetry signal is the one that gives people the answer that they find most useful for the question that they find most meaningful. + +Traces. Traces are my favorite signal because, like, they give a full, you know, a full picture of everything going on in the system and you can easily spot on errors. + +Favorite OpenTelemetry signal. This is a tough one. I think traces has to take the win at this point of time because again, just thinking about how things connect well. I'm also very, very keenly pursuing profiling. I think that's going to be potentially the winner in the next conversation we have because I think performance is a big area for a lot of organizations and especially when, as an API gateway, when we are working with different components, we have one part of the API platform stack to know if there are potential bottlenecks. + +Are we a bottleneck? Are there other elements that are potential bottlenecks there? How do we improve performance? How do we actually put our money where the numbers are, essentially? That's what profiling again, sort of promises to a point? + +Because of the background, Elastic, I gotta say logs. But of course, you know, it's... The logs are, you know, they bring like deep contextual insights, but at the end of the day, you need them all. Like metrics are gonna let us know there is a problem. Tracing is gonna help us to understand where the problem is, and logs are gonna help us understand what the problem is. + +My favorite signal is traces because I fell in love with observability and OpenTelemetry because of traces. I would have to say that I got the most value out of tracing. But recently I started to correlate traces with metrics, and I think that is like the golden flow. + +I have been a huge fan of distributed tracing in general. I think it gives you the understanding of how big, like, services interact with each other. But I've been growing to like profiling. I think it gives interesting, exciting opportunities on how people understand even deeper how their systems behave, especially how their systems behave under different flow, different conditions, and to be able to adjust, improve their architectures and the scale of their systems to cater to future loads. + +My favorite OpenTelemetry signal right now is logs, because even though I'm fully immersed in OpenTelemetry now and I know what all three of the signals mean, I started on logs because logs are easy. It just makes so much sense and I understand where people are coming from, coming from with observability, second wave, you know, everything should just be trace or lot wide events. I understand the value of that, but I just feel like logs aren't ever going away. + +My favorite signal coming from an infrastructure and systems background. I really like metrics, and this is something that actually is my personal goal for the next months. Coming to help a lot stabilizing metrics like system metrics in the semantic conventions and Kubernetes metrics as well, and make the collector providing more confidence to our users because having the semantic stable, that will help us. + +My favorite signal. You know, I want to say traces, because they were kind of the first thing I learned when I got into the world of observability to begin with, and I think that was kind of what my mind understood. And I really like the trace waterfalls, so I'll go with that. ## Raw YouTube Transcript diff --git a/video-transcripts/transcripts/2024-12-20T07:03:14Z-otel-me-an-end-user-conversation-with-ariel-valentin.md b/video-transcripts/transcripts/2024-12-20T07:03:14Z-otel-me-an-end-user-conversation-with-ariel-valentin.md index a608827..1f0454e 100644 --- a/video-transcripts/transcripts/2024-12-20T07:03:14Z-otel-me-an-end-user-conversation-with-ariel-valentin.md +++ b/video-transcripts/transcripts/2024-12-20T07:03:14Z-otel-me-an-end-user-conversation-with-ariel-valentin.md @@ -10,154 +10,288 @@ URL: https://www.youtube.com/watch?v=vB9_SiTV5CI ## Summary -In this episode of the "Oh Tell Me" end user Q&A, host Reys, a Senior Developer Relations Engineer at New Relic, introduces Ariel Valentin, a Staff Software Engineer at GitHub focused on observability. The discussion centers on GitHub's adoption of OpenTelemetry, with Ariel sharing insights into the challenges and processes involved in transitioning from proprietary SDKs to OpenTelemetry for distributed tracing. Key topics include the importance of consistent terminology in observability, the architecture landscape at GitHub, the benefits of using OpenTelemetry for tracing, and the collaborative spirit within the community for contributions. Ariel also emphasizes the need for better communication regarding updates in specifications and encourages more involvement from end users in the OpenTelemetry community. The episode concludes with a light-hearted exchange about holiday plans, showcasing the friendly dynamic between the host and guest. +In this episode of "Oh Tell Me," hosted by Reys, a Senior Developer Relations Engineer at New Relic, the focus is on open telemetry and observability, featuring guest Ariel Valentin, a Staff Software Engineer at GitHub. The discussion revolves around Ariel's experiences with observability, the adoption of open telemetry at GitHub, and the challenges faced in transitioning from proprietary SDKs to open standards. Key topics include the importance of distributed tracing, the need for a shared vocabulary in observability, and the architectural landscape at GitHub, which utilizes a customized OpenTelemetry collector. Ariel shares insights on the impact of tracing on service monitoring, the growth in usage post-adoption, and the collaboration within the open telemetry community. The episode also emphasizes the importance of community involvement, encouraging viewers to participate in SIG meetings and contribute feedback. ## Chapters -Here are the key moments from the livestream with timestamps: +00:00:00 Welcome and intro +00:01:37 Guest introduction +00:03:00 Format overview +00:04:10 Warmup questions +00:05:40 Observability discussion +00:09:40 Adoption challenges +00:10:50 GitHub's OpenTelemetry adoption +00:12:10 Architecture landscape +00:15:00 Telemetry capture methods +00:19:01 Community contribution process -00:00:00 Introductions and new format overview -00:01:30 Reys introduces himself and mentions the goal of the session -00:02:45 Ariel Valentin joins and introduces himself -00:04:15 Overview of the session format and audience interaction -00:06:00 Ariel discusses his journey into observability and open telemetry -00:09:00 Discussion on challenges organizations face with observability -00:12:00 GitHub's adoption process of open telemetry -00:15:30 Description of GitHub's architecture and telemetry capturing -00:20:00 Discussion on sampling techniques used at GitHub -00:25:00 Ariel shares GitHub's previous observability tools and changes since switching to open telemetry -00:30:15 Audience questions and interactive discussion on contributions to the project -00:35:00 Reys provides ways for new contributors to get involved in the end user Sig -00:40:00 Ariel asks about expectations for participation in the Sig -00:45:00 Conversation about holiday plans and cultural traditions +**Reys:** Hello everyone! Welcome to a brand new episode of Oh Tell Me, an end user Q&A. If you have been to one of these sessions in the past, you might notice that this looks a little bit different and it has a bit of a different name. That's right, we have done some growing up since the last few times and we're very excited to debut this new look. But don't worry, almost everything else is going to be the same. We have a great interview here for you today to learn from. -Feel free to reach out if you need more information! +But before I introduce myself and our guest, I would love to know where everyone is connecting from. I am online from Portland, Oregon today and would love to see where people are from. It looks like someone is here from Brooklyn, New York, so thank you so much for joining out in New York. -# Oh Tell Me: End User Q&A Episode Transcript +[00:01:37] So my name is Reys. I am a senior developer relations engineer at New Relic. I also co-lead the Ner Sig, which is our cute little logo that you see up in the right corner. At the end of your Sig, we really focus on connecting directly with users and we are dedicated to helping the rest of the SIGs get feedback from end users so they can help improve the project. To that end, this is one of those events that we host to do that. -**Host:** Hello everyone, welcome to a brand new episode of Oh Tell Me, an end user Q&A! If you have attended one of these sessions in the past, you might notice that this looks a bit different and has a new name. That's right, we’ve grown since the last few times, and we’re excited to debut this new look! But don’t worry, almost everything else is going to be the same. We have a great interview for you today to learn from. +We are going to have Ariel on. Ariel, if you would like to join us and introduce yourself. -Before I introduce myself and our guest, I’d love to know where everyone is connecting from. I am online from Portland, Oregon today, and I’d love to see where others are from. It looks like someone is here from Brooklyn, New York. Thank you so much for joining us! +**Ariel:** Hey, how's it going, everybody? I'm Ariel Valentin. I'm a Staff Software Engineer at GitHub, working on observability. Thanks so much, Rey, for inviting me here to chat with you today. It's an absolute honor to be the first in the new format. Also, a big shout out to everybody who's done work to try to get this new platform up and running, so thank you. -My name is Reyes, and I am a Senior Developer Relations Engineer at New Relic. I also co-lead the Ner SIG, which is our cute little logo that you see in the right corner. At the end user SIG, we focus on connecting directly with users and are dedicated to helping the rest of the SIGs receive feedback from end users to improve the project. This event is one of those initiatives. +[00:03:00] **Reys:** Oh no, thank you for being here! I'm really excited. Just to get you and everyone up to speed on the format, it's similar to the ones if you've been on one of these before, but we're going to start with some warmup questions where we'll kind of massage Ariel into being comfortable, and then we'll hit him with some meaty questions. Then we will actually also get into questions more around the Open Telemetry community, where he'll have an opportunity to share feedback about his experiences with contributing, using the project, stuff like that. Then he'll get the chance to ask us questions as well in a little section we call "turn the tables." At the very end, if there are any audience questions—oh actually, scratch that—if you have questions that come up during our conversation, feel free to put them into the chat. If you're watching from YouTube, then just put them in the live chat, and then I think LinkedIn will also have its own chat, so put in your question there, and we will get to them as we can throughout the conversation. Then at the end, if you have questions at the end as well, then we will get to those. But yeah, if you have any questions that pop up as Ariel is telling us his story, we definitely want to get to those as they come in. -So, we are going to have Ariel on. Ariel, if you would like to join us and introduce yourself. +All right, so I think we are good to get started. Ariel, tell us a little bit about your role at the company. How did you get started with observability? How did you get started with Open Telemetry? -**Ariel:** Hey, how's it going, everybody? I’m Ariel Valentin, a Staff Software Engineer at GitHub, working on observability. Thank you so much, Reyes, for inviting me here to chat with you today. It's an absolute honor to be the first in this new format! Also, a big shout out to everyone who has worked to get this new platform up and running. So, thank you! +**Ariel:** Oh, those are great questions. I mean, I should have mentioned that I'm here in Austin, Texas—Sunny Austin, Texas—so you know where I am in the world. I think a lot of us started originally, and I don't think my experience is unique, is working with sort of like APM tools, which are vendor proprietary tools, generally speaking, that didn't have distributed tracing in place at that time. As the community evolved and Open Tracing became a standard, that was my first experience with working with distributed tracing was the Open Tracing specification and working and trying out different vendors, different experiences with Open Tracing. -**Reyes:** Thank you for being here! I’m really excited. Just to get you and everyone up to speed on the format, it’s similar to the ones if you’ve been on one of these before. We’ll start with some warm-up questions to help Ariel get comfortable, and then we’ll hit him with some meaty questions. We will also get into questions around the OpenTelemetry community, where he will have an opportunity to share feedback about his experiences contributing to the project. He’ll also get a chance to ask us questions in a section we call “Turn the Tables.” At the very end, we’ll address any audience questions. +[00:05:40] By the time that I had gotten to GitHub, I was a champion for distributed tracing because I saw the power that was in there. Around that same time, folks were already moving towards developing and transitioning away from Open Tracing and Open Census to Open Telemetry, so I got really involved very early on in trying to spread the word and learning more and working with my team, which was the observability team, to start to adopt tracing more, to embrace it more, and to make Open Telemetry sort of our North Star for all of our telemetry signals. -If you have questions that come up during our conversation, feel free to put them in the chat. If you're watching from YouTube, just put them in the live chat, and LinkedIn will also have its own chat, so put your questions there. We’ll get to them as we can throughout the conversation. If you have any questions at the end, we’ll get to those as well! +I feel like I keep saying the word telemetry over and over again. I'm going to have to find a synonym for that. -Alright, I think we are good to get started. Ariel, tell us a little bit about your role at GitHub. How did you get started with observability and OpenTelemetry? +**Reys:** Yeah, me too! We're both going to stumble over Open Telemetry at some point or distributed tracing at some point. It's all right; this is a safe space. -**Ariel:** Those are great questions! I should have mentioned that I’m here in sunny Austin, Texas, so now you know where I am in the world. I think a lot of us originally started working with APM tools, which are vendor-proprietary tools that didn’t have distributed tracing in place at that time. As the community evolved, and OpenTracing became a standard, that was my first experience with distributed tracing. +What do you think is a main challenge that most organizations face when it comes to observability? Is it not just understanding the value of distributed tracing? -When I got to GitHub, I was a champion for distributed tracing because I saw the power in it. Around the same time, folks were developing and transitioning away from OpenTracing and OpenCensus to OpenTelemetry. I got involved early on in trying to spread the word, learning more, and working with my observability team to adopt tracing more and embrace OpenTelemetry as our North Star for all of our telemetry signals. +**Ariel:** Yeah, because these are all sort of new concepts to folks, right? Folks have their different levels of understanding of the tools that are available to them, and there's all this vocabulary that you hear that's a little bit hard to parse through. Sometimes it's like, "Oh, when you say trace, do you mean like a trace log? Log level is at the trace level? When you say trace, do you mean the samples taken from a profiler?" There's a lot of this sort of language that, even though we're converging on a lot of this language and we have these dictionaries that are defined and published everywhere, there's still sort of like this hump that we have to get over or like a challenge that we have with trying to get everybody speaking the same language within the same context. Very similar to in domain-driven design, we have these bounded contexts where the same term means a different thing. That flows over into sort of our domain language when it comes to observability and SRE practices, and I'm sure many folks have faced those challenges as well. It's kind of like, "Let's all get on the same page about what we mean." -I feel like I keep saying the word telemetry over and over again, and I’m going to have to find a way to break that habit! +**Reys:** Oh, absolutely. What are currently some of the most interesting problems that you are facing in your role? -**Reyes:** No worries, we’re both going to stumble over OpenTelemetry or distributed tracing at some point. This is a safe space! You mentioned you became a champion for distributed tracing. What do you think is the main challenge most organizations face when it comes to observability? +**Ariel:** Oh well, I mean one of those things is that transition, right? It's really hard for an organization like us, who's been around for a long time. I say a long time, but you know, whatever it is—10 years—where the system has grown and evolved. There have been acquisitions that have been brought together; there's disparate kind of backends where we're collecting this data. It's trying to transition from one way of doing things to a new way of doing things, right? So it's learning something new. That's always going to be a big challenge. Folks are trying to do their job every day; they're not trying to learn new SDKs or trying to learn new vocabulary or trying to learn how to be proficient in their backends. What they're trying to do is keep the system up and running and keep our customers happy, right? -**Ariel:** That’s a good question. There’s a lot of new language and concepts to parse through. When you say "trace," do you mean a trace log? When you say "trace," do you mean the samples taken from a profiler? There's a lot of language that even though we’re converging on it, there’s still a hump we have to get over to get everyone speaking the same language within the same context. It’s similar to domain-driven design, where the same term can mean different things. Many folks have faced these challenges as well; it’s about getting everyone on the same page. +I think those are some of the challenges that we all face. I know I face it, and I'm sure others do, which is making these transitions with the fewest pain points as possible, trying to avoid these pain points. There's so many more we can go on and on, Reys, but I think right now that's kind of like one of the biggest challenges for adoption overall. -**Reyes:** Absolutely. What are currently some of the most interesting problems you are facing in your role? +[00:09:40] **Reys:** That's actually a great segue into the meaty section. What was the process like for GitHub to adopt Open Telemetry? -**Ariel:** One of the significant challenges is that transition. It’s really hard for an organization like us, which has been around for about ten years, where the system has grown and evolved with various acquisitions. There are disparate backends where we collect data, and transitioning from one way of doing things to a new way is a challenge. People are trying to keep the system running and keep our customers happy, so making these transitions with the fewest pain points possible is a big challenge. +**Ariel:** For us, it was the role of the advocate, right? I acted as an advocate and was a champion for Open Telemetry at the company. I was specifically brought in to help advance the mission of tracing, and it was something that I had pitched. There were other challenges that we faced too, which was, "Hey, let's get everybody building a data dictionary. Let's get everybody agreeing to the same language when it comes to what our attributes were going to be." As you can imagine, as a system of walls or acquisitions or other teams are rolled in, everybody has their own log attributes or their own metric attributes or whatever it is. -**Reyes:** That’s a great segue into the meat of our discussion. What was the process like for GitHub to adopt OpenTelemetry? +[00:10:50] I said, "Look, here's a North Star. Here's semantic conventions from Open Telemetry. Let's anchor onto this and let's follow the rules around semantic convention so that we can all build up our own internal dictionary." That comes with its own challenges—schema migrations and trying to keep up to date with the spec and what the instrumentations are doing, right? As an advocate, I was also a lead on the rollout. One of the things I wanted to do was sunset all of our old SDKs and move over to our open-source SDKs. -**Ariel:** For us, it was a lot of advocacy. I acted as an advocate and champion for OpenTelemetry at the company. It was something I pitched, and we also faced challenges like building a data dictionary and getting everyone to agree on the same language regarding our attributes. As you can imagine, as systems grow, every team has its own log attributes or metric attributes. We anchored onto the semantic conventions from OpenTelemetry to build our internal dictionary. +My team and I were all involved in Open Telemetry Ruby, for example, because we're a big Ruby shop. We got involved there to help the instrumentations get better and to help test different releases of the SDK and get that rolled down into our GitHub monolith. We were, like I said, very early adopters, ran into some challenges, and continued to give feedback to the community that way. I'm not only an end user, but like I said, I'm also a maintainer, so I'm playing multiple roles there, trying to help the community along. -This came with its own challenges, such as schema migrations and keeping up to date with the spec. I wanted to sunset all of our old SDKs and move over to our open-source SDKs. My team and I got involved in OpenTelemetry Ruby, for example, since we are a big Ruby shop. We helped make the instrumentations better, tested different releases, and got that rolled into our GitHub monolith. We were early adopters, ran into challenges, and continued to give feedback to the community. +**Reys:** That's a really interesting position to be in as an end user and contributor. I definitely want to circle back on that when we get to the OpenTelemetry community questions. Tell us about the architecture landscape at GitHub and the telemetry that you're capturing. -**Reyes:** That’s a really interesting position to be in as both an end user and contributor. I’d love to circle back on that when we get to the OpenTelemetry community questions. Tell us about the architecture landscape at GitHub and the telemetry you’re capturing. +[00:12:10] **Ariel:** Sure! I've got this little slide that I put together in markdown, so it's not anything official. We can bring that up now so we can take a look at it. I imagine that for a lot of folks out there who are working with virtual machines where they're running systemd units or if they're running Kubernetes, we have different deployment styles here. -**Ariel:** Sure! I have a slide I put together, which we can bring up now. For many folks working with virtual machines or Kubernetes, we have different deployment styles. Some of our main workloads are running in Kubernetes, and we run our own custom meshing. On every worker node, we’re running a deployment of the OpenTelemetry collector that we build using OCB. We chose that route to ensure we had the most secure build possible with the minimum number of dependencies and the ability to build our custom processors. +Some of our main workloads are running in Kubernetes, and the way that we've got everything set up, you can see here as part of our Kubernetes cluster, we run our own custom messaging graph. On every worker node, we're running a deployment of the OpenTelemetry collector, which we build using OCB. We have our own version of the OpenTelemetry collector that is specific to us. We chose that route because we wanted to ensure that we had the most secure build possible with the minimum number of dependencies, and we also wanted the ability to build our own custom processors so that we can address any issues that we might have that are specific to our needs. -In each pod, we have a mesh sidecar. Ingress traffic comes into the OpenTelemetry collector over HTTP. If you have another application running your service, it sends traces to the mesh and sends them to our OpenTelemetry collectors, which generate span metrics and sample traces before sending them to a SaaS provider for aggregation. +Inside of each of these pods, we also have a mesh sidecar. Ingress traffic is going to come into the OpenTelemetry collector over HTTP. If you have another application that's running your service, it's shooting over OTel traces over to the mesh and sending it to our OpenTelemetry collectors. From there, we generate span metrics and sample traces and send those off to a SaaS provider where we aggregate all this data. -We are currently leveraging OpenTelemetry for traces and generating span metric data, but we still collect custom metrics and logs using non-OpenTelemetry formats. On each worker node, we have a metrics agent that can speak various protocols. We have Fluent Bit running to collect logs, which get streamed out through Azure Event Hubs, processed by consumers, and sent off to our log store and search system. +On each individual worker, we're running in this hybrid world right now. We're only leveraging OTel for traces and for generating span metric data. We're still living in a world where we're using non-OTel formats for collecting things like custom metrics, system metrics, and for collecting logs. On each one of our worker nodes, we have a metrics agent that can speak various different protocols, but mostly it's either using Open Metrics or it's using StatsD to collect data and aggregate it and send it off to the SaaS provider. -For our trace data, we’ve rolled out OpenTelemetry SDKs for different programming languages, including Ruby, Go, JavaScript, Node.js, and .NET. We had some experimental Rust and Java usage but had to scale that back. We started before automatic instrumentation was available for some of these languages, so we’re deploying them through wrapper libraries, which we maintain and update periodically. +[00:15:00] In addition to that, on all of our worker nodes, we have Fluent Bit running, and that's what we're using to collect our logs. All of our logs end up getting streamed out through Azure Event Hubs, which are processed by a bunch of consumers and then sent off to our log store and search system. -**Reyes:** Wow, that’s a lot of information! I have many follow-up questions. You mentioned that you’re leveraging OpenTelemetry for traces right now. Is that mainly because you’re a Ruby shop and the signals for metrics and logs aren’t as mature yet? +In addition to these Kubernetes workloads, we have our own virtual machines where we run systemd units. You have to think about the Git file system services. Those are all running on systemd, and we have the metrics agent and Fluent running there. We don't yet have the OpenTelemetry collector deployed there, but that's where we want to get to. We want to get to a world where essentially our entire platform is running open-source software, the OpenTelemetry collector, and we're converging on OTel for all these formats. So those services are also using StatsD, Open Metrics, and Fluent Bit's pulling things out of JournalD because that's where we're streaming all of our logs of the system journal. -**Ariel:** That’s one of the reasons. There’s also a lag between when something is published in the OpenTelemetry spec and when vendors or open-source platforms can leverage those things. We ran into challenges rolling out tracing that we didn’t want to take on yet, especially as some SDKs are ahead of others. With Ruby, we recently saw improvements to the metric SDK, which is something we wanted to use. +All of that stuff again goes right through to the same channels, and it all gets aggregated in these places where we do our work. A little bit about some facts about our trace data is that we've rolled out the OpenTelemetry SDKs for different programming languages. We have them for Ruby, Go, JavaScript, Node.js, .NET. We had some experimental Rust usage, but we've had to scale that back, and we had some Java experiments that didn't pan out; we didn't roll those out to production. -**Reyes:** So, it sounds like the plan is to migrate your other signals over once they reach GA in languages? +A lot of the stuff that we do, we started before any of the automatic instrumentation was available for some of these languages, so we're still deploying them through wrapper libraries. We maintain a set of wrapper libraries. You install those; it gives you the sort of the what we consider to be the minimal defaults that we require for our needs, and we'll do periodic updates of those through Dependabot. As we roll out new versions of those things, I feel like I've said a lot so far, and I don't know, Rey, if you had any follow-up questions for me. -**Ariel:** Certainly! Once we have more time on our calendar, we want to migrate everything over. GitHub is constantly growing. We just hit 150 million users, so it’s amazing growth, and we’re here to support that volume growth. +**Reys:** I actually have so many! -**Reyes:** That absolutely answers my question! For those who joined us later, feel free to pop any questions into the live chat of whatever platform you're watching. +**Ariel:** Oh, okay, great! -I noticed on your second slide you mentioned probabilistic sampling. It sounds like you're not doing any kind of tail sampling. +**Reys:** You mentioned you are leveraging OpenTelemetry for traces right now, and you talked a little bit about the plans and some of the stuff that you tried. I was curious to find out more about is that mainly because you're primarily a Ruby shop and the SDKs for metrics and logs aren't quite as mature yet? Is that the reason? -**Ariel:** Not at the moment. One of the hard things about tail sampling is that traces have to go to the same collector to make that decision. We wanted to reduce the burden in complexity immediately, so we started with probabilistic sampling. More advanced remote sampling is something we want to pursue in the future. +**Ariel:** That's one of the reasons. One of the other challenges that I didn't discuss when you asked about adoption challenges is even though OTel says, "Hey, this is what the signals look like; this is what the semantic attributes are like," there's a lag between the time that something is published or declared in the OTel spec to the time that vendors or open-source platforms are able to leverage those things. There's sort of like this feedback loop as we go through OTs and say, "Hey, we want to try this new thing out." -**Reyes:** At what point do you think you might get to the point where you could implement tail sampling? +[00:19:01] We do a couple things in experimental languages; we'll do this stuff, and it was a big enough challenge for us to start to roll tracing out that we didn't want to take yet another thing on, which was to say, "Okay, now we're going to switch over to native OTel metrics as well." Like you said, some of the SDKs are ahead of others. So with Ruby, we're just recently, with the help of our amazing maintainers, working diligently to get the metric SDK up to speed for Ruby. So that's something that wasn't available to us to use. -**Ariel:** That’s on the pile of things I want to do. It’s hard to say right now, but it’s something I’d love to explore. +One of the biggest advantages that I see in tracing is the ability to generate span metrics from traces. It's like, "Hey, the less things that we have to impose on our users for them to try to figure out how to do, the better for us." -**Reyes:** I’m curious about the experiments you mentioned in Java and Rust that didn’t pan out. What didn’t quite work out for those? +**Reys:** Are you using the span metrics connector? Is that what you're using? -**Ariel:** We reduced the number of Java workloads we’re running, migrating them over to different programming languages. We didn’t see the return on investment we wanted. Similarly, with Rust, we have a limited set of applications that run Rust, and they ran into performance issues that impacted their latency. I wasn’t able to get my hands dirty to help address those problems, so we had to put that on pause. +**Ariel:** We're using a custom connector right now. I'd like to be able to get us to the point where we're using a span metrics connector, but we don't have egress in OTel right now. We're still doing sort of like vendor proprietary formats for export. Where I want to get to is, you know, that's one of the biggest strengths of OTel is that OTel is that standard format that makes things portable for us. -**Reyes:** What was GitHub’s observability tool before migrating to OpenTelemetry? +**Reys:** Absolutely. So that sounds like the plan is to migrate your other signals over once those reach GA in languages. -**Ariel:** We were using proprietary SDKs for OpenTracing. GitHub is also a heavy StatsD metrics user, and logs are a big part of what we utilize. We were piecing together logs and metrics for debugging and started using distributed tracing as an additional tool in our toolbox. +**Ariel:** Oh, certainly. And once I have more time on the calendar too, right? We have so many projects that we have to do as engineers and companies, and it's like, "Hey, where do we fit these in?" As you imagine, GitHub is constantly growing every day. We just hit 150 million users, so it's like just an amazing growth of the company. Shout out to all the people at GitHub who've been working so diligently to make this happen. We're here to support that volume growth and support our end users. I hope that answers your question. -**Reyes:** How have things changed since GitHub switched to OpenTelemetry? +**Reys:** Absolutely. I also want to mention again real quick for those who joined us a little bit later, feel free if you have questions that come up. You want to learn more about something that Ariel has gone over, feel free to pop it into the live chat of whatever platform you're watching from, whether it's YouTube or LinkedIn. Go ahead and pop the question in, and we will try to get to them throughout the show as you can. -**Ariel:** Just this year alone, we’ve seen a dramatic increase in the number of people using tracing and the number of services instrumented. When we started, around 80 services were instrumented, and now we’re closer to 300. We’ve gone from about 5 million spans per second to 32 million spans per second, which is a huge increase in usage. +I noticed on your second slide you mentioned probabilistic sampling, so it sounds like you're not doing any kind of tail sampling. -**Reyes:** How has that impacted how your services are running and the speed at which your team can debug issues? +**Ariel:** No, not at the moment. Would you mind bringing that up again—the second slide? -**Ariel:** It’s a mix of education. We have teams dedicated to improving the experience and helping identify new workflows. We try to provide insights during incidents and help teams identify issues even before they go to production. It’s been a mix of results—some teams have identified issues before they go live, while others have leveraged it during incidents to find the root cause. +**Reys:** Yeah, that's awesome. -**Reyes:** What would prevent you from implementing better telemetry? +**Ariel:** Yeah, so we started with probabilistic sampling. One of the hard things about tail sampling, as you can imagine, is that the traces all have to go to the same collector in order to make that decision if you're using the collector for tail sampling. Right now, we wanted to reduce that burden in complexity immediately. When we started with probabilistic sampling, some of the things that we want to do in the future is more advanced remote sampling so that we can have more fine-grain control of what we're looking at. We want to be able to leverage tail sampling rules, but as you know, that's very difficult to implement and a little bit difficult to scale in our case because we have about 2,000 collectors that are supporting everything right now across all of our fleet of like 14,000 hosts or something like that. I'm making that number up off the top of my head; I think that's the last number we had. -**Ariel:** There’s a lot in early stages. We were early adopters, but some tools are risky to roll out. For example, I’m excited about the continuous profiler, but it’s still in the early stages, and we don’t want to take that risk just yet. There’s also a challenge in migrating semantic conventions from pre-1.0 to 1.x, as we’ve already sent data out, and we need to figure out a way to upgrade it. +Right now, we're doing about 26 million spans per second. Just yesterday during our peak times, we hit our all-time high of 32 million spans per second as we continue to grow our volume. This is one of the challenges that we've had, and so on my list of all the things that I want to do, tail sampling is definitely on there. -**Reyes:** Let's shift to some community questions. What was the contribution process for you and your team like to OpenTelemetry? +**Reys:** At what point do you think you might get to the point where you could implement tail sampling? -**Ariel:** It was a short process with a lot of observation. We looked at the code of conduct and previous PRs to ensure the behaviors matched our expectations. We participated in SIG meetings, provided feedback as end users, and were able to contribute some custom propagators and instrumentations. We focused our contributions on libraries that are popular at GitHub and aimed to create an environment that lowers the barrier to entry for collaboration. +**Ariel:** That would be more like when the processor has been more developed. I don't know how to answer that question because it's on the pile of the list of things that I want to do towards the bottom. -**Reyes:** Do you have any feedback for the SIG on ways to improve based on your experiences with Ruby and other SIGs? +**Reys:** Got it. -**Ariel:** I’d like to see better communication about changes in specs. If a change happens, it should automatically open issues for every maintainer repo to track updates. Right now, it’s hard to keep up with changes and deprecations. +**Ariel:** Although I am curious about the other. -**Reyes:** What SIGs do you interact with most right now? +**Reys:** But that's okay! That’s all right. -**Ariel:** I mostly participate in the Ruby SIG and have limited interaction with other SIGs these days. I recently met the end user SIG at KubeCon, so I’m looking to get more involved there. +**Ariel:** Questions that I want to get to as well. -**Reyes:** For our “Turn the Tables” section, what questions do you have for me or anyone on the call? +**Reys:** Sure! Also, just going back, you mentioned trying some experiments in Java and Rust that didn't quite pan out, and I was just curious what it was that didn't work out or didn't meet your expectations. -**Ariel:** What are the best ways for newcomers to get involved in the end user SIG? +**Ariel:** Well, we've reduced the number—I'm sorry, all the JVM people out there. We've reduced the number of Java workloads that we're running, so those have been migrated over to different programming languages. That's one of the reasons why we didn't go through and say, "Oh, let me continue to roll out JVM work." We just didn't have the return on investment that we wanted to, you know, try. -**Reyes:** There are many ways to contribute. Joining SIG meetings and sharing your experiences is a great way to start. We do surveys, open Q&A sessions, and even documentation. Joining the CNCF Slack instance is also essential. +Also, we don't have a lot of the same thing for Rust. We have a very limited set of applications that run Rust, and those are performing for performance reasons. They ran into some challenges with how it impacted their latency when they introduced the use of the SDK. I'm going to be honest with you: me not being a Rust expert and being able to get in there and get my hands dirty to try to help out with addressing some of those problems and reporting them upstream, that was something that we had to put on pause. Again, it's like one program versus all of these other services that are running in Go and Ruby and JavaScript that we need to pay attention to. -**Ariel:** What kind of expectations of commitment do you have from folks who want to participate? +I think that's really where it was, where sort of like a critical mass of application services that use these programming languages, and it was like, "Hey, we have to focus our attention on those that are going to get us a higher return on investment for this." -**Reyes:** We welcome contributions in various forms. We hope people can join at least once a month, but if not, that’s okay! We understand everyone has commitments. We want to make it as welcoming as possible for everyone. +**Reys:** Absolutely. Okay, so what was GitHub's observability tool before migrating to OpenTelemetry? -**Ariel:** Thank you for answering my questions, Reyes! +**Ariel:** Yeah, I mean, we were using a proprietary SDK for OpenTracing. OpenTracing was how we collected traces before, but generally speaking, GitHub is a huge StatsD metrics user still to this day. Logs are a big part of what we utilize here. Folks are looking at exception stack traces a lot of the times to try to understand where errors are coming from. They're looking at access log streams, and they were trying to piece together, "Hey, where's the request going, and where is it slowing down?" I showed up with my magic tool—distributed tracing. I'm like, "Hey, look, here it is on the flame graph or an icicle graph, or here's a waterfall view of this thing." It's like, "Oh, that's pretty cool!" People started looking into that as an additional tool in their toolbox for them to help try to debug things during an incident. -**Reyes:** You’re welcome! If anyone has any last-minute questions, we’ll hang out for a minute. Otherwise, I’ll ask about your holiday plans! +I hope that answers your question there. -**Ariel:** We're spending it with family, traveling on Christmas Eve. It should be fun! +**Reys:** Yes, and kind of along those lines, how have things changed since GitHub switched to OpenTelemetry? -**Reyes:** I’ll be working on house projects that have been on my to-do list for three years! +**Ariel:** What I'll tell you is that just this year alone, we've seen a dramatic—I wish I had these statistics off hand—but we've seen a dramatic increase in the number of people using tracing and the number of services that have been instrumented. Part of that comes from the fact that I was like, "Hey, everybody, we're all moving to this new SDK, so everybody install it." Let Dependabot go ahead and do these installations on your apps, and people started seeing the value of this investment as well. -**Ariel:** That sounds like a plan! +I want to say that when we started this adventure, only about 80—and I'm going to use the word services in air quotes—only about 80 services were instrumented, and now we're closer to about 300 of those services, which are effectively like Kubernetes deployment types or systemd types. As you can imagine, our monolith has, you know, maybe a thousand services within it, so there are like sub-services in there. But the monolith itself has broken down into like eight services—like web UI, API, GraphQL, background workers, stream processors, and whatnot. That's why I put them in air quotes as these are services. -**Reyes:** Thank you so much, Ariel! You were a fantastic guest. I’m excited to learn more from you. +We saw that that was quite an increase in the number of services that came in. We were doing something like 5 million spans per second to now we're up to 32 million spans per second. It's just a huge increase in volume and usage. -If anyone wants to reach out, you can find us in the CNCF Slack's OpenTelemetry SIG end user channel. We’ll have the information in the show notes. Thank you all for joining us, and we look forward to seeing you again next time! Happy everything! +**Reys:** Along with that volume increase in usage and data volume, how would you say that's impacted how your services are running and how quickly your team is able to debug issues as they arise? -**Ariel:** Thank you for having me, Reyes! This was awesome. Thanks, everyone, for watching, and I hope to see you in a SIG room sometime! +**Ariel:** Part of it is an education thing. A lot of things that—I mentioned that I'm on the observability team, but really, we're two groups. One of the groups is called The Experience Team that works directly with teams. They operate in a role sort of like developer relations and advocacy, but also working on cost control and improving your experience, helping you with identifying new workflows and introducing them into your incident command experience. -**Reyes:** Adios! +Those teams set up sessions, education sessions, to bring folks on board. We try to do them not during an incident because that puts some stress. What we try to do is, if an incident is happening, it's like, "Hey, here's some insights that we're gaining, and we'll share with you that we're seeing from the traces that you might not be able to see somewhere else." That has helped in a lot of cases where we couldn't exactly pinpoint what was going on. + +Then there are also these spots where not every system has been instrumented, so we have to rely a lot on sort of like client metrics or client trace data and say, "Hey, look, this client is experiencing this problem. Can we take a look deeper at this other service that hasn't yet been instrumented?" It's been sort of, I'm going to say, mixed results. Some teams have identified issues even before they go out to production, and other teams have been able to leverage it when it's like, "Oh, this, we're having an incident right now. Oh, here's the reason why this is failing." + +We've been able to do things like identify bottlenecks and mistakes, like little coding mistakes—"Oh, I forgot to ack the message before I pulled it out of Kafka. I'm just retrying that same message millions of times," or "Oh, the client timeout doesn't match the server timeout, so the client times out, and the server is continuing to turn along to try to do this request." We're identifying things like that that we couldn't identify before easily. + +**Reys:** Oh no, that's awesome! I think I will probably be asking you more about that because I think this is really interesting. One more question from our meaty section: what would prevent you from implementing better telemetry? + +**Ariel:** What would prevent me from doing that? I think it's because there's so much of the stuff that's in early stages. We were early adopters on a lot of things, but then there's a lot of stuff that's a lot more risky for us to try to roll out. For example, one of the things I'm really excited about when I came back from KubeCon is the continuous profiler. + +That's one of the things on my wish list that we'd be able to use today if we could. That's the thing that I would want to do, but because we're still in the early stages of the profiler and the specification, and there's still some churn with the data model, I think that there are—when it comes to the resource-intensive or sort of like introspective tools like that, we don't want to take that risk going a little bit too early to adopt those tools. + +Also, there's not a lot of support from our current vendors that will be able to leverage that. It's kind of like we would be experimenting with that to go to nowhere. That's kind of, you know, once vendors start to support the OpenTelemetry profiling format, data model, I should say, and once the profiler is a little bit more stable, I'd love to jump in there and be able to roll that out a little more widely. + +I think that a little bit of a bumpy road for us is still migrating Semantic conventions from pre-1.0 because we were early adopters of Semantic conventions, so we're at this pre-1.0 stage. Getting ourselves to migrate towards a 1.x version of Semantic conventions is another big challenge because we've already sent all of this data out to our backends. We have to figure out a way to upgrade it or to say, "Oh, this is version X." I kind of feel like a lot of the progress was stalled there for the moment. + +**Reys:** If that answers your question. + +**Ariel:** Oh absolutely! I think that is another interesting topic—migrating some semantic conventions, so I might jump back if we have time later. But I do want to get to some of these community questions. + +**Reys:** Yes, ma'am! + +**Ariel:** Yeah, so one of the big things I think working in the community that you've seen, I've seen, is people want to contribute to the project, but they're not really sure where to get started. We have resources such as, you know, I think in the documentation we added getting started, and you have various SIG channels and CNCF Slack. I think we all have the problem of how do we reach these people because there are still so many. What was the contribution process for you and your team like to OpenTelemetry? + +**Ariel:** For us, it was a short process with a lot of observation. The things that we looked for—myself and my teammates at the time—when we approached the community, we were looking at the code of conduct: what were the expectations of the code of conduct? We went through and actually looked at issues, previous PRs, the feedback that was in those PRs to see if the behaviors expected in the code of conduct were reflected in the PRs and in the issues. + +There's nothing worse than you wanting to be part of this community and finding that those two things don't match. To see that that happened, then we participated in the SIGs, which was very nice for us to be able and convenient for us in the U.S. because the Ruby SIG was in U.S. hours. We joined those meetings during the day and started to provide some feedback as end users and say, "Hey, look, we ran into this challenge." If we identified something that was a challenge for us, we would contribute a PR. + +The maintainers were very generous with their time, did their reviews, and we were able to get a couple of custom propagators merged. We got some instrumentations merged, and we identified other gaps for the instrumentations that we use because one of the biggest challenges, I think, as a maintainer is there's so much that you have to do. You've got the SDK, you've got the API, you've got documentation to do, and you have to have language-specific instrumentations. That's daunting, especially with all of the popular libraries that are out. + +For us, we contributed back to the libraries that we use heavily. We have a subset of libraries that are very popular at GitHub and sort of blessed by their popularity. That's where we focused our energy and our contributions. Then we reached out to people. Anytime somebody comes to me and says, "Hey, I have this challenge with this thing. It's missing this attribute, or I'd really like it to work like this, or I've identified this bug," the first thing I do is say, "Let's open up an issue and let's work on this together. I'd love to review your PR." + +Creating an environment that lowers the barrier to entry for folks to collaborate and submit contributions is important. I want them to feel like this is yours, this is ours—this is not mine, and I'm gatekeeping. We still have some standards for quality, obviously, and some expectations from you as a maintainer, but we try to make this a painless experience for you, at least in the Ruby community. That’s what the contribution process was like for me and getting involved in the community. + +**Reys:** That's awesome! So you started basically joining the SIG meetings as an end user and sharing feedback and then finding information about how you can open an issue and then building custom stuff and helping the community or sorry, the Ruby SIG build out more of their components. + +**Ariel:** Okay, yeah, certainly. And like, you know, with the home for open source, right? For us, we open-sourced, for example, new database drivers in Ruby for MySQL. We had that instrumentation in-house before all of that became public, and as soon as we released that new driver, I went right to the SIG and said, "Ta-da, I have a donation for you!" It's this thing where there's just this spirit of collaboration and supporting open source that’s important to our mission. + +**Reys:** Okay, so it sounds like you had a pretty positive experience from contributing to Ruby. Do you have any feedback for the SIG on ways to improve how things work based on your experiences with Ruby and then other SIGs? I know you mentioned also some other components that you've used, but we'll come back to that. + +**Ariel:** Yeah, for sure. Again, I want to thank everybody for their amazing work that they've done contributing to the collector in particular because we release our OCB build, or sorry, our custom build, every time a new version of the collector package rolls out. Every week, Dependabot is telling us, or every two weeks, it's telling us, "Hey, it's time to upgrade; a new release has rolled out." + +We ran into some challenges where we identified some performance issues in the collector, and we worked with the team, provided profiles, provided feedback, and showed them our configurations, and they were so responsive to our concerns. They were happy to see that we were able to contribute back just in providing feedback, just giving them actual data of production workloads, which is very difficult to do as a maintainer. + +It's really hard for me to know, "Hey, what's going on in your customer's deployment or this user's deployment?" We can't replicate it in our own environment. That spirit of collaboration was really great, and that's the thing that I'd like to see happen in all the other SIGs. I have limited experience with other SIGs. + +The other part was contributing to semantic conventions, and I contributed to semantic conventions, trying to introduce some new attributes, but there's just so much churn in that repository that it was hard for me to keep up with changes and deprecations. That was a little bit on me to be able to keep up and find out that the stuff I had submitted was deprecated in favor of something else. It's those kinds of challenges that I like—I have to find a better way for us to all be able to keep up with each other. + +One suggestion I might say is, "Hey, look, a change happened. Here's an OTOP—a change happened in the spec that should automatically open issues for every maintainer repo and say, 'Here's a new thing that has changed. This is a new feature we expect that the SDK should support.'" That way, we're able to track it because right now, the way it works is, "Hey, we have somebody who's a representative at the SIG spec; they collect some data, they come back to us, and they tell us, 'Hey, we need to implement this other thing.'" Sometimes we're not diligent about project management, right? We're ICs who are doing this in our spare time, so we need to improve overall at being able to keep each other informed and tracking work. + +**Reys:** Got it. So, making sure everyone's on the same page at the same time, which I can see obviously is a bit daunting. + +**Ariel:** Yeah. + +**Reys:** I know you mentioned some of the SIGs that you have worked with in the past. What would you say are the things you interact with most right now? + +**Ariel:** Right now, I almost exclusively participate in the Ruby SIG. I don't have a lot of interaction with other SIGs at the moment, other than through an occasional issue or a discussion that'll pop up, or in Slack, I'll jump into a channel and say, "Hey, I've run into this problem," or "I have this question for y'all. Can you help me?" + +I have limited interactions now these days with other SIGs. Just recently, I'm meeting the end user SIG at KubeCon, so I'm going to probably get more involved in the end user SIG now that I have this special invitation from you. + +**Reys:** Oh, I'm so excited! + +**Ariel:** This brings us to our "turn the table" section where you get to ask me or anyone on the call questions, and we'll see if I can answer them or if anyone else can answer them. + +**Ariel:** Okay, so for me, Rey, it's like that's how I got involved, but maybe you know a special sort of formula or the best way to have someone who's new get involved, in particular in the end user SIG, to provide feedback, or how do they get involved in other SIGs that you've been involved in? + +**Reys:** Oh, for sure! So there are a lot of different ways to contribute, right? I think a lot of people, when they think of contribute, they think it means, "Oh, I got to open PRs. I got to be writing code." That's not necessarily the case. There are so many different ways you can provide feedback. One of the ways you mentioned was you just started out by going to a SIG meeting and sharing some stuff that you're seeing. That is an absolutely great way to contribute to the project, and that's what the end user SIG is all about. + +One of the things we're all about is gathering feedback to give back to the appropriate SIG so we can help improve things. To that end, we do things like surveys, open solar Q&A interview sessions where we kind of dive deeper into the adoption implementation process and find out your pain points and specific feedback. You can contribute blogs. If you love writing, documentation is a great way. + +If you are not already a part of the CNCF Slack instance, I would join, and I just realized I did not include that link, but we will have it in the show notes. Once you are a member of the CNCF Slack instance, you can scan the QR code and join the OTel Sig end user channel, and we will be happy to direct you if you have questions about your implementation or you just kind of want to know, "Hey, where do I get started?" + +We are happy to help direct you. We currently have a survey going, which I also just realized I did not share the link. I'm so sorry; I'm going to see if I can get that right now, Henrik. + +So yes, join the SIG meetings where they can access the calendar. It’s through the community repository; it has links to the calendar. + +**Ariel:** Perfect! + +**Reys:** Yes, if there's a specific language that you are already working in, check out the calendar to find out when the SIG meets and just hop on. When I first started, I was like, "I'm just going to check out a few different SIGs." I went to the Python SIG, and everyone there was so welcoming and so nice. I was so intimidated at first, and then I was like, "Oh wow, these people are so lovely!" That's been my experience with almost every SIG meeting that I've jumped into. Everyone is so open to answering questions, and they want your help. They want to help you help them. + +One of the things is just, you know, letting people know that it doesn't have to be contributions. Of course, if that's what you want to do and are able to, we would love that, but there are so many other different ways—feedback, donation, blogs, doing interviews, and surveys. + +Every little bit helps, right? Every little bit counts. If you find yourself interested in trying to contribute back or you don't know where to get started or you feel just like intimidated, just know that all of us, we do our best to make the most welcoming communities possible in our SIGs. I love that the end user SIG creates these opportunities for folks to come and just provide feedback, a space for them to say, "Hey, I'm having trouble with this," or "I like this," or "This was a great experience for me." + +This part has been very enjoyable for me to be able to share my personal experience and my team's experience with you. I really hope that folks who are watching at home, hey, come on in; the water's fine! Don't worry! We have floaties, we have all kinds of gear to help you get started, right? + +**Ariel:** Exactly! It's just the little things. Every little contribution can help us. If your contribution is there to help you, it'll help out the community. So come if you're having trouble with a specific problem. I guarantee you at least a dozen other people are having the same problem. If you don't understand something, I guarantee you probably at least a hundred other people don't understand it either. + +**Reys:** For sure! + +**Ariel:** I really appreciate you answering my questions here. + +**Reys:** Oh, you are so welcome! I guess I kind of put this at right about time in case anyone has any last-minute questions. We'll hang out here for another minute and just kind of chitchat and give people an opportunity to share if they would like. Otherwise, I will just ask about any fun Christmas or New Year plans. + +**Ariel:** Yeah, we're going to be just spending it with family, so that's going to be really nice. We're going to be traveling, so that might be a little bit hectic. We have to travel; we're going to go visit my father, but we are traveling on Christmas Eve, so that should be fun, right? I'm going to show up there, and we'll be singing Christmas carols at the top of our lungs and waking up the neighbors as soon as we arrive. + +**Reys:** Oh my gosh! How about yourself? + +**Ariel:** Whoa, whoa, whoa! What are these Puerto Rican Christmas songs, and where can I listen to them? + +**Reys:** Oh, okay! You can look at them on YouTube, on TikTok; they're all over the place. In Puerto Rico, folks and in other Caribbean islands will join and make something called the "parranda" and have a "paranda," which is a party basically. It's like a Katamari ball, so you go from house to house knocking on doors, and then you start singing these songs. + +The songs are like, "Hey, open up the door! I'm here to say hello!" "Oh, you turned the lights on; I can see you in there! Let me in!" You have to feed the guests and give them beverages and stuff. What you do is you take those people whose home you invaded, and we call it an "asalto," which is like—we're showing up uninvited. We break into your home and then we bring you and add you to the "parranda," and we move to the next house, and we come and, you know, we'll sing and stuff like that. + +It's really great; it's a lot of fun, and so we're looking forward to doing that with our family! + +**Ariel:** That sounds so fun! What did you say that was called? + +**Reys:** The "parranda" and the "asalto." + +**Ariel:** That sounds lovely! Honestly, that sounds fantastic! + +**Reys:** It's a lot of fun! I'm so excited! You have to share pictures! + +**Ariel:** I will, thank you! Maybe videos too—who knows what kind of trouble we can get ourselves into! + +**Reys:** Yes! I would love to have a video of you singing one of these songs! + +**Ariel:** There are videos of me on the internet singing songs, but not now! + +**Reys:** What are you going to be doing this holiday season? + +**Ariel:** I am going to be at home busting out hopefully a bunch of house projects that have been on my to-do list for three years. + +**Reys:** Nothing like it, right? You're going to get to the tail sampling project, right? + +**Ariel:** Oh my gosh! I have! But I'm really excited about it because I moved into this house about three years ago, and it's like, you know, 85% there. There are still—I just—don't look in the cabinets. That's a project; I need to organize all the inside stuff. + +**Ariel:** That's the thing I tell people all the time: "Hey, don't look on the inside of the repositories. You may not know; you may not like how the way things are arranged, but you know where things are, so that's totally fine." + +**Reys:** Well, all right! Thank you so much, Ariel! You were a fantastic guest! I'm really excited to learn more. As I said, I did have some follow-up questions, so I'm sure I'll be slacking you to learn more about some of your adoption processes as well as some of the other components you mentioned you had used, such as the OCB. + +But yeah, if anyone wants to reach out to either of us, you can find us in the CNCF Slack's OTel SIG end user channel. Again, this link is up here for you to scan, and I believe we will have the information in the show notes as well, which will be posted, I think, right after this. + +Thank you so much for joining us, and we look forward to seeing you all again next time! + +**Ariel:** Yeah, thank you for having me again, Rey! This was really awesome. Thanks everybody for watching, wherever you are and whenever you are! I hope to see y’all in a SIG room sometime. + +**Reys:** Oh yes! And happy everything to everyone! Adios! ## Raw YouTube Transcript diff --git a/video-transcripts/transcripts/2025-01-31T05:58:14Z-cfp-writing-q-a-livestream.md b/video-transcripts/transcripts/2025-01-31T05:58:14Z-cfp-writing-q-a-livestream.md index 6a66886..07d74c2 100644 --- a/video-transcripts/transcripts/2025-01-31T05:58:14Z-cfp-writing-q-a-livestream.md +++ b/video-transcripts/transcripts/2025-01-31T05:58:14Z-cfp-writing-q-a-livestream.md @@ -10,168 +10,224 @@ URL: https://www.youtube.com/watch?v=6SmO4yKjmCs ## Summary -In this YouTube panel hosted by Dan, experts from the open Telemetry community discuss how to effectively apply to speak at tech conferences, sharing insights on crafting compelling call for proposals (CFPs). Panelists include Adriana Vela, Henrik Rexed, Josh, and Ree, who provide valuable tips on choosing topics, overcoming imposter syndrome, and preparing for presentations. They emphasize the importance of unique, engaging content and the value of end-user stories in tech talks. The conversation also touches on the significance of networking, the challenges of public speaking, and how to handle rejection after submitting proposals. The panelists collectively stress that while rejection is common, it can be a learning opportunity, and they encourage aspiring speakers to persist and refine their submissions. The session concludes with personal anecdotes from the panelists about their experiences at various conferences, underscoring the community aspect of speaking engagements. +In this YouTube video, Dan hosts a panel discussion about effectively writing conference proposals (CFPs) and preparing for speaking engagements, particularly within the context of open telemetry and observability. Panelists include Adriana Vela, Henrik Rexed, Josh, and Ree, who share their insights on choosing topics, overcoming imposter syndrome, and enhancing presentation skills. Key points include the importance of submitting unique and relevant topics, the value of end-user stories in presentations, and strategies for engaging audiences. The panelists also discuss the common experience of rejection when submitting CFPs, emphasizing the need to manage expectations and learn from feedback. They highlight their own conference experiences, including the challenges of live demos and the significance of networking at events. The video encourages viewers to participate in the open telemetry community and share their knowledge through speaking opportunities. ## Chapters -Sure! Here are 10 key moments from the livestream transcript along with their timestamps: +00:00:00 Welcome and introduction +00:02:20 Meet the panelists +00:04:30 What is a CFP? +00:06:50 Finding conferences to speak at +00:09:17 Motivation for speaking +00:12:51 Overcoming imposter syndrome +00:15:12 Importance of unique topics +00:18:10 End user stories value +00:22:40 Choosing topics for CFP +00:29:36 Hot topics in observability -00:00:00 Introductions to the panelists and the topic of discussion. -00:02:30 Explanation of what a CFP (Call for Papers) is by Adriana. -00:05:15 How to find conferences to speak at, with Josh sharing resources. -00:09:00 Ree discusses motivations for wanting to speak at conferences. -00:12:45 Overcoming imposter syndrome when applying to speak, with insights from Adriana. -00:16:30 Tips on selecting topics for CFPs, including the importance of uniqueness. -00:21:00 Discussion on knowing your audience and adjusting your talk accordingly. -00:25:00 The importance of catchy titles for CFPs and how they affect acceptance. -00:30:15 Advice on what to do after your CFP is accepted, including rehearsal tips. -00:35:00 Panelists share their horror stories from past speaking experiences. +**Dan:** Hello everybody, good morning, good afternoon, good evening. I'm Dan, and as part of the Open Telemetry and End User SIG, today I have the absolute pleasure of hosting this panel on end users. We normally focus on Open Telemetry — I mean, it's in the name. We host end users telling us how they're adopting observability best practices across the industry. If you want to know more, you can follow the QR code on the screen and you'll find how to get in touch. -Feel free to ask if you need any further information! +However, this time we're going to be getting some tips and tricks from true veterans in the tech conference arena. We'll be talking about how to write good CFPs, what topics to choose, how to prepare for a talk, or even why going through the whole process of applying to speak at a conference is well worth it. So if you have recently applied to speak at KubeCon or Observability Day in London or another conference, it doesn't matter if you made it or not; I think these experts will have some advice that you can take home and use in your future talks. -# Panel Discussion on Speaking at Conferences +[00:02:20] We want this to be an interactive panel, so if you're watching live on YouTube or LinkedIn and you have any questions related to the topics that we're discussing, please drop them in the chat, and I will try to incorporate them into the discussion as best I can. Okay, so it's time to meet our panelists. If you're watching live on YouTube or LinkedIn, I would love to know where you're watching from, so drop a comment in the chat and tell us where you're watching from. Myself, I'm in Edinburgh, Scotland, where it is currently 5:02 PM. -**Dan:** -Hello everybody! Good morning, good afternoon, good evening. I'm Dan, and as part of the OpenTelemetry End User SIG, I have the absolute pleasure of hosting this panel today. We normally focus on OpenTelemetry, as it's in the name, but this time, we’re going to be getting some tips and tricks from true veterans in the tech conference arena. We'll discuss how to write good CFPs, what topics to choose, how to prepare for a talk, and why going through the whole process of applying to speak at a conference is worth it. +Let's meet our first panelist, Adriana. Hello! Can you tell us where you're connecting from and a little bit about yourself? -If you've recently applied to speak at KubeCon, Observability Day in London, or another conference, it doesn't matter whether you got accepted or not. I believe our experts will have valuable advice that you can use for your future talks. +**Adriana:** Hey Dan, nice to see you! My name is Adriana Vela. I am connecting from Toronto, Canada. It is noon here, and I work alongside Dan as one of the maintainers of the Open Telemetry End User SIG. Thank you very much! -We want this to be an interactive panel, so if you're watching live on YouTube or LinkedIn, please drop any questions related to our topics in the chat, and I'll do my best to incorporate them into the discussion. +**Henrik:** Hello, Henrik here. Hey, pleasure to be here. My name is Henrik Rexed, as it looks like it's written at the bottom of my video. I'm based in the south of France, the beautiful sunny south of France, even if it's still 10-15 degrees at the moment Celsius. It's 6:03 PM local time in France. Otherwise, I'm trying to be involved in a lot of observability topics, so Tag Observ, and I try to give a hand as well in the End User SIG. Thank you very much. -Now, let’s meet our panelists. If you’re watching live, I’d love to know where you’re tuning in from, so please drop a comment in the chat. I’m in Edinburgh, Scotland, where it's currently 5:02 PM. +**Josh:** Hello, yes, thanks for having me! I'm Josh. I've been a developer advocate and product manager in all kinds of things around Open Telemetry, and I'm connecting from Brussels, where I'm here for FUM, starting on Saturday. Looking forward to that! I like the European representation here. -**Adriana:** -Hey Dan, nice to see you! I'm Adriana Vela, connecting from Toronto, Canada. It’s noon here, and I work alongside Dan as one of the maintainers of the OpenTelemetry End User SIG. +**Ree:** Hi everyone, I'm so excited to be here! I am joining from Vancouver, Washington, not to be confused with Vancouver, BC. It's like 10 minutes north of Portland. I work in developer relations at New Relic, and I also work with these lovely folks in the Open Telemetry community, primarily as part of the End User SIG. -**Henrik:** -Hello! My name is Henrik Rexed, based in the beautiful sunny south of France. It's currently 6:03 PM here, and I'm involved in a lot of observability topics and I help out with the End User SIG as well. +[00:04:30] **Dan:** Thanks to everyone for being here! So let's start with the first question from me, which is going to be directed to Adriana. What is a CFP in the first place? Can you tell us more about it? -**Josh:** -Hello! Thanks for having me. I’m Josh, a Developer Advocate and Product Manager working in the realm of OpenTelemetry. I’m connecting from Brussels, where I’m here for FOSDEM, starting on Saturday. +**Adriana:** Okay, CFP, if I got this right, stands for Call for Proposals or Call for Papers, depending on the context. It's basically a request for a proposal for a talk in the context of conferences, where you basically have an idea for a talk, and you give the details for said talk. Depending on what conference you're applying to, the details will vary from conference to conference. For example, if anyone's ever applied to SREcon, it is a song and a dance to apply to SREcon because I feel like they really want you to hash out all of the details of your proposal ahead of time compared to, say, KubeCon, where you can be a little bit more high-level. So yeah, that is CFP in a nutshell. I don't know if anyone else wants to chime in. -**Ree:** -Hi everyone! I'm so excited to be here. I'm joining from Vancouver, Washington, not to be confused with Vancouver, BC, which is just 10 minutes north of Portland. I work in Developer Relations at New Relic and also engage with the OpenTelemetry community, primarily as part of the End User SIG. +**Henrik:** For me, a CFP is the opportunity to be in a conference to meet people and to network, because at the end, in my role, to be able to reach out to that conference is a great excuse to talk. For me, a CFP is like, "Oh, maybe I will have some miles with my air because I'm an Air France member if I'm accepted." That would be great! ---- +**Dan:** Okay, so we'll move to the next one. If you're new to conferences, you're probably thinking, "Okay, so how do I find conferences to talk at?" Josh, can you tell us a bit more about how you go about finding what conferences you want to apply to speak at? -## What is a CFP? +**Josh:** Absolutely! A really good way is to use the aggregator sites that a lot of the conferences will post their CFPS on. I think the two big ones would be Session Eyes and Paper Call. And then what's the third one? Anyone can help me out with the name of that third one? -**Dan:** -Let's kick things off with the first question directed to Adriana. What is a CFP? +**Adriana:** Pre-Talks! -**Adriana:** -Great question! CFP stands for Call for Proposals or Call for Papers, depending on the context. It’s essentially a request for a proposal for a talk at a conference. You submit an idea along with the details, and the specifics can vary from conference to conference. For instance, applying to SREcon requires more detailed information compared to KubeCon, where you can be a bit more high-level. +[00:06:50] **Josh:** Pre-Talks is the third one! You can kind of check out those or Skedge, right? Those are not necessarily going to be specific to the area that you want to speak at. Of course, for the CNCF, you can look at all the CNCF events; that's going to be relevant for open topics. There are also a lot of aggregators. I think we can maybe add some of those to this resources list after this is done. There are some GitHub repos and some Airtable databases that are maintained fairly well that just sort of keep an up-to-date list of all of the ongoing CFPS sorted by their due dates. -**Henrik:** -I’d add that for me, a CFP is an opportunity to network and connect with people at the conference. If I get accepted, it can also help me rack up some air miles! +**Dan:** And then from those, do you normally choose by the topic of the conference? ---- +**Josh:** Sometimes it's... I think the topic helps narrow down the initial list, and then from there... Oh, maybe Henrik has something to say on this. -## Finding Conferences to Speak At +**Henrik:** No, no, I'm changing the display. -**Dan:** -If you're new to conferences, how do you find the right ones to speak at? Josh, can you share some insights? +**Josh:** I forget what I was saying about that. Oh yeah! The topic narrows down the list, but especially if your topic is as broad as DevOps or even observability, that's not going to narrow it down that much. There are still a lot of conferences, so then you have to pick: where am I okay to travel? Do I like to travel far, like Henrik and rack up those miles, or do I want to maybe get my start somewhere that I can drive to and I don't have to get a hotel? Right? And how big will the conference be? I think there's a really big difference between speaking at a single-track conference as a first-time speaker versus speaking at a multi-track conference. I think we'll maybe get into that in some of the other questions, but yeah, I think just the topic alone is not enough to narrow it down. -**Josh:** -Absolutely! A great way to find conferences is to use aggregator sites that list CFPs, such as SessionEyes, PaperCall, and Pre-Talks. There are also GitHub repos and Airtable databases that maintain up-to-date lists of ongoing CFPs sorted by their due dates. +**Dan:** Nice! That's cool. All right, so we found the conference and we've got some resources there that people can check out to find where to speak. I guess, you know, a question for Ree: what makes you want to speak? Why do you apply to speak at conferences? -**Dan:** -Do you usually choose conferences based on the topic? +[00:09:17] **Ree:** One thing that I kind of found out early on was, because my job is so busy, it sometimes can be hard to find time to learn something new or, you know, something maybe more niche. So if I can submit a topic and it gets accepted on something that I want to learn more about, then that automatically gives me the time and bandwidth to work on that topic and learn about it. So that's one reason. And now that I've been doing it a while, just the benefits from the physical interactions of being able to be in the same space as people who want to learn about these things and having conversations about it — I'm able to learn more. There are a lot of other people that also want to learn the same thing that I wanted to learn, and so it just becomes like this really cool collective, I guess. You get to meet a lot of cool people that way too, and not just for professional networking, but also some have become personal friends. It's just really great; it's a great community exposure. -**Josh:** -Yes, the topic usually helps narrow down the list, but sometimes it also depends on the travel logistics and the size of the conference. Speaking at a single-track conference as a first-time speaker versus a multi-track conference can make a big difference. +**Adriana:** I was going to say, sometimes, you know, for me personally, speaking is almost like a challenge. I love speaking in front of audiences, but I also find it a little bit terrifying. This kind of forces me to get past that discomfort. Another thing I wanted to point out, especially for those of us who are in an underrepresented group in tech, I think the more of us in underrepresented groups that go out and do talks, the more we can show those folks in underrepresented groups that we exist, and we can empower them to go out and speak as well. I think that's so, so important. ---- +**Henrik:** Just to add as well, I think all of us, we all work in technical environments, and sometimes we think that what we are working on is quite normal and nothing special, but there's always something to learn. So if you like the community and you want to share your work, you want to share your experience, you share your journey from being a complete beginner to being a complete expert, and the advice that you can share, I think it's the best opportunity. Sharing a talk is a lot of sharing knowledge and education with the community. I think it's an amazing opportunity for all. -## Motivation to Speak +**Josh:** Yeah, and as well, like for someone that's not a dev advocate, for example, you know, what motivates you? -**Dan:** -Ree, what motivates you to speak at conferences? +**Henrik:** Well, I'm a dev, so I get the same excuse as Ree, right? If I get a talk accepted, that's brownie points for my boss, and it justifies my existence a little bit. But for people who don't have that motivation, there's another thing that devs actually do that’s really, really important, right? Which is bring feedback back from the community. So that's another thing. If you're an engineer on an engineering team, it can get a little bit insular in our bubbles. Going and talking at a conference is a chance to kick the tires on some ideas with some people that are new to you and gather that feedback from the community as well. -**Ree:** -I found that speaking at conferences forces me to take the time to learn something new, especially topics I want to dig into. Plus, the physical interactions and conversations are invaluable. The community aspect is incredible, and it helps me learn more while connecting with others. +**Dan:** I think that's it as well. Sometimes you forget that what you're doing might be something that is ahead of the curve, and you want to give back. That's great. -**Adriana:** -For me, speaking is a challenge I enjoy, even though it can be terrifying. It also empowers those in underrepresented groups to share their experiences and show that we exist in tech. +[00:12:51] Moving on, they say that, you know, now basically I'm convinced that I want to talk, but sometimes, you know, I think a lot of us suffer from imposter syndrome, and you think that you're not good enough to talk about a certain topic. So I'll go back to you, Adriana. How do you get past that imposter syndrome and that sense of, "I don't know enough to talk about a topic?" -**Josh:** -For me, it adds value to my role and gives me feedback from the community. It’s a great opportunity to gather insights from people outside my usual circle. +**Adriana:** I feel like you just have to force yourself past it — just force yourself to do it anyway. Because you know what, if you don't do it, someone else is going to do it. So why not you, right? A lot of times, I think this is advice that someone gave me early on when I started applying for talks, which is like you don't have to be the expert. I think Ree touched on this too: you don't necessarily have to be the expert; you can use this as an opportunity to force yourself to learn something about a topic. Or the other thing is, like, I have to be the expert to be able to speak intelligently about it, but there's something to be said for having kind of a newbie's point of view as well. We're so much more relatable that way when we talk about our experiences as newbies because there are so many people who are new to things. To show people that you're human and you're not some perfect being up on this pedestal, I think it makes it super relatable and more fun for people to learn. ---- +**Henrik:** I think it's important to note that I have not gotten to the point where I don't have a little bit of impostor syndrome or stage fright. It just never goes away completely; I think I'm with you. -## Overcoming Imposter Syndrome +**Adriana:** You really do just have to push through it because it's always there. But I think it's the talk, like you mentioned before; it's the opportunity to learn and to be more of an expert on that side. But even if you're not the maintainer or the main contributor of the project or whatever topic you decided to present, I think if you have a way of presenting that is super interesting, brings a lot of entertainment, and people will probably listen more to you. If you are someone who is very boring on stage, you don't need to be the full expert; it's just sharing a passion in a very funny way or in your way, in fact, and that brings value for the audience. -**Dan:** -Imposter syndrome can be a hurdle. Adriana, how do you get past that feeling of not being “good enough”? +[00:15:12] **Josh:** That's such a good point! I've seen talks where you put subject matter experts who are really, really knowledgeable about the subject, but they weren't necessarily presenting it in the most absorbable way. I've seen people who were newer to the subject do really creative things that, for me, I was able to say, "Oh my gosh, yes, this makes complete sense!" I think that is a really good point. You can bring your own perspective into things, and the way you share your knowledge will resonate with somebody in the audience. -**Adriana:** -You just have to force yourself to push through. Remember, if you don’t do it, someone else will. You don’t have to be the expert; you can use this as a chance to learn and share relatable experiences. +**Ree:** I will also say that this is why Ree and I have cats on our slides! -**Henrik:** -I agree! It’s important to remember that you don’t need to be a perfect expert. Sharing your journey can be just as valuable. +**Adriana:** You mean that you bought cats just for conferences, or you had cats in the beginning? ---- +**Ree:** Oh, so like Ree's cat Taco is like the most photogenic cat ever, and so we just feature her on a bunch of our slides whenever we do talks together. She's a pretty short hair; she's ridiculous. Real animal! -## Choosing Topics +**Dan:** I think we talked about end users and how, you know, in Open Telemetry and observability, for example, you may think as an end user, or you know, people will want to know about the latest thing that maintainers have been working on in Open Telemetry or the latest thing that this particular vendor is delivering. But I think there's a lot of value from users telling their stories as well. Do you agree with that, Henrik, for example? I think you talked about that previously. -**Dan:** -How do you decide what topics to present in your CFP? Ree, can you share your thoughts? +**Henrik:** Yeah, I think having vendors presenting will always bring their angle, and I think having a user share their story offers a different angle and journey or an experience that is always super interesting. It's like a book where you have an adventure, and you just follow the adventure. I think it brings a lot of value to the audience. Sometimes, even more value to have someone like me, like a dev, going on stage because I won't necessarily have this experience and this journey. I will bring the same topic from a different angle, so I think a user has a real experience story. I think it's one of the best ones from my perspective. -**Ree:** -It can be challenging. I try to find a niche, especially since OpenTelemetry is becoming more popular. Unique perspectives are essential to stand out among the many submissions. +[00:18:10] **Josh:** Absolutely! We recently organized the Open Source Analytics Conference at my company, where we participated in organizing it. At the beginning of the process, right, choosing the talks, we specifically separated out all the end-user talks because we didn't have enough of them, and we wanted to make sure that they were well represented in the schedule. We definitely gave preferential treatment to those. The reason for that is for everything that Henrik said, right? Other people want to get these stories that they can follow along with and relate to that don't have that vendor spin necessarily. -**Adriana:** -Yes, and it’s important to ensure your submission is unique. While AI topics are popular, they need to offer a fresh angle to be accepted. +**Dan:** I think we got that basically from one of our audience, saying that end users just give a different weight to what they're saying because they are using it in production; they're using it in their systems. ---- +Okay, I think we can move on to another topic, which is the topic topic! So we've got, you know, we've got an idea that we want to speak at a conference, and then how do you actually decide what topics to go through? I guess, you know, what topics to put in that CFP? I will ask that to Ree. -## Preparing Your Talk +**Ree:** This is an interesting question because when I first submitted my very first CFP to KubeCon in 2022, I think Open Telemetry was still pretty new as a topic at KubeCon at that time. For the first couple of years, I was having pretty good success getting my topics selected. I've found that in the last year, it's been getting harder, one because there are more people talking about it, but also there are more and more things being covered. I think it's been interesting to figure out now it's time to get really kind of niche. I mean, there's still a place for intro-level talks, but I'm kind of in that space where I'm trying to figure out what are interesting aspects and topics related to Open Telemetry and observability in general to submit for. -**Dan:** -Once accepted, how do you prepare for your talk? Ree, what’s your process? +**Adriana:** I did, yeah! I was going to say on that same vein because, yeah, like Ree said, it's getting harder and harder especially in the observability space. It's interesting too because I think some of the folks on this panel have reviewed CFPS as well for KubeCon. I've done a number of CFP reviews for KubeCon, and it's interesting to see what topics come up over and over and over. I will tell you AI comes up a lot. It's almost to the point where you're like, "Oh my God, not another freaking AI CFP for the love of God!" They're not unique because here's the deal: there's nothing wrong with submitting a CFP on AI, but make sure it's unique at this point because a lot of the stuff that's out there is different permutations of the same thing. It's really about what's your unique take on it. Also, we were talking about end-user stories; I think those are still extremely useful and relevant, but again, after a while, you start seeing a lot of the same end-user stories. So again, when you're submitting an end-user story, what is special? What makes you a snowflake when you're telling your end-user story? -**Ree:** -I found that working with a public speaking coach helped me a lot. I also practice as much as possible to ensure I’m comfortable with the material. +The other thing I would say is some folks try to submit project updates as part of a CFP for KubeCon or whatever, and it's like, dude, a project update does not automatically get you accepted! Save that for the relevant venue, because I think there's special project update sessions, for example. Those are a couple of my pet peeves. I don't know if anyone else wants to chime in. -**Henrik:** -For me, I focus on timing and ensure I’m not trying to cram too much content into a limited timeframe. +**Josh:** I agree with that, right? Like, how do you be trendy without being too trendy? ---- +[00:22:40] **Adriana:** I see in the chat there's a thing about people not understanding the basics of OpenTelemetry. When you were speaking about trying to find your niche, do other people call it OTel or is it just me? -## Handling Rejection +**Josh:** I love it! OTel! -**Dan:** -How do you manage expectations around rejection? Josh, what are your thoughts? +**Adriana:** Yes! -**Josh:** -It’s completely normal to get rejected. A good rule of thumb is to expect about a 10% acceptance rate. It’s crucial to keep submitting, and don’t be afraid to ask for feedback on your proposals. +**Josh:** Anyways, maybe at KubeCon, right? That's true, you need to have something unique and novel to bring to a KubeCon conference. But outside of our bubble, right? Open Telemetry is still a very niche topic. I've had my introduction to Open Telemetry, right? The smaller regional conferences are clamoring for those introductory topics. -**Adriana:** -I agree. It’s okay to feel disappointed, but don’t let it deter you. Reflect on your submission and consider resubmitting to other conferences or tweaking your proposal for the next opportunity. +**Ree:** Introductory talks on these niche topics that are our niche, right? ---- +**Josh:** Yeah, and in my case, I usually try to avoid those introduction talks because I think many people will do the same, so it's just a matter of being different or bringing a different angle. When the people review your talk, then you get an interesting angle that brings value to the community, in my perspective. -## Closing Thoughts and Horror Stories +**Dan:** I do talks that are very expensive in terms of preparation because I love benchmarking. I think I like to have those studies where you show numbers, you show things. It's like an experience, and then you show that to the stage because I think it brings another value because sometimes you don't show that in the normal track. Having that difference, you probably have more chance to be selected. -**Dan:** -As we wrap up, let’s share some memorable conference experiences. Ree, do you have a horror story? +**Dan:** I guess, you know, related to this, I think the introductory topics and the more in-depth topics sometimes, you know, when you're applying to speak at a conference, you don't know what level of detail you want to go through either in the CFP or in the talk itself. Is it good to—how do you know your audience, and how do you know what level of depth you want to apply? Ree, how do you normally go about it? -**Ree:** -My first talk included a live demo that failed right in front of the audience. It was terrifying, but I managed to get it working later. +**Ree:** I still find that tough sometimes. Sometimes I'll start with like, "Okay, I kind of want to be more high-level about this topic," but in order for someone to come to this topic, they might need, you know, XYZ knowledge. So even though it's more high-level on that specific thing, it might be considered like an intermediate, if that makes sense. It really depends. I'll also try to consider how much, you know, am I talking about something newer, you know, that was more newly developed, so there's less info about it, or is it something that's been around for a while? I don't have a really good answer; I still find it tough honestly. I try to stick to either beginner or intermediate level in general because of the topics that I typically choose to do. -**Henrik:** -I had a demo where the screen mirroring didn’t work. I ended up scrambling to present without visuals, which was quite stressful! +**Josh:** The thing also is, it's a personal judgment as well. Sometimes you say, "Oh, I think it's beginner," or "I think it's..." I have no idea; it's complicated. -**Josh:** -Similar experience! The audience could only see half my slides, and I had to rely on my memory for the rest. +**Ree:** What may seem basic to you might be very advanced to others or the opposite, right? -**Adriana:** -I haven’t had any major mishaps, but I do prefer pre-recorded demos to avoid the stress of live presentations. +**Dan:** Okay, Henrik, I'm going to go back to you because I know that you're everywhere in Open Telemetry as well. I know that there are a lot of hot topics in Open Telemetry. What are the hot topics that you recommend people be writing about right now? I know that it doesn't have to be Open Telemetry; I know that we all love OTel, but what are the sort of hot topics that you would recommend thinking about writing about or talking about? ---- +**Henrik:** What I would suggest is that, by the way, AI is not valid; it is! In general, you have to look at previous conferences and what has been presented and covered so far. If you start to say, "I want to do Open Telemetry on instrumentation," there's plenty of them out there; there's plenty of video out there, and then there's no hot topic in general. Usually, I try to bring the problem statement. For example, how do I sample properly? How do I optimize? Because there is a big concern about observability being expensive, so how can I control that cost? Going through those directions, I think also there is another trend that I think is really important that people need to be educated a bit more on: sustainability. How can we make it green, make it better? Talking about AI is not going to be green because you're going to consume more resources. So maybe having another approach where you say, "We need to be good citizens of the world and save energy," I think that's a pretty good one. Also, I think there are new projects coming in. For example, today, as we speak, Open Telemetry profiling — there are a few talks out there in KubeCon, and it's going to be more and more popular. So there's a big chance that profiling will bring new problems and challenges, so covering them could also be interesting. Try to follow what happens in the industry and think about, "If I start digging this, I may have problems, so how can I resolve them and maybe share that solution or this approach with a larger group in the community?" -**Dan:** -Thank you all for sharing your insights and experiences! It’s been incredibly valuable. If you have more questions, feel free to reach out in the End User SIG channel. We’re always open to discussions about OpenTelemetry and beyond. Thank you, everyone, and see you at a conference! +**Adriana:** I just add, Henrik, you're really good at this, right? You mentioned sampling, and I saw that video in my feed, and I was like, "Oh, that's a great topic! I'm going to check that out later when I have time." + +**Dan:** I think we have some hot topics here. Any other takes on what some of the hot topics are? + +**Ree:** I would like to add something. You know, I think piggybacking on what Henrik was saying on the topics of sustainability, there are some really cool CNCF projects out there on tech sustainability, and I feel like this is a super hot topic for the year just because we're seeing a lot of stuff around wacky environmental things happening, right? Like an increase in forest fires, bizarre temperature swings. We're inherently in an industry that is contributing to the problem. Writing talks about how we can use technology to lessen the problem, I think, can be really compelling and very timely. There's a hot topic for y'all for anyone considering it. + +I would also mention, we talk a lot about KubeCons, and I think Josh made a point earlier about there being a lot of conferences where things like Open Telemetry are still kind of not super well known. Especially like a lot of these open-source conferences, like Scale, for example, I think just added an observability track this year. FUM I think would probably be another great one. State of Open Con would be another great one where we probably don't have enough talks on observability. Getting into those sort of more niche conferences, I think, would be a good place to start, especially if you're looking to do a talk on observability. + +**Dan:** Oh yeah, all things open as well. + +**Ree:** Yes! + +**Dan:** Right, I'm going to take one question from the audience. Thank you so much for your questions. Again, if you've got any questions on what we're talking about, drop them in the chat on YouTube or LinkedIn. The question is: should we go for a catchy title, a clickbaity title, when you write your CFP, or should you do something more perhaps descriptive and accurate? Josh, what do you think about that? + +**Josh:** I almost always do a clickbaity title, or at least I did. That was always sort of my way. But I've only been doing this for a couple of years, and someone who's been doing it for longer than me told me, "Actually, it's a pendulum, right? It swings." What the conference organizers are looking for is going to swing back and forth almost like a cultural zeitgeist. There are times when they want it to be super, super specific, and we're not in one of those times right now. Maybe we're swinging in that direction, but for me right now, it does feel very much like the clickbaity titles are in, and maybe we'll all get tired of them because of the AI topic, and we'll be like, "No, you need to tell me exactly what's in your talk so that I know that it's not a surprise AI talk." That'll be where we're at a year from now. + +I think if you can come up with a catchy title that also gives the audience an idea of what to expect or what your topic is about, that is a great way to go. I also do really like just clear, straightforward titles as well. I think they both have their place, and the content for sure is going to matter more, I say. Because you could have a great catchy title, and then the abstract is kind of, you know, maybe doesn't fully flesh out the idea. The content is still king, but yeah, the title is important as well. + +**Henrik:** When I prepare my talks, I usually try to bring the technical aspects and then try to find a funny angle or an analogy to a movie, to video games, to do whatever. Then in the title, I try to bring that fun angle because at the end, I think that could be more attractive. When you have the schedule and you see a title that sounds more fun, then maybe more people will join your session for some reasons. + +**Adriana:** I think the title is as important as the abstract, from my perspective. + +**Josh:** I think, going back to what you just said, it's important to note that some titles sound like they were generated by ChatGPT. + +**Ree:** Yes! + +**Josh:** And B, make sure that your titles aren't so pop culture where it ends up alienating potentially the CFP reviewers. Like, I don't know if someone is making a Game of Thrones reference in a talk title. I'll be like, "I have no idea what you're talking about," and I'm kind of super put off by it. + +**Dan:** So I would just caution, it's a fine line to tread. + +**Dan:** And I guess that's part of—there are a couple of questions from YouTube which are related to that, basically to knowing your audience, right? Who's going to be reading your title, your CFP? So how do you know that? How do you know how to approach the audience in a way that they would understand or engage with? I'll ask that to Henrik. + +**Henrik:** It's a very, very good question. I usually go to KubeCons and to Open Source-friendly conferences, so I know that the people that will be in that conference have at least a technical background. I know that my talk would not be perceived as too technical. But yeah, if you go to a conference that is more salesy, then yes, you may have to adjust and say, "Okay, what are the type of personas that will be in this conference? Do I have the normal audience that I'm talking to in general, or should I reduce the complexity of my talk so that people can follow the idea behind this talk?" + +**Dan:** Cool. Right, I'm going to move on to another topic, which is the post-acceptance. So you got accepted — hooray! So what happens now? How do you train? How do you rehearse? What can you give people to get ready for the day? I know that was a very wide question, so I'm going to start with Ree. How do you prepare for it? + +**Ree:** Well, so when I got my first talk, funny story: my very first CFP that I submitted ever was also my first one that was ever accepted, which was huge and terrifying. My manager at the time actually hired a—well, she got me a few lessons, virtual lessons, with a public speaking coach, and I found that that was super helpful. It was just three virtual sessions, and I came away with techniques that I still use. But I also know people who are great speakers who haven't gone through that professional training. I say it's at least worth it if you can, if you have the resources to. I recommend it, but if not, there are also YouTube videos you can watch. Of course, rehearse as much as you can so that you feel comfortable and it feels natural when you're presenting. + +**Josh:** Any other tips or tricks to basically get to the training rehearsing stage? + +**Henrik:** I usually don't. I do just the rehearsal, not a full-fledged rehearsal. I'm just checking that my talk is basically respecting the duration because sometimes I put a lot of content in, and then I realize, "Oh, maybe too much here." So I need to time myself, and then you know when you will be on stage, there's a big chance that you will take probably more time. So I had a few minutes to be always on time. I think that's the best thing. But what I usually do, I'm trying to put a lot of graphics, make it nicer. I put a lot of effort into making a smoother experience for the audience, so that's something that I prepare a lot in the background. + +**Dan:** I'm going to take now a question from the audience. I think I'm going to choose—I'm going to pick Josh to answer this question. If a conference offers multiple formats, for example, lightning talks or a deep dive, should I spam the organizers with multiple responses, or is there a better way? + +**Josh:** In my opinion, that’s something that will work if you know the organizer and can reach out to them directly, right? Or if they're having office hours and you can talk to them about it and you can get that feedback, that's fine. But I do think that if you're a first-time speaker at the conference or sort of unknown to the conference, you have to sell the talk. You have to be a little bit more confident in the talk that you're proposing because they're going to be evaluating you based on the talk more than how they know you as a speaker. + +[00:29:36] **Ree:** Yeah, I wanted to just mention something. Especially depending on the conference, some conferences don't have limits as to how many talks you can submit. Sometimes, you know, especially if you're a newbie speaker, go for it and just see how far you can take it. But other conferences, like KubeCon, where there is a limit on the number of sessions, I would caution against submitting two versions of the same session where one's a lightning talk and the other one's a longer-form talk. I think you probably have a better chance of just submitting two unique topics because you never know. Also, I will say, don't be afraid to recycle submissions at other conferences. It doesn't always have to be a unique topic. Super, super important! It saves you a lot of mental energy because putting together a talk is a lot of work. + +**Henrik:** I would just add to the fact that for the reviewers, if you have a talk and usually you design your abstract for a 30-minute talk, for example, then the reviewer would expect some details. Then you say, "Okay, so he’s going to cover this and that, and it makes sense in 30 minutes because it's going to be well covered." If you do the same CFP in a lightning talk, then you say, "Wow, how is it going to cover that in five or ten minutes?" I think you need to adjust, of course, the way you're going to present the abstracts. + +**Ree:** I would definitely recommend doing, like Adriana mentioned, to have two different submissions: one lighter for the lightning talks and one bigger for the normal track. + +**Dan:** Cool! Okay, I think it's time to start closing thoughts. One of the questions that people normally get is: I apply to many conferences, and then you get rejected. How normal is it to get rejected? How do you manage your expectations for getting your CFPs accepted? Josh, do you want to give us your thoughts? + +**Josh:** Sure! I think someone, again, you know, getting advice from mentors is great. Someone told me to expect a 10% acceptance rate. If you want to talk at one conference a year, that means you need to submit 10. I think that holds even for experienced speakers to some extent as well as new, at least from what I've seen. I don't know how you all feel about that ratio, but it is absolutely normal to get rejected. You think about conferences; some conferences get, especially the bigger ones, thousands, sometimes, but usually at least tens or dozens or hundreds of submissions. Just because they rejected it this time doesn't mean that they won't next time. I know people who've submitted the same one a few times to the same conference before it got accepted. Sometimes it's just timing; sometimes it's just, you know, they just have a lot of topics that were really good. Sometimes too, you can also ask for feedback from the conference. KubeCon, for example, they're pretty good about sharing feedback on your proposal; they'll be happy to share it. + +**Ree:** I also wanted to add, it's okay to mourn your rejection because especially when you're really invested in a topic, and you think, "I've got such a great chance of getting this in!" and you get rejected. You got a great title, and you're just... + +**Josh:** Yes! + +**Ree:** Yeah, it's okay to mourn and take that time to mourn. As Ree said, if possible, ask for feedback. Resubmit it to that same conference or another conference or, like in the case of KubeCons, there are so many across the globe. Submit it to a different KubeCon. You know, if you didn't get in for NA, submit for EU. There's Open Source Summit NA, there's Open Source Summit EMEA, and also take some time to reflect on your CFP and see if there are areas where you see improvement. Sometimes I'll write out a CFP, and then I'll realize, "Oh, yeah, I can kind of see why they didn't like that," and just make a few tweaks. + +**Dan:** I agree! + +Well, thank you so much! I think now I've got one last question from you, and I think, as well, if everyone's been following the Golden Globes and the Academy Awards, you'll see that we've got quite a lot of horror films coming up in the awards. I've got a question for you. I think you've been to many conferences, and I would like to hear some of the conferences that you've been to because I think we didn't cover that in the intro. I would like to know some of the conferences that people could go and rewatch your videos, but also if you've got a quick anecdote or something that failed that was a bit of a horror story — right? The mic doesn't work or, you know, something that doesn't work. Can you tell us about it? I'll go for Ree first because she's got her hands raised. + +**Ree:** My very first talk that I did, it was about tail sampling in Open Telemetry. I had a live demo, and everything was working. I checked it right before my talk; everything was working. I got up on the stage, started the demo, and it didn't work! I still don't know what it was because I got it to work a few minutes later, and I know there are some people that still remember that too! So anyways, that's my—I have other ones too, but that one sticks close. + +**Henrik:** What was my worst experience? I would say it happened once where I had to connect my laptop, and the screen mirroring didn't work. I suddenly did the demo, and then you end up having nothing on your screen, and you're like this, trying to type and looking at it. It's like a nightmare! I was saying, "Okay, the demo is going to be a nightmare; it's impossible to achieve!" So then I think the demo, I tried to skip it very fast, but I was not very happy. People didn't see so much because I tried to react as fast as possible to avoid having that blank effect where stress is coming up to you. At the end, I was unhappy because I said, "Ah, my performance was just not normal and not acceptable!" I had to improve that. + +But yeah, where can we see you next? + +**Henrik:** I was rejected for KubeCon, so I've been crying! So now I think I have no tears anymore. + +**Ree:** It happens to the best! + +**Henrik:** It does! So yeah, the next I have a virtual conference that I don't know when it's going to happen in a few weeks. I'm presenting a talk, and then I submitted lots of talks for the season. I have a lot of KCDs and KubeCon China and Japan. We'll see; I didn't have the feedback yet, so cross your fingers. But otherwise, if you want to watch any content from my end, you can find it on KubeCon Europe, KubeCon North America, and I have plenty of other KCDs where I have presented talks last year. + +**Josh:** I think it's quite similar to Henrik’s. My very first talk, the audience could only see half of my slides. I could see all of my slides, but none of my notes. My lesson that I took away from that is I never rely on my notes ever again! Plus one to that! You have to have it all in your head. You have to be able to talk about the thing for 20 minutes with no notes and no slides if it comes down to it. I think not to scare anyone else off from doing this, right? You can do that; it's not as hard as it sounds. But yeah, that was my worst horror story. Although I would say I feel like at least something small goes wrong pretty much every time. + +And then where you can find me? A lot of DevOps Days; I'm really a big fan of those. I love that the KCDs are starting to adopt that community format and sort of the open discussion format more. I really love those conferences, and I love to get the combination of the people who are coming from all over like you would see at the bigger conferences, and then just also the local representatives of the tech companies that are employers in that area. + +**Adriana:** I think I know what my horror story is! I think I had a similar experience where I had a demo that was not working, and the audience could see it! But I think for me, the worst is when you give a talk, and you're like, "H, that wasn't my best work." You feel like you're a little bit off. Sometimes my voice is a little shaky or I forget my talking points. I think that's getting rattled by that, and it's interesting because you come off a talk, and people are like, "Oh, that was so great," and you're like, "Oh, I sucked!" You kind of just have to get over yourself and tell yourself it happens, and it'll happen again, and you just have to be okay with it. I would also suggest when you're practicing talks, practice. As you're practicing, if you stumble, just power through it because that can happen in real life. + +As far as conferences that I've spoken at, and I'm just sending Henrik a link to my YouTube channel because I have a playlist of all my talks. Recently, I've done a bunch of talks; we've spoken at KubeCon North America and EU a few times together. We've spoken at Observability Day North America and EU together. I've spoken at Platform Engineering Day most recently in North America. We've done All Things Open together. We did Open Source Summit, and I'm speaking at KubeCon EU coming up in London. I got in for Observability Day EU as well. I did DevOps Days in Montreal, and one of my favorites was KCD BUU, where I got to give a keynote. Unfortunately, there's no recording of the keynote, but I have a blog post version of the talk if anyone wants to check out my Medium channel. + +**Dan:** Awesome! Well, thanks everybody so much. You are really the dream team—really all the veterans from all the conference dos. You can see that from the amount of conferences you've been to! Thank you so much for your tips, tricks, and advice. I think it's been really, really useful. But I think that's all we had time for. I just want to say thanks to the panelists and thanks to the people that asked questions in the audience. I think we didn't get to cover all the questions, but we are always open in the End User SIG. You can come to the CNCF Slack; you can go to the resources that are here, basically, and get in touch with us. I think we'd love to hear more about your—well, in general, anything about Open Telemetry, but also about topics like this one: how do you go and talk about Open Telemetry everywhere? + +With that, thank you very much, and see you at a conference! Get the questions that were unanswered in the Slack channel. + +**Dan:** Thank you! Bye-bye! ## Raw YouTube Transcript diff --git a/video-transcripts/transcripts/2025-02-26T15:40:58Z-what-is-otel-otel-for-beginners-the-javascript-journey.md b/video-transcripts/transcripts/2025-02-26T15:40:58Z-what-is-otel-otel-for-beginners-the-javascript-journey.md index 9938318..7b9c883 100644 --- a/video-transcripts/transcripts/2025-02-26T15:40:58Z-what-is-otel-otel-for-beginners-the-javascript-journey.md +++ b/video-transcripts/transcripts/2025-02-26T15:40:58Z-what-is-otel-otel-for-beginners-the-javascript-journey.md @@ -10,100 +10,58 @@ URL: https://www.youtube.com/watch?v=iEEIabOha8U ## Summary -In this introductory video titled "OTel for Beginners," Lisa Jung, a member of the OpenTelemetry (OTel) Communication and End User Special Interest Groups, shares her personal journey of learning OTel and aims to guide viewers through the process. The video explains the importance of observability in modern software systems and how OTel serves as an open-source framework to generate, collect, manage, and export telemetry data, thereby preventing vendor lock-in. Lisa emphasizes the advantages of using OTel over proprietary solutions, including reduced switching costs and ease of use across different observability backends. She also highlights the resources available for getting started with OTel, including official documentation, YouTube tutorials, and community support on Slack. The next episode will focus on beginning the JavaScript journey with OTel. +In the video "OTel for Beginners," Lisa Jung, a member of the OpenTelemetry (OTel) Communication and End User Special Interest Groups, shares her journey as a newcomer to OTel and aims to guide viewers through the basics of using this open-source framework, particularly through a JavaScript lens. Lisa first explains what OTel is and emphasizes the importance of observability in modern software systems, likening a non-observable system to a "black box." She discusses how OTel helps to convert this black box into a "glass box" by generating, collecting, managing, and exporting telemetry data, which is crucial for understanding system performance. The video also addresses the issue of vendor lock-in in observability, illustrating how OTel provides a universal standard that allows for easier transitions between different observability backends, thus reducing the costs associated with switching vendors. Lisa encourages viewers to explore various resources for learning OTel, including documentation, the official OTel YouTube channel, and community support through the CNCF Slack Channel. The next episode will focus on starting the JavaScript journey with OTel. ## Chapters -Sure! Here are the key moments from the livestream along with their timestamps: +00:00:00 Welcome and intro +00:00:30 Observability explanation +00:01:00 Importance of telemetry data +00:02:00 OTel framework overview +00:03:46 Vendor lock-in discussion +00:04:10 USB-C analogy +00:05:00 Switching vendors challenges +00:06:00 OTel as an open standard +00:07:00 Benefits of using OTel +00:08:00 Resources for getting started -00:00:00 Introductions to OTel for Beginners -00:01:30 What is OpenTelemetry (OTel)? -00:02:45 Importance of Observability -00:04:00 Turning a Black Box into a Glass Box -00:05:30 Types of Telemetry Data: Metrics, Logs, and Traces -00:07:15 Instrumentation and Data Collection in OTel -00:09:00 The Problem of Vendor Lock-In -00:11:30 Comparison of OTel to USB-C Standard -00:14:00 Benefits of Using OTel for Observability -00:16:15 Resources to Get Started with OTel +**Lisa:** Hi! Welcome to OTel for Beginners. My name is Lisa Jung and I'm a member of the OTel Communication SIG and End User SIG. I recently began my journey with OTel and as a newbie, I had a tough time figuring out how to get started. I want you to have a different experience, so I'll learn alongside you and share what I'm learning through the series. Depending on which programming language you're working with, you'll follow a language-specific journey. In this series, I'll go through the JavaScript journey to get you started with OTel. -Feel free to ask if you need more details on any specific section! +[00:00:30] Before we get our hands dirty, let's talk about what OTel is and why you should consider using it and the resources to get started. OTel stands for OpenTelemetry and it plays an important role in observing your system. So let's talk about observability first, then delve into how OTel fits into all of this. -# OTel for Beginners +[00:01:00] Our modern software systems can consist of complex, multi-layered, and distributed systems with many interdependencies. A system without observability is like a black box; we have no idea what's going on inside, so if something goes wrong, it's going to be more difficult and more time-consuming to solve the problem. With observability, we turn this black box into a glass box. As a matter of fact, it helps you collect the data necessary to visualize and understand what's going on in your system. -Hi! Welcome to OTel for Beginners. My name is Lisa Jung, and I'm a member of the OTel Communication SIG and the End User SIG. I recently began my journey with OTel, and as a newbie, I had a tough time figuring out how to get started. I want you to have a different experience, so I'll learn alongside you and share what I'm discovering through this series. +How do we make this possible? First, you have your infrastructure or applications that you want to observe. You'll collect data from it and send the data to the observability backend of your choosing. Then connect the backend to a visualization front end where you can query and use the data that you're interested in. The most common types of data collected for observability are metrics, logs, and traces. These are known as telemetry data. -Depending on which programming language you're working with, you'll follow a language-specific journey. In this series, I'll guide you through the JavaScript journey to help you get started with OTel. +[00:02:00] Getting the telemetry data into the backend is an important part of understanding your infrastructure or applications, and this is where OTel comes in. OTel is an open-source framework. Using OTel, you can add software to your applications or systems to generate telemetry data. This process is known as instrumentation. Then it collects, manages, and exports telemetry data to an observability backend and the database for storage. Different aspects of OTel make this process possible, and we're going to talk about that in more detail as we go through the JavaScript Journey. For now, remember that OTel is focused on the generation, collection, management, and export of telemetry data. -## What is OTel? +You can easily instrument your applications or systems, no matter their language, infrastructure, or runtime environment, and the storage and visualization of telemetry are intentionally left to other tools. So why should we consider using OTel? Before we answer this question, let's talk about something that almost all of us have experienced. -Before we get our hands dirty, let's talk about what OTel is, why you should consider using it, and the resources available to get started. OTel stands for **OpenTelemetry**, and it plays an important role in observing your system. +Now, if we dig through our drawers at home, we could probably find a bunch of cables of different types. Why? Because the devices we own come from different vendors, and each vendor has a specific type of cable and port to charge or connect your gadget. We're all familiar with buying multiple products from the same vendor and investing in additional accessories and apps specifically designed for that product. Before we know it, we get so used to using the line of products that switching over to another vendor could get pretty difficult. -Let’s discuss observability first, then delve into how OTel fits into this concept. Our modern software systems can be complex, multi-layered, and distributed, with many interdependencies. A system without observability is like a black box; we have no idea what's going on inside. If something goes wrong, it becomes more difficult and time-consuming to solve the problem. +[00:03:46] The product from another vendor may operate differently, which will take some time to learn and get used to. It would also cost us more money because the additional accessories or apps we invested in don't work with a product from a new vendor. This is a problem known as vendor lock-in, where the costs of switching vendors are so high that customers feel stuck with what they have. Then things are slowly changing with USB-C ports becoming the universal standard. -With observability, we turn this black box into a glass box. It helps you collect the data necessary to visualize and understand what's happening in your system. +[00:04:10] Let's take the newest model of phones as an example. It doesn't matter if you're using an iPhone or an Android. You could charge or connect many of these phones with a USB-C cable because a universal standard has been set. Users can use the same cable regardless of how many times they charge their phones. OTel has similar implications in observability as a USB-C does for phones. -### Making Observability Possible +Let's do a quick review. We have our infrastructure or applications we want to observe. We want to collect telemetry data and send it to an observability backend for storage, so this data can be queried and visualized with an observability frontend. Say you have three observability backends to choose from: vendors A, B, and C. Each vendor often has proprietary instrumentations, agents, and or collectors. -How do we make this possible? First, you have your infrastructure or applications that you want to observe. You'll collect data from it and send that data to the observability backend of your choosing. Then, you connect the backend to a visualization frontend where you can query and use the data that interests you. +[00:05:00] Now, let's say you picked vendor A. You're using their proprietary instrumentations, agents, or collectors and sending the data to their backend. But your needs change down the road; you want to switch your backend to vendor B. But switching vendors is not as simple as you might think. You can't just send the existing data from vendor A to B, because vendor B requires its own instrumentation, agents, or collectors, so you can't accept data from vendor A's proprietary instrumentation. -The most common types of data collected for observability are **metrics, logs,** and **traces**. These are known as telemetry data. Getting this telemetry data into the backend is crucial for understanding your infrastructure or applications, and this is where OTel comes in. +Now your development team has to change their instrumentation, possibly to a new proprietary instrumentation of vendor B. Now imagine doing this for thousands of Linux machines or dozens of applications. Is the cost of money, time, and effort worth the benefits of changing vendors? As you could see with this setup, the cost of switching vendors can get so high that customers can be effectively locked in by their choices. -OTel is an open-source framework. Using OTel, you can add software to your applications or systems to generate telemetry data. This process is known as **instrumentation**. OTel collects, manages, and exports telemetry data to an observability backend and the database for storage. +[00:06:00] Now, imagine if all vendors accepted the same standard to send or receive telemetry data, similar to many phone companies accepting USB-C as a standard. When you switch vendors, you don't need to learn proprietary instrumentations, agents, or collectors each time. You just need one technology and learn a single set of APIs and conventions associated with it. Whatever data you generate with this technology is yours. You could send the data to any observability backend that accepts the standard, so you could easily switch vendors with substantially reduced cost, time, and effort. -Different aspects of OTel make this process possible, and we'll discuss these in more detail as we go through the JavaScript journey. For now, remember that OTel focuses on the **generation, collection, management,** and **export** of telemetry data. You can easily instrument your applications or systems, regardless of their language, infrastructure, or runtime environment, with the storage and visualization of telemetry intentionally left to other tools. +[00:07:00] This is exactly what OTel does for you. It creates an open standard, a set of guidelines, rules, or specifications to send and receive data. There's quite an incentive for the vendors to accept OTel. Many customers now prefer OTel to avoid vendor lock-in. They want to learn a single set of APIs and conventions rather than having to learn a new one every time they change a vendor. They also want to own their data and send the data to any observability backend that accepts these standards. -## Why Should We Use OTel? +Now, the vendors benefit from accepting OTel as well. Customers may already be familiar with using OTel. There's a vibrant OTel community that serves as a great resource because of that. Accepting OTel helps the vendors reduce their support and implementation costs. On top of that, there is even more drive for innovation. Vendors are receiving the same data, so they have to innovate to stand out from the competitors. -Now, why should we consider using OTel? Before we answer this question, let's talk about something many of us have experienced: the issue of **vendor lock-in**. +As you can see, there are many benefits to accepting open standards, and you can take advantage of these benefits by using OTel. Now that we covered what OTel is and why you should use it, let's go over the resources to get started. The links to all the resources are included in the description of the video. -If we dig through our drawers at home, we could probably find a bunch of cables of different types. Why? Because the devices we own come from different vendors, and each vendor has a specific type of cable and port to charge or connect your gadget. +[00:08:00] The best place to get started is the OTel documentation. The documentation is continuously being improved, so the page may look different by the time you watch this video. So use a link in the description to check out the latest page. The first place in the doc you should start with is the Language APIs and SDKs. As I mentioned earlier, your OTel journey will differ depending on the programming language you're working with. -We're all familiar with buying multiple products from the same vendor and investing in additional accessories and apps specifically designed for that product. Before we know it, we become so accustomed to using a line of products that switching to another vendor can become quite difficult. +Select the language of your choice and you should end up on a page that lists all the resources to get started. Next, we have the official OTel YouTube channel. You'll find helpful videos along with the OTel for Beginners series on this channel. Last but not least, as you start your OTel journey, you'll come across a lot of questions. We have a huge community of OTel users on Slack. -A product from another vendor may operate differently, requiring time to learn and adapt. It would also cost us more money because the additional accessories or apps we've invested in don't work with a product from a new vendor. This challenge is known as **vendor lock-in**, where the costs of switching vendors are so high that customers feel stuck with what they have. - -### The Shift Towards Universal Standards - -However, things are slowly changing with **USB-C ports** becoming the universal standard. For example, with the newest model of phones—regardless of whether you're using an iPhone or an Android—you can charge or connect many of these phones with a USB-C cable because a universal standard has been established. This allows users to use the same cable, no matter how many times they charge their phones. - -OTel has similar implications in observability as USB-C does for phones. - -### A Quick Review - -Let’s review briefly. We have our infrastructure or applications that we want to observe. We want to collect telemetry data and send it to an observability backend for storage, enabling the data to be queried and visualized with an observability frontend. - -Imagine you have three observability backends to choose from: vendors A, B, and C. Each vendor often has proprietary instrumentation, agents, and/or collectors. - -Let’s say you choose vendor A and start using their proprietary instrumentation, agents, or collectors to send data to their backend. But what if your needs change down the road, and you want to switch your backend to vendor B? Switching vendors isn't as simple as it seems. You can't just send the existing data from vendor A to B because vendor B requires its own instrumentation, agents, or collectors that cannot accept data from vendor A's proprietary instrumentation. - -Now your development team has to change their instrumentation, possibly to a new proprietary solution from vendor B. Imagine doing this for thousands of Linux machines or dozens of applications. Is the cost in money, time, and effort worth the benefits of changing vendors? - -As you can see, the cost of switching vendors can become so high that customers can be effectively locked in by their choices. - -### The Solution: OpenTelemetry - -Now, imagine if all vendors accepted the same standard for sending or receiving telemetry data, similar to how many phone companies accept USB-C as a standard. When you switch vendors, you wouldn't need to learn proprietary instrumentation, agents, or collectors each time. You just need one technology and a single set of APIs and conventions associated with it. - -Whatever data you generate with this technology is yours. You could send the data to any observability backend that accepts the standard, significantly reducing the cost, time, and effort involved in switching vendors. This is exactly what OTel offers. - -OTel creates an **open standard**, a set of guidelines, rules, or specifications for sending and receiving data. There’s a strong incentive for vendors to accept OTel. Many customers now prefer OTel to avoid vendor lock-in, wanting to learn a single set of APIs and conventions instead of a new one every time they change vendors. They also want to own their data and send it to any observability backend that accepts these standards. - -Vendors benefit from accepting OTel as well. Since customers may already be familiar with using OTel, there's a vibrant OTel community that serves as a great resource. Accepting OTel helps vendors reduce their support and implementation costs. Moreover, it drives innovation because vendors receive the same data, compelling them to innovate to stand out from competitors. - -## Getting Started with OTel - -Now that we've covered what OTel is and why you should use it, let's go over the resources to get started. The links to all the resources are included in the description of the video. - -1. **OTel Documentation**: The best place to start is the OTel documentation, which is continuously being improved. The page may look different by the time you watch this video, so use the link in the description to check out the latest version. Start with the **Language APIs and SDKs** section. Your OTel journey will differ depending on the programming language you're using, so select your language of choice to find all the necessary resources. - -2. **Official OTel YouTube Channel**: You'll find helpful videos, including the OTel for Beginners series on this channel. - -3. **Community Support**: As you embark on your OTel journey, you may have many questions. We have a huge community of OTel users on Slack. Join the **CNCF Slack Channel**, post your questions in the **OpenTelemetry channel**, and connect with other community members. - -Again, check the description of this video to access all these resources. - -In the next episode, we'll talk about how to get started with the JavaScript journey, so stay tuned for that. Thank you for watching, and I'll see you in the next episode! +So join the CNCF Slack Channel, post your questions on the OpenTelemetry channel, and connect with other community members. Again, check the description of this video to access all these resources. In the next episode, we'll talk about how to get started with a JavaScript Journey, so stay tuned for that. Thank you for watching, and I'll see you in the next episode. ## Raw YouTube Transcript diff --git a/video-transcripts/transcripts/2025-03-21T05:53:37Z-otel-me-with-jerome-johnson-cal-loomis.md b/video-transcripts/transcripts/2025-03-21T05:53:37Z-otel-me-with-jerome-johnson-cal-loomis.md index 44a9779..117a0c2 100644 --- a/video-transcripts/transcripts/2025-03-21T05:53:37Z-otel-me-with-jerome-johnson-cal-loomis.md +++ b/video-transcripts/transcripts/2025-03-21T05:53:37Z-otel-me-with-jerome-johnson-cal-loomis.md @@ -10,130 +10,134 @@ URL: https://www.youtube.com/watch?v=DrD35XxTDsY ## Summary -In this episode of "Hotel Me," hosts Ree and Adriana welcome guests Cal and Jerome from Relativity, who discuss their journey with open telemetry and their complex tech stack. The conversation highlights Relativity's early adoption of open telemetry and how it has significantly improved their observability practices. They delve into their architecture, which includes a network of over 200 to 500 collectors managed via Kubernetes and Helm charts, and share insights into the challenges they faced with legacy systems. The guests emphasize the importance of standardizing telemetry data and the cultural shift within their organization towards embracing observability. They also address audience questions regarding data consumption, collector management, and the benefits of auto instrumentation. Overall, the discussion showcases Relativity's commitment to leveraging open telemetry for enhanced operational insights while providing constructive feedback for further improvements in the open telemetry project. +In this episode of "Hotel Me," hosts Ree and Adriana, broadcasting from Vancouver and Toronto respectively, engage in a discussion about observability and OpenTelemetry with guests Cal and Jerome from Relativity. The conversation dives into Relativity's journey as one of the early adopters of OpenTelemetry, exploring their tech stack, architecture, and the complexities of managing a vast number of services and telemetry data. They share insights on how they transitioned from a bespoke telemetry system to OpenTelemetry, the challenges faced, and the cultural shifts within their organization towards a more observability-driven approach. The guests discuss their experience with deploying custom OpenTelemetry collectors, their strategies for data management, and the importance of standardizing telemetry data for downstream consumers like AI divisions and management reporting. They also provide feedback for the OpenTelemetry project, emphasizing the need for improved documentation and better integration of semantic conventions. The episode concludes with audience questions regarding data consumption and auto instrumentation, highlighting the ongoing evolution in telemetry practices. ## Chapters -Here are the key moments from the livestream along with their timestamps: +00:00:00 Welcome and intro +00:01:50 Guest introductions +00:04:10 Relativity overview +00:06:00 Tech stack insights +00:09:50 OpenTelemetry adoption discussion +00:12:30 Collector setup explanation +00:15:00 Managing collector fleet +00:20:00 Observability team structure +00:21:50 Implementation evolution +00:30:00 Data consumption and management -00:00:00 Introductions and welcome to the stream -00:02:20 Introduction of guests from Relativity -00:05:30 Overview of Relativity's tech stack and use of OpenTelemetry -00:12:00 Discussion on the decision to adopt OpenTelemetry -00:18:00 Insights into the complexity of the observability platform -00:25:00 Explanation of the collector setup and management -00:32:00 Discussion on maintaining the collector pipeline and configurations -00:39:00 Evolution of the OpenTelemetry implementation over the years -00:45:00 Discussion on data volume and budget management for telemetry -00:52:00 Sharing feedback on the OpenTelemetry project and its documentation +**Ree:** Hello. Welcome everyone to a new episode of Hotel Me, formerly known as the end user Q&A. I'm very excited to be here. If you have been here before, I am Ree and I have Adriana here with me. I am coming to you live from Vancouver, Washington, not BC. Adriana, where are you coming from? -Feel free to reach out if you need further assistance! +**Adriana:** I am coming to you live from Toronto, Canada. -# Hotel Me Episode Transcript +**Ree:** So, we're at opposite ends of the coast, which is pretty— -**Hello and welcome everyone to a new episode of Hotel Me**, formerly known as the End User Q&A. I'm very excited to be here. If you've been here before, I am Ree, and I have Adriana here with me. I am coming to you live from Vancouver, Washington, not BC. Adriana, where are you coming from? +**Adriana:** We are. Yeah. -**Adriana:** I am coming to you live from Toronto, Canada. So, we're at opposite ends of the coast, which is pretty cool! +[00:01:50] **Ree:** And so, for those of you who are familiar with the flow, hang tight. We are going to bring our guests on pretty soon. For those of you who are new, welcome. So, how this is going to go is we are going to bring on our two special guests from an end user organization shortly. We're going to have a quick introduction. They'll introduce us to their tech stack and how long they've been using open telemetry. This one's gonna be a really interesting one because they were actually one of the first guests we talked to when we started this segment back in 2022. So, I'm pretty excited to follow up with them and see how far they've come in their journey, what kind of struggles they've had along the way, and what they've figured out. And then we'll get into some time for them to give us some feedback about the project. We should also have some time at the end for any audience members to ask questions. If you do have questions that come up as we go, feel free to pop them in the chat, whether you're on LinkedIn or YouTube, and we will get to them as appropriate during the conversation. If not, we will wait until the end. And with that, I believe we can bring our guests on. Oh, and Adriana, do you want to talk about our resources just a little bit? -**Ree:** Yes, it is! For those of you who are familiar with the flow, hang tight. We will bring our guests on pretty soon. For those of you who are new, welcome! Here's how this will go: we will bring on our two special guests from an end-user organization shortly. We’ll have a quick introduction, and they’ll introduce us to their tech stack and how long they’ve been using OpenTelemetry. This segment is particularly interesting because they were actually one of the first guests we spoke with when we started this segment back in 2022. I'm excited to follow up with them and see how far they've come in their journey, what struggles they've faced along the way, and what they've figured out. +**Adriana:** Oh, yeah. So, our resources, I believe this will—I'm going to scan the QR code myself because I don't remember what this does. This will take us to—this takes us to open telemetry.io. We have a page in the hotel site all about our end user SIG. If you want to learn more about the end user SIG, we welcome you to join. I believe there's a link on there also for joining our Slack channel on CNCF Slack. So, if you're already on CNCF Slack, please come find us. We welcome everyone. People come along to ask questions. We might not have all the answers, but we can definitely direct you to where you need to be. Oh, and also, we would love to know where you are listening from. So feel free to put that in the chat as well so we can—we just like to know where people are listening from. -Then, we'll have some time for them to give us feedback about the project, and we should also have time at the end for any audience members to ask questions. If you have questions that come up as we go, feel free to pop them in the chat, whether you’re on LinkedIn or YouTube, and we will address them as appropriate during the conversation. If not, we will wait until the end. +**Ree:** And so the guests we have for you today are Jerome Johnson and Cal—columnist from Relativity. -With that, I believe we can bring our guests on. Oh, Adriana, do you want to talk about our resources just a little bit? +**Jerome:** Hello. -**Adriana:** Sure! Our resources will take us to [OpenTelemetry.io](https://opentelemetry.io). We have a page on the Hotel site dedicated to our End User SIG. If you want to learn more about the End User SIG, we welcome you to join. There’s a link on there for joining our Slack channel on CNCF Slack. If you’re already on CNCF Slack, please come find us. We welcome everyone. People come along to ask questions. We might not have all the answers, but we can definitely direct you to where you need to be. +**Cal:** Hi everyone. Howdy howdy. -We would also love to know where you are listening from, so feel free to put that in the chat as well. Now, the guests we have for you today are Jerome Johnson and Calumnus from Relativity. +**Ree:** We would love to learn about you. Can you tell us a little bit about your role in the company and kind of what Relativity is focused on? -**Jerome:** Hello! +**Cal:** You can go ahead, Jerome. Alphabetical order and whatnot. -**Calumnus:** Hi everyone! +[00:04:10] **Jerome:** Oh, okay. All right. Excellent. Very democratic. Yeah. So, a little bit about Relativity. Relativity is a company that has predominantly a SaaS product that provides services to the legal community. So, we automate essentially all the processes around legal e-discovery, so the exchange of documents and all of that. It is a SaaS platform. We have a lot of companies around the world that actually use it, and we as observability are really tuned toward how our SaaS project is actually working at any particular point in time. As far as me, I started with Relativity about four—almost five years ago now. I was brought in basically to upgrade the way we do telemetry, and as Ree mentioned, we were one of the probably the first ones to have a chat with you folks but also one of the first adopters of open telemetry in production. So we were very, very early adopters of open telemetry. I'm an architect here, and then I'll let Jerome introduce himself. -**Ree:** We would love to learn about you. Can you tell us a little bit about your role in the company and what Relativity is focused on? +**Jerome:** Hello. As Cal said, we're on the observability team. My name is Jerome. I've been at the company for about 10 years or so now, but I've only been on the observability team, specifically the one that's dealing with open telemetry, for a little over a year now. So hopefully I can bring some perspective on how easy it is to use from an internal side and how nice it is—so looking forward to chatting it up with you guys. -**Calumnus:** Sure! A little bit about Relativity: we predominantly have a SaaS product that provides services to the legal community. We automate essentially all the processes around legal e-discovery, such as the exchange of documents. It’s a SaaS platform used by many companies around the world. Our observability is tuned towards how our SaaS product is actually working at any given time. +[00:06:00] **Ree:** Cool. That's so exciting. Well, I think this is a great opportunity for us to start digging in. Can either of you or both of you give us some insights into your tech stack and architecture? -As for me, I started with Relativity about four, almost five years ago now. I was brought in to upgrade our telemetry. As Ree mentioned, we were one of the first to adopt OpenTelemetry in production. I’m an architect here. Jerome, would you like to introduce yourself? +**Cal:** Yeah, so I could talk about that a bit. Like I said, we were an early adopter of open telemetry. What we had before that was an entirely bespoke, I would say, system for collecting telemetry from a lot of different sources. In addition to that, we did use New Relic at the time, but we also had a lot of other vendors in there too. So, it was kind of a nightmare from an operational standpoint to actually do any troubleshooting and debugging because you had to go to lots of different sources to sort of put all that telemetry together in your head. At the time when I was hired, the company made sort of a decision to try to unify everything. We looked around to see what would be the best platform for doing that. The one that sort of checked all the boxes, even though it was really immature, was open telemetry. So that was the one which we pointed to to do that. Now internally we have a very complicated platform. Maybe this is time to bring up the diagram on what we actually do now with open telemetry. We have 1500 plus different services running on our platform, and that is a huge mix of things. We are predominantly a Windows and .NET shop, so most of that is actually Windows and .NET. But we have a mix of basically every other language in the world. So we have a very, very complicated tech stack and we need to pull in the telemetry from all those different things. Most of the complexity in our observability platform—and I'll probably mention at various points—if I say RELI, that is our observability platform that's built over open telemetry. Most of the complexity is just being able to pull those signals in from lots of different sources, and you see most of them here. We really prefer that people use the open telemetry SDKs now, and we push that as hard as we can. But we still have people using our old platform, which is relatively APM, which we translate into open telemetry now. We have things like SNMP. We pull logging from lots and lots of different places, and that's probably the part which is most difficult and most heterogeneous in terms of the platform itself. We have a whole fleet of open telemetry collectors that we run in 20 different regions around the world. Each one of those is load balanced. So we're running anywhere between, depending on the time of day and all that, between 200 and probably 800 individual collectors around the world to pull all this telemetry in. Then we send that to various data stores for operations. This is really New Relic at this point. So that's where most of our engineers go to actually see what the telemetry is doing. We have a subset of people who use LaunchDarkly and use that for future releases. So we send some set of telemetry there. We actually archive all of our telemetry for compliance reasons for a year and for forensic analysis and stuff like that. We also have a reporting platform that uses the Azure blob storage basically as its backend to make that reporting for managers and things like that. This makes it look simpler than it is. It's a lot more gnarly than this diagram lets on. But we do have a large scale—I mean, 20 plus regions around the world, 1500 services—all feeding into this is a fairly complicated platform. -**Jerome:** Sure! As Cal mentioned, we’re on the observability team. I’ve been with the company for about 10 years now, but I’ve only been on the observability team, specifically dealing with OpenTelemetry, for a little over a year. I’m looking forward to chatting with you all! +[00:09:50] **Ree:** Dang, that was really cool actually. Questions. So many questions. I do actually have a lot of follow-up questions, but I also want to get into why open telemetry. How, like when exactly—I know you mentioned, you know, kind of back in 2022, your team was early adopters. How did you come upon open telemetry and like why did you decide, like, "Oh, this makes sense for us?" -**Ree:** That’s exciting! I think this is a great opportunity for us to start digging in. Can either of you, or both of you, give us some insights into your tech stack and architecture? +[00:12:30] **Cal:** Yeah. So, you know, I think the thing which sort of ticked all the boxes for us—we came from a place where we were having to maintain our own observability libraries, and given the fact that we were .NET and then trying to diversify into lots of different languages meant that we were having to translate that thing again and again and again. On top of that, we also had—well, we were using New Relic at the time and we had a lot of the New Relic agents in the mix as well. So we had this weird mix of our own libraries for some stuff. We had a mix of things from New Relic with the agents and all of that kind of stuff. We had a lot of people who were writing to New Relic directly from their code as well as to other sources. One of the things which was really complicated on our side was every time we wanted to move something or we wanted to change something, it meant code changes. That's something we wanted to avoid going forward. So we wanted to standardize on something which was ideally open source that we could include in our services one time and maintain that and then change where we put things but didn't have to change the code every time. So we were really looking for, you know, something that would give us stability in terms of the codebase but allow us the flexibility to send things to other places as we needed to do that. Open telemetry was the only one which sort of ticked those boxes for us at the time. The only worry we really had at that point was that it was a new project. There was a lot of risk in going that direction. We decided in the end—it wasn't an easy decision. We actually discussed this a long time internally, but we decided that it was worth the risk to go ahead and adopt that because it looked like it was going to tick the boxes for what we wanted to do there. Basically, it's to isolate basically our codebase from what we do with the telemetry afterwards, which was sort of the selling point for what we wanted to do there. -**Calumnus:** Certainly! As I mentioned, we were an early adopter of OpenTelemetry. Before that, we had a completely bespoke system for collecting telemetry from various sources. We were using New Relic at the time, but we also had a lot of other vendors, which made troubleshooting and debugging quite complicated. +**Jerome:** Oh yeah, just to add on to that, I can give somewhat of a perspective to what it was like before we had open telemetry. I was serving on the performance team for a good number of years, and it was very challenging to do analysis of specific applications just because, like Cal had mentioned, it was kind of the wild west on how we handled metrics. Performance metrics and monitoring are somewhat look similar, a little different, but enough to where every time we had to analyze a specific application, we would have to do a deep dive on that application. Like, "What telemetry are you guys putting out?" "Oh, you're not putting out any at all." Then we'd have to dig into the code, and that would really muck up how much time we could actually drill down into what the core problem of said application from a performance perspective was. Usually, it would take half the time we would do an analysis. It would be like trying to understand what the product was and then actually coming up with a solution would be the latter 50 or 40%. So since joining observability, even though Cal showed a simplified version of our RELI stack—and like you said, it's a bit more complicated once you drill down into it—I was amazed at just how much cleaner it is to understand what our Relativity is doing as a whole. Take that with what you want. -When I was hired, the company decided to unify everything. We looked for the best platform for that, and even though OpenTelemetry was immature at the time, it checked all the boxes for us. Internally, we have a very complicated platform with over 1,500 different services running, predominantly on Windows and .NET, but also incorporating many other languages. +**Ree:** That's very cool. Now as a follow-up question, one of the things that we are always curious about in the SIG is collector setup. What kind of—you mentioned, I believe, Cal mentioned that you guys manage a fleet of collectors. How do you manage that fleet of collectors? Can you give us an idea of how many collectors? Is there a particular setup that you use? We would love to know. -We prefer that people use the OpenTelemetry SDKs, but we still have some people using our old platform. We run a fleet of OpenTelemetry collectors in over 20 different regions worldwide, with load balancing. Depending on the time of day, we run between 200 to 800 individual collectors globally to pull all this telemetry in. We send that telemetry to various data stores for operations, primarily New Relic, which is where most of our engineers go to see what the telemetry is doing. +[00:15:00] **Cal:** Yeah. So, you know, I think in terms of how things are usually deployed, I think you call it a gateway setup. All of our collectors are basically in a gateway setup. People send their telemetry to us, we process it, and send it on. Over the years, we've had more or less complicated pipelines, but in general, we have essentially a single set of collectors with a common configuration that handles all of that telemetry. In terms of the numbers of things, we are running over 20 plus regions. Each region has somewhere between, depending on the load, between 10 and probably 50 to 100 instances themselves. So we're running at any point in time somewhere between 200 and 500 individual collectors. These are all running in Kubernetes. We are very focused on compute and Kubernetes. So that's where almost all these things are running. In terms of the deployment itself, we actually handle all of our deployments via Helm charts. We use Helm charts basically to deploy these all out to the Kubernetes clusters. We do take pains to make sure that essentially all of our configurations are isolated. Our collectors, except in a very few cases, do not reach out to external services. We manage the configurations all locally to make sure that they are as available and robust as possible. So that's kind of how we do all of that. In terms of the pipelines themselves, one of the other reasons why we went with open telemetry is we have some internal attribute enrichments that we do specifically to our product and the way we use things. We actually implement those inside of the collector itself. So we have some custom processors which enhance the data in various ways. We actually validate it. We actually drop data which is not—contains things which shouldn't be there, for instance. So we do a lot of validation on the fly as things go through the collectors themselves before we write it out to our various data stores. I think I think that hit most of the points you're asking for. Did I miss any of the things you're asking? -We also archive all our telemetry for compliance reasons and have a reporting platform that uses Azure Blob Storage as its backend. Overall, we have a large-scale system supporting 1,500 services across the globe. +**Ree:** I think you got most of them. And yeah, actually, one follow-up question I had was, since you mentioned you deploy your collectors in Kubernetes via Helm charts, have you used the open telemetry operator to manage any of your collectors? Is that something you played with? -**Ree:** That sounds really cool! I have so many questions. I’d like to learn more about why you chose OpenTelemetry. When did you first come upon it, and why did you decide it made sense for you? +**Cal:** We have not. The open telemetry operator sort of came out after we were already doing stuff. In fact, we don't—but not for any particularly good reason. No. -**Calumnus:** The main reason for us was the need to maintain our own observability libraries while trying to diversify into various languages, which required constant translations. We were also using New Relic, so we had a mixed setup that complicated things further. +**Ree:** Okay. Fair enough. -We were looking for something open source that we could include in our services once and maintain without changing the code every time we needed to move something. OpenTelemetry was the only option that ticked those boxes. There was a lot of risk in going in that direction, but we decided it was worth it. +**Cal:** So, it's that we had—you can consider it sort of tech debt. You know, we had the Helm chart and all of that beforehand. So we've just continued doing that, and we're kind of used to managing things that way. We actually will have an internal shift in our deployment tooling coming up for the organization. So that might be an opportunity to switch over how we're actually managing things. But at the moment, yeah, we don't have any experience with the open telemetry operator at the moment. -The primary selling point was to isolate our code base from telemetry processing, allowing us to adapt without constant code changes. +**Ree:** Got it. Got it. And I have another similar question. Since you do manage a fleet of collectors, have you played around with the opamp protocol for doing that? -**Jerome:** To add to that, before OpenTelemetry, we had a very challenging time analyzing specific applications. We had to deep dive into each application to understand what telemetry they were emitting, which was often very inconsistent. Since adopting OpenTelemetry, it’s been much cleaner and easier to understand what Relativity is doing as a whole. +**Cal:** We've not. Again, we’ve tried very hard to make our fleet essentially standalone. So even for configurations and things like that, we don't actually reach out to anything else. We really deployed sort of static config maps for that. That was basically a choice that we made just to—because we have such a large deployment and because it's distributed so widely, we wanted to make sure we were as resilient against networking problems as possible. So even for things like configuration changes, we push out new configs for those things rather than doing something that— -**Ree:** That’s fantastic! Now, regarding your collector setup, how do you manage your fleet of collectors? +**Ree:** Got it. Got it. Do you have a—it's not to—not to say that's a bad idea. It's just it's a choice. It's a choice that we made. -**Calumnus:** We have a gateway setup for our collectors. They receive telemetry from users, process it, and send it on. We have a common configuration for our collectors, which handles all the telemetry. +**Cal:** Of course. Of course. That makes sense. What about maintaining the collector pipeline? Is there a dedicated team? Is that just whoever is on the observability team, or how do you maintain the local configurations and the pipelines? -We run over 20 regions, each with 10 to 100 instances, so we usually have between 200 to 500 individual collectors running in Kubernetes. We deploy everything via Helm charts and make sure all configurations are isolated. The collectors typically don’t reach out to external services. +[00:20:00] **Jerome:** Yeah, so we do have an observability team. So we are split between Poland and here. Our other big engineering office is actually in Krakow, Poland. So we have two observability teams split between the two regions. Each one has I think around five people at the moment. But collectively we are responsible for maintaining those pipelines and that configuration. Almost, I would say almost entirely, we are responsible for that. There are a few areas where we share responsibility with certain places where we pull in configurations. One of them is our Kubernetes team. They actually define part of our configuration for which metrics we actually pull into the global system rather than just keeping it local to their Prometheus instances. But by and large, we are responsible for that entire configuration. We try to avoid shared responsibility there just because we want to make sure that, you know, shared responsibility always means no one's responsible. So we try to really avoid that. -We also implement custom processors to enhance the data, validate it, and drop any invalid data on the fly before writing it to our data stores. +**Ree:** So, as you know, the project has matured, and it sounds like you've got a pretty good setup that works well for you guys right now. How has the implementation evolved over the years, you know, as things have become GA? Have you added additional instrumentation signals? How has that looked as you've kind of grown alongside the project? -**Ree:** That’s an impressive setup! Have you used the OpenTelemetry operator to manage any of your collectors? +[00:21:50] **Cal:** Yeah. So, yeah, it's actually an interesting question in the sense that the thing that surprised me overall is that the deployment that we have hasn't really changed in terms of its architecture since the beginning. It's been the same architecture the entire time. Now, we've made tweaks here and there, and we've simplified things, you know, as new features have come out and that kind of stuff, but by and large, it's stayed the same. I think that is amazing given the fact that it was really sort of pre-alpha when we started. So we've gone through, you know, we sort of gone through the alpha, beta, you know, and things are starting to become stable now, and we've not really noticed any really heavy changes that we've needed to make in the platform. Now, that being said, I mean, we have added new signals. So at the beginning, of course, it was metrics and traces that were there at the beginning. Logs came in afterwards. We had logs coming into our system even earlier than that, so we had some of our own receivers to do some of that before it was an official signal. We've sort of folded those changes into the platform as they come along, and it hasn't really been terribly painful in doing that. As far as the community goes, the alpha, beta, stable kind of thing, we've not seen really any negative consequences from that. We've had to update on occasion, but by and large, those updates are pretty easy. Jerome has been doing a lot of the following for the collectors. We have our own build of the collector, which of course we try to keep in sync with the external one, and Jerome has been doing quite a lot of that, so you can probably give some feedback on how difficult that process is and where we've run into issues. -**Calumnus:** We have not, mainly because the operator came out after we had already set things up. We have our Helm chart and are used to managing things that way. However, we are considering an internal shift in our deployment tooling, which might be an opportunity to switch over how we manage things. +**Jerome:** Oh yeah, I'm sure. I mean, for the most part, I don't know if this speaks to open telemetry or the automation we put around it, but as far as just updating the collectors, it's been a relatively painless process. I mean, sure, there have been a few changes that we've had to team up and just file down on as a team and like, "Hey, what path do we want to go down?" But as far as just like pushing out those changes, they've been pretty simple, especially with how elaborate and detailed the white papers or the change logs have been out of open telemetry, just explicitly telling, "Hey, these libraries or modules you've been using—here's an introduction to breaking changes or an API that's going to be deprecated." But overall, it's been quite an easy experience, and getting more people on our team to actually utilize that process has been fairly easy as well. -**Jerome:** To add to that, we have focused on keeping our fleet standalone to avoid networking issues, so we push out new configurations rather than relying on external management. +**Ree:** I have a follow-up actually on building your collectors because I think out of all the people we've talked to, I think you guys might be the ones that have built your own collector, which makes me very excited. In terms of building your own—did you have—what was your experience around that, like with as far as the documentation provided on the open telemetry site? Because that's another thing that we want to hear from practitioners of OTEL is how usable is the documentation, and have you noticed an improvement in the documentation? Even building your own collector, how useful was that documentation? Did you have to do a little extra Sherlock Holmesing to figure stuff out? Also, were you able to build a nice streamlined process for building the collector that—did you have to go outside of the confines of what was already documented to do that? -**Ree:** That makes sense. Who maintains the collector pipeline? Is there a dedicated team? +**Cal:** Yeah. So, I'm not sure I'm going to be able to provide necessarily documentation feedback on the documentation, but the process as a whole—I mean, one of the things we started with was we had to start with our own build of the open telemetry collector mainly because we do have custom receivers, we have custom processors, we have custom exporters as well, and we kind of needed that at the beginning to cover some of our use cases. So we were forced basically at the beginning to build our own collector, right? At the beginning, it was very painful. It was basically us cloning the entire repository and then making the changes we needed to build it. Once the collector builder came along, that made that whole process so much easier, and it was a great addition to that process. Right now, basically we have moved to a situation where we have our own repository, and basically we have the single config file for the collector builder that just tells us what we pull in from contrib and from the main open telemetry collector, and the only thing we have in our repo at this point is our own custom stuff. Once the builder was there, everything since then has been really, really easy and smooth to build our own stuff. We did that right at the beginning, so before there was documentation, so it's worked since then. I can't say whether the current documentation is good there or not, but we use the builder and it works seamlessly for us. -**Calumnus:** Yes, we have an observability team split between our offices in Poland and here. Each team has about five people who are responsible for maintaining our pipelines and configurations, avoiding shared responsibility to ensure accountability. +**Jerome:** And just as a follow-up because you mentioned you built your own components. How was that experience of building your own components? Was that tricky? What was the—because that’s definitely something that I would say personally feels a little bit lacking in the open telemetry documentation for building your own receivers, processors, exporters, that kind of thing. How was that for you? -**Ree:** As the project has matured, how has your implementation evolved? Have you added additional instrumentation signals? +**Cal:** Yeah. So I think in terms of the team itself, I think the biggest hurdle there was learning Go. We weren’t—so I think from the team standpoint that was sort of the biggest initial obstacle for that. All of us, I think, have some good experience with Go now, so I don't think that's an obstacle anymore. In terms of how to do it, we again were doing this very early. So it came at a time when it was basically, "Well, let's take one which we know works, copy it over, and then make the changes you need to make our own stuff." Since then, the documentation has gotten much better, and it's much easier to sort of start with those things and get them done. But you know, I must say the process of just copying something that works and moving it over and making the changes you need also worked pretty easily as well. For an experienced programmer, I don't think it's a huge barrier to do those things. Now, there are some things which I would love to have better documentation on, like in terms of internal telemetry—how to hook in better to the internal telemetry and that kind of stuff. We sort of worked it out, you know, by reverse engineering what was there, but some of the things like that would be much more helpful if they were better documented, for instance. -**Calumnus:** Interestingly, the overall architecture of our deployment hasn’t changed much since we started. We’ve made tweaks and simplified things as new features have come out, but the core architecture has remained stable. +**Ree:** Thanks. Oh, I was just going to say, yes, I remember that when we were catching up a bit yesterday, and we definitely want to get into more of the feedback that you have for the project. I know you talked a little bit about the issue with conventions for internal data schemas. So if you want to go into a little bit more about that and anything else that you would like to see improvements in or feature requests, things like that. -We initially started with metrics and traces, then added logs. We’ve folded in changes as they come along, and updates have been relatively easy to implement. Jerome has been handling a lot of the collector updates, and it’s been a smooth process. +[00:30:00] **Cal:** Yeah. So, first let me preface this by saying I think probably from what I've said already, I am 100% in with open telemetry, and I think it's a great platform. What the maintainers are doing is fantastic, both on the documentation level and the code level, right? So that's all great. This isn't feedback necessarily in terms of the actual collector implementations or open telemetry project as a whole, but I think it's sort of feedback on I think where we're going in our own journey, and probably that has some bearing on other people as well. As we're generating more and more telemetry and it's becoming more and more useful for people, we have a lot of downstream consumers now. So we have an entire AI division who's now looking at our telemetry. We have people managers who are trying to do reporting off of this, and getting standard data is becoming more and more important. One of the reasons we had to have a custom processor is we do have standards for the attributes which come into the system. So we enforce a sort of a global schema on everyone, even though that's extensible. Now how that fits in with the semantic conventions, how that fits in with the Elastic Common Schema, in general, how this fits in with sort of data contracts and how you have contracts between the producers of the data and the consumers of it. All those things are things we're thinking very hard about right now, and we don't have good ideas on how to apply that across the board and good ways and the level at which we need to do that. I think discussions about the semantic conventions and the ECS and how to do things like data contracts with open telemetry data I think is going to become more and more important. I see that in our own organization, and it would be good to come up with sort of common themes there and common tooling and common ideas. A little more convergence, I think, within the community there I think would go a long way. In terms of other feedback, the tech stack, I think in general, is really, really solid, and the way things are managed I think is great. So I don't have a whole lot of feedback there. -**Jerome:** Yes, updating the collectors has been relatively painless. The detailed change logs from OpenTelemetry have been very helpful in navigating updates and breaking changes. +**Ree:** Yeah, so I think it's mainly looking forward and seeing how we can sort of use the technological base that we have and come up with better ways of standardizing the data which is produced, making it more useful for downstream consumers, I think is the main thing I would like to see discussed more. -**Ree:** That’s great to hear! Can you tell us about your experience building your own collector and components? +**Adriana:** Right on. Thank you, and I think we have a bit of time to catch up on some audience questions. So let's see, one of them is from Doug. Doug on YouTube wants to know, with regards to collectors, their dream is that cloud providers will offer as a boring text similar to an SMTP or SFTP. Is Doug crazy to want that? -**Calumnus:** We had to build our own collector because we needed custom receivers, processors, and exporters. Initially, it was quite painful, but once the collector builder came along, the process became much easier. Now we have our own repository with a single config file that tells us what we need and pulls in from the OpenTelemetry collector and contrib. +**Cal:** We—the first question we ask any vendor now is do they support OTLP? So, you know, we treat that as our standard protocol, and it's a negative check mark against someone if they're not supporting it already. So we're pushing all of our vendors and various tech products and whatever to support that as much as possible. I don't think that that's a pipe dream. I think pushing people in that direction is a good thing. As I said, we are very heavily a Microsoft shop, so we use Azure very much, and the feedback we've consistently given them is we want Azure Monitor and those types of things to produce open telemetry signals because we don't want to have to do that translation ourselves. -**Jerome:** In terms of building our own components, the biggest hurdle was learning Go, but now it’s not an issue. The process of copying existing components and modifying them worked well for us, and the documentation has improved over time. +**Adriana:** Thank you. And Manas, I see you have a question about OPM tutorials. We will respond directly with some resources for you in the LinkedIn chat. In the meantime, there was another question. Did you ever feel that there were classes of telemetry data that open telemetry didn't collect well and you needed to add another collection method, such as eBPF? -**Ree:** That’s insightful! Regarding data volume, how much data do you consume, and have you found the need to reduce it via methods like tail sampling? +**Cal:** We've had to come up with some custom receivers and things like that that weren't covered beforehand. One of the early things we did was there was no way to sort of provide a web hook into open telemetry at the beginning. There is now. So that's one of our legacy receivers that we've had, and that was something that was not difficult to write but something that was sort of lacking. In terms of other classes of telemetry, I would say that one of the biggest gaps we've had internally as an organization is front-end telemetry. That's something that we're actually closing now. We actually have a project currently going on basically to collect more of our front-end telemetry with the open telemetry SDK—the JavaScript one. So that was something that we internally found difficult to do for many reasons, and we're now sort of closing that gap. In terms of other types of telemetry that's been difficult, I don't think we've had any real issues. Most of the things that we've had to collect, if it's not OTLP or things like Splunk, Hack, or Fluent Bit and things like that, we've been able to get things into our collectors fairly easily. -**Calumnus:** We typically write between 10 to 15 terabytes a day. We haven’t had scaling issues with our collectors, but we do manage the data volume we push to New Relic for budget reasons. We sample down traces and manage log levels dynamically to control what we send. +**Adriana:** Perfect. Thank you. And I think our time is just about up. Were there any last—well, not last parting words? -**Ree:** It sounds like your organization is very supportive of observability and OpenTelemetry adoption. Can you talk about the culture of observability within your organization? +**Cal:** No, just thanks. Thanks for inviting us, I think is the last thing from our side. If anyone wants to reach out or whatever, I am in the Slack, so feel free to reach out to me directly if you have questions or whatever. Happy to discuss what we've done in detail if it's helpful. -**Calumnus:** When I started, I was tasked with changing how we approached observability. The technical aspects have been relatively easy, but the cultural changes have been more challenging. There was resistance to migration at first, but as people see the benefits of OpenTelemetry, that resistance has lessened. +**Ree:** Thank you so much for offering that, and thank you so much, Cal and Jerome, for being on and sharing your journey with us. We will link to our hotel SIG and user channel as well, so you can find all of us in there. You can also reach out to Cal directly via the CNCF Slack as well. And so we do have a link, Manas, for you that we'll put up here in a second that links to OPE documentation, and—oh, well, we might have time for one more quick question. This is kind of—what do you think about auto instrumentation? -The ability to write telemetry to various platforms has enabled teams to do interesting things, which has encouraged adoption. +**Cal:** Me personally, I think it's a great thing. We use it where we can, and in fact, for the front-end stuff, we are planning to use some of that auto instrumentation there. The downside we've seen—we have tried to use it, and we do use it on the .NET side in the .NET SDK for HTTP requests and things like that. The problem we've run into there is volume. So it tends to be difficult to sort of scale down to exactly which ones you want. The auto instrumentation tends to be overly broad sometimes, but that's the only sort of downside we've seen with it. So in general, we think it's a great thing, and if it works for you, then yeah, definitely use it. -**Jerome:** To add to that, I’ve seen a significant shift in attitudes toward telemetry. Teams are now more responsive to adopting telemetry practices and seeing the benefits, which is a positive change. +**Adriana:** Awesome. Thank you. Well, Ree and I would like to thank you again so much for being on here. This has been really great, and I'm sure I'll have myself and other people will have more questions to learn more about open telemetry implementation since you guys were early adopters. We do have a couple things we want to share real quick before we head out. Hotel Community Day is coming up on June 25th, I believe, and we are currently accepting talk proposals. So, if you would like to submit a talk, we definitely encourage you to submit a topic for open telemetry day. It's going to be in Colorado as part of Open Source in Denver. It's a collocated event of Open Source Summit North America in Denver, Colorado. -**Ree:** Before we wrap up, do you have any feedback for the project maintainers? +**Ree:** Yes. So if you want to come out—well, I guess that's too late to ski, but the beautiful mountains, I guess. -**Calumnus:** Absolutely! I think OpenTelemetry is a fantastic platform, and the maintainers are doing a great job. However, as we generate more telemetry, standardizing data for downstream consumers is becoming increasingly important. Discussions around semantic conventions and data contracts will be vital for organizations moving forward. +**Adriana:** Yes. Also, if you are going to—or if you are already in Europe and you're planning to head out to KubeCon EU in London in two weeks—less than two weeks. -Overall, the tech stack is solid, and I’m excited to see how it evolves. +**Ree:** Yes. -**Ree:** Thank you so much for sharing your insights! We appreciate you both being here today. +**Adriana:** We would love to see you there. Ree and I will be speaking at the event, and we have a blog post about the event that outlines all the open telemetry specific talks. The recordings, if you're not able to make it, will be available on the CNCF YouTube channel, I think about one to two weeks after, I think—depending. -Before we go, I want to remind everyone about the upcoming Hotel Community Day on June 25th, where we are accepting talk proposals. Also, if you’re attending KubeCon EU in London soon, Adriana and I will be speaking there. +**Ree:** Yeah, they're usually pretty fast. Sometimes they get it within a couple of days, so just keep an eye out. -If you missed this recording but have friends who couldn’t make it, you can replay it on our LinkedIn or find it on our YouTube channel. +**Adriana:** Yes, so if you can make it in person, you are definitely welcome to check out the recordings on YouTube. And I believe that's it. Do you want to mention also if you are at KubeCon EU, we are going to be hanging out at the hotel observatory off and on. So if you're there, come say hi. The observatory is always lots of fun because there's usually tons of hotel contributors, maintainers, etc. hanging out. It's great to have ad hoc conversations with folks. There's also going to be—if you're a KubeCon hotel project updates, I strongly encourage folks to join that as well if they want to see what's new and exciting in hotel. That's always a fun session to attend. I think it's usually like half an hour that it's slated for. -Thank you all so much, and we will see you next time! Goodbye! +**Ree:** And also, if you made this recording but you have a friend who couldn't make it, you can replay on our LinkedIn—just share the same link for the live recording with your friends, or this will also be available on our hotel YouTube channel. So tell your friends—officially tell your friends. + +**Adriana:** Tell your friends. + +**Ree:** All right. Thank you all so much, and we will see you next time. Bye. ## Raw YouTube Transcript diff --git a/video-transcripts/transcripts/2025-04-04T01:10:32Z-humans-of-otel-streaming-live-from-kubecon-with-austin-parker-marylia-gutierrez.md b/video-transcripts/transcripts/2025-04-04T01:10:32Z-humans-of-otel-streaming-live-from-kubecon-with-austin-parker-marylia-gutierrez.md index e5a5159..38c8304 100644 --- a/video-transcripts/transcripts/2025-04-04T01:10:32Z-humans-of-otel-streaming-live-from-kubecon-with-austin-parker-marylia-gutierrez.md +++ b/video-transcripts/transcripts/2025-04-04T01:10:32Z-humans-of-otel-streaming-live-from-kubecon-with-austin-parker-marylia-gutierrez.md @@ -10,78 +10,308 @@ URL: https://www.youtube.com/watch?v=EL0UkhvFAmY ## Summary -In this live stream from CubeCon EU, host Reese, a senior developer relations engineer at New Relic, is joined by colleagues Adriana Vila and Marilia, a staff engineer at Grafana Labs. The discussion centers around the OpenTelemetry project, its community, and the importance of localization in technology. Marilia shares her journey from being an end user to contributing to OpenTelemetry, emphasizing the significance of semantic conventions and the roles within the community, such as contributor and maintainer. The conversation also highlights the recent advancements in OpenTelemetry, including a new database monitoring release, ongoing efforts in localization for non-English speakers, and the impact of AI on observability tools. Austin Parker, the community manager for OpenTelemetry, joins later to discuss the project's growth, upcoming community events, and the evolution of logging infrastructure. Overall, the stream showcases the vibrant OpenTelemetry community and the collaborative efforts in improving observability practices. +In this live stream of "Humans of OA," host Reese, a senior developer relations engineer at New Relic, is joined by colleagues Adriana Vila and Marilia, a staff engineer at Grafana Labs. The discussion centers around their experiences with OpenTelemetry, including Marilia's journey from an end user to a significant contributor. They explore the importance of contributor roles within the open-source community, the recent release of a database monitoring tool, and the significance of localization efforts to make technical content accessible in multiple languages. Additionally, they touch upon the evolving role of AI in observability and OpenTelemetry, emphasizing its potential to reduce toil in software development and improve documentation practices. The conversation highlights the collaborative nature of the tech community, ongoing projects, and upcoming events like the OpenTelemetry Community Day. ## Chapters -00:00:00 Introductions to the livestream and guests -00:02:30 Marilia discusses her background and role at Grafana Labs -00:05:15 Overview of OpenTelemetry's focus and recent updates -00:08:00 Discussion on the Database Semantic Convention Special Interest Group (SIG) -00:12:30 Explanation of contributor roles in the OpenTelemetry community -00:16:00 Marilia talks about the importance of localization in documentation -00:20:00 Overview of the Contributor Experience SIG and its goals -00:24:00 Austin Parker joins the discussion as the community manager -00:30:00 Discussion on AI's potential impact on OpenTelemetry -00:35:00 Overview of upcoming OpenTelemetry events and community days +00:00:00 Welcome and intro +00:00:30 Guest introduction: Marilia +00:02:00 Database monitoring talk +00:03:30 Semantic conventions discussion +00:05:11 Contributor roles overview +00:07:40 Localization importance +00:09:30 Localization SIG details +00:11:50 Contributor experience SIG +00:14:42 Guest introduction: Austin Parker +00:16:40 OpenTelemetry project updates -# Humans of OA Live Stream Transcript +**Reese:** Hello, welcome to a live stream of humans of OA. We are coming to you live from what had a little bit of subtitle issues, but we are here and we're so excited. My name is Reese. I am a senior developer relations engineer at New Relic and I am joined here by my lovely industry colleague Adriana and Marilia who is our first guest. -**Hello, welcome to a live stream of Humans of OA!** We are excited to be here despite some subtitle issues. My name is Reese, and I am a Senior Developer Relations Engineer at New Relic. I’m joined by my lovely industry colleagues, Adriana and Marilia, who is our first guest. +**Adriana:** Hey everyone. Thanks for tuning in. My name is Adriana Vila. I work alongside Reese. -**Adriana:** Hey everyone, thanks for tuning in! My name is Adriana Vila, and I work alongside Reese. +[00:00:30] **Marilia:** Hi everyone. My name is Marilia. I am a staff engineer at Grafana Labs and also in OpenTelemetry. I work in a few different groups. I am a maintainer for the contributor experience and I'm an approver for... -**Marilia:** Hi everyone! My name is Marilia, and I am a Staff Engineer at Grafana Labs, working with OpenTelemetry. I’m involved in a few different groups; I am a maintainer for the contributor experience and an approver. +**Reese:** I was earlier about all the different parts that you're involved in and we're really curious because you started as an end user and then eventually started contributing. -**Reese:** That’s amazing! You started as an end user and eventually began contributing. Can you tell us more about your journey? +**Marilia:** Yeah. And so we would love to... on my prior job I was responsible for the observability. I used to work at Cockroach Labs and I was the manager for the observability there. So I started learning about OpenTelemetry specifically because there we did would be able to use that. Maybe someone want to see some of those things outside of the database. So we created like some endpoints here and there, but it was like very basic stuff. -**Marilia:** Sure! In my prior job at Cockroach Labs, I was responsible for observability. I began learning about OpenTelemetry because we wanted to use it outside of the database. We created some basic endpoints, but my team focused solely on OpenTelemetry. Now, I get to have fun with it every day! +**Reese:** So this team is to work only on OpenTelemetry. So your focus is just OpenTelemetry. -**Reese:** That’s awesome! What are some of the programming languages and SDKs you work with? +**Marilia:** So this is my day-to-day. I can have fun and just be on it. -**Marilia:** Currently, I’m focusing on Java and .NET, ensuring that all SDKs are on the same level. I also manage the Postgres plugin, which I have been working on. +**Reese:** That's awesome. I didn't realize that. That's so cool. And what would that new like Java and .NET and I want to make sure that all SDKs are on the same level and the other two were a little here. What are the things that we can add? -**Reese:** You recently gave a talk about database monitoring, right? +**Marilia:** So we have owner for the Postgres plugin that I was already like touching and there. That's awesome. And actually you gave a talk yesterday about database monitoring. -**Marilia:** Yes! We just released the release candidate this week, and by the end of the month, it will be marked as stable. This means people can start using it without concerns. +[00:02:00] **Marilia:** Exactly. So one of the other big ones. So fresh news. We just this week released the release candidate two. So that means by the end of the month we're going to mark it as stable. So people can really start using it without any concerns. -**Reese:** You mentioned you’re part of the semantic convention SIG for databases. Can you explain how that differs from the general semantic conventions SIG? +**Reese:** You mentioned that you're in the semantic convention SIG for databases and there's like a general semantic convention SIG, correct? What is it about the database semantic convention SIG that spun off? -**Marilia:** Typically, the general semantic conventions include representatives from various areas, which can lead to discussions that may not be relevant to everyone. By creating separate SIGs for specific areas like databases, HTTP, and RPC, we can focus on topics relevant to those who have experience in that field. This helps keep the work focused and easier to prioritize. +**Marilia:** Yeah. So usually you have the general semantic convention and on that one you have sometimes people from all of the other semantics that can give updates or discuss this particular name of the metric. And spend an hour when you have a lot of people on the code they're not related to databases, so they don't have any input to give. It might be a waste of time. So it's kind of like you spin up like different semantic conventions for each of like, oh we want to focus on database, we want to focus on HTTP, we want to focus on RPC. So now you can bring people that have experience in that area and talk about it. The same way you can think about if you would have a SIG for SDK, you have so many languages. So we have now one SIG for each language. So it's kind of like the same idea. -**Reese:** That makes sense! You have a lot of responsibilities in the community. Could you explain the distinctions between your roles as a contributor, approver, and maintainer? +**Reese:** Help like keep it like the work focused and easy to prioritize. -**Marilia:** Absolutely! You can become a contributor without being a member of the CNCF. After making a few contributions, you can ask to become an official member. Once you start helping out by reviewing PRs and commenting, you can achieve the status of triager, where you help manage incoming issues. From there, you can become an approver, meaning your approval counts towards merging PRs. Finally, maintainers are responsible for merging changes and creating releases, while also gathering input from contributors. +[00:03:30] **Marilia:** Yeah. And then we can always have the updates that we can bring to the main semantic one because when they generate a version, it's for all semantics. It is not as specific for the database. So you just merge that repo and whenever a new version comes in, it might be bringing updates for the database, might be from the HTTP, and so on. So we just like can bring the update like to the main ones like, hey this one we're marking as stable. We have these updates and things like that. -**Reese:** That’s a great overview! Speaking of contributions, I know you’re also involved in Portuguese localization. Can you tell us why it's important and how you got involved? +**Reese:** And then you mentioned, and I'm glad you were able to rattle off the long list of titles that you have in the community because I was like I am not going to remember all these but they're so cool. So there's like different titles. There's approver, then approver, then maintainer. -**Marilia:** Localization is crucial because language can be a barrier. Many people learning tech concepts may not speak English as their first language, making documentation less accessible. I started a blog to help with translations and worked on reviewing materials. It’s important to decide what to translate and what to leave in English, as some concepts don’t have a direct translation. This work helps break down barriers and brings more people into the community. +**Marilia:** Okay perfect. And then could you explain to us briefly like what each, what's the distinction between each role? -**Reese:** It sounds like a lot of work! Has this experience helped improve your own Portuguese? +[00:05:11] **Marilia:** Yeah. So you started first as a contributor. So you can become a contributor not being a member of CNCF at all. So when you do a couple of contributions, you ask to become a member, which is something I think everybody should do if you are doing it because it's very easy to join. You just have to open a PR, say like, hey I want to become an official member of CNCF and then OpenTelemetry and then you are officially a contributor. Then you pick something that is more interesting to you. So for example, I really like this language or I really like this specific component. Start joining the SIGs just to learn more about it, see how you can help. And when you start really helping out, start reviewing PRs, putting comments, creating issues for things that you mind. And with time you get the status of triager, which is somebody that is helping when a lot of issues come up. You help exactly triage those issues and see like things that make sense, things that actually they were not using correctly or there are actual bugs. Then from that, you can become an approver. When you approve a PR, your approval actually counts because people can approve, but to be able to merge you need to have people with permission giving the approval and being able to merge. So the approver now has this, whatever we approve it counts and somebody who has the permission can just merge it in. And then you have the final one, which is maintainer. So these are the people that can actually merge, usually the ones creating the releases, creating like road maps, but they do get input from everybody that is contributing. -**Marilia:** Yes, it has! Living abroad for so long has made it a challenge to keep my language skills sharp, especially when giving talks in Portuguese. We also have contributors working on Spanish, French, and Japanese localization, so we’re constantly learning from each other. +**Reese:** That's so exciting. That is really cool. Thanks for the overview. -**Reese:** That's fantastic! As we wrap up, what advice would you give to someone looking to start contributing? +**Marilia:** Yeah. The other thing that we wanted to ask, you know, you mentioned that you're working in the Portuguese localization. Talk a little bit about that, like talk about why it's important to have localization, how you got involved with that, specifically how that's been. -**Marilia:** There’s always something for everyone, so just try to get involved! Open source is different from a daily job, so it's important to keep that in mind. Be proactive, and don’t hesitate to reach out to the community for guidance. +**Marilia:** Yeah. So I think it's really important because it's a barrier, like the language. And it's already hard for you to like completely learn something completely new like on tech, but imagine if you don't even know the language. And a lot of other countries like their first language is not English. It's not always you're going to find this documentation and not easily accessible for everybody. So actually I started like a while ago. I created like my own blog post and I was like everything that I'm creating there, I'm going to create and where people that actually met when I did like conference days in Brazil and there was the bad group. So I started like, oh I'm going to help them out. So I like to like review a lot of things and the good things a lot of things in English that you learn like just experience living. So I can help out with like the translation, the localization, and also decisions like what are the things that you don't translate at all because we're going to say tracer. We don't, that makes sense because at the same time you don't want to translate everything because then when they're going to search those words don't exist anywhere. -**Reese:** That’s great advice. Thank you so much for sharing your insights today, Marilia! We’ll continue our conversation shortly as we bring on our next guest, Austin Parker, the community manager for OpenTelemetry. +**Reese:** Yeah, but yeah, I think it's like really good to just break this barrier and bring a lot of people they would have no idea. -**Austin:** Hi everyone! I'm excited to be here. I’ve been managing the community and working on various projects, including the OpenTelemetry website. +**Marilia:** Yeah, that's so great. That's a lot of work. I guess in some ways has it like improved your Portuguese as a result? -**Reese:** It's great to have you! What changes have you seen in the community since it started? +[00:07:40] **Marilia:** So yeah, it's always funny because like living outside for so long and sometimes you, I'm going to give a talk like in Portuguese and I was like here so I know the right thing to use but it's always always a challenge. But we also try because there's not only Portuguese. We also have like people working on the Spanish, French, and Japanese. So do completely different than the French know. We also have this thing like which is like similar to the same. So we have the documentation one, but we have different groups. So every time a docs page comes out, you like the localization SIG translates. -**Austin:** There’s been significant growth! When we first came to KubeCon, many people were using OpenTelemetry, and the project updates room has grown substantially over the years. It's exciting to see new vendors and commercial solutions built on top of OpenTelemetry. +**Reese:** Yeah. So that is the idea. So right now that people are accessing more things like that. So we start with those ones and like the same thing for SDKs. I see which ones people are like downloading more. -**Reese:** It sounds like the excitement is palpable! So many people are getting involved. What do you see as the future of OpenTelemetry? +**Marilia:** So we do have something to... because part of the when you do a localization there is a new kind of like tag that you had to put in what was the commit that you translated. Because it's really hard for people to know that somebody did an update somewhere. You had to really be paying attention. So the idea is to have something that would tell us, hey those are behind just catch up and things like that. So we were always up to date. -**Austin:** AI is going to play a big role. We’re looking into how to improve documentation for LLMs, as well as using AI to reduce toil in observability tasks, like migrations. This will help make the process smoother for developers. +**Reese:** Oh, sorry. Go ahead. -**Reese:** That’s a fascinating perspective! As we look ahead, what upcoming events should we be aware of? +**Marilia:** I was curious because when I was asking you all your different roles, that was like the first time I'd heard of that specifically because I know we have translated some of our documentation, but I didn't realize there were specific teams working on these languages. -**Austin:** We have the OpenTelemetry Community Day in Denver scheduled for June, alongside the Open Source Summit. We're also planning to have a community day in Europe, so stay tuned for more information! +**Marilia:** So the localization SIGs, they are only translating documentation or what all involves? -**Reese:** Fantastic! Thank you both for joining us today, and thank you to our behind-the-scenes crew for making this possible. We appreciate everyone tuning in, and we hope to see you next time! +[00:09:30] **Marilia:** So like the official page like the OpenTelemetry.io. All the pages there are the ones. If you look at the now at the top that didn't have anything. Now if you're looking at the page, the top you have a select there that you can change the language that you want. + +**Reese:** So whenever the page is translated now you have... + +**Marilia:** So cool, learned something new today. I then joke that I have a PR with the translation to the Japanese even though I don't speak any Japanese and sometimes it goes wrong there too. + +**Reese:** Uh-huh. + +**Marilia:** So it's a way to catch up. So like when they was like, "Okay, so the..." I was like, "Oh, let me see if any other languages translated the same thing wrong." Async or something. + +**Reese:** Okay. + +**Marilia:** Yeah, that's pretty big. + +**Marilia:** And then it was good that it was just like a word and I translated and I put a PR and I tagged the reviewers for Japanese. I was like, I don't speak any Japanese. I put this thing that I think is the right translation. Like there was a part for the translation, but we even like looking for we want to add also more examples because we don't have a lot of examples. If there is something like for example you never did, so you try it yourself and you might for the future ones that are coming that they know like more examples, more things to try on. + +**Reese:** Yeah, man. This is so exciting. I'm learning all kinds of new things. + +**Marilia:** I know. Yeah. + +**Marilia:** So yeah, so many things that we're like when people ask like how do I start, like so many of you're going to be able to find somewhere something that is interesting. There's always something for you, right? + +**Reese:** Yeah, it's really hard and just try joining just like listening. But at the same time, keep in mind that open source is different than a day-to-day job because sometimes what I see people find things was like how I found the things that like, oh, they need help with this, they need help with that. And then the things that I already have priorities like from even coming like from, hey can you take a look at this up for grabs? A few repos have them, so we have things like for example contribute fest that we did here and a lot of them got tagged with contribute fest, but not all of them got actually worked on. + +**Marilia:** So those are the ones that you know and you're going to find something and people in the community are very helpful like tag them on Slack, on the issue itself, they are happy to give you guidance as well. + +[00:11:50] **Reese:** Yeah, and I'll do a shameless plug for the end user SIG as well. We are always happy to have contributors. You can find more information in our show notes. Part of like the end user SIG. The other thing I want to ask is the contributor experience. So if you want to contribute like what is missing? So for example, one of the projects that I did that was so like how do I start? I don't know how to set up locally. There was no like what is the dependency that you need. So there was like nothing there. So I worked like as a mentor for the AI program template that all repos could use and started creating like the PR so people have at least the basic things that they need to just at least start having things. + +**Marilia:** Oh, that's amazing. + +**Reese:** And really quickly, actually, I want to ask about the contributor experience SIG because that one's a relatively new SIG, right? + +**Marilia:** Yes. + +**Reese:** Can you tell us a little bit about that SIG, how it got started, and how you became a member? + +**Marilia:** It was because we saw this need of people complaining that they don't know how to start. They don't always understand like the path for like triager or like how like there is nobody there to guide. Okay, if nobody is going to be there for you, are you able to find all the documentation that you need to be able to follow the path that you want? Or like we didn't even know the challenges people were having a few of the things and we do like also surveys to find out people like how are you feeling like being a contributor and we started tackle the things that were the most concerned. Like documentation was the top one, so this is why I started that project and container almost right away. + +**Reese:** What are some of the upcoming things for the contributor experience? + +**Marilia:** So for this one, I just finished this part like this month for the template. So my next meeting is going to be actually picking up what is going to be the next one. But we do have like issues open that is just helping out like clarify things to people. And if anyone like also if you're listening, if you have something that is really like bothering you and you need, feel free to like message us or like open an issue directly on the repo there. We can prioritize because we have a list there, but we are just prioritizing ourselves because we don't have necessarily feedback things but react like thumbs up or really want this kind of thing and we will be able to put it in first. + +**Reese:** Awesome. And I think we're almost ready for our next guest. + +[00:14:42] **Marilia:** Yeah, I think we're so I guess we will continue our conversation. I guess as we get ready to bring on our next guest experience, they will be able to share like people just joining and people that have a lot of experience will be able to guide you as well just as I join the calls. + +**Reese:** And they are Austin Parker. I'm just going to get the surprise out of the way. Community manager for OpenTelemetry and we are so excited to have him on. + +**Austin:** How am I still community manager? + +**Reese:** Wait, are you no longer the community manager? + +**Austin:** What? You're still community manager? + +**Reese:** Yeah, I think I am. If you go and check the repo, I probably am. I mean, I do both jobs still. So, I am a member of the OpenTelemetry governance committee. I also have been a maintainer on the OpenTelemetry website, OpenTelemetry demo. I've working group I guess you could say like where we combine OpenCensus maintainers and OpenTracing maintainers where we, you know, came in and worked with some people. + +**Reese:** What have you seen as like the biggest changes since things started? + +**Austin:** Like when we first came to KubeCon, zero day event, you know, ask people who's how many people are using OpenTelemetry and it's like every hand in the room is going up, right? Like the growth of OpenTelemetry... + +**Reese:** Which was 2022. + +[00:16:40] **Austin:** The project updates room, you know, the project was given like a really small room. It was like pretty packed. Yeah. And over the years I've seen that room like get bigger and bigger. + +**Reese:** Yeah. I know they probably like a 200 seat room and there are still people standing in the back, right? + +**Austin:** There was also a lot of, you know, there was a keynote yesterday that was actually the thing that I've been most excited about. The growth of OpenTelemetry isn't OpenTelemetry necessarily, but the opportunities that we see other people kind of taking and running with, right? Projects like Percy’s, new vendors, new commercial solutions built on top of OpenTelemetry have been really exciting to see. + +**Reese:** And I think, you know, that that's the sort of stuff that you... + +**Austin:** Yeah. Yeah, definitely. Yeah. It's been exciting to see it grow. I remember even like my first KubeCon was Detroit and there was an Open Observability Day and then there was an OpenTelemetry unplugged with KubeCon North America and EU, which I'm super stoked about. + +**Reese:** What super busy too. We have hundreds and hundreds of people showing up for those. + +**Austin:** Yeah. It was great. + +**Reese:** Yeah. Thank you. I mean, I feel like it's also one of those things where it's kind of like you can't really do three tracks, right? Like that's a bit like two is good. Seems like a good number, but there's observability talks, you know, throughout the week at KubeCon. I don't know. This used to be the Kubernetes club, right? This is the operations, the SRE kids. And now, you know, security has a big presence in the CNCF. + +**Austin:** I think what we're seeing... + +**Austin:** What I think is important is you in a lot of ways if you think about how a lot of technology that we still rely on today, you know, was built and maintained and came right. Maybe someone, maybe someone and if that's you, more power to you. + +**Reese:** To America, we might need your help. + +**Austin:** Things became the de facto standards through whatever, you know, combination of factors happened, right? Like I think what's cool about C4 vendor, you happen to use or so on and so forth. + +**Reese:** Now the downside is, you know, in 25 years my kid's going to copy me and be like, "What the, what's this OpenTelemetry crap dad?" + +**Austin:** Don't metaverse OpenTelemetry now. + +**Reese:** Oh man. + +**Austin:** Also speaking of the future, yeah, we were curious about how do you think AI might impact OpenTelemetry? + +**Austin:** How it might, how it might impact or have an impact or... + +**Austin:** Yeah. There's a lot of ways. I think one thing I was, so it's funny 'cause I was just talking to someone about this. A project I've been working on kind of on the side is how to make better docs for LLMs because the documentation that we write for human beings is really good for human beings and LLMs are not human beings. They are something else. So you need to kind of give them documentation in a way that is similar to how you would give it to a human but distinct enough that it's not, you know, it's trickier than just saying like, oh go read this web page, right? + +**Reese:** 'Cause the web page has a lot of stuff that's for humans. + +**Austin:** We care about things like font sizes and colors and we care a lot about sort of the organization of information and we want to have pages that are single concept, right? So that you can focus on like what is the goal of this page. But then LLMs, there's a distinction you need. You can, like one interesting thing if you think about writing a document, you know, writing documentation is you're not supposed to use, even if there's a more specific word, like use more words, not less, right? You have to kind of hit it at whatever reading level you expect people to be at, even for that kind of stuff. But with LLMs, actually, they don't have that disadvantage, right? Like you can give it a very large flowery word and if it's the most specific word, that's actually helpful because of the way the semantic search works. + +**Reese:** Precision in language is actually really important. + +**Austin:** So you wind up needing documentation that is the same documentation you give to humans because the concepts all need to match but is organized and structured in a different way, is much longer, has kind of everything in one big chunk, is optimized for the amount of tokens and the semantic values of those tokens. But the advantage of doing this right, of thinking about how do I give the LLM documentation is that LLMs are remarkable, you know, especially if you're using them as part of like AI-assisted coding, are remarkable at reducing toil. + +**Austin:** One of the things that I think, you know, all of us here have been working in observability for years and what is like the one thing nobody wants to do? Nobody wants to do a migration, right? No, you go tell someone like, "Oh, here's the new thing." They have to rewrite all this instrumentation, all these logs, all this stuff. They just be like, "Thanks, I'll pass." It's 'cause it's a toil. It doesn't make sense, right? The existing stuff works well enough. It's maybe it could be better, but you know, we're not going to go dedicate however many engineers' lives for three months to rewrite all of our logging statements. + +**Reese:** But what if you just have an AI do it, right? The AI doesn't care. It doesn't complain. If you give it good instructions and good rules, it's able to, you know, make good decisions. And it's not like you're replacing human effort, right? You're not replacing the programmer. You're not replacing the people that are responsible for maintaining the system. You're easing the burden of modernizing what they're trying to do. + +**Austin:** And so you're actually benefiting those people a lot 'cause even with improvements in AI-powered anomaly detection, whatever, right? Like we've been doing AI in observability for a while. LLMs just advance it a little bit, maybe a lot, we'll see. But even non-AI stuff, right? If you think about traditional sort of anomaly detection and heuristic-based detection, it's still machine learning. LLM machine learning. It's all machine learning. We've been doing machine learning for a while. And when you get to a certain size and complexity of your systems, you have to have it because there's just too much data for humans to process and go through. + +**Reese:** So let's figure out how we can, you know, that to me is like the impact of AI on observability and on OpenTelemetry is we can make it easier for the AI, make it easier for your observability tooling to understand OpenTelemetry and interpret what's happening in your system better and at the end of the day give you more time to do... + +**Austin:** We had talked about OpenTelemetry project updates, which took place yesterday, right? + +**Reese:** That sounds right. I'm losing track. I'm losing... + +**Austin:** Yeah, it is. It was yesterday. + +**Reese:** Milk. That doesn't sound like a good bagel. I wish it said bagel clock. + +**Austin:** This is probably a great visual bit for the... + +**Reese:** And it's been... it attracted my attention ever since I came up here and now it's like I've been obsessed. I've been waiting for the appropriate moment to drop the bagel clock into the conversation. + +**Austin:** You did it. There you go. This is what we call commitment to the bit. + +**Reese:** Yes. I update. + +**Austin:** Yeah, project update. So, OpenTelemetry right now is the second or first biggest project in the CNCF, depending on how you count it, but contributing across, you know, dozens and dozens of repositories. We're maintaining APIs, SDKs, tools in, you know, dozen plus languages. A lot's going on, right? And at this point, the project is really too big almost to have kind of a single narrative of whatever we're doing, but there's a few areas that we wanted to focus on. + +**Austin:** We're starting to see a lot of great adoption of OpenTelemetry by the sort of broader community outside of just OpenTelemetry. We're starting to see more CNCF projects natively integrate OpenTelemetry. We're starting to see are integrating OpenTelemetry into their frameworks and into the, in Dino's case, into the runtime itself, right? So if you're writing a JavaScript app and you're using Dino, you pass in a config, you don't have to do anything, which is great. That's the vision for the project. + +**Reese:** So beyond that, you know, beyond the kind of growth we're seeing in adoption, we're seeing a few, you know, longer-term projects that we are proceeding along. + +**Austin:** So one thing that we've been working on over the past six to eight months, we've had a lot of progress there. We have a system-level profiler that is being worked on. It uses eBPF and various other technologies to let you use profiling across your services on a node. That is still in alpha, like it's not done. It's not ready, but pretty soon that should be ready for people to start banging on. + +**Austin:** Another important thing we're doing is that we are evolving our logging infrastructure or logging APIs. So traditionally we would just bridge to your existing logging API because there's a lot of those. There's Log4j, there's various facades in Go and .NET and wherever, but one of the pieces of feedback we were going out and finding these sort of consistent metadata across services and libraries and domains is people need a structured way to emit structured events, right? Things like that you and I would probably call the LOG. Some people would call it an event. + +**Reese:** And one thing I have learned is that the third rail of observability is talking about logging at all because people are very, very precious about what the word logs means to them. + +**Austin:** I've noticed. + +**Reese:** Oh, true. Yeah. It's you don't mess with people's logs. + +**Austin:** Yes. So what we've kind of come to realize through this whole process is that we need some sort of API level answer to that and we're pretty close to have, you know, we have some OTAs and some specs in flight on this, but the idea is that there will be an OpenTelemetry logging API that will exist to let you emit structured events. + +**Austin:** A structured event is really just a fancy way of saying a structured log that has a known schema, right? In the same way that semantic conventions in OpenTelemetry let you apply schemas to your telemetry, to your logs, or sorry, to your metrics and traces. You'll be able to say, hey, here's a client-side ROM event, or here's a, you know, out of memory exception, or any of the various things that can happen. You'll be able to say, "Hey, here's a generative AI prompt, for example." And what we'll do is you'll be able to either take that and use it like you would use a span event today and bundle it in with the span or you'll be able to emit it separately through the log record signal and then have your backend either stitch them together or process them independently or do whatever, right? Like once it's out of our hands, we don't really care what you do with it. + +**Austin:** But that's probably the two big in-flight things I would say. Beyond that, a lot of work is happening on other things. Stabilizing the collector, stabilizing various other SDKs and APIs. Shout out to our JavaScript SIG which just released JS SDK 2.0. My understanding is this fixes a lot of problems that people have had with especially with bundling it and things around ESM modules. I don't quite know what all that is. It sounds very scary. The JS devs assure me it's very important, but seriously I think it's actually a really good sign of the health of that project, right? That they have been able to get enough feedback about like, hey, these are the decisions that worked and didn't work to create a 2.0. And then for an end user, I was actually talking to someone Monday, Sunday at Cloud Native Rejects about this who maintains an integration into OpenTelemetry into his company's product and he was like, oh yeah, the migration was like five minutes, right? + +**Reese:** Wow. + +**Austin:** Because the API and the SDK are independent, so an SDK change is really very, you know, it's not a huge deal, it's not a lot that you have to do to take those updates. So that's something that for obviously other maintainers may or may not decide to do it, but we're, you know, it certainly seems that a lot of maintainers are thinking about, well maybe it's a good idea to go back. It's been five, six years, right? Like you can learn a lot, you get a lot of great feedback over that time and there's things that we would probably do differently in every language if we had a do-over. So thanks to the OpenTelemetry architecture, you can kind of get that do-over, which is cool. + +**Reese:** That's great. How are we on time? I'm not sure 'cause we have... + +**Austin:** Are we on time, producer? + +**Reese:** Okay. Well, I guess this could be a good opportunity for us to plug some upcoming stuff like OpenTelemetry Community Day. + +**Austin:** Yes, OpenTelemetry Community Day in Denver, Colorado coming up this summer. June 25th or 26th. It's the same week as Open Source Summit. Go look on the web. The CFP is closed unfortunately, but we will be, we should be announcing the schedule on that here pretty soon. Even if you aren't planning on, you know, plan on speaking, highly recommend everyone that's in the US, North America to come out to that. It's going to be great. + +**Austin:** And we are also, no promises, but we're trying to do a community day in Europe this year. So stay tuned. If not this year, definitely next year. + +**Reese:** Is it going to be part of Open Source Summit EU or that would be the plan, but nothing's set in stone. But we've definitely, one of the fun facts at the project update is that about 50% of our contributors are actually not in the US in OpenTelemetry and we've seen now that's US and then everywhere else, right? So, but if you look at, if you break it down by like region, so you go like North America, EMIA, APAC, other, then we like the line for EMIA is just like doing this and the US one is kind of doing this a little bit. + +**Austin:** So they're starting to get closer and closer together. But we've definitely, you know, one of the things I always love coming to KubeCon EU is we have so many, you know, our user community is so vibrant here. We have so many maintainers whose work is just fantastic and we really want to support our European end user and contributor community. So we're very strongly going to be figuring out how to do a European community day. + +**Reese:** Awesome. Oh, that's awesome. + +**Austin:** Yeah. I always love the vibe at the KubeCon EU. It's just very vibrant. And I think at the keynote, they mentioned this was the biggest KubeCon so far, 12,000. + +**Reese:** Yeah. Over 13,000 people. + +**Austin:** Over 13,000. That's the number I heard. Damn, that is wild. + +**Reese:** Yeah. No, they... I mean I... + +**Austin:** And it's like how's your KubeCon? And they say it's like, "Oh, it's my first." I'm like, "You're going to have that. Bring lots of water. Bring lots of water." + +**Reese:** But I love that we're still that new people are still coming into this community, right? That we're really cool to see the growth of, you know, this community, right? Like to see it expand, to see it bring in new people, right? + +**Austin:** Yeah. Oh my god. There's a ton of people that we know that all the three of us here know that careers, friendships, right? Friendships like that's given people the opportunity to really, you know. + +**Reese:** Yeah. + +**Austin:** That's neat. + +**Reese:** Yeah. Also, it feels like a family reunion every time we're together, right? + +**Austin:** Yeah. No, it's great. It's like a family reunion with like 10,000 people. + +**Reese:** Yeah, a bunch of batteries and then we get on the other side. They have the retail technology exhibition. + +**Austin:** Yep. They actually get a red carpet. Very loud burgers walking around which was exciting. It was definitely unexpected. + +**Reese:** Yeah, I know. We were walking yesterday and we're like well the tube it was like there's like two stops. + +**Austin:** Yeah, there's a station either end. + +**Reese:** Yeah. + +**Austin:** And I heard that it's actually faster if you time it right to take the tube. + +**Reese:** If you hit... if you hit the... + +**Austin:** It also depends on where you start from, but if you're from door to door, it's definitely faster. + +**Reese:** Yeah, it's nice. You just tap in and out. + +**Austin:** Oh, I know. + +**Reese:** Yeah, it's really card. Discovered that. + +**Austin:** Oh, is that better than the... + +**Reese:** I just use Google Pay. + +**Austin:** Yeah, you can use your phone. + +**Reese:** Yeah. + +**Austin:** Yeah. Works well. + +**Reese:** Yeah, we have that in Toronto. I mean, we have tap in New York. They finally... it's Omni now. Omny instead of... + +**Austin:** This for another episode. + +**Reese:** Live stream for you from KubeCon EU. Sorry guys, it's been a long week. Thank you so much for joining us. We will have our lovely guests. Again, I'm Reese. This is Adriana. Thank you so much Austin for being here and Marilia for being here earlier. And also shout out to our behind the scenes. He is the one producing all this for you, all of our streams. Yes. Henrik said from Dana Trace. Thank you so much and we will see you next time. ## Raw YouTube Transcript diff --git a/video-transcripts/transcripts/2025-05-01T06:00:52Z-otel-me-with-eromosele-akhigbe.md b/video-transcripts/transcripts/2025-05-01T06:00:52Z-otel-me-with-eromosele-akhigbe.md index d8096ac..92f3132 100644 --- a/video-transcripts/transcripts/2025-05-01T06:00:52Z-otel-me-with-eromosele-akhigbe.md +++ b/video-transcripts/transcripts/2025-05-01T06:00:52Z-otel-me-with-eromosele-akhigbe.md @@ -10,143 +10,218 @@ URL: https://www.youtube.com/watch?v=KzqY4roXhHs ## Summary -In this episode of "O Tell Me," host Adriana Vila, along with co-host Victoria, engages in a discussion with guest Mole Aigbe, a developer advocate at Step Security and a member of the OpenTelemetry community. They explore Mole's journey into open source through the Outreachy program, which supports underrepresented communities in contributing to open source projects. Mole shares insights on the challenges of applying for Outreachy, the application process, and his experiences learning Go and contributing to OpenTelemetry, including building an exporter for the OpenTelemetry Collector. He emphasizes the importance of supportive mentors and the community's welcoming nature, as well as the value of studying existing code to accelerate learning. The conversation also touches on the need for improved documentation and resources for newcomers to OpenTelemetry. The episode concludes with announcements about upcoming events and opportunities for community engagement. +In the latest episode of "O Tell Me," host Adriana Vila, along with co-host Victoria, welcomes guest Mole Aigbe, a developer advocate at Step Security and an OpenTelemetry community member. The discussion revolves around Mole's journey into open source through the Outreachy program, which aims to support underrepresented communities in tech. Mole shares his initial hesitations about contributing to open source, his learning process in Golang, and the challenges faced while creating an OpenTelemetry exporter. He emphasizes the importance of mentorship and community support, particularly from figures like Adriana and his mentors, who encouraged him throughout his journey. The conversation highlights the value of community, advocacy for open source, and the resources that helped Mole, such as YouTube channels and blog posts. The episode wraps up with mentions of upcoming events and initiatives within the OpenTelemetry community. ## Chapters -Here are 10 key moments from the livestream along with their timestamps: +00:00:00 Welcome and introductions +00:02:30 Outreachy program overview +00:05:20 Application process discussion +00:10:00 Internship duration and mentors +00:11:30 Contributions to OpenTelemetry +00:14:02 Securing first job experience +00:15:50 Learning resources and methods +00:20:01 Documentation feedback and suggestions +00:25:00 Experience creating first PR +00:30:00 Building an OpenTelemetry exporter -00:00:00 Introductions and welcome message -00:02:30 Introduction of co-host Victoria -00:04:00 Introduction of guest Mole Aigbe -00:06:00 Overview of the Outreachy program -00:10:30 Mole's initial thoughts on open source before Outreachy -00:15:00 Discussion on the application process for Outreachy -00:20:00 Mole shares his first contribution experience -00:30:00 Mole talks about building an exporter for OpenTelemetry -00:40:00 Insights on learning Go programming language -00:50:00 Discussion about the OpenTelemetry community and its support +**Adriana:** Hey everyone, welcome to the latest edition of O tellme. My name is Adriana Vila. I am one of the maintainers of the open telemetry enduser SIG. Super excited to have folks joining us here, and please feel free in the chat to just say where you're joining from. I'm joining from Toronto, Canada. I have some very special folks here joining today. First of all, I have Victoria who is co-hosting with me. So, Victoria, why don't you introduce yourself? -Feel free to ask if you need any more details! +**Victoria:** Hi everyone. I'm Victoria. I'm a user experience designer. I'm currently interning with the Prometheus project as an LFM. My project has got me interfacing with a lot of hotel members, which is how I learned about this program. I'm looking forward to interviewing and learning all about his experience. -# O Tellme Episode Transcript +**Adriana:** Awesome. And now it's time to introduce our guest for the day. -**Adriana:** -Hey everyone, welcome to the latest edition of O Tellme. My name is Adriana Vila, and I am one of the maintainers of the OpenTelemetry End User SIG. I'm super excited to have folks joining us here, and please feel free in the chat to just say where you're joining from. I'm joining from Toronto, Canada. +**Mole:** Hello everyone. It's so nice to be here. Thank you so much, Adriana. My name is Mole Aigbe. I am a developer advocate at step security. I'm also an open telemetry member. I was a former algi intern and that's where I got introduced to open telemetry, and I'm so excited to share my experience with hotel here today. Thank you for having me. -I have some very special folks here joining today. First of all, I have Victoria, who is co-hosting with me. So, Victoria, why don't you introduce yourself? +[00:02:30] **Adriana:** Amazing. Yeah, super excited. Well, you mentioned outreachy in your intro. Why don't you tell folks a little bit about the outreachy program? -**Victoria:** -Hi everyone! I'm Victoria. I'm a user experience designer, currently interning with the Prometheus project as an LFM. My project has got me interfacing with a lot of OpenTelemetry members, which is how I learned about this program. I'm looking forward to interviewing and learning all about this experience. +**Mole:** Okay. So, the outreach program is an initiative that aims to help people from underrepresented communities, you know, marginalized, looked down on communities, get into open source. They encourage them to put their first leg into open source and they also give some kind of incentive to assist them, right? To get their first leg into the open source community at large. So, that is what outreachy is. It's not just for open source; it's for both open source and open science. They have projects that cover biological things, projects that cover geomaps, you know, different diverse projects—not just open source projects. But my love, since I'm a tech guy, I love open source, and that's where I got to know about open source and the amazing open telemetry community. -**Adriana:** -Awesome. And now it's time to introduce our guest, Mole. +**Adriana:** Did you know about open source before you applied for outreachy, or was outreachy your awareness into open source then? -**Mole:** -Hello everyone, it's so nice to be here. Thank you so much, Adriana. My name is Mole Aigbe. I am a developer advocate at Sematext and also an OpenTelemetry member. I was a former Outreachy intern, and that's where I got introduced to OpenTelemetry. I'm so excited to share my experience with OpenTelemetry here today. Thank you for having me. +[00:14:02] **Mole:** Okay. So, yeah. I always knew that open source existed, right? I always knew that, okay, there's something out there in the world called open source that some cool dudes, very smart, very intelligent, are doing awesome work in, right? But I just never believed that someone like me could contribute to open source because I was like, bro, what do you know? You do not work in Microsoft. You don't work in Google. You don't have that knowledge, man. You don't have experience. So just chill when you're skilled enough, maybe you can start playing in the big leagues, right? So open source was something I remember, you know, several times I would use YouTube to learn how to contribute to open source, and then I would try to watch some videos, I’d get scared and then I'd run away, right? But outreachy provided that platform. We had mentors that actually directed and showed us the step-by-step process on how to make your first contribution. And then once you make your first contribution, you realize that wait a second man, it's not that hard. It's all in your head, right? And once you make that first contribution, you just keep making and making, and then it just became so cool. So, yeah, that's how I got to learn about open source. I knew about open source, but outreachy gave me the platform to take that first step into open source. -**Adriana:** -Amazing! Yeah, I'm super excited. You mentioned Outreachy in your intro. Why don't you tell folks a little bit about the Outreachy program? +**Adriana:** Yeah, really interesting. Could you tell us more about the application process? What was it like? -**Mole:** -Sure! The Outreachy program is an initiative aimed at helping people from underrepresented communities—marginalized and overlooked communities—get into open source. They encourage them to take their first steps into open source and provide incentives to assist them in doing so. It's not just for open source; it's also for open science. They have projects covering biological topics, geomaps, and various diverse areas, not just open source projects. However, my love is in tech, and I love open source. That's where I got to know about OpenTelemetry and the amazing community. +[00:05:20] **Mole:** Okay. So first of all, I almost did not apply for outreachy because I also had that whole mentality that, okay man, this is open source. This is for the cool kids. I'm not a cool kid, right? So I was like, you know, my very good friend was like, no, you can do it. Just apply. What's the worst that can happen? They tell you no, you go back, you cry, and you're fine. So, I decided to take that step. The application process is that you have to write like five essays on why you believe that you should be awarded the internship. Because, like I said, the internship is not about they’re not looking for very skilled people. They're looking to actually support, you know, people that come from the minorities of society, like people that you would hear about on a regular day, right? Like people that are looked down on, you know, that people don’t even like—the big leagues don’t even look at in society. So that's their goal, right? Those are the people they are trying to support. So, you can be quote unquote very smart or high and mighty, and they won’t accept you. But you know, if you feel like man, who will hear my voice, no one cares about me, I’ll say go for outreachy. They care. The people actually sit down and read people's essays to see and to look for those kinds of people that they know they can support. So, I know people have heard a lot of things about job applications that they use one AI tracker to just wipe people's applications, but I can tell you from experience that outreachy is not like that. Real people actually read your essays and you can tell your story, and through your story, you know, you might get a chance to walk into and begin your own open source journey. -**Adriana:** -Were you aware of open source before you applied for Outreachy, or was Outreachy like your introduction to open source? +[00:11:30] **Mole:** So you have to write five essays. Once you write the five essays, you get into the first phase. There are actually two stages. The first phase is the essay phase where you have to prove that you need the internship. The second phase is where you actually have to now do the work. In doing that work, I’ll say there were a million times I wanted to give up because I did not know Golang. That's the language that most cloud-native applications are written in. I remember in December 2023, two months before I was learning about DevOps. So I just in that February I started learning Golang. I was trying to learn open telemetry; I was trying to learn Golang; and I was trying to contribute—all in one month. So it was a whole lot. But you know, I always say shout out to Henrik and Adriana. Their content really, you know, ramped me up to speed on how to first understand the project and then make some reasonable contributions. I was able to get like 70 hours merged, and then my mentors were like, "You're so cool. Come on board." My mentors, by the way, Jurassi and Yuri are the coolest guys out there. Shout out to them. I love them so much with all my heart, and I'm always grateful to them for the forever they will be part of my tech journey. I always mention their names everywhere I go, you know, as the ones that helped me like a baby and trained me up to be this man that I am now. -**Mole:** -I always knew open source existed. I knew there were smart people doing awesome work in that space. But I never believed someone like me could contribute to open source. I thought, "What do I know?" I didn't work at Microsoft or Google; I didn't have that knowledge or experience. I would often use YouTube to figure out how to contribute to open source, but I'd get scared and run away. Outreachy provided a platform with mentors who directed us step-by-step on how to make our first contributions. Once I made my first contribution, I realized it wasn't that hard; it was all in my head. +**Adriana:** That's like a summary of the application process. -**Adriana:** -That’s amazing! Could you tell us more about the application process? What was it like? +**Mole:** Yeah, that's so great. And you know, I can't underscore enough the importance of programs like outreachy because they give opportunities to people who otherwise wouldn't necessarily have gotten the opportunity. And especially like nowadays, I'm going to get on my little soap box and talk about the fact that so many DEI programs are getting cut and there's so much, I don't know, there's like global hate towards DEI programs. So it's so nice to have a reminder that programs like outreach exist. They benefit folks. They elevate folks who are awesome who we wouldn't necessarily have known about. I love that you got so much out of the program. I love that the program exists. I wanted to ask you, how long was your internship? -**Mole:** -Sure! I almost didn't apply because of the same mentality I had—that open source was for the "cool kids." But my good friend encouraged me to apply, saying, "What's the worst that can happen? They tell you no, and you move on." So, I decided to take that step. The application process involves writing five essays about why you believe you should be awarded the internship. They're not looking for highly skilled people; they want to support those from minority backgrounds. +[00:10:00] **Mole:** Okay, so the internship was for three months. But if I want to combine the whole application process, everything together was like six months, right? But officially, the internship is just for three months. -Once you write the essays, there's a two-phase process. The first phase is the essay phase, followed by a phase where you have to do the work. I remember wanting to give up many times because I didn't know Go, which is the language most cloud-native applications are written in. I started learning Go just two months before the internship while trying to learn about OpenTelemetry. It was hectic, but with the help of mentors and resources, I managed to get my contributions in. +**Adriana:** Awesome. You talked a little bit about your mentors. How do you get paired up with a mentor? -**Adriana:** -That's really interesting. I can't underscore enough the importance of programs like Outreachy. They provide opportunities to people who might not otherwise have them. How long was your internship? +**Mole:** Okay, so when you pick a project, every project has mentors for the project, right? So when I picked the project open telemetry, it was Jurassi and Yuri that actually volunteered to be mentors for a particular project, right? And so they became my mentors. -**Mole:** -The internship lasted for three months. However, if you include the entire application process, it was about six months in total. +**Adriana:** Cool. I would like to ask—I know that hotel is a very big project—what specific areas did you contribute to? -**Adriana:** -How do you get paired up with a mentor? +**Mole:** Okay. So the, I'll go before the actual internship. I remember scrolling through the different repos on GitHub and I was like, oh my god, where do I even start from? I found solace in the collector contrib. In outreachy, the project that I worked on was building a logging bridge in Golang, and by the end of the internship, I was able to build the zero log bridge for Golang. So those are the projects that I'm working on, and currently I still contribute to the collector contributions. -**Mole:** -When you pick a project, each project has mentors who volunteer to guide participants. In my case, Jurassi and Yuri volunteered to be mentors for the OpenTelemetry project, so they became my mentors. +**Adriana:** Wow! But like it's funny to see how you went from not knowing about school to completing your internship to building something that the community is making. That's huge. Could you share with us how the outreachy internship helped you secure your first job? -**Adriana:** -What specific areas did you contribute to in OpenTelemetry? +**Mole:** Okay. So what happened was I had done a lot of work and then I was writing articles for Signals. Signals is a very cool collab observability platform that also supports open telemetry natively. I think my former boss, Otus, the CEO of Semantex, saw some of the articles that I wrote and, you know, had a conversation with me that, oh hey man, you worked on open telemetry, right? I'm like yeah, I did. What did you do? I showed him what I did. We got on a call, I did some demos on how auto instrumentation works. He was like, wow, this is so cool. I was like, yeah, it's so cool. Oh, open telemetry is so cool. And he was like, okay, they want to integrate their platform with open telemetry and am I up for it? I'm like, okay, sure, I'm up for it. Let's go. And at Semantex, what I was working on was building an exporter for their platform so that they can receive data from open telemetry data into their platform directly using the open telemetry collector. So that was the project I worked on while at Semantex, and it was very stressful but it was very cool. -**Mole:** -Before the internship, I explored various repos on GitHub, but I found solace in the Collector contrib. The project I worked on during Outreachy was building a logging bridge in Go. By the end of the internship, I successfully built the Zero log bridge for Go. +**Adriana:** That's amazing. I think it really underscores like two things. First of all, outreachy kind of gave you the tools to contribute to open source, contribute to open telemetry, but then you did another awesome thing, which is putting yourself out there, writing these blog posts, which is so important. It's so hard sometimes to put ourselves out there. And you know, I think we all in tech suffer from impostor syndrome at some point or another. Maybe constantly feel it comes in waves for me. And you just have to push past that impostor syndrome and do it anyway. Mega kudos to what you're doing. Now you've made a few references about learning open telemetry, learning Golang. What did you use for—can you dig a little bit deeper on how you learned open telemetry? -**Adriana:** -Can you share how the Outreachy internship helped you secure your first job? +[00:15:50] **Mole:** Okay, yeah. I'll start from my first introduction to open telemetry. I just Googled open telemetry, entered, and I saw the documentation. I was like, oh, this looks so cool. And then I tried to read the docs and my brain wanted to blow. So I went to—I'm more of a visual learner, right? So I went to YouTube and I typed "what is open telemetry?" and Henrik, he is the owner of the channel—is it observable?—and I was just watching a lot of his videos at that time. I would say those videos gave me a lot of context because how I learn is I like specific information before general information. So I used his videos a lot to ramp up my knowledge on open telemetry. And then our wonderful co-host here, Adriana—her blogs are so awesome. I read a lot of her blogs at that time just to ramp up because the documentation was very overwhelming for me. After reading those two things, I then began to go to the docs and actually narrow down on what I was looking for. I really knew what I was looking for because documentation sometimes can be very overwhelming because there's so much information packed in the documentation and it's trying to talk to different classes of people—both beginners, intermediate, and advanced people. So beginners sometimes always feel overwhelmed right from those documentation. But I'll say, when it came to running the open telemetry demo, the docs did a good job. -**Mole:** -I had done a lot of work and was writing articles for Sematext. The CEO, Otus, saw some of my articles and reached out to me about integrating their platform with OpenTelemetry. I showed him what I did, and he offered me a position to build an exporter for their platform using the OpenTelemetry Collector. That’s how I got my first job. +**Adriana:** Wow, that's great. Personally, I'm more of a reader. I prefer to consume text-based resources than watching. -**Adriana:** -That’s amazing! It really underscores how Outreachy gave you the tools to contribute to OpenTelemetry, but you also put yourself out there by writing blog posts. What were your resources for learning OpenTelemetry? +**Mole:** Cool. I'm with you on that. -**Mole:** -I initially stumbled upon the OpenTelemetry documentation, but it felt overwhelming. As a visual learner, I turned to YouTube and found Henrik's channel, which provided a lot of context. I also read Adriana's blogs, which were immensely helpful. After getting a foundational understanding, I could go back to the documentation, knowing what I was looking for. +**Adriana:** That's interesting to know. Good for you. Of all these resources that you mentioned, the YouTube videos, which of them would you say was the most useful for you to learn about? -**Adriana:** -What specific resources were most useful for you? +**Mole:** So you want me to pick one? If you tell me to pick one, I'm very biased. I always go for videos. So, if you tell me to pick one, I'll go for Henrik's channel. Hands up. -**Mole:** -For videos, I would definitely go with Henrik’s channel. For text, I would recommend Adriana's blog on Medium. +**Adriana:** I mean, Henrik produces amazing content, so I can definitely vouch for that. -**Adriana:** -What was your experience like creating your first PR? +**Mole:** Yep. So, for video, I'll say Henrik's channel, Is it observable? If you haven't, go and check it out. It's awesome. And then for text, Adriana's blog on Medium. I don't know if she's hearing me right now, but Adriana's blog, yeah. So, you should also check that out. -**Mole:** -Creating my first PR was a whole experience. I spent about two weeks learning and then realized I needed to make my first PR. I studied other people's PRs and found an issue that I could tackle. With the guidance of others, I was able to submit my first PR, and when it got merged, I was ecstatic! +**Adriana:** That's cool. -**Adriana:** -Can you tell us about your experience creating the OpenTelemetry exporter? +**Mole:** Yeah. -**Mole:** -When I started working on the exporter, I found that there was no documentation on how to create one. I had to study existing exporters and use the OpenTelemetry Collector Builder. It was a lot of trial and error, but eventually, after much effort, I succeeded in building the exporter. +**Adriana:** Now, you said you have a preference for the videos, right? Is there a reason why? -**Adriana:** -What was your experience learning Go for this project? +**Mole:** Okay. So how I reason is I'm a very imaginative thinker. I can sit down and just go on and just be imagining a lot of things. I like pictures, visual things because if I'm just reading a bunch of text of a concept I haven't understood yet, I get bored. I will just easily get bored and zone off. So, if I'm watching a video, you know, the sound of the person, the tone, the infographics, the pictures, you know, just keeps me excited to keep going. Before I know it, I've tricked my brain into learning a new concept, right? So, yeah. So that's just what works for me. I've observed it over time. -**Mole:** -Learning Go was challenging because I had to apply it directly to real-life projects. I made many mistakes, but my team at Sematext was supportive. I took a course that provided context, which helped me understand the language better. It was a tough but rewarding experience. +[00:20:01] **Adriana:** That's awesome. Yeah, makes sense. The next question that we were wondering about, you mentioned that you use the hotel docs as a reference. One of the goals of these sessions that we have like with hotel me is also to provide feedback to the hotel community on areas for improvement. Is there anything that you found about the third-party documentation more useful than the official hotel docs and what would you do to improve the docs experience if it were up to you as a new learner? -**Adriana:** -What plans do you have for documenting the exporter for future users? +**Mole:** Okay. So, sorry, this is an issue with multiple questions. The first question is what made me prefer the docs or maybe leave the docs? -**Mole:** -While there is a template for exporter documentation, I think it would be more beneficial to write about the thought process rather than a step-by-step guide, as each context is different. Additionally, I believe sharing experiences and practical advice would be more helpful for new learners. +**Adriana:** Yeah. Why did you choose third-party documentation over the hotel docs for starters? -**Adriana:** -What feedback do you have for the OpenTelemetry documentation? +**Mole:** Okay. Well, because number one, there’s no video documentation yet. That's why. -**Mole:** -I think creating video documentation would be beneficial for visual learners. The documentation can be overwhelming, so having video resources would help bridge the gap for different kinds of learners. +**Adriana:** Fair. -**Adriana:** -What was your first impression of the OpenTelemetry community? +**Mole:** But also, when I'm reading someone's content, right? Like I'm learning your content or someone else out there, I am not just reading. I can feel human connection. You can say something like, "Ah, I got stuck at this." This is what I had to do, and then this got solved. But in a documentation, it's just man, it's like reading a textbook, you know? Do this, do this, do this, do that, move on, you know? But when you're writing your content, you can say, "Oh, do this. If this does not work, try this." So, I'd say that's why I preferred. When I entered the documentation, I got overwhelmed because I was scrolling through a lot of things. I didn't understand what instrumentation was, and documentation has so many links that link you to so many things, and before you know, you're lost in the rabbit hole of the documentation. So, I’d be reading something, I'm like, what is instrumentation? I'll see a link, click the link, and I'm going to a different entire part, and then before you know, I'm like, where am I? What am I learning? So, I got overwhelmed. -**Mole:** -I found the community to be very welcoming and supportive. I felt encouraged from the start, especially from my mentors. The OpenTelemetry community is caring, and I love being a part of it. Meeting everyone in person at KubeCon was a fantastic experience. +**Mole:** What made me go back to the docs? I already had a foundation. So I was not overwhelmed when I saw what instrumentation was or when I saw what a collector was. So imagine someone that is new that does not know anything about a collector trying to learn open telemetry. They're seeing open telemetry. What is open telemetry? You can't really define open telemetry without the four components: instrumentation, collector, and you’re like, what? What is an open telemetry operator? Oh my gosh, they're seeing Kubernetes—like where am I, you know? So, I already got that foundational knowledge. I could now go and head for the docs. Because now I already knew what I was looking for in the documentation. So, let's say I want to do a particular task, I can just search for that particular part in the docs and focus on just that side. -**Adriana:** -Have you made any recent contributions to the OpenTelemetry community? +**Adriana:** And what would you do to improve the doc experience? -**Mole:** -Yes! I’ve been advocating for OpenTelemetry in Nigeria. I realized it wasn't a widely discussed topic here, so I started attending conferences to share knowledge about OpenTelemetry and how to get started with open source. I'm also still working on pushing the exporter into the OpenTelemetry Collector contrib. +**Mole:** I’d say I'd make video documentation for those who do follow the hotel content. We do have our first video, like instructional video, on the hotel YouTube channel. There will be more to come. You can also satisfy both at the same time. ---- +**Adriana:** That is a super fair point. It's interesting too, like what you were saying, that basically you use the hotel docs for more of a directed search, right? Like first understand stuff from people's experiences and then, okay, I want to learn about this, I will go into the docs to learn about that. -**Adriana:** -Thank you so much, Mole, for sharing your experiences. And thank you, Victoria, for co-hosting with me today. We will see everyone at the next event! +**Mole:** That’s the approach that has worked for me to learn everything and that’s what I’ll keep using. By the way, Adriana, I started learning Kubernetes—just a side note. + +**Adriana:** Oh, right on! Good luck! + +**Mole:** Thank you! + +[00:25:00] **Adriana:** Yeah. Could you tell us what your experience was like creating your first PR? + +**Mole:** Ah, okay. We're here. So, creating my first PR, I’ll say it was a whole experience, right? After watching some videos, reading some documentation, I think I did this for two weeks. I was just learning a whole lot and then I was like, you know what? You need to make your first PR. You can't waste any more time. I went to the repos and I was like, okay, so which repo do I pick? I think that was the first issue I had. Open telemetry has a bunch of repos, so I was like, which one do I go for? For a while, I was just lost. Do I contribute to NodeJS? Do I know Node? I don't know Node. Let me go to collector. I don't know—I'm like, okay. + +**Mole:** Then, what I started doing was I started reading other people's PRs. I would literally go to an issue that is closed, go under the PR, you know, because when an issue is closed, the link to the PR that closed the issue will always be there. I would literally go and be studying their code. That's why I did a lot. I don't think I've said this anywhere before, but that's why I did a lot. I would go and literally be studying people's PRs, studying their code, studying how they framed things. Okay, this is how you write this in Go because remember I was still learning Go at this point. So, it was like a—I was learning so much in one month because I was studying people's code, I was doing a course at the same time, right? + +**Mole:** After studying people's PRs for a while, I gained some confidence. I found an issue that someone had done something similar. That's the beautiful thing about open source. Sometimes you want to update a bunch of components at the same time. I saw a similar issue to one of the PRs I had studied and I was like, I can tackle this. I asked, “Hello, my name is this. Can I please be assigned to this issue?” I didn’t think the person would answer me, and he was like, "Oh yeah, sure, why not? Go ahead." I think his username is MX-Psi. I don't know his real name. + +**Mole:** He was like, "Why not go ahead?" I tried it out. I made mistakes, obviously, because I was new to Go. I didn't understand the whole lint check thing. But he, you know, like I said, the open telemetry community is so wonderful. They do not look down on you. They don't look down on where you are as long as you're honest, right? Don't come and prove to be a senior developer when you're actually not. Just come clean. I told the guy, man, I haven't done this before. Please help my life. I went to his DMs and he was like, "Okay, do this, do this, do this." At the end of, you know, I think after two days, it got merged. I was like, "Yay, my first PR!" And then from there, it was just a full journey, right? It was a full journey, you know, from there. Really, really nice journey. + +**Adriana:** I have to agree with you that like that first PR getting merged is like the most glorious experience ever. But I will tell you, like every time I have a PR merged, I do like a little happy dance at home. It's like, oh my god, I'm up for something. By the way, I gotta say hats off to you in your approach to working on issues too, studying the PRs and the code behind the PRs. You have the heart and soul of a software engineer. It's like you were made for this. That's amazing. + +**Adriana:** I think this is a good lead-in to our next area that we want to talk about, which is—I believe you mentioned earlier that you did some work on creating an open telemetry exporter for the collector. Do talk about that. What was that experience like? + +[00:30:00] **Mole:** Okay, I have a lot of funny experiences. Sorry. When I got the project with my former company, Semantex, I was like, okay, let's do this, right? How hard can it be? So I went to the documentation and there was no documentation on how to create an exporter, right? The only documentation was how to create a receiver, and I was trying to understand it, and it wasn't working. One thing I realized later that I didn't know then was that there's not really a standard way of creating an exporter. Well, there’s a template you have to follow, but there’s no standardized way because exporters depend on your vendor back end. So at Semantex, we were receiving logs over influx 5 protocol and we were receiving metrics over elastic bulk index format, right? So I had to build an exporter that would convert hotel data to these two sources. + +**Mole:** What I did, like I did for my other PRs, was I started studying other people's exporters. I studied the influx DB exporter because that way our metrics would look like. Then I studied the elastic search exporter. I did a lot of study. I tried to break down a lot of things with the help of my assistant—great assistant. I tried to break it down. I tried to fit things in our own context. So how do I take this part from here, put it here, remove this part, you know? It was a whole trial and error. There was really no content that I could look to; okay, this is what assisted me in this journey. Oh, sorry, pardon me on that. I remember the open telemetry collector builder. + +**Mole:** While learning how to build the exporter, I had to use the OCB, the open telemetry collector builder, and that's when I came across Bind Plane's content on YouTube. They have a full series on open telemetry collector—a very wonderful series. I enjoyed every bit. That's why I learned how to use the open telemetry collector builder. The OCB is very fun. It's so cool, man. It's like you can build your own custom collector, you know, with whatever component you want. It's cool. It's like play-doh, right? You can build as you want, play with it as you want, connect things, join things, break things, and you know, it's fine. + +**Mole:** I learned how to use the OCB. After lots of trial and error, I got my first metric into Semantex, and it was an exciting feeling, right? After a while again, I got the logs in, and then awesome. Then I built the exporter. That was the summarized version of the experience of building the exporter. + +**Adriana:** It's really interesting. Believe me, I'm sharing in your excitement right now. I can only imagine. So for our next question, could you tell us a bit about learning Go to be able to work on this hotel exporter that you just talked about? + +**Mole:** Learning Go has been a very interesting journey because I did not learn Go the normal way, right? The normal way is just watching a course and just building some interesting projects, you know, maybe to-do app stuff. I learned Go the hard way because I was actually learning it and had to apply it directly into real-life projects that go into production, right? I remember I had a lot of help from the people I worked with at Semantex—Bora and Akshhat, really cool guys, part of my team. They were so—I’d make mistakes, and they would be like, no, this is not the right way to do this. I know this is the YouTube way, this is the course way, but in production, this would break, this would fail. + +**Mole:** I say I learned Go hands-on, you know, contributing to things, right? It was a very interesting journey. I actually had a course from Code Cloud that gave me a lot of context. I could understand code; if you give me any Golang code, I can know okay this is what this is, what this is, this is a struct, this is an interface, this is what this does, this is what it should do. But then I had a lot of building things in real life. I feel like that’s really the best way to learn, you know, although it’s very stressful. You feel very dumb sometimes because some things—I remember sometimes at night I’d literally be up by 2:00 a.m. and the test kept failing, and I’m like please just run, please just run. I’d literally be praying. I’d be praying over and over and over again. Finally, it would run, and I’d be like yay, let’s go! Another milestone achieved! + +**Adriana:** Cool. Throughout our conversation, you’ve talked a lot about the resources that helped you to learn open telemetry, that helped you to learn Golang, helped you to build the auto exporter. Now, could you tell me what plan you had for documenting this exporter that you built so that other people could use it? + +**Mole:** What do you mean document it? Do you mean like you want me to document my journey or how I built the exporter? Can you clarify documentation for me? + +**Adriana:** Documentation for the exporter itself, for those who would be using it. + +**Mole:** Okay. So, I think when you're contributing an exporter, there's actually a template of documentation that you have to do. I’ll answer your question from the standpoint because I think Adriana and I spoke about this last time. Even if I document it, and she actually put an idea that I should write an article on how it's not really possible to show you how to build an exporter because I tried. I actually sat down after building the exporter, and I was like, okay, how can I help the community with this? Even if I show you how I did it, it's not going to be useful to you because your context might be different, right? + +**Mole:** So what I would say is maybe the article I can write on or maybe the documentation will be: don't give up. Try, try, try again. Even when it fails, keep trying. It's going to work. Another thing I'll say is study other code. Studying other people's code—like I’ll say that’s one thing that has helped me in my whole coding journey and has really accelerated my speed. I always studied people's code and understood the patterns in the code. From understanding those patterns, I can now create my own, you know, find how to remove parts and then join to what I'm looking for. + +**Adriana:** That's great. It sounds like a great plan. + +**Mole:** Because I guess from the sounds of it, like the other two components—receivers and processors—I think there is some documentation around creating those because it's more formulaic, right? Whereas exporters are not. But the thing that's in common is like what's the thought process that you need for creating it. + +**Adriana:** But not just this is the exporter template, this is step by step. + +**Mole:** Now, yeah, I think that would be such a great blog post to write because I think, you know, again, it all goes back to what you were saying too about some of these third-party sources being about personal experiences and having that human touch, human connection. I think that would be so beneficial. It would also be interesting—I'm just giving an idea—to put in the docs something along the lines of like why is it that we don't have a template for creating open telemetry exporters? Because that was like, I remember at some point I was looking into that as well, and I'm like, I don't get it. Why is there no template for this? + +**Mole:** Yeah, absolutely. There’s no official template. I think it’s when you start and then you speak to the maintainers, they tell you, oh no, no, no, this should be here, this shouldn’t be here. I’d be like, okay, let’s change things up, right? + +**Adriana:** Yeah, I think that’s great. + +**Mole:** That’s great feedback. + +**Adriana:** The other thing that you mentioned that I want to dig into is the open telemetry collector builder. Specifically, what was your experience around that? Because I’ve played around with the builder; you’ve played around with the builder. Was it as easy as you thought it would be? If not, what would you say is some feedback that you would like to give on what can be done to make it easier? + +**Mole:** Okay. You know, in my journey—and I’m going to be very honest here—in my journey with learning to try to get the understanding of the exporter, I didn’t really get any good information from the documentation. I’ll say for a while I just did not look at the docs anymore. I just went to YouTube and was doing the research. That’s why I found Bind Plane, right? They did a whole video on how to build your own custom collector—a 45, 46-minute video—and that basically showed me, okay, yeah, this works, this can do this. + +**Mole:** I think also the readme files—some of the open telemetry collector builder are more documented in the readme file than well at that time than the documentation itself. So I think I got more value from the readme files at that time than the actual documentation. I don't know if it has been updated yet. I'll probably check, but yeah. + +**Adriana:** I did just get a PR merged in the hotel docs about—because when I was doing it, I’m like, okay, I built this DRO—how the hell do I containerize it? And there was no documentation around that. I had to ask, and then I wrote a blog post about it. But I’m like, it should be in the docs too. I just had that PR merged, I think today or yesterday. + +**Mole:** That’s cool. That’s so cool. + +**Adriana:** Yeah. So that was my experience with the open telemetry collector. + +**Mole:** Awesome. + +**Adriana:** I think I have just one more question. Could you tell us a bit about the open telemetry community? What’s your first impression of the community? How have you been able to collaborate with the community from the time you contributed to it as an intern and beyond that? + +**Mole:** Okay, I love the open telemetry community so much. My first experience with them was, I introduced myself: hello guys, I am—so I used to be introverted in some sort of way, but the people don’t really believe it because I’m very outspoken. But I used to be introverted, right? I was kind of shy because I was like, okay, I’m a newbie; I don’t really know much about stuff; I don’t know anything about open source. Who’s going to listen to me, right? I was like hello, my name is Mole from Nigeria. Nice to meet you all. And people said hi back. People waved back. Oh, nice to meet you, welcome to the open telemetry community. I was like, oh, that’s so cool. + +**Mole:** When I stumbled into a lot of like—back to my mentors, right? So Jurassi was my major mentor for the outreach internship. We had a lot of calls, right? In those calls, he just kept encouraging me that I know this is tough, but don’t worry, you’re going to get it. Just keep pushing, you know, keep trying. I believe in you. He’d come and say, I believe in you, I know you can do this. + +**Mole:** It was like a collective effort. I’d go to some people's DMs and I’d be like I’d be crying that please help me. I don’t understand this issue. How do I solve it? You’ve already assigned it to me. I tried this; it did not work. They’d be like, "Okay, you know what? Try this. Make this change." I remember when I wrote my first blog, my first blog post on open telemetry, and I had already read some of Adriana’s blog posts. I was like I’m going to send a DM to Adriana because I like to hear her thoughts on my blog post. I was like hello Adriana, hi, I’m Mole. Please can you review my blog post for me? She replied, and she reviewed my blog post. I was like wow, she doesn’t know me from anywhere, you know? We are not in the same continent or in different tribes, and she’s actually willing to assist, right? + +**Mole:** I would say the open telemetry community is very loving. I can’t really vouch for a lot of open source communities, but I can vouch for the open telemetry community. They’re really cool and it’s a very nice community to join, by the way. I’m now a member of the open telemetry community. You can also text me. I’m very cool if you have questions about it, and she can text me. I’m cool. They are also cool, right? The community is loving, nice, really caring, and I love it. But then when I got to KubeCon and I met everyone in person, it was cooler. I met Adriana in person. I met Henrik. I met Tras and some other really awesome people. It was really nice. + +**Adriana:** I’m going to completely agree with everything you just said because I just recently joined the community too, and everyone has been amazing. Plus one to that; Adriana is very cool. It’s been nice getting to interact with her. + +**Adriana:** One more question before we go. Have you made any recent—I believe as an open telemetry community member now, you have to contribute to the open telemetry community. Have you had any recent contributions to the community that you’d like to talk about? + +**Mole:** Yeah. Okay. So what I have been doing majorly right now is—when I came back from KubeCon, I think when I started learning the whole open telemetry thing, I realized that open telemetry was not a topic that was spoken about a lot in my country locally, you know, in Africa. Kubernetes is very popular here; everyone knows Kubernetes, right? So, open telemetry is not really heard a lot. I started advocating a lot about it. I think that’s how I even slided into my developer advocate role. + +**Mole:** Currently, I’m a developer advocate. I’m not a full software engineer anymore. I started attending conferences, talking about open telemetry, showing people how to get started with open telemetry, how to get into open source, you know, because I experienced the dramas, the struggles, you know? I understand when people come to me and tell me I’m so confused about open source; how do I start? I can also give them practical steps I took to get started. I would say I’ve been doing more of that, and then I’m actually still—the exporter has not been built fully, but I’m still pushing it into the open telemetry collector contrib. So, I’d say that’s the—aside from related to code, those are the contributions that I’ve been making, but then I’ve been doing more of advocacy about open telemetry in my country. + +**Adriana:** That’s amazing. And I think that makes a really good point as well because contributing to open telemetry isn't just about the PRs that you're making; it's also about the advocacy, the bringing that awareness. I think that's great. + +**Adriana:** I think this wraps up the main Q&A, and we do have a few minutes left. If anyone from the audience has any questions that they would like to ask at this point, we are happy to take your questions. + +**Mole:** I feel like we need some music. This is where you take out the guitar for this. + +**Adriana:** Dang it! Who’s the music? Is that you? Was that—I think that was Henrik, our amazing producer. + +**Mole:** Well, it doesn’t look like we have any questions. So, what I will say is we’ll wrap up, but before we go, a few quick announcements, promotions. First of all, for anyone who is planning on attending OpenSource Summit in Denver, Colorado, there’s also going to be as a collocated event, there’s an open observability con and hotel community day that is going on. The CFPs for hotel community day are closed, but open observability con CFPs are still open. That’s going to be at the end of June. Feel free to submit CFPs to attend if you’re in the area or if you’re planning on attending. OpenSource Summit also is like lots of fun. It’s like KubeCon lite. It’s all the awesome people who attend without the flash, so it’s tons of fun. + +**Adriana:** The other thing, for anyone who is interested, as I mentioned, I’m one of the maintainers of the hotel end user SIG. Victoria is a newly joined member of the hotel end user SIG. If you’re interested in joining us, seeing what we’re up to, Henrik just flashed a link for our GitHub repo. We do a bunch of things like these types of events. We have hotel me. We have hotel in practice, where if anyone who's watching is interested, if you have a topic—an observability hotel topic that you’d like to present on, maybe you’re testing out a topic for a conference for a talk, want to flesh it out, use us. Just DM us on the hotel end user SIG Slack and just let us know if you’re interested in doing a presentation for hotel in practice. + +**Adriana:** We also run surveys in collaboration with the various hotel SIGs. Stay tuned; there will be more surveys heading your way. The surveys are a great way for us to get feedback from our user community on ways to improve the SIGs. That information is super important, super valuable to the community. Thank you to the folks who have responded to the surveys. Thank you to the folks who have engaged in the hotel end user SIG. + +**Adriana:** Also, for those who aren’t able to attend and for those who were able to attend, tell your friends that this video will be posted on our hotel YouTube channel. It’s hotel-official on YouTube. You can catch all of our previous hotel in practice and hotel me sessions. We do have our first how-to hotel video on the hotel YouTube channel, so for those who are visual learners, this is your opportunity to check that out. + +**Adriana:** Thank you so much, Mole, for joining and for Victoria for co-hosting with me today. This has been lots of fun. I always enjoy having these conversations, and we will see everyone at the next event. + +**Mole:** Sure. Thanks, Adriana. + +**Victoria:** Thank you. + +**Mole:** Thanks, Adriana. Thanks, Victoria. It was nice meeting you. ## Raw YouTube Transcript diff --git a/video-transcripts/transcripts/2025-05-02T18:18:11Z-even-more-humans-of-opentelemetry-kubecon-eu-2025.md b/video-transcripts/transcripts/2025-05-02T18:18:11Z-even-more-humans-of-opentelemetry-kubecon-eu-2025.md index f9a2c59..43c0296 100644 --- a/video-transcripts/transcripts/2025-05-02T18:18:11Z-even-more-humans-of-opentelemetry-kubecon-eu-2025.md +++ b/video-transcripts/transcripts/2025-05-02T18:18:11Z-even-more-humans-of-opentelemetry-kubecon-eu-2025.md @@ -10,80 +10,116 @@ URL: https://www.youtube.com/watch?v=eZ3OrhxUAmU ## Summary -In this YouTube video, a diverse group of professionals, including Marylia Gutierrez from Grafana Labs, Adriel Perkins from Liatrio, and Alolita Sharma from Apple, discuss their experiences and insights regarding OpenTelemetry and observability in software engineering. They explore the importance of OpenTelemetry as a vendor-neutral standard that enhances collaboration among various companies and contributes to a more effective observability ecosystem. Key topics include the evolution of their involvement in OpenTelemetry, the significance of observability in understanding complex systems, and the role of different telemetry signals like metrics and traces in improving application performance. The participants emphasize the community aspect of OpenTelemetry, highlighting how collaboration and shared goals drive innovation and problem-solving in the field of observability. +In this YouTube video, several experts in the field of observability and OpenTelemetry come together to share their experiences and insights. Notable participants include Marylia Gutierrez from Grafana Labs, Adriel Perkins from Liatrio, Jamie Danielson from Honeycomb, and Alolita Sharma from Apple, among others. The discussion revolves around the significance of OpenTelemetry as a community-driven, vendor-neutral standard for observability, which allows for better data collection, sharing, and interoperability across different systems and platforms. Participants reflect on their journeys in OpenTelemetry, emphasizing the importance of observability in understanding complex systems and improving engineering practices. They also discuss their favorite telemetry signals, highlighting the roles of metrics and tracing in diagnosing system issues and enhancing performance. Overall, the video showcases the collaborative spirit of the OpenTelemetry community and the critical role observability plays in modern software development. ## Chapters -Here are 10 key moments from the livestream with their corresponding timestamps: +00:00:00 Introductions of speakers +00:02:40 First involvement with OpenTelemetry +00:06:31 Observability definitions and insights +00:10:30 Personal experiences with observability +00:12:39 Importance of OpenTelemetry +00:15:30 Community and collaboration in OpenTelemetry +00:18:36 OpenTelemetry protocol significance +00:19:20 Working on OpenTelemetry Collector +00:21:40 Favorite telemetry signals discussion +00:24:00 Closing thoughts on observability -00:00:00 Introductions of speakers and their roles in OpenTelemetry -00:05:30 Personal journeys into OpenTelemetry -00:10:15 The importance of OpenTelemetry in centralizing data sources -00:14:20 Observability: definition and personal significance -00:20:40 How OpenTelemetry facilitates collaboration across companies -00:25:10 The role of OpenTelemetry in cloud-native applications -00:30:00 Discussion on the OpenTelemetry protocol and its impact -00:35:45 Favorite telemetry signals: metrics vs. traces -00:40:00 The community aspect of OpenTelemetry and its collaborative nature -00:45:30 Insights on the future of observability and OpenTelemetry's role in it +**Marylia:** My name is Marylia Gutierrez. I'm a staff software engineer at Grafana Labs. And I also work on a few different groups on OpenTelemetry. -# OpenTelemetry Community Insights +**Adriel:** My name is Adriel Perkins. I'm a principal engineer at a consulting company in the United States called Liatrio. We're both an end user, but I'm also a contributor in the OpenTelemetry project. I'm co-lead of the CI/CD SIG, with Dotan Horvitz, a CNCF Ambassador. And we work in the Collector as well as the specification repository. -## Introductions -- **Marylia Gutierrez**: Staff Software Engineer at Grafana Labs, involved with various groups in OpenTelemetry. -- **Adriel Perkins**: Principal Engineer at Liatrio, contributor to the OpenTelemetry project, co-lead of the CI/CD SIG alongside Dotan Horvitz. -- **Hanson Ho**: Works on Android observability at Embrace. -- **Jamie Danielson**: Engineer at Honeycomb, focusing on instrumentation libraries and OpenTelemetry. -- **Mikko Viitanen**: Product Manager at Dynatrace, maintainer of the OTel Demo App. -- **Damien Matheiu**: Maintainer for OpenTelemetry Go, code owner for profiling in the Collector, and approver for the eBPF profiler. -- **Jacob Aronoff**: CTO at Omelet, a startup focused on observability and telemetry pipelines. -- **Alolita Sharma**: Leads AI/ML observability engineering at Apple. +**Hanson:** My name is Hanson Ho and I do Android stuff. Specifically observing the Android stuff, at Embrace. -## Early Involvement in OpenTelemetry -Alolita Sharma mentioned her first encounter with OpenTelemetry while exploring an enterprise observability solution. She found it to be an excellent way to centralize data from various sources, allowing for a holistic view of observability. +**Jamie:** My name is Jamie Danielson. I'm an engineer at Honeycomb, and I work on instrumentation libraries and OpenTelemetry. -At Embrace, the team sought to better serve the community, leading them to OpenTelemetry as an open framework for contributing to a common standard. This transition allowed them to expand their data collection capabilities without drastically modifying their SDK. +**Mikko:** I’m Mikko Viitanen. I work as a product manager for Dynatrace. Then I'm also a maintainer in the OTel Demo App. -Jamie Danielson began working with OpenTelemetry more extensively in 2021 at Honeycomb, focusing on the Collector and instrumentation libraries. +**Damien:** I am Damien Matheiu, and I do many things at OpenTelemetry. I am a maintainer for OpenTelemetry Go. I'm also a code owner for everything profiling on the Collector. And I'm an approver for the eBPF profiler. -Damien Matheiu started his journey in OpenTelemetry around three years ago, contributing to the OTel Demo App which provided a hands-on learning experience with instrumentation and the Collector configuration. +**Jacob:** My name is Jacob Aronoff. I am the CTO at Omelet. We're a new startup that does observability, telemetry pipelines, and OpenTelemetry very generally. -Jacob Aronoff began his work with OpenTelemetry while at Lightstep, contributing to the telemetry pipelines team and focusing on Kubernetes aspects of the OpenTelemetry Operator. +[00:02:40] **Alolita:** Hi, everyone. I'm Alolita Sharma, and I lead AIML at Apple, for observability engineering, and observability infrastructure. I started in OpenTelemetry. I had actually been working on the observability world for a little while, and eventually you end up finding out about OpenTelemetry, and you find that this is, like, a really cool way to work and do not have dependency. My first involvement with OpenTelemetry was actually, I got asked to look at an observability enterprise solution, and that was when I discovered OpenTelemetry, and specifically the Collector. This discovery was because we had a lot of different data sources from different places, and we wanted to centralize them so that you could get a holistic view. So that was my like, first introduction when I was finding OpenTelemetry and its Collector. And I said, this stuff is really, really, really cool. -## Perspectives on Observability -### What Does Observability Mean? -Observability allows engineers to discover insights they didn't initially know they needed. It enables continuous learning and improvement by providing visibility into the system's health. +**Hanson:** Embrace looked to see how it could better serve the community. So OpenTelemetry was an obvious thing to look at being an open framework folks contribute to Common Standard. So when I saw it, I was like, this is great. This is kind of what we need. So we had a proprietary SDK collecting proprietary signals sending to our own servers. With OpenTelemetry, we were able to expand where we send this data to open source open Collectors, without changing, you know, a ton of stuff in the guts of the SDK. -**Key Insights:** -- Observability is about asking insightful questions and acting on the answers. -- It helps understand application behavior and operational health. -- Observability is crucial in pinpointing issues within complex systems, such as telecom networks, where it was essential to resolve call drop issues. +**Jamie:** So when I started looking at it, I was like, hey, look at mobile. Mobile's great. And at that point, not a lot of people were looking at mobile world of OpenTelemetry with a few exceptions. And I think now even like since a year or so that I've been involved, things have grown a lot, and a lot more interest. So, I'm really happy to get involved. -### Personal Experiences -Many speakers shared their frustrations with traditional logging and the challenges of convincing teams to adopt more holistic observability practices. They emphasized the importance of continuous learning and the ability to interpret data effectively. +**Adriel:** When I started at Honeycomb in 2021, the team that I was on started working on OpenTelemetry more, and working through observability and instrumentation libraries. So I started working on the Collector and a little bit of Java before sort of settling into OpenTelemetry JavaScript. And so I've been there, you know the last three years, becoming an approver and more recently a maintainer of the project. -**Examples:** -- **Damien** shared his transition from SRE roles to focusing on observability to improve incident response. -- **Alolita** highlighted her commitment to learning and improving through observability. +**Damien:** I started with OpenTelemetry around three years ago, and I started doing small contributions to the OTel Demo App. And I found that’s a great place to learn the basics of instrumentation and a little bit of the Collector, configuration and, and kind of I found the Demo App actually provides a little bit of everything around code and hands on. So I find it well. -## The Role of OpenTelemetry -OpenTelemetry is viewed as a central component in observability stacks. It provides a vendor-neutral standard that allows different contributors to collaborate and build solutions that benefit the entire community. +**Mikko:** I was working at a different company in an observability team, and I was rather frustrated because I had a hard time. I was convinced already that we should not just be using logs, but we should also do tracing and therefore use OpenTelemetry. And it was very hard to convince some of our folks in engineering there at the time. And so kind of as a New Year resolution in 2022, I decided that I would start watching the OpenTelemetry Go repository and start contributing. And one thing led to another. And, a year later, I got a full time job working on OpenTelemetry. -**Highlights:** -- OpenTelemetry fosters collaboration among competitors, creating a shared understanding and vocabulary for observability. -- It allows end users to instrument their applications seamlessly across different vendors. +**Jacob:** I started my OTel journey at Lightstep, my former employer. I started on the telemetry pipelines team, working with some amazing contributors to the OTel ecosystem. I started on the Kubernetes side of things with the OpenTelemetry Operator, working on upgrading the target allocator, which does horizontally sharding of, scrape targets for Prometheus, for the Collector. Very technical, but very important part of the ecosystem. -## Favorite Telemetry Signals -The community shared their preferences for telemetry signals, often highlighting the importance of tracing and metrics: +[00:06:31] **Damien:** I got started with OpenTelemetry more than six years ago now. And I was at AWS at that time, and I've been always very involved in open source projects all the way from the beginning Linux. And, in my journey in the cloud native world, as I was building platform services at AWS, we decided to build out a new generation of Kubernetes native services. And, it was really exciting that we took the opportunity to get involved, as a team in all the new shiny of open source observability projects. And, of course, OpenTelemetry was at the forefront. This is right after OpenTracing and OpenCensus got together and combined to form OpenTelemetry, and it was an exciting change to now see this beautiful brand new project with all contributors from both two projects combining together and getting new contributors like me involved. -- **Tracing** is favored for its ability to provide end-to-end visibility in applications, helping teams to pinpoint issues effectively. -- **Metrics** serve as a foundational signal, offering a straightforward way to introduce observability concepts to new users. +**Marylia:** Observability, to me, means that you are able to find the things that you didn't even know that you wanted to know, because you have a lot of information. But just having information doesn't mean anything, if you don't know how to interpret. So is a as like they say that it actually comes from like mechanic engineering. That was just trying to understand like the system where you can just extrapolate this for everything. So finding a way to understand everything from your system can even bring to your life observability, understand what is going on. -**Conclusions:** -- Many believe that while all telemetry signals are important, **tracing** stands out due to its powerful insights and versatility. -- The combination of **metrics** and **traces** is crucial for understanding application behavior and infrastructure performance. +**Adriel:** What does observability mean to me personally? It enables me to find things out that I didn't know and improve. And I've always been someone who loves continuously learning and continuous improvement and observability is the thing that helps me do that, because there's a lot of things that I don't know. I think the more that I find out that I do now, the more I realize there's more things I don't know. -In conclusion, the OpenTelemetry community is marked by collaboration, shared learning, and a commitment to improving observability across diverse systems. The participants view OpenTelemetry as an integral tool that not only enhances their work but also contributes to the broader observability landscape. +**Hanson:** So observability has really helped me to, like, discover that. Both the technical level of like various different applications and services. But it's also helped me do that in the socio-technical aspect. Right. So, the telemetry that I've been able to find and discover as part of just like software development lifecycle has enabled me to be a better engineer. And so it's just help for discovery in general for myself. + +**Jamie:** Observability. I mean, to boil Hazel Weekly's definition, it's about asking questions and getting answers, specifically ones that you didn't really think needed to ask initially, then doing something with that. Being able to act and learn from your data. It's not just telemetry. It's about understanding your system through data. + +**Mikko:** Observability means having insights into your system. It's about understanding how your applications work, how your systems are working, and having visibility into things you don't even know necessarily are important until something comes up. So being able to find, you know, those unknown unknowns in your services and be able to make sense of things that maybe don't make sense otherwise. + +**Adriel:** I actually associated that many years back... I was working with the telecom networks and yeah, observability was really, really crucial. So consider you make a phone call from here or for example for US. And the call goes through the ends of nodes and even multiple operators. So without observability, you couldn't pinpoint the problems. So the customer calls, hey, why are my calls dropped? So definitely you have to have a really strong observability in order to find out what's happening and pinpoint the issues. + +[00:10:30] **Jacob:** Before working on observability, I worked for many years on, I guess, SRE roles or equivalents. I've also done like incident management and, like, yeah, I've been an incidents commander as well. So I've been in many incidents trying to figure out its root causes. Very hard to figure out because it's on a Sunday morning and things like that. And yeah, I don't want to have to go through that again. And, working on observability ensures that, if I go back to being an SRE and like, operations role, things should be better. + +**Mikko:** And if they should be better for others. For me personally, it is understanding what is going on in your system at any given time. I think of observability generally as, you know, knowing the health of your running servers. The analogy that I use is when you're driving a car, you have a dashboard in front of you that has, you know, lots of instruments, lots of measurements that are telling you if you're operating the car effectively. Observability is the same thing. But for servers at a much, much larger scale than a single car. + +**Adriel:** Observability means a lot because I think that, you know, observability as a discipline, especially for cloud native infrastructure and applications, is a very essential part of guaranteeing that your applications and your infrastructure observable they work. And in the, you know, as a software engineer, especially as a distributed systems engineer, if you are looking in building applications, and using cloud native infrastructure, whether that is public cloud or, you know, non-public cloud on prem, Kubernetes based infrastructure, you are, inevitably, dealing with and working with a lot of complex microservices, which is absolutely essential for you to have observability baked into your application as well as your infrastructure day one. + +**Jamie:** And, in the case of observability, as it stands today, it’s not only collection of telemetry, which we have, you know, literally trillions and trillions of petabytes of data being generated by not only applications but models now as well as infra and but also, looking at how the whole solution works, in terms of storage, performance, analysis, as well as visualization. + +[00:12:39] **Damien:** To me, OpenTelemetry means that you don't have to have the dependency on anyone and is also about the community. So it's a way for everybody to work together that you normally find your competitor, but actually you work together. You go to a meeting or do pair programming, and you just want to see that community to grow. + +**Jacob:** What does OpenTelemetry mean? So many things, because it does so much. And having been able to touch all the different parts has really given this huge meaning to me, but it's really like it to me, it means that it's the central thing in any observability stack. If I have OpenTelemetry, then I can go figure out exactly the things I need to do. No matter what vendor I'm using as a backend. + +**Mikko:** I think OpenTelemetry is an opportunity for everybody to kind of work together, in, you know, maybe slightly different goals, but working on the same thing that will achieve it for everyone. I believe it's important to have an open standard so we speak the same language. We need a lingua franca, for observability. Instead of using proprietary things that really don't add a ton of value. + +**Adriel:** Let's agree on the vocabulary. Let's agree on the letters. And I believe OpenTelemetry gives us an opportunity so we can all understand each other without having to literally be behind the same, you know, platform wall. + +**Jamie:** OpenTelemetry is sort of special to me. Obviously I've been involved in OpenTelemetry for a few years, but I love this idea of this vendor neutral standard that people can use, that everyone sort of benefits from the standard being there and everyone is contributing from all over the place. From vendors, from end users to different people in the community who are passionate about it. And it's sort of like being around a lot of friends and people who work really hard and hold each other accountable and just try to make the best of this project. + +**Damien:** And I love the idea of vendors competing on their baguettes and competing on the features that they have in their products, and it's less difficult for end users to just instrument their applications just to get visibility into their system without having to deal with a different agent if they decide they want to switch to another vendor. It's just very open and a lot easier for people to use. + +[00:15:30] **Mikko:** OpenTelemetry, personally, I feel it's such a great example of an open source project, but we are in a competitive industry and, and having so many companies, like +100 companies contributing to OTel and solving common problems and collaborating every day. So it feels really amazing. I feel it's about community and collaboration. + +**Jacob:** To me, I think it's not just about like, solving engineering problems, but, I think really, like the global community is extremely nice and welcoming. I think it's extremely impressive what we have been able to succeed doing. Like having a common and shared understanding of how things should work across over 15 languages with very different needs and problems and ways of solving problems depending on the languages and the fact that as human beings, we have been able to solve that is extremely encouraging, I think. + +**Adriel:** And I would also add that it's extremely rare in the current environments to see like multiple companies, competitors aligned together to build something so that they can all be more, like, provide better value together rather than just, like, stick to their own little corner. And so that's, that's kind of the huge things are great things I find about OpenTelemetry. + +**Jamie:** OpenTelemetry to me is the way that we get all of that data. Right? It's this really vast ecosystem of people who have agreed that there should be one way to do something, you know, that is new, generic to the field. Back to that car analogy. You know, you don't learn how to drive a Nissan or a Volvo. You learn how to drive a car. Right. And so in the same way that when you're learning how to engineer, it's important for there to be standards so that you don't have to relearn everything every time you go to a different company. And to me that is what OpenTelemetry means. + +**Damien:** OpenTelemetry has almost 80 repos today on the project. And, as many of you may know, OpenTelemetry is a very large project. It is not only a collection of components, but also an amazing community in terms of the partnerships and the collaboration that vendors and end users work together on, in terms of solving technical challenges and building the best components in OpenTelemetry. + +[00:18:36] **Adriel:** To me, OpenTelemetry is not only an integral part of building collection architectures for cloud native applications, but it is also an amazing community where you actually see the use of interoperability across the different components in the project, as well as open standards such as the OpenTelemetry protocol being, implemented end to end, which really is a game changer for the industry. + +**Hanson:** OpenTelemetry protocol. The reason I call that out is because it really enables end users to be able to build solutions for observability, end to end and use observability out of the box without having to think about, oh, what's my protocol going to be for metrics or logs or traces or profiles at all of the data that we collect? Right. But open. + +[00:19:20] **Jacob:** I love OpenTelemetry as a project and as a community. And, I love working on all the different pieces I worked on. I focused on working on the Collector. Collector contrib components, where we have added integrations, improving the operators for OpenTelemetry Collector, adding more metrics, performance features such as the targets allocator improvements in the operator, and also working on improving tracing, and logging. + +**Mikko:** My favorite telemetry signal... I still like metrics. I know that you will start to find out a little bit about traces, but I find that metrics is still almost like the gateway for the signals, because it's very simple to explain to people. It’s a number. Then on top of the number you can add, for example, attributes and get more data on top of it. So this way people don't get scared with like a big trace or big span as soon as you start, but it's a way to at least get more people into this area. + +**Jamie:** Favorite telemetry signal. That's such a hard thing to pick one because you can combine them so well. I think I started off as having metrics as my favorite because they were the thing that I looked at first when it came to the SDLC. But as I've gotten more into traces and tracing pipelines and all that stuff, I just realized how powerful it was. And then I can just derive, like any of the signals from those traces, and I can embed them directly in there. + +**Damien:** So I think I have to say right now, tracing is my favourite. + +**Adriel:** Well, traces really is the most powerful. So I would now pick that because you can. Yeah. Let's go spans. Yeah let’s go spans. Love spans. + +[00:21:40] **Mikko:** My favorite OpenTelemetry signal is probably going to be traces. I really like traces. I like the idea of starting in one place in your application, starting in one service, and seeing a request go from end to end service to service, and get an understanding of how that is flowing that you might not otherwise have visibility into because it's a full connected trace. So, yeah, traces. Traces would be my favorite signal I think. + +**Jamie:** My favorite telemetry signal. They're all important, but I mostly I would select distributed traces. I find it special because with the single view, single waterfall view, you can easily get the overview. You can pinpoint problems. So if your request gets rejected, you can quite often see it's already promising. The view of what was the service causing the reject. Or if your request is returned really slowly or the system is running really slowly. You can see how each service is adding up, just from the distributed trace. So that's lots of insight. + +**Adriel:** Oh, my colleagues are going to hate me for this, but it's tracing. And I'm saying that because, I work on profiling a lot, and the people that I work with really live and think by profiling. I think both are very important. But profiling is more like for, whatever you don't, everything works fine. And you want to improve things. And tracing is for, things are broken, and you need to figure out why. And that's kind of how I came into OpenTelemetry. And that's kind of my own work experience. So that's why my favourite signal is tracing. + +**Jamie:** My favourite telemetry signal by far is tracing. Tracing is, you know, the best of every world, in my opinion. You can derive metrics, you can derive logs, you can do lots of really important visualizations that help you understand both, you know, the high level observability goals that you might have, as well as the really low level debugging that you might have to do. It is the most important one, and I think the most misunderstood as well. + +[00:24:00] **Damien:** My favorite telemetry signal, I would say top of my list today. And that's metrics and traces. And the reason I say that is because when you're looking at observability real time, especially for AI applications, you know, in the new generation of applications coming in, tracing is very valuable, along with profiling, to be able to understand, you know, model behavior as well as software application behavior. And combined with metrics, you know, which actually are usually the standard way of getting telemetry and understanding of your infrastructure systems, it provides a very nice way of actually providing an end to end view of observability and observable components, all the way up the stack. ## Raw YouTube Transcript diff --git a/video-transcripts/transcripts/2025-05-07T18:47:21Z-otel-night-berlin-v2025-05-spring-starter-for-opentelemetry.md b/video-transcripts/transcripts/2025-05-07T18:47:21Z-otel-night-berlin-v2025-05-spring-starter-for-opentelemetry.md index 8bc9a43..f734595 100644 --- a/video-transcripts/transcripts/2025-05-07T18:47:21Z-otel-night-berlin-v2025-05-spring-starter-for-opentelemetry.md +++ b/video-transcripts/transcripts/2025-05-07T18:47:21Z-otel-night-berlin-v2025-05-spring-starter-for-opentelemetry.md @@ -10,81 +10,239 @@ URL: https://www.youtube.com/watch?v=R0n6Ny588dk ## Summary -In this YouTube video, the speaker, who has extensive experience in Java development and is currently an approver for OpenTelemetry Java instrumentation at Grafana, discusses the Spring Boot starter for OpenTelemetry. The presentation covers the current state of OpenTelemetry as of 2025, including its significant advancements in Java and comparisons with existing solutions like Micrometer. The speaker emphasizes the importance of a seamless integration experience for Spring Boot users and discusses the challenges of matching the large number of libraries supported by OpenTelemetry. The video features a demo showcasing how to implement the Spring Boot starter, including configuring the application and observing telemetry data. The speaker addresses various technical questions from the audience regarding context propagation, service naming, and instrumentation challenges, concluding the session with a Q&A segment. +In this video, the speaker discusses the Spring Starter for OpenTelemetry, sharing insights from their extensive experience with Java and Spring Boot. They explain the motivations behind creating a Spring Boot starter for OpenTelemetry, highlighting its ease of use compared to existing solutions like Micrometer. The speaker also provides an overview of the current state of OpenTelemetry in Java, including the stability of various libraries and the importance of semantic conventions. A significant portion of the presentation is dedicated to a live demo, showcasing how the starter integrates seamlessly with Spring Boot applications for observability. Throughout the session, they engage with the audience, addressing questions and clarifying technical concepts related to the implementation and functionality of OpenTelemetry in Java applications. ## Chapters -00:00:00 Introductions and background of the speaker -00:03:00 Overview of OpenTelemetry and its state in 2025 -00:05:30 Motivation for building a Spring Boot starter for OpenTelemetry -00:10:15 Comparison of Spring Boot starter with Micrometer -00:15:00 Explanation of how OpenTelemetry works, focusing on Java -00:20:00 Discussion of semantic conventions and their importance -00:25:00 Introduction to the demo environment setup -00:30:00 Starting the demo application and observability stack -00:35:00 Demonstrating data collection and trace visibility in Grafana -00:40:00 Q&A session on context propagation and service naming issues -00:45:00 Conclusion and final thoughts on OpenTelemetry and Spring Boot integration +00:00:00 Introduction and background +00:01:30 Overview of OpenTelemetry +00:03:10 State of OpenTelemetry in Java +00:04:37 Spring Boot starter motivation +00:06:05 Spring Boot experience expectations +00:08:00 Protocol and semantic conventions +00:10:41 Comparison with Micrometer +00:12:00 Spring Boot starter vs existing technologies +00:15:40 Demo introduction +00:15:40 Starting the demo application +00:28:00 Demo results and Q&A -# Spring Boot Starter for OpenTelemetry +**Speaker 1:** All right. Yeah, I want to talk about the spring starter for open telemetry. And what I'll try to do is it just works, but except when it doesn't. So, that's for the demo gods and because it's spring boot. -## Introduction -Hello everyone! Today, I want to share my insights about the Spring starter for OpenTelemetry. The aim is to demonstrate how it works, and I'm going to cover a few key points. Just a heads up, the demo gods might play a role since it’s all about Spring Boot! +**Speaker 1:** About myself. Actually, what I didn't put on here but 20 years ago I started my first job as a Java developer in this building. Just realized today I... Yeah. It was not the first location but the first company and we moved here from a different location in Berlin to this one and I worked here for about three years. -## About Me -A bit about myself: I began my career as a Java developer right here in this building 20 years ago. While it wasn’t the original location, it was the first company I worked for. After about three years, I moved on. Ten years later, I started using Spring Boot. I still don’t know everything about it! Fast forward five years, I began working with OpenTelemetry and realized I would never know everything there is to know about it. Two years ago, I joined Grafana, where I'm currently an approver for OpenTelemetry Java instrumentation, which involves sending all the data you’ll see later. I also work on other areas in OpenTelemetry, such as the operator, collector, and specifications. +**Speaker 1:** Yeah, 10 years later I started using Spring Boot. I still don't know everything about it. Five years ago I started working with hotel and I definitely know that I will never know everything about it. Yeah, and then I started working for Grafana two years ago and now I'm an approver for hotel Java instrumentation. So the stuff that sends all the data but you will see that later. And I work a bit on some other areas in hotel, operator, collector, and specification. -## Overview -I want to provide you with a quick overview of the state of OpenTelemetry in 2025, highlight what’s new, and discuss the motivation behind building a Spring Boot starter in the first place. We’ll also compare it briefly to other solutions in Spring Boot, namely Micrometer. Most of our time will be spent on the demo to show you how easy it is to use, and I’ll leave some time for Q&A, but feel free to ask questions anytime. +[00:01:30] **Speaker 1:** I want to give you like a quick overview of the state of hotel in 2025, what is new. Then the motivation why building a spring boot startup in the first place. I also want to compare it quickly to other solutions in spring boot which is called micrometer. I want to spend most of the time in the demo to actually see and show you that it is easy and leave time for Q&A, but you can also ask in between. I hope that we have enough time and if we don't then it's better if you ask before rather than I'm trying to finish the demo and it doesn't work. -### State of OpenTelemetry -As you can see, there are many languages supported in OpenTelemetry, and most have moved to stable releases for traces, logs, and metrics. This is significant progress from last year. We’re focusing on Java, PHP, .NET, and C++. Profiling is a new topic that’s actively being developed and might be included in the future. +**Speaker 1:** As you can see, we have quite a lot of languages in hotel and a lot of them have actually moved to stable for traces, logs, metrics. I think that's quite a bit of change for last year but I haven't recorded it. So Java, which we are talking about, PHP, .NET, and C++. For some reason, profiling is not listed here but this is like a new topic that is an active development. I guess in a year or two this will also be listed here. -#### Java and OpenTelemetry -Java is one of the more mature areas in OpenTelemetry, likely due to the prevalence of e-commerce and business applications written in Java. There’s a high demand for libraries, with over 100 supported libraries available. Java's compatibility with older versions is possible because of a technology called bytecode manipulation, allowing software to modify methods and classes at runtime. +[00:03:10] **Speaker 1:** And I will not show that. Okay, state of hotel in Java. Java is one of the more mature areas in hotel probably because a lot of e-commerce and business applications are written in Java and there's a lot of demand and there are many libraries. I just counted yesterday over 100 supported libraries are supported and it's also compatible with ancient versions of Java. So, Java 8 was released long more than 10 years ago but I did not count. -### Why Build a Spring Boot Starter? -The question arises: Why did we spend significant time last year creating a Spring Boot starter? One reason is that we can’t match the large number of libraries because we cannot alter existing code, and it would take a lot of time to reach the same level. As a Spring Boot user, I want a native Spring Boot experience. It should be straightforward to add a library as a dependency without any special steps, similar to adding a database. +**Speaker 1:** And that is actually only possible because Java has a technology called byte code manipulation which means that you can write software that at runtime looks at all the methods and classes that are loaded and can basically change anything. It can change a return value and can look at parameters similar to how Python can do monkey patching. -When using a Java agent, it feels somewhat magical and detached from the integrated experience Spring Boot provides. You should be able to add properties in the `application.yaml` file easily, with names and values automatically suggested. However, if you add a Java agent, it can complicate things because adding multiple Java agents usually doesn’t work well together. +[00:04:37] **Speaker 1:** Now that I explained how the spring starter works, the question is, isn't that good enough? So the question is why did we or I and another engineer from Microsoft spend a huge amount of time last year working on making a spring boot startup? -### Comparing with Micrometer -Micrometer is an older solution that has been used in many applications. It added a module for exporting metrics using OTLP, but since Spring is older than OpenTelemetry, there are limitations on changing the names of their metrics. Micrometer tracing, previously known as Sleuth, is also relatively new and lacks semantic conventions, leading to inconsistencies when combining libraries. +**Speaker 1:** Let's start with what we cannot do or cannot yet do. We cannot match the huge number of libraries, more than 100, simply because we cannot change old code. Also, because there's a lot of work being done and it would take a lot of time to have the same level. -### Key Concepts in OpenTelemetry -1. **Protocol**: The protocol is crucial as it ensures data is sent out correctly. -2. **Semantic Conventions**: These define names and attributes for metrics, traces, and logs. -3. **API**: Stable APIs are essential for library authors to maintain compatibility with OpenTelemetry. -4. **SDK**: The SDK implements the API and provides configuration options. -5. **Tooling**: The collector and operator are significant for managing telemetry data. +**Speaker 1:** Then the question is what we can do, and as a spring boot user, I like to have a native spring boot experience. That means I can just select a library, add it as a dependency, and it does not anything special and that is exactly what I wanted to have for observability. It should not be anything special. It should be built into the development like you have a database. Well, it's probably kind of hard because without a database, you could not do anything. You could not serve customer requests. But it should be as close as possible to just adding a database. -## Moving to the Demo -Before diving into the demo, are there any questions? +[00:06:05] **Speaker 1:** And that is not possible. If you are adding a Java agent, then it's kind of this magic. As a spring boot user, you also are used to have a very integrated experience. That means you can take your configuration file called application YAML and you can just add properties and the values and names of those properties are automatically suggested. -### Q&A -One question raised was about sending Zipkin spans to the collector and whether they would show up as one trace. The response is that while both could be sent, they might not maintain the correct parent-child relationship due to the different internal mechanisms used by Spring. +**Speaker 1:** You can basically just select service name because we talked about service name a bit before and hopefully I can show you that this is very easy. And then you can also use another Java agent to do your other magic work because adding two Java agents doesn’t usually work well. And you can also use a GraalVM native image which compiles your entire application into an executable which starts faster but it takes ages to compile. I'm too impatient to show that to you. But there is a full working example that where you can try that out. -## Starting the Demo -Now, let’s move on to the demo. I’m using IntelliJ and am currently in a project called `docker-hotel-lgtm`, which has examples of instrumentation and a complete observability stack based on Grafana technology. +**Speaker 1:** As I said, I spent a lot of time last year and it also became stable last year. Now I can talk about it. Before going into exactly how the spring starter works, this is a recap of how it works. This is not Java specific and not spring specific but in the next slide you will see how it relates to what I'm trying to show you. -I have two build materials set up: one for OpenTelemetry and the other for Spring Boot dependencies. First, let’s start the application, and while it’s starting, I’ll also start the observability stack via Docker. +[00:08:00] **Speaker 1:** The order is my personal preference and from most important to least important for me. The most important one is the protocol because once you have that protocol and you have your application, you know data is sent out. I can switch it somewhere else I don't have to recompile the application even if I don't have any developers. -### Application Setup -The observability stack consists of an application sending data to the OpenTelemetry collector, which then sends data to databases. The goal here is to create an easy trial experience, not optimized for scalability but for startup time. +**Speaker 1:** As we heard in the previous post, even if the data is kind of crappy you can use AI to make it better but you need to have the data. The next one is semantic conventions because semantic conventions define the names and attributes. So for a metric, it would be the metric name of the duration of a request for example, and also the units. -### Application Traffic -Now that the application is up, I’ll generate some traffic using a simple curl script. Let’s see if we can retrieve some data. +**Speaker 1:** So metrics and traces for logs. There is a bit of semantic conventions but not much. The third one is an API. The API is more for library authors who want to be compatible with hotel and they don't want to spend time in five years rebuilding their observability stack and for that, the APIs are stable. -### Analyzing Data -We have a few dice rolls recorded, and we can check the traces generated. The application is not calling any other services or databases, which is why not much data is available. +**Speaker 1:** In hotel, they are stable as you have seen in like here what it means stable is means the API among others but one is that the API is stable. One special thing is that for tracing the API is also important so that different libraries in the same process know which parent span belongs to which child span. -### Adding Improvements -During the demo, I asked if anyone had suggestions for improvements. The common response was to add more spans or metrics. Adding the service name correctly is also essential, and I’ll demonstrate how to do that. +**Speaker 1:** Because there are two cases. The easy case is you have a thread in Java and the thread is for a request and the thing that is first in the thread is done first. This is a parent and the second one is the child. But sometimes you have something that is asynchronous because it has an executor framework or something like that and then you need to have some kind of object that says this is a parent, that is a child, and that is what the API is for. -## Conclusion -To wrap up, we discussed the significance of the Spring Boot starter for OpenTelemetry, compared it to Micrometer, and walked through a demo showcasing its functionality. If you have any further questions about the implementation or concepts, feel free to ask! +**Speaker 1:** Lastly, the SDK. The SDK is an implementation of the API and it also gives you configuration options for example with environment variables. The last one is not the least important one. It's just more like at the side, the tooling, the collector, and operator is actually very important and very helpful. -Thank you for your time! +[00:10:41] **Speaker 1:** Now I want to compare the spring boot starter that I've worked on to the existing technologies in spring boot to explain a little bit why we built something where kind of something similar is already there. In micrometer, you have two things. Metrics is quite old and used in many applications as far as I know. More recently, a module has been added where you can export the metrics using OTLP, that's why OTLP the first one. + +**Speaker 1:** But as spring boot is older than hotel, they cannot just change the names of their metrics. I mean they could, but for spring, they hold it very dear to not change anything for old applications because there are many old spring applications. Changes are only in major versions and it takes a very long time and that's why they did not add yet support for semantic conventions. + +**Speaker 1:** In our spring starter, when I say our, it's not Grafana where I work for but it’s hotel as because this is something that I worked on just as a hotel contributor, you can import those metrics because there are many applications and libraries that emit those metrics and in some cases can be useful to use those micrometer tracing. + +**Speaker 1:** The second part that is a bit newer was called something different, Sleuth. I think before it was called micrometer tracing but it's still newer than metrics. In tracing, you can use the hotel SDK to emit traces but it also does not have the semantic conventions even though it's newer. + +**Speaker 1:** Here you have the problem that it is not using the same API. That's why you have this problem here on the right side. That has actually created some questions and probably even more exploding heads that I'm not aware of for users who are trying to combine some library with some other library. They usually don't know that one is using micrometer tracing and the other one is using hotel and sometimes works, sometimes doesn't, and sometimes works kind of. + +**Speaker 1:** That's basically it. I would love to have a common approach where there would be only either the micrometer APIs or the hotel APIs but this has not happened because the spring folks have a rich tradition and they know when they create an API they also want to keep it. On the other hand, the hotel folks don’t want to say we just take another API because it is for many languages and picking a different API makes it very difficult for hotel users who actually love to have the same experience across all languages. + +**Speaker 1:** This has been a question that has been asked many times but so far nobody has really given an answer to this. This is basically what I said just more in a data flow kind of way. In hotel, we have instrumentation libraries and in spring, there are instrumentation libraries in spring. They are called native because they are not maintained by hotel and they are not repositories, but they are done by the authors of different libraries themselves. + +**Speaker 1:** In spring, this is sometimes also not, they are also sometimes doing it for other libraries. Semantic conventions, I mentioned semantic conventions and later we will hopefully see why this is important. + +[00:15:40] **Speaker 1:** That's actually it. Now I want to start the demo, but if you have any questions, it probably makes sense to ask them now before we go to the demo. + +**Speaker 2:** All right. So, I used to work with Zipkin, right? + +**Speaker 2:** Yes. And we have a Zipkin receiver in the collector. So, would it work to just send Zipkin to the collector and like does it work if I send Zipkin spans to the collector and hotel from my Python application to the collector and do they show up as one trace at the back end? I mean, I guess the question is on the context propagation side of things, right? + +**Speaker 2:** You could send both and they would both show up, but they would not have the correct parent-child relationship because internally Spring does not use Zipkin. It uses something else which is like a common denominator of both Zipkin and hotel because those are the two targets for micrometer tracing. + +**Speaker 2:** If they would say okay, we just want to optimize for hotel then it would work. Doesn't that work with the B3 headers? + +**Speaker 1:** The headers are basically where it’s between processes. + +**Speaker 2:** Yeah. I mean, I have a Python application that is sending data to my collector and hopefully that one generated a span that originates the Java spans. + +**Speaker 1:** Yeah, then it would work. So if here between parent and child you have a process boundary, then it would work. If it's in the same process, then it only works in some cases. So if it's in the same Java thread, then I think it will work. But if it's in a different Java thread because you have an executor then it will not work because the context is actually stored somewhere else as a matter object and Java does not have a context object like Go for example. + +**Speaker 2:** Is that a property of Java or spring like kind of is that could this also happen just with general Java applications because I'm just thinking like we sometimes see that there are incorrect parent-child relationships in spam and we're mostly using Java but I'm not sure if they only use spring. + +**Speaker 1:** There are many scenarios where this can happen. Even if you just use the Java agent which is like one distribution, it usually is because there is some other context propagation mechanism that is not supported or a library version upgrade changes the version how context is propagated. That sometimes happens. But if it's between threads but in the same process, then it couldn't be, you said like if it's in the same process, it could still lead to this mismatch. + +**Speaker 1:** Yes. But if it's in the same process, it cannot be like different methods of propagation at least I understand propagation is in between processes. + +**Speaker 1:** Oh, there propagation is the term that is both used between processes and also within the same process. Between processes you have headers like B3 headers for example that are added to HTTP and in the same process it is some language-specific passing. + +**Speaker 1:** In Java, by default, the context is stored in a thread local because you can store a thread local without modifying the application. You don't have to pass a parameter. But that only works so long as the entire call is in one thread and new applications typically don't do that because this is not very efficient. + +**Speaker 1:** In general, you have to expect that there are linkage breaking between parents and child with Java. + +**Speaker 2:** No, no, no, no. The Java agent authors have support for those technologies or libraries where it's not easy. So Java executor for example is a framework where you can pass a runnable to a different pool and then it will be picked up by a number of different threads. They do round robins so that you don't need a thread for every request. + +**Speaker 2:** This byte code manipulation thing of the Java agent can see when you put an object into this pool and then it will attach some metadata into that object. So maybe there's a map where you can put things into and then you can add a context ID, a trace ID, and a parent span ID, and then you can form the correct parent-child relationship. + +**Speaker 1:** But if that support is missing in the Java agent and here in the spring boot starter it would be the same. If the spring boot starter does not support this then it would not work. What is even more difficult is if there are two different instrumentation technologies that don't use the same API then they also don't know what they should expect as a context object and that is what is making it so difficult to combine micrometer and hotel instrumentation. + +**Speaker 1:** This might be the beard talking, but I miss EJB context. + +**Speaker 2:** EJB. I mean, EJB context was perfect for this kind of situation. Does EJB even still exist? + +**Speaker 1:** I don't. Maybe it would have been the solution. How old I am. The last time I used EJB was when I was in university. And as I just said, it's more than 20 years ago. + +**Speaker 1:** Let me try with the demo before we run out of time. How much time do we have left? + +**Speaker 2:** You have... + +**Speaker 1:** Okay, good. Good. As long as you still have some energy left. So, yeah, this is my IntelliJ. Weirdly, it does not look black even though I have the black theme. Oh, no. Okay. Let's see if it does. Yep. + +**Speaker 1:** Oh, wow. Should have checked that table. You just turned it off and turned it on. Yeah, it's all you have to know as an S. + +**Speaker 1:** Here I'm not just in any project but I'm in the project called docker hotel lgtm which both has examples of instrumenting and also has a complete observability stack based on Grafana technology of course because I'm working there. And I have created a new example just so that it to get us started a bit faster. + +**Speaker 1:** Here I have two bill of materials. In we are using Maven, this is a dependency management section and first we have open telemetry, this is the latest version and then we have spring boot dependencies. We have the open telemetry first so that if there are libraries with the same name the one from the first one win and since spring boot has a micrometer which has the hotel exporter but in an older version I want to make sure that we have the newer version and that they are compatible. + +**Speaker 1:** Then I just have a starter web which is from spring and then I have the open telemetry spring boot starter and then there's really nothing here. I don't know why we have repackage we have a spring boot app but there's nothing interesting here and then we have a small controller that is just doing rolls and it has a player name. + +**Speaker 1:** So, first I want to start this. And while this is starting, I'm going to switch here where I have two windows. The first one is actually starting the image where we have the complete observability stack. So this one is actually just a docker script but just to show you what this is really doing. + +**Speaker 1:** MIS is just a task runner. So like make file but a bit nicer. And this one is just a shell script. The shell script is just Docker. The only thing why I don't call Docker is that I probably forget some of the ports. By the way, I did not mention the ports. + +**Speaker 1:** I did not mention the whole slide because it said demo. This is explaining the project much better than I could do. On the left side is the application that we're doing that is sending all the data to the hotel collector which is sending the data to all the databases. The profiling one is not used here and then we can view it with Grafana and that's why we have two ports on the left and one port on the right side. + +**Speaker 1:** This is just passing the ports and some people like to actually persist the data but the goal of the project is more to have an easy try-out experience. So it's not optimized for scalability but it's optimized for startup time. Did it work out? Yeah, it started in 3 seconds. That was the latest iteration that I did because it was taking half a minute before. + +**Speaker 1:** And then it also has an environment file which it's not really needed but I set it here to an external endpoint. So I can also view the data in Grafana cloud but this is actually not needed. + +[00:28:00] **Speaker 1:** Okay, so back to our application as it started now. Yeah, I think in the beginning it sent some errors because the observability stack was not started and now I will also put some traffic. This is just a curl script that pulls the endpoint. Sometimes it gives an error because the error is simulated. See if we actually get some data now. + +**Speaker 1:** Yeah, we have dice rolls. Now let's actually see if we can see some data. Yeah, we have some data. We have some traces. Are the traces? Oh, cannot scroll down. Okay, so nothing really interesting. Why is it not interesting? Well, because the service is not calling any other service and it is not calling a database just sometimes has errors. + +**Speaker 1:** Do we see errors? Let's see if that at least works. So this is by the way the Grafana 12 that was just released today. I hope I'm using that correctly. Error. Yeah, it's error. So, here we have an error. Let me see if that works. Copy that. + +**Speaker 1:** Does it work? Oh, I did not copy the entire thing here. There's copy. Yep, that's where the error is thrown. Okay, so that part works. + +**Speaker 1:** Oh, what I forgot to ask. How many of you have used Spring Boot? Okay, at least a couple of people. How many have instrumented Open Telemetry in any language? Even more. Good. How many have used the two together? Oh, one. Okay, good. + +**Speaker 1:** But the next question is not about instrumenting. Do you see anything that could be improved that we should make better or that you want to have added? More spans? More spans. + +**Speaker 2:** Yeah. An unknown service. Exactly. Unknown service. And I promise that this should be easy to do. So let's see if we can do that. Where's my directory here? Very slow today. + +**Speaker 1:** Let me see. Service service hotel service name. No, but it's a different one. Okay. Let me see if I have hotel. Oh, is it not recognizing? Maybe I not I have to reload. Maven did work today. But the goat... + +**Speaker 2:** Yeah, I did not sacrifice a goat. + +**Speaker 1:** So, I guess I have a question. Why do you look it up? The new way of doing things is using hotel config files that can be reused across services. That is right. + +**Speaker 1:** Would that come to the spring boot starter as well? Like can I could I just add my hotel.yaml file there instead of configuring to the partition? + +**Speaker 1:** Before answering, just a bit on the background. Right now in hotel, the standard way of doing things is environment variables and that is quite limited. So the most fancy thing is if I write hotel resource at... Oh, it's even completing that but this is not spring, this is the AI. So I can say service name, service version, something like that. + +**Speaker 1:** Anything more complicated is specific to any language and is currently blocked. What Drusi is saying is that the plan currently is to have the same experience, well not in spring syntax but in YAML or in any structured data, to have the same experience. + +**Speaker 1:** For spring, we would probably have a major version where we would tell you to switch from the syntax that I'm trying to show here to the one that is compatible. I have spent some time trying to figure out if this can be integrated with spring because here you can also use a spring expression. + +**Speaker 1:** In spring, you have a language where you can reference environment variables. I don't know if I can get it right if that is actually correct or if this is just an AI hallucination but there's a spring feature where you can reference other parts in spring and you can even reference Java beans and so on. + +**Speaker 1:** What I want for the future is that you can also have this because you're used to it as a spring user but with the common YAML format. I'm not sure if that is possible because this is already stretching the limits of spring and it will be even more so with the new format. So I hope it's possible but no promises. + +**Speaker 1:** Since I'm trying to do that here, I'll just look in my stash to see if I'm just too stupid to find the right format. Resource attributes, hotel service name. Yeah, the autocomplete just didn't work for some reason. It's not suggesting it. + +**Speaker 1:** So show up as a plugin on the right-hand side. There's no plugin, right? Actually, this is a feature of the dependency here. So it basically just has a JSON file where it lists all the properties. + +**Speaker 1:** Yeah, it should work. Let's see if it's working. If we restart the service. Tracing is always the tracing locks is the fastest to show because it's processed right away. Metrics is ingested every minute. You can change it to more often but for this demo it's not necessary. + +**Speaker 1:** See the last one. No, it still has unknown service. Well, what I had before is that when I started in spring, it's not working. But when I use Maven to start it then it is working. So that can be tricky. + +**Speaker 1:** Let me see if that is the case. Oh, it's still unknown service. They look for unknown service. I think that's too old by now. That's still a node price that you were looking at. + +**Speaker 2:** Can you filter by service? + +**Speaker 1:** It should. It actually says service name here. So, it's automatically grouping by service name. That is not the problem. I'm probably doing something wrong because I put it in a wrong directory or something. + +**Speaker 1:** Something that is a typical spring problem that happens every time when I try to do something new. Okay. I'm just going to delete that file and apply the stash as I had it before. + +**Speaker 1:** Oh yeah, it's in a different directory. That's why my error. Yeah. Good that I had the stash. Let me see if I can also type service. Yep, now it works. Okay, at least that worked. + +**Speaker 1:** But now I made the mistake that I started it here and also in the other one. It's probably saying that the port is already used. Started it here and no seems that it is loaded. + +**Speaker 1:** Okay, see if it works better here. Here it is. Okay, here's rolled dice. Okay, then let's just look at that instead. + +**Speaker 1:** Here now I can show you where the service name is. Here is service name and then you also have a bunch of stuff that we did not specify and that is because there are resource detectors so that you don't have to do all the work yourself. + +**Speaker 1:** For example, you have the process ID that was taken from the operating system and you can also see that it was created by the spring boot starter. So if you're reporting a bug, then it's easier for the authors to find where this is coming from. + +**Speaker 1:** You have also this correlation that you have between some metric types. You can actually see the trace that was the trace that started before the log message was emitted. + +**Speaker 1:** Now that we have that, let's look at some metrics. There is this duration metric that I talked about before and that has the same name across all languages. That is really useful for having an APM system where you can see what the request rates for all of your services are. + +**Speaker 1:** Here this is the open-source version of Grafana. So you don't have the APM tool here. I would have some things that we could do. We could add a manual metric or an additional metric where you can count the number of requests or players. + +**Speaker 1:** We can add more spans, I heard that before. Or we can add a database or we do none of it and we just leave more time for questions. So if you don't have anything that you want then I think we can just cut it here. But if you want something then I will add it. + +**Speaker 2:** I just have a quick question. Why is the service name not on the resource attribute? + +**Speaker 1:** Oh yeah, that's a good one. Let's I think we saw it here. That is the visualization of Loki. Loki shows everything flat and it's actually you're not the first one to notice that it would be more useful if that was structured in traces. + +**Speaker 1:** It is structured. Yeah. It's a good point. I think it would be better. And now it's also showing up here for some reason didn't before. So here it is in resource attributes and it's down there. It's there. Service name, right? + +**Speaker 1:** Yeah, here it is. But in the configuration, you have to add it to a different place or is that kind of free? You can... Oh, service name is special. In open telemetry, service name is special. It internally translates into resource attribute but it is like the single one that is required by every telemetry data that I could also add it here by the way. + +**Speaker 1:** I also find that confusing that there are two ways and I just had a customer call today where we found out that in some cases it does matter where you put it even though it should not. + +**Speaker 1:** Yeah, I mean I think it's an implementation detail. Basically, service name is the only required attribute for open telemetry and it is top level. An implementation detail is it is a resource attribute and so that's why sometimes it works sometimes it doesn't. + +**Speaker 1:** I mean in theory it should only work if it is at top level right because if you add it as resource attribute then it's probably not recognized at the top level. + +**Speaker 1:** Confusion in OTLP, it is only resource attributes so it's only within the resource attributes that is a special one called service name and that's it, so that's why some if you specify an environment variable with the service name it might work but then depending on the library if you don't have the proper service name environment variable it overrides the one that you would override with the resource attributes with a default one so that's why sometimes it doesn't work sometimes it does. + +**Speaker 1:** So if as a user you should only use it at the top level not at the resource attributes. + +**Speaker 2:** So when I see in Grafana on the resource attributes that's just the dual thing basically. + +**Speaker 1:** Yeah. Is that really true? It is how it is internally stored. In Grafana, I know if you go like to the traceQL query instead of using traceQL itself it has this kind of building query by the same then there's like the top two or so. + +**Speaker 1:** Yeah, exactly. So if you go to search now, not the trace. So service name, span name, and status are like the special ones. + +**Speaker 1:** For open telemetry, only the service name is special. The folks at Tempo, they said you know span name is interesting to have it as part of the trace syntax and so it's there. But from the hotel perspective, the only thing that we require is a service name. Even then we don't throw an error. + +**Speaker 1:** The specification says that if a service name is not specified the unknown service should be used which is what we've seen before, right? So the spring boot application was working without the service name and it was showing unknown service but it is against this spec it should resources then in the application YAML you should not put it under resources. + +**Speaker 1:** No, it's perfectly fine. It works. But as a user you should not, as a user you should probably not do that. + +**Speaker 2:** Got it. Okay. I mean, it might work for spring but it might not work for only lambdas. I mean I think per spec it should work but the case I had today was where it actually did not work. They were putting it in the environment variable but it's basically the same. Most people I think most authors expected in service name that's why it's safer to put it in there. + +**Speaker 1:** This is also just my last obligatory slide. That's why I put it on here. Cool. All right. Thanks for your time. ## Raw YouTube Transcript diff --git a/video-transcripts/transcripts/2025-05-07T18:48:36Z-otel-night-berlin-v2025-05-leveraging-ai-for-opentelemetry-data.md b/video-transcripts/transcripts/2025-05-07T18:48:36Z-otel-night-berlin-v2025-05-leveraging-ai-for-opentelemetry-data.md index 67e477a..450bb02 100644 --- a/video-transcripts/transcripts/2025-05-07T18:48:36Z-otel-night-berlin-v2025-05-leveraging-ai-for-opentelemetry-data.md +++ b/video-transcripts/transcripts/2025-05-07T18:48:36Z-otel-night-berlin-v2025-05-leveraging-ai-for-opentelemetry-data.md @@ -10,101 +10,186 @@ URL: https://www.youtube.com/watch?v=8JlFuGTDCXQ ## Summary -In this YouTube video, Lariel, a principal AI engineer at Dash Zero, shares insights into applying AI to open telemetry data, focusing on a case study about log processing. Lariel discusses the challenges of parsing unstructured logs and the limitations of existing models, such as the drain model and large language models (LLMs). The presentation outlines a novel approach combining various techniques to efficiently predict log formats and patterns without requiring extensive human intervention. Key points include the importance of resource clustering, the integration of traditional machine learning with LLMs, and the evaluation of the system’s performance, which demonstrated a 98% success rate in log parsing. Lariel highlights the significance of context and semantic conventions in improving AI predictions. The video concludes with insights into future developments and the potential for further collaboration in the observability space. +In this video, Lariel, a principal AI engineer at Dash Zero, discusses his experience applying AI to open telemetry data, specifically focusing on a case study involving log parsing. He outlines the challenges of working with unstructured logs that lack the attributes needed for effective analysis and explains the motivation behind developing their log AI solution. Lariel explains their approach, which involves combining various methods for log pattern recognition without relying on custom regular expressions or large language models (LLMs) for every log. He details their process of clustering logs by resource attributes and using a combination of heuristics and prompt engineering to extract valuable information. The results show a high success rate in accurately parsing log severity and patterns, and he emphasizes the importance of context and evaluation in AI applications. The video concludes with key takeaways and an invitation for questions. ## Chapters -Sure! Here are the key moments from the livestream with their corresponding timestamps: +00:00:00 Welcome and intro +00:00:40 Overview of D-Zero +00:02:40 Log AI motivation +00:04:10 Limitations of existing models +00:06:40 Challenges with LLMs +00:09:46 Combining different approaches +00:10:40 Predicting log formats +00:12:00 Evaluation methodology +00:13:01 Results and success metrics +00:19:20 Key takeaways -00:00:00 Introductions and background of Lariel -00:02:15 Overview of Dash Zero and its focus on observability -00:04:00 Introduction to the case study on log AI -00:05:30 Challenges with unstructured logs in real-world applications -00:08:00 Limitations of existing models for log parsing -00:11:45 Exploring alternatives like LLMs and deep learning for log parsing -00:15:00 Explanation of the combined approach to log parsing -00:18:30 Overview of the log parsing pipeline and resource clustering -00:23:00 Evaluation of the log AI model and its success rates -00:27:00 Demonstration of results and visualization in the UI -00:30:30 Key takeaways and additional experiments with AI agents +**Lariel:** All right. So, hello everyone. I'm Lariel. Today I'm going to share a little bit of my experience in applying AI to open telemetry data. So, short intro about myself. I'm from Brazil, moved to Berlin a couple of years ago. Nowadays, I work as principal AI engineer at Dash Zero with these guys here. Before that, I had been around playing different roles in data and AI, ranging from data engineering to data scientist, ML ops. My CV kind of looks like a mess nowadays. -Feel free to ask if you need any further assistance! +[00:00:40] Also, for those of you who have not heard of D-Zero, just a quick introduction: we are an open telemetry native SAS solution for observability. We work with metrics, traces, logs, we do dashboarding, alerting. We're built on open standards like Prometheus and Persis. Obviously, we are building lots of cool AI capabilities to help our users get more value from the data that they send to us. And that brings me to the subject of today's talk. Actually, the main subject, which is a case study on our log AI. So how to make sense of unstructured logs. -# AI and Open Telemetry Data: A Case Study on Log AI +Many of you probably had a similar disappointment at some point. You plug in the open telemetry demo data set and then the logs look beautiful. The logs are shipped via OTLP. They have plenty of useful attributes. They are formatted as JSON, so they're very easy to work with, right? But in reality, most people's logs look like this. You have something running on Kubernetes, and you have those file listeners that get the logs from every node, and in the end, the result is something like this. Your log has some raw string body and some attributes saying what file it was collected from, and that's not very useful. But you know that there is information there. If you look into the log, there is something that looks like a module name, something that looks like a duration, a timestamp, a severity at debug level. How can we harness that information? That's the motivation behind the log AI that we developed. -**Introduction** +[00:02:40] We want to do log parsing and log abstraction at scale for any logs of any application, without a human in the loop, without having to write custom regular expressions. For example, if I have an application writing logs like this, I want to parse them by separating this prefix from the message. I have in the prefix timestamp, host, severity text, and we can see that the messages follow different patterns. I have some "ad lookup failed," "display this ad to this user." There are basically two different patterns of log messages that my application is writing, and there are some structured fields that I would like to be able to query. There is some ad ID in the middle, there is some user ID, and we want to abstract that in the form of a pattern without writing a regular expression manually. We want to do that for every log of every application. That's the challenge and the motivation behind our log AI. -Hello everyone, I'm Lariel. Today, I'm going to share my experience in applying AI to open telemetry data. A brief introduction about myself: I'm from Brazil and moved to Berlin a couple of years ago. Currently, I work as a Principal AI Engineer at Dash Zero. Before that, I held various roles in data and AI, ranging from data engineering to data science and ML Ops. My CV looks a bit chaotic nowadays! +[00:04:10] We started looking into the available options to do that. The most famous one is the drain model that some of you might have seen already. Drain comes from this family of natural language processing models that work with word frequency. They basically compare the parts of the text that change frequently with the parts that are often the same. They can identify where the variables are and what the patterns should be. But those algorithms have plenty of limitations. First, they assume that the log is already prepared, so you have already separated that prefix from the message that can change from one pattern to another. For that, you need custom regular expressions written by hand for each log source. That is already a deal breaker. Secondly, those algorithms are very dependent on pre-processing. -For those who haven't heard of Dash Zero, we are an open telemetry native SaaS solution for observability, working with metrics, traces, and logs. We provide dashboarding and alerting services, built on open standards like Prometheus and Persis. We're also developing several AI capabilities to help our users derive more value from the data they send to us. +There is this paper called "pre-processing is all you need," where they describe this manual iterative process of coming up with tokenization rules, variable identification rules, text cleaning rules for each of your applications. Unless you do that, you cannot expect these models to produce any decent results for your custom logs. Believe me or not, the screenshot here on the left side is real code from the repository where they have the benchmarks for these models. You can see that for each log data set that they are benchmarking on, they have hardcoded rules; otherwise, the model doesn't work. That's a real deal breaker for us. We cannot just apply this drain model to every log and expect it to work. -**Main Subject: Log AI Case Study** +We started looking into other alternatives. What's very popular nowadays is just throwing things at a large language model. If you ask GPT to parse a log, it's going to get everything right on the first try. You get the fields really nice. The problem is, with hundreds of customers instrumenting thousands of applications and sending billions of logs every day, using an LLM for everything would be very cost ineffective and slow. So what do we do? -The main subject of today's talk is a case study on our Log AI, focusing on how to make sense of unstructured logs. +[00:06:40] Yet another option is, like this paper suggests, using a deep learning model that is trained to predict the pattern on every log. But in order to train a model in the first place, you need to have lots of labeled data, logs where you already know what the correct pattern should be, so you can teach the model how to infer that. We cannot possibly have such a dataset for all of our customers. Yet another problem with these models is that although they have nice benchmarks, those can be interpreted as a form of data contamination because the logs used to benchmark the model were produced from the same applications with the same templates as the logs that were used to train the model in the first place. This indicates that the model can actually only identify patterns that it is already familiar with or parse logs of applications that it is already familiar with. It doesn't generalize for any logs of any applications. -Many of you may have experienced disappointment when using open telemetry demo datasets. Initially, the logs look beautiful, shipped via OTLP with useful attributes and formatted as JSON, which makes them easy to work with. However, in reality, most people's logs look quite different. For instance, when running applications on Kubernetes, logs collected from each node often result in raw string bodies with minimal useful attributes. While there is information embedded in these logs, such as module names, durations, timestamps, and severity levels, the challenge is finding a way to harness that information effectively. +This was the status quo: plenty of challenges, different options, each one with their limitations. How we solved this in the end was no magic at all. We actually just combined different approaches, trying to balance out the limitations of some with the advantages of others. The first thing we did was figure out the right level of granularity for predicting log formats and patterns. I don't know if anyone here has already worked with predictive AI in production, but you usually have the concept of an instance. For example, if you are doing customer churn prediction, then every customer is an instance. If you're doing product sales forecast, then every product is an instance. It's quite straightforward. -This is the motivation behind our Log AI. We aim to perform log parsing and abstraction at scale for any logs from any application without requiring human intervention or custom regular expressions. For example, when parsing logs, we want to separate the prefix (containing timestamps, host, and severity) from the message itself. Within the messages, there are patterns that we want to identify without manual regex. +When it comes to open telemetry data, it's quite challenging because there are different levels of granularity, and the schema is very nested. What is an instance? We started with the concept of the open telemetry resource. We tried to predict the log format and patterns for every resource, but it wasn't efficient at all. If you think for example of a Kubernetes deployment with autoscale, it is creating new pods and killing old pods all the time, and every new pod has different resource attributes. So it's treated as a new resource. If we would predict log formats and patterns for every resource, our model would be working all the time, which would not be efficient at all. -**Challenges and Alternatives** +Instead of doing this, we came up with resource clustering rules that we apply to resource attributes of different resource types. This allows us to scope and cache and reuse the predicted log formats and log patterns between different resources that belong in the same group. To make this work in production, we also came up with a recommended configuration for the open telemetry collector that guarantees that the logs that we get from our customers always contain all the resource attributes that are relevant for our resource clustering rules. -We explored various methods to tackle these challenges. One of the most well-known is the Drain model, which comes from the family of natural language processing models that analyze word frequency. However, these algorithms have limitations: +[00:09:46] Now we have a concept of an instance that we want to make predictions for. Next step, how do we predict the patterns? We came up with this pipeline where we start with all the logs, and then we first cluster by resource attributes, like I said before. Then we focus on parsing. We use a combination of heuristics and prompt engineering to identify what is the prefix, what is the message, what parts of the prefix are worth extracting, like the log severity, and we come up with log formats for each resource cluster. -1. They assume that logs are already prepared—meaning the prefix is separated from the message—requiring custom regular expressions for each log source. -2. They are highly dependent on pre-processing and manual tokenization rules. +[00:10:40] After the logs are parsed, we apply those traditional word frequency models like drain in order to cluster the log messages by structure. Every log message that has kind of the same structure goes into the same cluster. Looking at each log cluster, we take the output of the drain model and inject it into the prompt for a language model in order to identify the final variables and also give names and types to the variables. At this stage, we observe that including semantic convention information and resource attributes in the prompt helps a lot in getting variable names and types that make sense for each application. -The reality is that these models cannot be applied to every log uniformly and expect good results. +Then, in the end, we just apply some heuristics to optimize the patterns that we get so they match as best as possible the respective log cluster. At ingestion time, it's actually really straightforward. In our open telemetry collector, the processor is going to match the logs against patterns from the respective cluster and attach the attributes with the variables that it finds. -Another option is using large language models (LLMs) to parse logs. While they can produce impressive results, processing billions of logs daily across numerous customers would be very costly and slow. +[00:12:00] This is an overview of our solution. But before I show you what the results look like in our beautiful UI, I'm just going to have a quick word about evaluation. This is really important. Before we applied this to every log of every customer all the time in production, we needed to build some confidence to know that it would produce the expected results. We came up with an evaluation dataset that combines logs from community datasets with logs from our own proprietary data. The focus was not to have too many logs or quantity, but rather to have diversity of logs so we could really stress the model and confirm that it was working as expected for many different edge cases. -We also considered deep learning models trained to predict log patterns. However, this approach requires substantial labeled data—logs with known patterns—which we cannot obtain for all our customers. Additionally, benchmarked models often suffer from data contamination, meaning they perform well on familiar patterns but fail to generalize to new ones. +[00:13:01] Our main goal with this evaluation was to confirm that our approach had this kind of built-in graceful degradation. What does that mean? It means that the model doesn't need to work all the time. It doesn't need to work for every log from the billions of logs that we are ingesting. Whenever it works, it needs to produce correct information. When it doesn't work, then it cannot produce any unwanted side effects. That’s graceful degradation, and that's basically what we observed in the evaluation. We had a 98% success rate at the log parsing step with 100% log severity accuracy among the cases that are successfully parsed. The model almost always understands the log format, and when it does, the log severity that it extracts is always correct. When it does not, then it doesn't extract anything, and the log stays as it was before. -**Our Solution** +Besides that, the patterns that we extracted for each application's logs cover an average of 85% of that application's logs. That means we successfully abstracted almost all the information that is available to be abstracted. -After analyzing various options, we decided to combine different approaches to address the limitations of each. +Oh, sorry. Ask a question right at the end. Let's save the questions for the end if you don't mind. Don't forget it. I'm just quickly going to show what the results look like in our UI. This is a histogram of log severities over time. The gray bars are logs with unknown severity. As soon as we plug in the new log processor that applies the log formats and patterns, we start having colorful logs in the histogram indicating the information and warning and error logs. -First, we established the right level of granularity for predicting log formats and patterns. In open telemetry data, the schema is highly nested, making it challenging to define an instance. We initially attempted to predict log formats for every resource but realized this was inefficient. Instead, we implemented resource clustering rules that allow us to cache and reuse predicted log formats across similar resources. +We also observe a couple of examples here of errors and warnings that would have passed by unnoticed if it wasn't for the correct log parsing. We come up with this dedicated visualization for the patterns. This is data from the open telemetry demo, and we have different patterns where the variable is in the middle, things that say, for example, "targeted ad request received for ad category," and then ad category is a variable or a method name called with user ID. The method name and the user ID are variables. If we open one log record like this one with product ID name, then we know which pattern it matches, and we have the product ID and product name as dedicated variables. This is structured data that can be referenced in queries, in filters, in group by expressions, also in Prometheus query language. -Next, we built a pipeline that starts by clustering logs based on resource attributes. We then use a combination of heuristics and prompt engineering to identify log prefixes and messages, extracting valuable parts like log severity. After parsing, we apply traditional word frequency models, such as Drain, to cluster log messages with similar structures. +For example, if I want to create an alert based on log frequency using the patterns, I have some logs that say "product ID lookup failed," and then it has some ID in the end. This one matches a pattern, and the product ID is a variable. So I can create a log frequency alert with the filter saying that the pattern should be that one and grouping by the product ID field that comes from the semi-structured text. -At this stage, we inject the outputs from the Drain model into the prompts for a language model to identify final variables and assign appropriate names and types. Including semantic convention information and resource attributes significantly improves the model's ability to generate meaningful variable names. +This is the Prometheus query language expression to alert on that log frequency, and the result is when I'm having too many lookup errors for that product, I get a nice failed check with the custom message up here that says "lookup failures increasing for product with the ID of the product." So basically, alerting based on semi-structured information that was abstracted from logs. -Finally, during ingestion, our open telemetry collector matches logs against cached patterns and attaches relevant attributes to the identified variables. +This happens without any metrics, without needing an extra metric, without needing to write any regular expression, without depending on traces and anything else, just based on the logs. This is really cool. So yeah, this was the case study on the log AI that we have been developing. -**Evaluation** +I'm just going to quickly comment on a couple of other experiments that we have been running. We have been experimenting with AI agents to write and debug Prometheus query language expressions and also to diagnose and find the root cause of failed checks and alerts. The agent that we work with has access to a model context protocol, MCP server that allows it to browse through every metric and log and trace, kind of like the same way that a human would do in our UI. -Before deploying this solution across all logs and customers, we needed to evaluate its effectiveness. We created a diverse evaluation dataset combining community and proprietary logs to stress-test the model and confirm its performance across different edge cases. Our goal was to ensure the model exhibited *graceful degradation*, meaning it doesn't need to work perfectly all the time, but when it does, it must produce accurate information without side effects. +In this context, we also observe that if we feed the agent with information about semantic conventions and resource attributes, then it gets way better at writing correct queries in the first try. It doesn't take so many iterations to figure out the correct metric names, the relevant attributes, and so on because it builds on top of the semantic conventions. Yet another use case is our trace AI, where we group duplicated spans in our trace explorer by their attribute similarity, and we also generate trace names and trace descriptions to clusters of similar traces. -We achieved a 98% success rate in log parsing, with 100% accuracy in identifying log severity among successfully parsed logs. Notably, our patterns covered an average of 85% of each application's logs, indicating our ability to abstract most available information. +In this context, we also observe that giving semantic convention information to the prompt increases the quality of the names and the descriptions that we get. Also, when clusterizing the traces by their semantic embeddings, if we include the resource attributes in the embedding, we get way more meaningful embeddings, therefore a better clustering of similar traces. -**Results and User Interface** +[00:19:20] Those were just a couple of other examples of use cases that we have been working on. Before we go on to the questions, I just wanted to leave you with four key takeaways. -Now, let me briefly share what the results look like in our user interface. +First, applying AI in production at scale can be expensive. It can get expensive very quickly. But remember that in the open telemetry schema, we have plenty of attributes that you can use to cluster similar resources. This way, you can scope, cache, and reuse AI predictions between different resources. -- We visualize log severities over time, with the introduction of our log processor leading to colorful histograms that reflect various log severities. -- We also developed a dedicated visualization for log patterns, allowing users to see variable names and values in structured data, which can be referenced in queries and filters. +Second, LLMs are cool, but they are expensive. We always try to see what we can do with traditional machine learning techniques or how we can combine them with LLMs to get the best of both worlds. -For instance, if we want to create an alert based on log frequency using extracted patterns, we can do so without requiring additional metrics or writing custom regular expressions. +Third, evaluation is very important. Even if you're building a simple LLM application and you're not really training or fine-tuning any model, you can always try to model the expected behavior of your application and measure how often and how close it gets to the expected results. You kind of quantify the limitations and the risks before shipping it to production. -**Conclusion and Key Takeaways** +Lastly, context is everything when it comes to LLMs. Always benefit from resource attributes from the rich open telemetry schema and the semantic conventions so you contextualize your prompts and get better results from language models. -In summary, this case study on Log AI demonstrates our approach to making unstructured logs actionable. +That was it. If you want to connect with us, please follow us on LinkedIn. We also have a blog on our website. We're always posting some cool stuff. We have the Code Red podcast on Spotify and Apple. We have a couple of open positions on our careers website too, including one for another AI engineer and for a product manager with a background in observability. -Here are four key takeaways: +That was it. Thanks for your attention. -1. **Cost Management**: Applying AI at scale can become expensive quickly. However, leveraging attributes in the open telemetry schema allows for clustering similar resources, making predictions efficient. - -2. **Balancing Techniques**: While LLMs are powerful, they are also costly. Combining traditional machine learning techniques with LLMs can yield better results. +**Audience Member:** I'm sorry. Do you remember your question? -3. **Importance of Evaluation**: Regardless of whether you're building a simple LLM application or a complex model, it's crucial to evaluate expected behaviors and measure performance before deploying. +**Lariel:** Yes. I think it was about the log levels. You had 98% with log levels, but there are some logs that do not have a log level. How is that working out? -4. **Context is Key**: Always utilize resource attributes and semantic conventions to contextualize prompts for better results with language models. +**Lariel:** We only measure the accuracy where there is a ground truth. For a log that really doesn't make sense to assign a level to, it doesn't have the level, even in an unstructured way or in the form of a status code or anything, then we do not assign it a log level. Actually, in that case, if we would assign a log level, let's say error, warning, or anything to a log that doesn't contain any textual information that references that, it would count as a false positive. We have a separate metric for measuring how often that occurs. It's the specificity metric. It's how often we avoid false positives, and that one is maximized as well, but it wasn't on the slide. -Thank you for your attention! If you wish to connect with us, please follow us on LinkedIn. We also have a blog on our website, a podcast on Spotify and Apple, and several open positions, including roles for AI engineers and product managers with a background in observability. +**Audience Member:** Okay. I think there's even an unspecified level for open telemetry, right? -**Q&A Session** +**Lariel:** Yes, but it is the default. At least when we ingest, if that field is not set, we set it to unspecified, and then after the AI runs, if we don't identify an explicit level, it remains unidentified. -Now, I would like to open the floor for any questions you may have! +**Audience Member:** Any other questions? + +**Audience Member:** I didn't get this specifically about the semantic conventions and resource. Did you use them to insert resource attributes and semantic conventions into logs that don't have it, or what was the... + +**Lariel:** No. The thing is, whenever we get any signal that has attributes or a span name, a metric name that matches the latest semantic conventions, then we have access to the documentation of that metric or attribute, and we include that documentation in the prompt. The model has context on what that metric means in which system it belongs. It is able to, for example, give an appropriate name to a variable or give a better description to a trace. So that's how we work with semantic conventions for prompt contextualization. + +**Audience Member:** That's very interesting. I was thinking it would also be nice to have these kinds of resource attributes and conventions. Sometimes the values are not correct, especially if you have your own semantic conventions on top of OpenTelemetry, like business-relevant semantic conventions. If you could use AI to rectify basically the discrepancies in what the users are putting in, because usually they put in a lot of stuff, a lot of different stuff. Even if you have it written down, please use these values, they make up their own values and their own writing systems. Instead of going there and telling them every time, "No, please lower case, not uppercase," or something along the lines, if we could use AI to understand the... + +**Lariel:** I'm not sure if AI is really necessary. Sometimes regular expressions do the trick, but sometimes they don't. That can actually be a challenge. Every case where I mentioned that we employed so many conventions, it was the official ones, like from the open source repositories. + +**Audience Member:** I have a follow-up question on this. A question I asked myself, I don't know how D-Zero works as an observability platform, but my assumption is that the log AI feature is the same for every customer because you used something like... + +**Audience Member:** Yeah, now someone is turning around if you're not... + +**Lariel:** You found an approach to leverage different kinds of AI technologies to have log AI enabled, and it's the same for every customer. + +**Audience Member:** With my own semantics, does my own and everyone have something like a slightly different implemented AI approach? + +**Lariel:** That's a good question. The approach right now is the same for everyone, but we work in a multi-tenant way. Even if two customers have, let's say, Oracle databases, we don't mix the log formats and patterns of one customer with the other. All of your data will be processed with formats and patterns that we inferred from your data only. It is pre-trained, yes. We are not customizing by customer, although not right now, maybe in the future. + +**Audience Member:** Thank you very much. Are there plans on open sourcing any of that? + +**Lariel:** I suppose that—I mean, I love the log AI part, but I suppose the Prometheus part would also have that kind of feature, and I think I heard other companies doing the same, right? + +**Lariel:** I think I heard some other companies doing something very similar. I see a lot of potential collaboration opportunities there. + +**Audience Member:** Regarding any kind of open source, you know, need to talk to our CTO, but what we are planning to do is to release an MCP, so anyone can connect their agents to D-Zero data if they want to over there. + +**Audience Member:** So you apply a model to every single log. Do you use an LLM? + +**Lariel:** No. + +**Audience Member:** Okay. So how do we apply? + +**Lariel:** We do it in a batch where the logs are already clustered by resource, and then they are clustered by structure. Within each log cluster, we identify the pattern with the LLM, and then we store that pattern. During ingestion time, we don't invoke the LLM at all; we just apply the patterns that we have already cached. + +**Audience Member:** So by pattern, you mean regular expression? + +**Lariel:** Yes. No. You can try that. Maybe the newer ones will succeed for some cases, but LLMs are usually bad at writing regular expressions on their first try. We actually use them to infer parts of the log, extract variables, get some insights on the log structure, and then we apply heuristics to these results to compose the regular expression ourselves. + +**Audience Member:** I hope that rests on the data at rest. + +**Lariel:** Yes, it was also part of the processor, which was done on the fly. + +**Audience Member:** On the fly during ingestion, we apply log formats and patterns that have already been cached. + +**Audience Member:** So at what frequency do you refresh the cache? + +**Lariel:** That's in our documentation. It should be every two hours. + +**Audience Member:** Every two hours? + +**Lariel:** Yes. It's in beta phase. I'm working on it, but yeah, it's meant to be every two hours. There is obviously a quota per customer. If you're sending logs of different formats and different patterns all the time, like generating random formats just to make us spend money, it's not going to work. Refreshes are for when patterns change, but actually, it's only needed when it changes. + +**Audience Member:** Sorry, you said the cache refreshes every two hours? + +**Lariel:** Yes. + +**Audience Member:** So that cache is for, you're caching the log patterns? + +**Lariel:** Yes. When the log pattern changes, the cache needs to be refreshed too. + +**Audience Member:** Yes. So then, yeah, it would be good if you could do this on demand. + +**Lariel:** It is on demand, every two hours. + +**Audience Member:** Okay. Seriously, that wasn't a joke. I mean, if they change, we realize that they changed at latest two hours after, and then update them. One very important aspect of Dash Zero is that we do not sell AI capabilities as separate modules. We provide them kind of for free. You only pay for the data that you send, and all the capabilities, all the cool stuff in the UI, you get out of the box. Whenever we implement something like this, we have to establish rate limits and optimize it as best as we can to make it cost-effective. + +**Audience Member:** But if I, as a customer, know that mine won't change, let's say, or I know when it will change, is there a possibility to tell Dash Zero that now there's a new pattern coming? + +**Lariel:** Right now, the way to do that would be you message our customer success people, and then they would ping me. But we wouldn't need to do anything if they change, and at latest two hours later, the pipeline would realize exactly. + +**Audience Member:** So what it means is, within the first two hours, I might not recognize all of the new fields or all of the new patterns that are flowing through the pipeline? + +**Lariel:** Yes. After two hours, it's going to be recognized. + +**Audience Member:** That graph you've shown with the gray bars, we could see a difference there for the first two hours, and then it goes back to the same level. + +**Lariel:** Exactly. That's very cool. + +**Audience Member:** Go on. + +**Audience Member:** One question is, you mentioned that you have some kind of prompt engineering in the pipeline detecting the patterns. Do customers somehow, can they involve, are they involved in this process? Can they change something like, for example, adding their own smart? + +**Lariel:** We use the same workflow for every customer, but the more information we have for each customer, the better we could contextualize the prompt. Maybe having custom semantic conventions for each customer's domain would be something to help improve the prompt. That's something we could build. + +**Audience Member:** And a second question, how do you monitor end to end, for example, how much money you spent on how many tokens and so on? + +**Lariel:** We use D-Zero to monitor. We also use the community libraries for instrumenting LLM applications, the ones from the OpenTelemetry Python contrib. From those, we get LLM traces and a cost for every LLM request that we make. The costs are also contextualized with the price per token of each vendor and model variant. We have a nice LLM trace view in our UI where we can debug LLM interactions, like with what tools were called and so on. + +**Lariel:** Folks, if anyone has other questions, we can chat later, here around in the kitchen. I'm going to give it to the next, and thanks for having me. ## Raw YouTube Transcript diff --git a/video-transcripts/transcripts/2025-06-11T05:54:28Z-otel-me-with-oluwatomisin-taiwo-and-andrei-morozov.md b/video-transcripts/transcripts/2025-06-11T05:54:28Z-otel-me-with-oluwatomisin-taiwo-and-andrei-morozov.md index 563552e..938b3ca 100644 --- a/video-transcripts/transcripts/2025-06-11T05:54:28Z-otel-me-with-oluwatomisin-taiwo-and-andrei-morozov.md +++ b/video-transcripts/transcripts/2025-06-11T05:54:28Z-otel-me-with-oluwatomisin-taiwo-and-andrei-morozov.md @@ -10,105 +10,262 @@ URL: https://www.youtube.com/watch?v=tTCuTAPE5aQ ## Summary -In this episode of Hotel Me, hosts Adriana Vila and Andre Kapolski discuss observability practices at Compass Digital with guests Tommy and Andre M. from the S3 team. They delve into the roles of observability in ensuring system reliability and scalability, touching on the use of AWS services like Lambda and ECS, and the integration of OpenTelemetry for distributed tracing and performance monitoring. Key challenges addressed include context propagation in microservices, manual instrumentation for business logic, and the need for effective log management. The conversation highlights the importance of community support, with participants sharing their experiences and encouraging more complex examples in OpenTelemetry resources. They also discuss future expectations for OpenTelemetry, emphasizing the potential for enhanced data insights and improved developer experiences. The session concludes with an invitation for audience engagement and a reminder of upcoming community events. +In this episode of "Hotel Me," hosts Adriana Vila and Andre Kapolski are joined by guests Tommy and Andre M. from Compass Digital, who discuss their experiences with observability in distributed systems using OpenTelemetry. Tommy focuses on the reliability and scalability of their web platform, while Andre M. specializes in developer experience and platform engineering. The conversation delves into the architecture of their systems, which primarily utilize AWS services like Lambdas and ECS, as well as the challenges they face with distributed tracing, performance monitoring, and error tracking. They share insights on their instrumentation practices, including the use of OpenTelemetry for logs, metrics, and traces, and discuss their approach to managing large amounts of telemetry data. Additionally, they highlight their journey towards adopting OpenTelemetry, the importance of community engagement, and the potential for future contributions to the project. The session concludes with a Q&A from the audience, covering topics such as instrumentation of legacy systems and reporting business value metrics. The hosts encourage viewers to participate in future sessions and community discussions. ## Chapters -Sure! Here are the key moments from the livestream along with their timestamps: +00:00:00 Welcome and intro +00:01:30 Guest introductions +00:03:35 System architecture overview +00:06:32 Observability challenges +00:10:01 Instrumentation methods +00:13:12 Context propagation issues +00:19:00 OpenTelemetry setup discussion +00:22:50 Data storage optimization +00:30:00 Collector setup and management +00:36:10 Future of OpenTelemetry -00:00:00 Introductions by Adriana Vila and Andre Kapolski -00:02:00 Guests Introduce Themselves: Tommy and Andre M -00:04:30 Discussion on Roles at Compass Digital -00:06:15 Overview of System Architecture and Technologies Used -00:10:00 Challenges Faced in Achieving Observability -00:13:45 Insights on System Instrumentation Techniques -00:18:30 Discussion on OpenTelemetry Setup and Configuration -00:25:00 Audience Questions on Metrics and Logging -00:30:00 Collecting and Managing Observability Data -00:35:00 Future of OpenTelemetry in Their Workflow -00:40:00 Closing Remarks and Upcoming Events +**Adriana:** Heat. Heat. Hello everyone and welcome to our latest hotel me. Super excited for those who are able to join and for those who are not, we do have this recording available after the fact both on LinkedIn and YouTube. My name is Adriana Vila and I am one of the maintainers of the hotel enduser SIG, and I am happy to introduce my co-presenter today, Andre. -Feel free to ask if you need further details! +**Andre:** Hi everyone and yeah, awesome to be here. My name is Andre Kapolski and I'm a fairly recent contributor to the end user SIG. So yeah, I'm super excited to hear more from our guests today. -# Hotel Me Session Transcript +[00:01:30] **Adriana:** Yeah, and I think this is a perfect segue as we bring on our guests. And as we do, folks on the chat, please feel free to say where you're watching from. -**Adriana:** Heat. Heat. Hello everyone and welcome to our latest Hotel Me. I'm super excited for those who are able to join, and for those who are not, we do have this recording available after the fact, both on LinkedIn and YouTube. My name is Adriana Vila, and I am one of the maintainers of the Hotel End User SIG. I am happy to introduce my co-presenter today, Andre. +**Andre:** Yeah, I can perhaps start. I'm watching from the Czech Republic, in the EU, specifically in Brno. Adriana, what about you? Where are you joining from? -**Andre:** Hi everyone! Yeah, awesome to be here. My name is Andre Kapolski, and I'm a fairly recent contributor to the End User SIG. I'm super excited to hear more from our guests today. +**Adriana:** Oh, yeah, that's right. I'm in Toronto, Canada, so it's, I guess, 1:00 for me and I guess it's evening for you, right? You're probably 7 o'clock in the... -**Adriana:** I think this is a perfect segue as we bring on our guests. Folks in the chat, please feel free to say where you're watching from. +**Andre:** That's correct. That's correct. -**Andre:** I can start. I'm joining from the Czech Republic, specifically from Brno. +**Adriana:** Yeah. Awesome. Awesome. All right. I guess let's bring our guests on. Hello everyone. -**Adriana:** Oh, that’s right! I'm in Toronto, Canada, so it's I guess 1:00 PM for me and evening for you, right? +**Tommy:** Hello. -**Andre:** That's correct! +**Andre M:** Hey. -**Adriana:** Awesome! All right, let's bring our guests on. Hello everyone! +**Adriana:** So, why don't you both introduce yourselves? Let's start with Tommy. -**Tommy:** Hello! +**Tommy:** My name is Tommy. I'm an SRE here at Compass Digital and my primary responsibility essentially is ensuring the reliability, scalability, and observability of our web platform. I work closely with the engineering team, product teams, and test engineers to ensure that our systems are robust and we have actionable insights into what's going on within our services. So yeah, that's me. -**Andre M:** Hey. +**Adriana:** And where are you calling from? + +**Tommy:** Oh yeah, I'm calling from Cambridge, Ontario in Canada. + +**Adriana:** Awesome. We're practically neighbors, a few hours away from each other. + +**Tommy:** Oh yeah. Awesome. + +**Adriana:** Okay. And then we have another Andre today. + +**Andre M:** Hi. I'm out from the Bonneville Durham region. So, same type of area. And for me, I've been in this industry for 11 years, but always been into the tech stuff since I was a kid. I work with Tommy on the same team, SRE at Compass Digital here, and very similar responsibilities across observability, on-call, PI, all the type of stuff for our systems. + +[00:03:35] **Adriana:** Awesome. Well, we're super excited to have you both on, and just a reminder for folks watching, if you do have questions along the way, please feel free to post them in the chat. And also, we will be taking questions at the end if anyone has questions. So I guess let us get started. So I know you guys both talked a little bit about your intros. Maybe you could talk a little bit about your respective roles at Compass Digital in a little bit more detail. + +**Tommy:** Sounds good. I'll start. + +**Andre M:** Okay. Yeah, sorry. I'll go first then. As I said, most of my role is primarily focused on ensuring the reliability and scalability of our systems. Very recently, or for the most part that I've been here, I've been working on one of our products. It's a company I actually bought over, it's e-club, to help get observability into what they're doing there. And largely, I work with things like AWS SDK, the CDK, TF rather, and PA duty. We have observability backend data trace where we send all our OpenTelemetry stuff to. So pretty much just SRE, there's nothing too crazy happening on there, but I believe as we go along on this podcast, I'll get more nuanced into what I do and how your architecture is structured and the stack there. + +**Adriana:** Sounds good. And Andre M, if you could elaborate a little bit on your... + +**Andre M:** Yeah, certainly. So when I started with Compass about five years now, I started off as a senior software engineer. So that's my background in software engineering, less so on the ops and infra side, but that's kind of been a new love with Terraform and services. But primarily about the developer experience, the platform engineering aspect of it. And more recently, it's more about understanding the system that you can worry about and they'll tell us that linkability, how potentially across that, and give us the insights with our platform and a vendor that we utilize that with. + +**Tommy:** So just to give you some pretty cool things is like identifying any services that are looping into each other, for instance, via API rather than just calling it internally with a function. So it's more about just setting up these guard rails and looking into the optimization of our system. And the other aspect is, because we are a microservice shop, it's been a challenge for us to get that context propagation and seeing all the services all together. And now we're able to kind of have that full aspect all together and actually run real, I guess, business impacts and analysis onto that. + +**Adriana:** Yeah, that's primarily an area I've been looking at more or less. + +**Tommy:** Cool. Folks, can you start with, can you tell us a bit more about the system that you are operating? What is its architecture? What programming languages are used and how it is deployed and so on? + +**Tommy:** Okay. So we have two products. So I'll talk about the one that I primarily focus on. So for us, our main infrastructure and architecture all runs, 90, 95% is all Lambdas. We have about 300 plus Lambdas in our ecosystem. We run a hybrid of DynamoDB with Aurora RDS and that's the exact thing progress, and we're slowly getting into Argate services. So that's a little bit different model, but that's the primary architecture of our ecosystem. You know the standard services like S3, API Gateway, just standard service stuff that the ecosystem provides. + +**Andre M:** Oh yeah. On the E-Club side, what we have is a blend of Python and Django on the backend. And then JavaScript, TypeScript on the frontend, and we run everything on that side on AWS ECS, which gives us that flexibility and consistency across all our environments. + +**Tommy:** Okay, thank you. Our CI/CD pipelines automate the process of building those Docker images and then deploying them onto ECS so we can roll out those changes quickly and safely. But to do a quick walkthrough across this little diagram that I have here, so imagine a user interacting with the application. What they hit first is the load balancers that we have here, and then it distributes traffic to the web service that we have on the backend, which runs ECS and is instrumented with the OpenTelemetry SDK. + +[00:10:01] As this web service or web services begin to process this request, what it does is offload some heavy tasks or async tasks to this Celery worker down here, which also runs on ECS and is also instrumented with the OpenTelemetry SDK. Thanks to the context—oops, typo—context propagation here, the trace context can flow seamlessly between the web service and the worker. With the introduction of the OpenTelemetry SDK, now we can follow a request end to end from inception to the end, right? Both these services, that's the web service and Celery worker, emit traces, logs, and metrics, and we sample them, make sure they have high cardinality and they are batched also for efficiency. All this data that I'm talking about is sent to OpenTelemetry collectors that run as a sidecar on ECS to send this data to our observability backend. So this is pretty much in a nutshell what the architecture on the E-Club side of what we are doing looks like. + +**Adriana:** Awesome. Thanks for that. As a follow-up question, you know, given your system architecture, what were the top three problems that you were facing that kind of compelled you to say, "Hey, I need observability"? + +[00:06:32] **Tommy:** Oh yeah, sure. I'm sure you know, and probably everyone listening knows that observability in a distributed system is never a solved problem. Some of the nuanced challenges that we faced was, again, as I said, distributed tracing. With microservices, a single user request can touch so many services and background jobs, and without distributed tracing, it's almost impossible to reconstruct that full journey of a request from the user's end to the response back to the user, especially when something goes wrong. For instance, if a user says that they had a slow order placement, we need to trace our request from the web frontend, the backend, the Celery workers, and even to third-party APIs. The introduction of OpenTelemetry helps us stitch all these things together, right? And also have consistent context propagation across all these boundaries. + +One of the things that we were also looking at was performance monitoring because we rely on Celery heavily on E-Club for background processing. However, these things can be black boxes if you don't instrument them properly. So, we need to know not just if a task has succeeded, but how long it took, what resources consumed, and whether it's causing bottlenecks along the way. Introducing Celery with, sorry, instrumenting Celery with OpenTelemetry required us to go with out-of-the-box solutions, again thanks for that. To be able to add custom spans and metrics to these things. Also, error tracking, you know, errors can manifest in logs, traces, or metrics, wherever. The most important thing for us at the time was to be able to correlate these things together, so that's having your trace ID and span ID in logs and being able to look at that trace ID in the log and correlate it with an actual trace. A spike in error log sometimes might correspond with a specific trace or a drop in a metric that probably we're tracking. What we've done is we've worked to ensure that our telemetry includes enough context, like I said, the trace ID, request ID, so we can pivot between the logs, traces, and metrics seamlessly. So yeah, that's what got us to where we are now. + +**Andre M:** Awesome. + +**Adriana:** Andre, do you have anything else to add? + +[00:13:12] **Andre M:** So for our end, I think the primary problem again really was that context propagation because again, the microservices and distributed nature of that. I think the other part that we weren't even really aware about is just how the system really behaves. So a big part of it is tying back into alerting. So depending on what vendor you use, it might have automatic alerting, but being able to provide that to our developers, in our case via Terraform. So developers can open up PR, create their own alerts, and now we have that across our ecosystems has really changed a lot of how we're able to empower the different developer teams to just take control of their own alerting and systems rather than just one team kind of owning that. + +**Adriana:** Can you tell us a bit more about how your systems are instrumented? What types of instrumentation are you using? + +**Andre M:** Certainly. So on our platform, currently, for us, we do use our vendor agent. So in our end, the agent's really nice for the most part. It works seamlessly with both of our infrastructure pieces for Lambda. Lambda has the concept of a Lambda layer where you can basically put on something. So their agent just goes onto that, kind of bolts right on and works essentially right out of the box. The second part to that that we had to play around with a little bit was our ECS Fargate. There is a solution from AWS called AWS FireLens, which allows you to kind of have your task logs, standard error, and send it out to basically be sent to a specific sidecar where then that sidecar will run something like Fluent Bit or they could be it, and essentially take your logs and ship them to where you need them. + +**Adriana:** Yeah, that makes sense. I would ask a follow-up right away. So are you using auto instrumentation or are you manually instrumenting your services? + +**Tommy:** So for us, because we do have the 300 Lambdas, each one need, we basically have like a core library that all these Lambdas will utilize, a very thin layer, but in there we do have OpenTelemetry instrumentation for Node.js. I believe it's just the auto instrument or I think we might have changed that for the fine-tuner because it needs to be small because Lambdas, you want to keep them really tiny. So I think we do have the custom one where it's not the auto instrument. + +**Adriana:** And how was your experience with doing that so far? + +**Tommy:** I haven't heard any issues. So I didn't do that one. It was actually our coworker Matt who did that one, but I didn't see any too many issues with that. PR went through pretty smoothly and the primary purpose was we use our own custom logger, one that isn't supported by OpenTelemetry or our vendor. So we had to use that to kind of pull in the trace ID from the headers and everything else and be able to actually populate the spans that we require. + +**Andre M:** Cool. So to clarify then, in a nutshell, you basically have like a wrapper around OpenTelemetry from what it sounds like. + +**Tommy:** Kind of, I guess so. + +**Andre M:** Yeah. + +**Adriana:** Cool. And a question for you around just to keep on the instrumentation thread. Who is responsible for instrumenting? + +**Andre M:** Well, that's an interesting journey. So we actually do have a new project upcoming. So the way our system has kind of evolved with Terraform in particular is, you know, we're IAC first for no matter what we do, infrastructure as code. A big part of that is we use the Cloud Development Kit extension on Terraform. So it gives us the ability to kind of have this inheritance model and allows us to build these L1, L2 constructs. So if anyone's familiar with that, it's a very useful way of building abstractions for your infrastructure and enabling teams to kind of handle that as well. + +In our case, what we're trying to build right now is kind of an L3 construct, which is like an application service level where we can give it to developers and they can essentially have all the nuts and bolts kind of already instrumented out of the box with the right permissions and the right access and the right agents and everything else. We kind of support this golden path for them. + +**Adriana:** Awesome. Tommy, do you have anything else that you want to add? + +**Tommy:** No, not really. As I said, what I think we're trying to get to a point where all of these things are, would I say, prepackaged for the devs to be able to use them, to leverage them, to do their own instrumentation on their own. I would say for the most part, we are automating many of our collector setup or the OpenTelemetry setup. But I think at the beginning, especially on the E-Club side, was to auto instrument so we can have quick wins and move quickly. But when we're getting down to things like business logic, salary tasks, and things that were unique to the E-Club side, we manually created spans along some critical code paths and added some custom attributes here and there so we can get more than just what OpenTelemetry offers out of the box. So I think for E-Club, it's just a little slightly different from what we have on the Centric side. + +**Adriana:** Nice. That's really cool because I think that's a path that a lot of organizations start with anyway. Like the auto instrumentation is that quick win and then it's like, oh yeah, we're lacking a little bit more. Let's go with the manual instrumentation. That's awesome. + +[00:19:00] We do have a question from someone in the audience, from Buddha. Hi Buddha, nice to have you join us. So, Buddha asks, are you currently using all OpenTelemetry signals: logs, metrics, traces, profiles, or a combination of them? + +**Tommy:** So we are using logs, metrics, and traces. Logs are coming through depending on which infrastructure piece. So if it's Fargate, it'll come through the AWS FireLens and Fluent Bit solution and then be sent to our vendor. Sorry, to our vendor. Otherwise, if it's Lambda, it comes straight from the Lambda layer agent. + +For metrics, that comes in through a different connector with our vendor. They have a solution we install in our cloud environment, and then it kind of pulls all the metrics over to that. Now profiles is interesting. We don't use profiles but the use case I think for profiles is enabling maybe common attributes or standardization, and we handle that through, again, our vendor has a solution out of the box that allows us to kind of set the schema and richer data as it gets ingested. + +**Adriana:** Great. And then another follow-up question from Buddha. And by the way, actually before I ask that, there was a question from Manish, who's asking if we are recording the session, and yes, we are recording the session. So it should be available on LinkedIn and YouTube on demand after the fact. So, great question. So the follow-up question from Buddha was, was this how you started or slowly built up to it? Who would like to take that one on? + +**Tommy:** Was that one for like the Terraform aspect or just in general? + +**Adriana:** I'm assuming it was for the instrumentation in general, I would say. + +**Tommy:** Yeah. At least one for Tommy. Tommy had a fun time doing that. + +**Tommy:** Okay. Well, this was not how we started. And for context, I joined about 10 months ago. And yeah, as time I joined, we did not have OpenTelemetry setup anywhere, at least best of my knowledge. I know that some of, yeah, a couple of the motivations for adopting OpenTelemetry on our side was standardization because we had like a patchwork of monitoring tools, like logs, metrics, everything scattered all over the place, but OpenTelemetry gave us a way to now unify all these and to reduce cognitive load and then the interaction overhead that comes with that. + +Additionally, we wanted to be vendor neutral, you know, using OpenTelemetry allows us not to be locked to a single vendor. I say, I know I'm getting into the motivations for why we adopted it, but it also cuts back into the fact that we did not start out that way, slowly built into it, and these were some of the decisions that drove us to using OpenTelemetry. + +**Adriana:** Yeah, I hope that answered the question. + +**Tommy:** I hope so as well. + +**Adriana:** How long did it take you to get to the point where you are at currently? + +**Tommy:** I'd say we're still getting there, right? If I remember correctly, we started somewhere around September, Andre, keep me honest here. I think by March, we closed the chapter on that one. From, I would say, from conceptualization, but I say from implementation to when we thought that we were in a good place, so roughly seven months trying out stuff. Yeah, I think we're in a good place now. + +**Adriana:** Alrighty. + +[00:22:50] **Tommy:** Yeah, we have a question in chat from Manish. When we talk of OpenTelemetry data, there is always a huge amount of data. How are you optimizing your storage and managing doing it? + +**Tommy:** Okay. I'll take this one a bit and then I'll let Andre finish off because he's the geek with that. One of the things we do is sampling, of course. I know when we query for stuff, we do a sampling ratio of I think 1 to 1,000 for traces at least on the E-Club side. This is tuned in this way to balance cost, performance, and visibility. Of course, we monitor the effectiveness of this sampling setting and adjust it dynamically during incidents or specific services that require deeper analysis. + +For logs, for querying and most of the other things we do with logs, we enforce a scan limit of 500 GB per query to keep the backend performant and cost-effective. But I know again, Andre geeks out on things like this, so I think you'll be able to give a better response. + +**Andre M:** Technically, we've got a really big budget for our vendors, so we actually have zero sampling at the moment. So we just grab it all. We're just trying to understand from that point and slim it down. You know, the first exercise that we kind of ran through was grab all the logs, run a pivot on it like the error logs, do I count on, you know, just general logs, and are there any useless logs that are being constantly created by the system or just noise? Can we remove that? And the answer was yes. So I think we had something like two billion, three billion logs coming in that are just like success that we just didn't need to have in place for it. + +Yeah, we could have been cleaning up the system a lot. We go through all our pods and kind of hand out tickets to them as we find through the investigation and be like, "Yeah, let's get rid of this," or "You make it a debug log potentially if you guys actually need it, enable it during your debug sessions." Otherwise, let's not try to have too much noise in our ecosystem. I will say though that one caveat is because we are running Lambda, that one agent and CloudWatch requires certain logs to come out. So we can't get away from certain noise. Right now, I think we're ingesting 54 million logs a day that we just can't get away from and they're just noise at the moment. So another motivation to move to Fargate where we have a bit more control over the system as we adapt a bit more of the observability ecosystem. + +**Adriana:** Cool. + +**Tommy:** Yeah, Manish, if you have any follow-ups, feel free to post them in chat. I would continue with the next question about collectors. Can you folks tell us a bit about the way how you set up your collector or collectors? What is the distribution and what is the architecture overall? Super curious about that. + +**Tommy:** Okay, well, I'll begin again with the E-Club side. And again, for context, as I said, I handle most of the E-Club side of things. For E-Club, what we do is try to make our collector environment aware. So in Sandbox, because what we have is Sandbox, staging, and production on E-Club. So on Sandbox, we sample a little more aggressively to capture as much data as possible for debugging, especially for me, right? To really know what's going on there and also help the devs really debug stuff that they have going on there. But in production, what we do is a little more probabilistic, I hope I got that right, to balance visibility with cost. + +What we do is we dynamically also adjust that sampling ratio if there's an incident, and then we temporarily increase that to capture more data. The collector definitely does all the batching, filtering, and exporting telemetry to the backend of our vendor. Of course, this is done in a YAML config to define our pipeline for traces, for metrics, and logs, and also set the environment-specific parameters like service name, the host type, and then all the OpenTelemetry options we want to set on that. So that's what we have for our collector on E-Club. As I said, it runs as a sidecar through the services that Celery, the Django application also. + +**Andre M:** So as a follow-up on your collector setup, do you have a way to manage a fleet of your... because it sounds like you have a number of collectors. Do you have a way to manage them effectively? Like do you use OpenTelemetry for that or do you use a homegrown thing? + +**Tommy:** And also, do you use a collector gateway to funnel all of your collectors into a central gateway before pushing that off to your observability? + +**Andre M:** To be honest, not really. We just manage the fleet using ECS, as I said, and then we run them as a sidecar to our standalone services. The environment variables that I said help us customize those things, I said earlier, which are the sampling rates, the exporter, and the service names. So we don't really manage the fleets independently or independent of our services. We just run them as a sidecar and let them do the rest for us. + +We also monitor the performance, right? Definitely. Things like the queue length and then we have some drop spans along the way, so we can scale them horizontally if we do need to. But for high throughput services, like I can't remember the name, one of our Celery, I think it's Celery Flower, right? We have, of course, a higher instance for that to manage the size and scale of how things can be and to avoid bottlenecks. + +**Adriana:** So just to clarify, do all your sidecar collectors then, do each of them go directly to your backend? + +**Andre M:** Oh, okay, got it. + +**Adriana:** And it sounds like Andre M had a screen share for us. + +**Andre M:** Yeah. So just, it's a visual essentially the same idea that Tommy was walking through there. Before we get started though, shout out to Skydraw for anyone using that, the best platform for brainstorming ideas and a little whiteboard place. But essentially the ECS Fargate, again I was mentioning the AWS FireLens configuration for us. The management all of it comes back to Terraform. So that allows us to easily manage our collectors because even the configurations are all in code. So it makes that part a little bit easier. + +[00:30:00] If that's what you're trying to guess, prelude to our vendor does have that instance within our account, but we don't utilize that as we've found that sometimes the logs are being dropped. Either we under-provisioned it, and we found that if we just directly send them straight to the vendor, there weren't any issues that we've noticed aside from losing some of the benefits of that additional collecting if we have to do some data masking or dropping those logs prior to being sent and ingested by the vendor there. So definitely an area we're trying to balance. + +**Adriana:** But the other part is, obviously, the Lambda, which again is super straightforward. It's just straight from the Lambda and straight to the vendor, again, without any additional pieces there. + +**Tommy:** So, that may be an area now that we're talking here, an area for improvement because if we have those 54 million logs that are just noise, we might benefit by going to that filtering agent first. So very useful. + +**Adriana:** Cool. Thanks for sharing. You already mentioned areas for improvement, and I'm wondering, do you have any challenges with OpenTelemetry currently and anything that perhaps the project could do better to serve you as its user? + +**Tommy:** Well, I'll say for me, one of the biggest challenges was just initial setup, right? Configuring the connector and ensuring context propagation across all the services, which required me to do manual instrumentation. Of course, there's a learning curve. Telemetry is powerful, but again, the API is just so large. Of course, in terms of best practices, we are still evolving. + +One of the things that I'll probably like to see, and I think I'm speaking to the community when I say this, is maybe more complex example projects, speaking to myself too, of course, putting it out there. But I would say the community has been really helpful, right? There were so many things that I benefited from, from GitHub mainly, and then a bunch of articles that people have written on Medium, right? For me, it's just more real-world examples, more complex setups like probably what we have. Maybe, you know, somebody just doing something that big and putting it out there for anybody to be able to use, but say for me that's just been the challenge. Nothing too crazy. + +**Andre M:** What about you? Do you have anything in mind? + +**Andre M:** Not so much I can give feedback to really improve. I mean, the whole time has just been learning, learning, learning, right? There's just a lot. I mean, it's my first exposure to it really for me. It's not even that I had experience with other observability tooling; it's just straight to our vendor and then also OpenTelemetry. So just learning the best of the best right off the bat basically, right? And then learn the history. For me, it's just been a constant uphill battle to learn all the different intricacies. Probably just want to echo Tommy's message: just more examples, right? Being able to pull things down, play with it, getting it into your hands really quickly. That's usually the best place to learn. + +**Adriana:** Yeah, I totally agree. I think we're at where OpenTelemetry has been around long enough now that I think many organizations are kind of past that 101 phase and are eager to get into the more gnarly use cases. And that's why, you know, we really appreciate folks like you jumping onto these Hotel Me sessions talking about your mature OpenTelemetry setups because it really helps others in the community. We definitely appreciate that. + +Switching gears a little bit, we were wondering if you could, if you have had any interactions with the OpenTelemetry community, things around, you know, like if you got stuck on a particular issue, like how did you go about it? Did you go on Slack or did you Google stuff? How did that work for you? -**Adriana:** Why don't you both introduce yourselves? Let's start with Tommy. +**Tommy:** I remember the first thing, it was for E-Club. When we were trying to instrument it originally, there’s an extension we use for, is it parallel processing, Tommy? Do you remember that one? It's not AWS Wake, there's another one. Well, it's a Django app that we run. The problem with that is that there's an extension we use, I think it optimizes performance. The problem with that is that it somehow was fighting OpenTelemetry. -**Tommy:** My name is Tommy. I'm an SRE here at Compass Digital, and my primary responsibility is ensuring the reliability, scalability, and observability of our web platform. I work closely with the engineering team, product teams, and test engineers to ensure that our systems are robust and we have actionable insights into what's going on within our services. I'm calling from Cambridge, Ontario, Canada. +**Andre M:** Yeah, I really can't remember the name, but I know they were fighting for the threading running in the background. I think that was one of the challenges that we had in the beginning. I really can't remember what specifically it was, but I know that was a big uphill battle back then. -**Adriana:** Awesome! We're practically neighbors, just a few hours away from each other. +**Tommy:** Yeah, so we had an open issue on GitHub with that. Luckily our vendor was able to get their agent working, so it worked for that environment. But I think the issue is still open, but wasn't sure if it was like upstream with Django or was it so much with OpenTelemetry itself. -**Andre M:** Hi, I'm Andre M, from Bonneville, Durham region, also in Ontario. I've been in this industry for 11 years, always interested in tech since I was a kid. I work with Tommy on the same SRE team at Compass Digital and have similar responsibilities focused on observability and on-call incidents for our systems. +**Adriana:** Cool. Cool. -**Adriana:** Well, we're super excited to have you both on! Just a reminder for folks watching: if you have questions along the way, please feel free to post them in the chat. We will also take questions at the end. +**Adriana:** As a follow-up, have you gotten to the point where you have made any contributions to OpenTelemetry or planning to make any contributions? -**Andre:** Let's get started! You both talked a little bit about your roles; could you talk a bit more about your respective roles at Compass Digital in more detail? +**Tommy:** 100%. Nothing yet in terms of contribution or commit, but 100% we'll probably be looking at something within the ecosystem of probably Node or Go, one of those two. -**Tommy:** Sure! Most of my role is focused on ensuring the reliability and scalability of our systems. Recently, I've been working on one of our products, which is E-Club. I'm helping to get observability into what they're doing there. I work with AWS SDK, CDK, Terraform, and PagerDuty. I also help with observability backend data tracing, where we send all our OpenTelemetry data. +[00:36:10] **Adriana:** Awesome. Final thing. I think we've covered most of the base questions. We do have a cool audience question that came in from Esther. It says, "How do you see OpenTelemetry evolving in your workflow over the next year?" Really great question. -**Andre M:** When I started with Compass about five years ago, I began as a senior software engineer. My background is primarily in software engineering, but I've developed an interest in ops and infrastructure. My focus is on developer experience and platform engineering. Recently, I've been working on understanding systems and promoting observability, especially given our microservices architecture. +**Tommy:** That is a tough question. Well, because I mean OpenTelemetry, I mean it's more about is it so much like the framework itself or is it more about the data we're getting out of it? Because for me, I think the value, I mean both are great, right? Because one gives us a standardized tooling and framework to be able to switch vendors or anything else we have the capability now. -**Adriana:** Folks, can you tell us a bit more about the system you are operating? What is its architecture and what programming languages are used? +For me, I think the real value comes into what type of data do we really pull out of that OpenTelemetry and what can we really get from it. I think that's where real value is. So being able to now take that data and apply user journeys and business events, like those things, I think are where our organization can really, I guess, expand or benefit from for the most part. -**Tommy:** We have two products, but I’ll focus on E-Club. Our main infrastructure is about 95% Lambda functions. We have over 300 Lambdas in our ecosystem, utilizing a hybrid of DynamoDB and Aurora RDS, and we're starting to use Fargate services. +**Andre M:** Tom, do you have anything to add? -**Andre M:** On the E-Club side, we have a blend of Python and Django on the backend and JavaScript/TypeScript on the frontend. Everything runs on AWS ECS, which gives us flexibility and consistency across environments. Our CI/CD pipelines automate the process of building Docker images and deploying them onto ECS. +**Tommy:** Not much, but I think just like OpenTelemetry also keeps maturing in our observability journey, especially around logs and for complex setups like Celery or serverless with Lambda. Because I know there were also some battles fought there. -**Adriana:** Thank you! As a follow-up, what were the top three problems that compelled you to implement observability? +I also think we are probably going to have more out-of-the-box integration and tools to simplify the configuration for our collector, right? I mean the ecosystem is growing so fast. So I would not be surprised to see us adopting it a little bit more. As Andre said, we are growing every day. We are learning every day. But I think we'll be leaning more towards probably much better or more standard industry practices when it comes to adopting OpenTelemetry and using it to really observe our system. -**Tommy:** Observability in a distributed system is never fully solved. One major challenge was distributed tracing. A single user request can touch many services, and without distributed tracing, reconstructing the journey of a request is nearly impossible, especially during issues. Another challenge was performance monitoring. For instance, we needed to know not just if a task succeeded, but how long it took and what resources were consumed. Finally, error tracking was crucial for us to correlate errors with logs and traces. +**Adriana:** So that's what I see happening over the next year, but who knows? -**Andre M:** For our end, context propagation was a primary concern due to our microservices and distributed nature. Additionally, we needed to provide developers with the ability to create their own alerts via Terraform instead of relying on a single team. +**Adriana:** Sounds good. Sounds good. -**Adriana:** Can you elaborate on the instrumentation of your systems and what types of instrumentation you are using? +**Adriana:** Yeah, we have more audience questions. Today we just talked over at the end with Adrian that we have a really, really engaged audience. So that’s amazing. So Manish is asking, how are you doing the instrumentation of legacy systems? Is anything like that happening? Do you have a use case for that? -**Tommy:** We use an agent for instrumentation on the E-Club side. It works well with our infrastructure, especially with Lambdas using Lambda layers. For ECS, we utilize AWS FireLens for log management, redirecting logs to a sidecar that handles exporting. +**Tommy:** I think we have some older .NET projects, but it seems as if our vendor supports out of the box, and anything I think we even with that solution, the vendor allows us to still have it in OpenTelemetry format as it comes through. So not really a concern for on our side from what I've seen so far. -**Andre M:** Our platform uses a vendor agent that integrates seamlessly with our Lambda functions. We manage our ECS Fargate tasks using Terraform, which allows us to customize configurations and manage the collectors effectively. +**Adriana:** Alrighty. Alrighty. -**Adriana:** Who is responsible for instrumentation in your teams? +**Adriana:** And one more from Buddha. How do you report on business value metrics for decision makers? What metrics do you report on? -**Tommy:** We're trying to build a pre-packaged solution for developers to leverage for their own instrumentation. For now, most of the initial instrumentation was done manually for critical code paths. +**Tommy:** Yeah, I know I think Tommy you mentioned it that for that business logic you had to do some manual instrumentation. Can you tell us a bit more about that? -**Andre M:** We’re evolving our processes and looking to provide more out-of-the-box solutions. The goal is to empower developers to manage their own instrumentation. +**Tommy:** Yeah. So, as I said earlier, there were certain business logic that we needed to especially on the Celery side because that's where we upload stuff to really see how data was flowing through that and then report on, I think specifically it was a reporting service or I think an order service at the time. -**Adriana:** Do you have any current challenges with OpenTelemetry that you think could be improved? +What we got out of that really was just the rise and fall or the spike of the, would I say the order process within that within a certain period or within a certain trace. But you know reporting back to business people, I think that one comes more out of the box with our vendor. And we, as Andre said, build stuff that aggregates the amount of logs that we use that translate into how much we pay for these things. Even though we have a big budget, we still don't want to be overshooting that. But I think again, Andre would have more context into this because he builds most of the stuff that I report back to business leaders and that's something he's so in love with. -**Tommy:** One major challenge was the initial setup and ensuring context propagation across services. The API can be overwhelming, and more complex examples would help us navigate this better. +**Andre M:** So I think for me, the biggest one at least it's been months in development is the most important key metric I've come to love recently is users. So what is a single user doing across a system and aggregate that over a certain period of time? That alone allowed us to find a particular leak where we had a single user, was a bug in our client and within our service that wasn't like a batch request where this one user within a short period of time was invoking over 200,000 requests into our system. -**Andre M:** I've mostly been learning through experience, and I agree that more real-world examples would be beneficial. +With Lambda, pay as you go, it's not a good model there. So without this observability in place, we wouldn't even know that there is, you know, one single user somehow invoking 200,000 requests within like a six-hour period. So these tips, it's small little things where now we can focus on a single user or aggregate it across a grouping and be able to find behaviors. And it's really good when you throw into like a time series or flame graph and you can see these kind of fat spots where you can just visually identify something doesn't look good here. Investigate. -**Adriana:** How do you see OpenTelemetry evolving in your workflow over the next year? +**Adriana:** That's great. Thank you so much, both of you, Andre and Tommy, for joining us. And thank you to my co-host, the other Andre, for joining. And also this is Andre K's first time on a stream, so big kudos. Awesome job as co-hosts. -**Tommy:** I believe the real value lies in the data we extract. We want to apply user journeys and business events to gain insights. +I want to give a shout-out, and you know, as always, we appreciate all the stories that we hear from our community members showing how they use OpenTelemetry out in the wild. It really helps us. And it helps show folks in the community, like, you know, OpenTelemetry is here to stay. We've got some gnarly use cases of it being used out in the wild and it's great to hear those stories. -**Andre M:** I see OpenTelemetry helping us mature our observability processes further, especially around logs and complex setups. More out-of-the-box integrations would simplify our configurations. +Coming up next, stay tuned. We should have another Hotel Me planned, I think, in July. So, stay tuned on the hotel socials for that. In the meantime, coming up at the end of this month, we have Hotel Community Day. It's part of Open Observability Con. It's combined with Hotel Community Day. So that's happening on the heels of Open Source Summit in Denver. Open Source Summit North America in Denver. So if you're around for Open Source Summit, check out Hotel Community Day and Open Observability Con that are happening together. -**Adriana:** Thank you both for sharing your insights today! Before we wrap up, do you have any final thoughts or shoutouts? +I think we've got a couple of KubeCons also coming up. We've got, I think, I want to say this week is KubeCon China. And then next week, I'm actually jetting off to Japan at the end of this week for KubeCon Japan. And it's the first KubeCon Japan, so very exciting stuff. So if anyone's around for KubeCon Japan, come say hello. We'd love to meet you and talk. -**Tommy:** Just a big thank you for having us! +I think that is a wrap. And again for those who attended, tell your friends if they missed it, the recording will be available both on LinkedIn and YouTube. Also, check out the hotel end user SIG. We also have a group on CNCF Slack we are called, I think we're called SIG-User, so come share your stories on the hotel end user SIG chat. Also, if you'd love to join us for one of our meetings, we meet every two weeks, so I think our next one is next week. -**Andre M:** Same here! It's great to share our experiences. +Thank you everyone once again for joining and we'll see you next time. -**Adriana:** Thank you to everyone who attended! The recording will be available on LinkedIn and YouTube. Be sure to check out the Hotel End User SIG group on CNCF Slack to share your stories and join us for our meetings every two weeks. Thanks again, and see you next time! +**Tommy:** Thanks so much for having us. ## Raw YouTube Transcript diff --git a/video-transcripts/transcripts/2025-07-16T20:50:15Z-otel-in-practice-alibabas-opentelemetry-journey.md b/video-transcripts/transcripts/2025-07-16T20:50:15Z-otel-in-practice-alibabas-opentelemetry-journey.md index 04d1a32..ac034e1 100644 --- a/video-transcripts/transcripts/2025-07-16T20:50:15Z-otel-in-practice-alibabas-opentelemetry-journey.md +++ b/video-transcripts/transcripts/2025-07-16T20:50:15Z-otel-in-practice-alibabas-opentelemetry-journey.md @@ -10,121 +10,120 @@ URL: https://www.youtube.com/watch?v=fgbB0HhVBq8 ## Summary -In this episode of "Hotel in Practice," Dan hosts Hushing Jang and Steve Ra from Alibaba to discuss the adoption of OpenTelemetry within their organization. Hushing and Steve share their experiences transitioning from Alibaba's internal instrumentation to OpenTelemetry, detailing the challenges they faced and the enhancements they made during the migration. They explain how Alibaba has developed Java and Go instrumentation, focusing on automatic context propagation and the integration of profiling capabilities. The discussion also touches on Alibaba's commitment to contributing back to the OpenTelemetry community and promoting its usage in the Asia-Pacific region. The video concludes with a Q&A session, addressing the ease of migration for teams and encouraging best practices for context propagation. +In this episode of "Hotel in Practice," Dan, a member of the End User Special Interest Group, hosts Hushing Jang and Steve Ra from Alibaba to discuss the adoption of OpenTelemetry within the company. Hushing, a staff engineer, and Steve, a senior software engineer, share their experiences transitioning Alibaba's observability tools from a legacy system to OpenTelemetry. They detail the migration process, emphasizing the challenges faced, the enhancements made to the OpenTelemetry Java agent, and their approach to compile-time instrumentation for Go applications. They highlight the benefits of the OpenTelemetry ecosystem, such as easier context propagation and better scalability, and discuss their contributions to the OpenTelemetry community. The session also addresses internal user adoption strategies and the importance of auto instrumentation in ensuring proper context propagation. The discussion concludes with an invitation for viewers to continue the conversation through community resources. ## Chapters -Here are the key moments from the livestream along with their timestamps: +00:00:00 Welcome and intro +00:01:36 Guest introductions +00:03:00 Presentation begins +00:05:00 Background on Alibaba's observability +00:07:40 Migration to OpenTelemetry +00:10:00 Enhancements on Java agent +00:12:40 Context propagation challenges +00:18:00 Go applications instrumentation +00:24:00 Adoption cases overview +00:30:24 Future work and contributions -00:00:00 Introductions and overview of the session -00:02:45 Introduction of Hushing Jang and Steve Ra from Alibaba -00:05:30 Background on Alibaba's observability capabilities -00:09:15 Discussion on the migration from Pinpoint to OpenTelemetry -00:12:00 Enhancements made during migration to OpenTelemetry -00:17:30 Introduction of Alibaba's Golang and Python instrumentation -00:22:00 Explanation of compile-time instrumentation approach -00:27:45 Use cases for OpenTelemetry in AI applications -00:33:15 Challenges of context propagation and solutions -00:38:00 Discussion on internal adoption and user support for migration -00:43:00 Closing remarks and invitation for further conversation +[00:01:36] **Dan:** Good morning, good afternoon and good evening whenever you're watching this or wherever you're watching this. Welcome to a new edition of hotel in practice where industry experts, end users and contributors tell us about a specific area of open telemetry and how it can benefit or it has already benefited end users. My name is Dan. I'm a member of the end user special interest group and I'm also a member of the governance committee and I'm very excited today to have Hushing and Steve to talk about adoption of open telemetry within Alibaba. I'm connecting from Edinburgh in Scotland. You can see the hills here. And if you're watching live on YouTube or LinkedIn, please say hi in the chat and tell us where you're watching from or where you're connecting from. Before we get started, I just wanted to cover a couple of housekeeping items. During the presentation, please drop your questions in the chat and although we'll make our best to get to all of your questions, I can't promise that we'll get to all of them. But fear not, if that's the case, if we can get to all of them and your question is not answered or you want to continue the conversation, you can go to the end user s and then we've got resources in this link and you can continue the conversation there. You find your way to Slack where we have a big community of end users including Hushing and Steve and we will be happy to provide some of their insights there. Also, please remember to keep the questions to date and topic of the presentation. And yeah, so with that out of the way, let's get started and welcome Hushing Jang, staff engineer at Alibaba and hotel compile instrumentation maintainer. Hi Hushing, can you tell us a bit more about what you do? -These timestamps provide a clear overview of the key topics discussed during the livestream. +**Hushing:** Hello Dan. Very nice to meet you and hello. I'm Hushing. I'm from Alibaba. Yeah, I'm working as an observability engineer in Alibaba cloud and we are also working on the client instrumentations. So we work very closely with the open telemetry community. We're working on different kinds of language instrumentations, collectors, and something like that. We also provide service both for internal and external clients, customers. Thanks. -# OpenTelemetry Adoption at Alibaba +**Dan:** That's awesome. And also welcome Steve, Steve Ra, senior software engineer at Alibaba and hotel Java instrumentation approver. Hi Steve. -Good morning, good afternoon, and good evening, whenever and wherever you're watching this. Welcome to a new edition of *Hotel in Practice*, where industry experts, end users, and contributors discuss specific areas of OpenTelemetry and how it can benefit, or has already benefited, end users. +**Steve:** Yeah. Hi Dan. Hi. I am. I'm Steve. I'm also from Alibaba cloud. I work in the observability team in Alibaba and I participate in Java agent and I also responsible for profiling in our team and provide out of box profiling ability for our users. Thank you. -My name is Dan, and I'm a member of the End User Special Interest Group and the Governance Committee. I'm excited today to have Hushing and Steve here to talk about the adoption of OpenTelemetry within Alibaba. I'm connecting from Edinburgh in Scotland, where you can see the hills in the background. If you're watching live on YouTube or LinkedIn, please say hi in the chat and share where you're connecting from. +**Dan:** Nice. Well, welcome both. I'm really excited to hear more about how you're approaching open telemetry in Alibaba and how you ended up contributing as well, being major contributors to open telemetry. So, now that we're all set, you've got a presentation and you'll guide us through it. I'm going to move out and then please take it away. -Before we get started, I want to cover a couple of housekeeping items. During the presentation, please drop your questions in the chat. While we'll do our best to answer all of them, I can't promise we'll get to every question. If your question remains unanswered or you'd like to continue the conversation, please visit the End User SIG resources linked in the chat. You can also find your way to Slack, where we have a large community of end users, including Hushing and Steve, who will be happy to share their insights. +[00:05:00] **Hushing:** Okay. Hello everyone. We are Steve and Hushing from Alibaba and we are very glad to share our talk Alibaba practice experience. Today we will explain our talk from the following five parts and I will introduce the first two parts and the remaining are given to my colleague. The reason for choosing hotel. Let me briefly introduce our background. In 2013, we provided observability capability for our internal users by manual instrumentation and we provided this ability into our internal software frameworks and we also designed a set of trace propagation protocol called Eagle I. In 2017, we developed the Alibaba Java agent based on the pinpoint Java agent. After several years of use, we found some problems such as pinpoint cannot support verifying the supporting version of a framework. For example, if a framework released a new version, the instrumentation may not take effect and we can perceive this immediately. Another thing sounds like pinpoint cannot support very well in some scenarios such as trace propagation synchronously. It doesn't support it automatically and there are also some other reasons due to the limited time I cannot list them one by one. -Please remember to keep your questions relevant to the topic of the presentation. With that said, let's get started and welcome Hushing Jang, a Staff Engineer at Alibaba and Hotel Compile Instrumentation Maintainer. Hi Hushing, can you tell us a bit more about what you do? +From 2020 to 2023, we researched some other solutions from open source. At the time, we found open telemetry is a very strong ecosystem. It has rich semantic conventions and it represents the standard in observability. After several internal discussions, we made up our mind to migrate our Alibaba Java agent from pinpoint to open telemetry. After several months of development, we launched our Alibaba Java agent based on open telemetry in 2024. We also launched our Golang and Python instrumentation based on the semantic convention or SDK. Currently we embraced open telemetry completely. -### Introduction of Speakers +[00:07:40] After introducing our background, during our migration, we really did a lot of enhancements on the hotel Java agent. The first one is instrumentation. At the time we found open telemetry not widely used in China because there are some frameworks that are not supported by the hotel Java agent but they are widely used in China. We had supported them before. So we needed to implement them based on the hotel Java agent architecture. The second point is profiles. As we know profile is a very powerful tool. It can help us to inspect the behavioral performance of our application at runtime. It can help us to know which code is responsible for the spikes in CPU memory usage. We have supported profiling for several years. So the migration of this feature is also very important for us. Other abilities such as abundant sampling or more metrics and we also need to debug a very popular debugging tool in China. Last but not least, we not only support observability by Java agent, we also support microservices governance such as traffic control or security management by this Java agent. -**Hushing:** Hello Dan, very nice to meet you! I’m Hushing from Alibaba. I work as an observability engineer in Alibaba Cloud, focusing on client instrumentations. We work closely with the OpenTelemetry community, developing various language instrumentations and collectors, providing services for both internal and external clients. +[00:10:00] So how to combine them in the hotel Java agent architecture? It's a toss-up we need to solve. Due to the limited time at the time and how to migrate them, it's challenging for us. In fact, at the time we weren't very familiar with telemetry and in order to achieve this goal, we came up with a two-step solution to solve this question. First one, we forked a copy of the Java agent repository and we promptly implemented all our commercial enhancements on the Java agent. In this process, it's not very easy because there are a lot of commercial features. If we migrate to the open telemetry, there are a lot of issues or problems and due to the rich business scenarios and during migration, we also found some problems and we needed to fix them and some of them are strongly related to the upstream and we also contributed them to the upstream. After about six months from research to launch the first release of our Alibaba Java agent based on open telemetry. After finishing this phase, we knew that it's not enough because we know the biggest charm of open telemetry is its stronger ecosystem. If we compare our code with open source, it's challenging for us to merge some upstream updates to our Alibaba Java agent. -**Dan:** Thanks, that’s awesome! Also, welcome Steve Ra, Senior Software Engineer at Alibaba and Hotel Java Instrumentation Approver. Hi Steve! +So on the one hand, we promoted our users to use our first version and then we also invited some developers to design some extensions or use some extensions from open source to migrate our commercial feature. This process is time-consuming because we need to design some extensions based on the Java agent and we also need to contribute them to the upstream. The other things we are currently doing, after this step we can see the open source hotel Java agent just like a dependency for us and we can decouple our commercial code and the open source code and we can merge upstream easily. -**Steve:** Hi Dan, I'm Steve, also from Alibaba Cloud. I work in the observability team and participate in Java agent development. I'm responsible for profiling in our team and providing out-of-the-box profiling capabilities for our users. +[00:12:40] This is our whole approach to achieve migration. After migration to open telemetry, we gained a lot from open source and we also got something we didn't anticipate at the beginning. The first one I want to share is the hotel semantic convention. I guess that is the soul of open telemetry and when we migrated to open telemetry and we told our users we achieved this Java agent based on open telemetry and they can do a lot of things by themselves such as they can search some materials, some documentation by themselves and they can learn how to use some extensions to write something by themselves and they can debug some problems by themselves. I think that is a very good point for us to embrace open telemetry and our users can do a lot of things by themselves and it rarely reduces users' learning burden. Because they just need to learn the standards, the semantic convention or implementation of open telemetry and they can do a lot of things by themselves. -**Dan:** Great, welcome both! I’m excited to hear more about how you approach OpenTelemetry at Alibaba and how you became major contributors. Now that we’re all set, I understand you have a presentation to guide us through. Please take it away! +This is the first point. The second point is good architectural design and better scalability of hotel instrumentation. It's very useful for us to promote our users to use our Java agent. How to understand this? For example, before Java hotel Java agent, if a user used a framework that is not widely used and we don't support that framework in our Java agent, they will create an issue for us and ask us to schedule resources to support this framework. But sometimes, we take the cost and energy into consideration, we are reluctant to support that. So how to communicate with our user that is a question at that time. Nowadays, this is not a big question. Because we can tell our users there are a lot of extensions; they can use this extension to support their own framework and they can do it by themselves and a lot of users are very interested in this because they can extend a lot of things by themselves and don't rely on us. -### Presentation by Hushing and Steve +This is all the things I want to share today. The remaining time, I will introduce something from other aspects. Thank you. -**Hushing:** Hello everyone, we are Steve and Hushing from Alibaba, and we are glad to share our experience with OpenTelemetry adoption. Today, we will explain our talk in five parts. I will introduce the first two parts, and the remaining will be presented by my colleague. +**Steve:** Thank you, Hushing. I will next introduce. Hello. Can you hear me very well? -#### Background of OpenTelemetry Adoption +Yes. Okay. Sorry. There's some network issues. Maybe I just lost the connections. I will go into for the remaining part. -In 2013, we provided observability capabilities for our internal users through manual instrumentation, integrating this ability into our internal software frameworks. We also designed a trace propagation protocol called Eagle I. In 2017, we developed the Alibaba Java agent based on the Pinpoint Java agent. After several years of usage, we encountered some issues. For example, Pinpoint could not verify the supporting version of a framework; if a framework released a new version, the instrumentation might not take effect. We also found that Pinpoint did not support scenarios like synchronous trace propagation automatically. +[00:18:00] Yes. Actually, this solution has been started in 2023 and at that time we wanted to look for a solution that can be easily applied for Go applications for the to-do instrumentations. Actually, when we were trying the eBPF based approach that is already been provided by the open telemetry community, but it turns out that there are some limitations. We finally figured out a way to do the instrumentations to satisfy our needs that is in a compile time instrumentation. If you're looking at the figure on the right, there's an approach that the injection has happened in the middle of the compilation process. Actually, we hooked the Golang compile command to make sure that can be done for users without any modifications to users' code. That's a key feature that we want to achieve. -From 2020 to 2023, we researched alternative solutions and found OpenTelemetry to be a robust ecosystem with reach semantic conventions, representing a standard in observability. After several internal discussions, we decided to migrate our Alibaba Java agent from Pinpoint to OpenTelemetry. After months of development, we launched our Alibaba Java agent based on OpenTelemetry in 2024, along with our Golang and Python instrumentations, all based on OpenTelemetry's semantic conventions and SDK. Currently, we fully embrace OpenTelemetry. +Next, I want to address several highlights of this approach. First is the async context propagations. Actually, we encourage our users to follow the guidelines, but actually not all the users follow the rules. Sometimes they do not pass the context object correctly and the traces may break and become incomplete. So this feature can actually do the context propagation automatically for us even when we are not passing the object correctly. The second one is that it is actually compatible with the old SDK. That means that users who have used manual instrumentations can work well with our approach. So the traces can be merged. The last one is the most powerful feature that we have done is kind of like customer extension. That means that you can instrument any code into any point, any line of that application. This allows us to solve some special requirements from users. Then they can do some customizations like traffic management and security. -#### Enhancements Made During Migration +Next, we will introduce some practices that we have in the LLM of observability. As we know, the LLM has been popular recently, especially this year a lot of AI agents have become active and there's a typical use case that we found. In the use cases, a user sends a request to the API gateway and the API gateway sends to the LLM applications, which are written in maybe Python, Java, or Go and the application will call different kinds of models, for example, DeepSea Queen or OpenAI. -During our migration, we made significant enhancements to the OpenTelemetry Java agent. The first enhancement was instrumentation. We discovered that OpenTelemetry was not widely used in China because some widely-used frameworks were not supported by the OpenTelemetry Java agent. Therefore, we implemented support for these frameworks based on the OpenTelemetry architecture. +We have adopted an AI gateway in the middle that can do a lot of things for us. For example, it can do unified access and token limitations, token cache, and other stuff like that. We have adopted the open source project called Highress that can do it very well for us. -The second enhancement was profiling. Profiling is a powerful tool that helps us inspect the behavioral performance of our applications at runtime, allowing us to identify which code is responsible for spikes in CPU and memory usage. We had provided this capability for several years, and its migration was crucial for us. +When we are building observability across the different components, we make sure we're following this latest JNI semantic convention. Because the convention is still in development, we're trying to keep pace with them as much as possible. We have done some instrumentations based on this semantic convention. This includes major AI agent frameworks like high code frameworks such as AgentScope, Agono, Lang, Spring, Alibaba, etc. -Other enhancements included abundant sampling, additional metrics, and popular debugging tools within China. Moreover, we not only supported observability through the Java agent but also contributed to microservices governance, such as traffic control and security management, through this Java agent. +For low code platforms, DIY is a very popular open source project to build AI agents with low code. We also do some instrumentation for the official MCP clients. For example, we want to capture the MCP2 courses; we want to capture the M32 responses and to make sure our observability is end-to-end. We want to put our instrumentations into the internal models that come up with serving frameworks like VM or SGAN, which are actually Python processes running there. So we can put our Python instrumentations into that Python process to capture traces and key metrics like TTFT (time to first token) and TBOT (time per output token), which are very important metrics for us to identify bottlenecks of LLM serving issues. -To achieve our migration goals, we devised a two-step solution. First, we forked a copy of the Java agent repository and quickly implemented our commercial enhancements. This process was challenging due to the many commercial features involved. We had to address numerous issues during migration, particularly those related to the upstream, and contributed fixes back to the upstream community. After about six months, we launched the first release of our Alibaba Java agent based on OpenTelemetry. +[00:24:00] Next, talking about the adoption cases. Actually, we have many adoption cases. I want to introduce several of them today. I divided them into two parts. The first part is the cloud service data flow. When users use cloud service, we make sure all the services that are in key positions of the application embrace the official W3 tracing goal by default. That means that balancer way or they support the W3 protocol by default. They will have the async messaging, like how a consumer is consuming the messages, to measure the latency or identify bottlenecks. They are using open telemetry to do that. -Following the initial release, we recognized that it was not enough to simply migrate; we also wanted to leverage OpenTelemetry's strong ecosystem. We encouraged users to adopt our initial version while inviting developers to design extensions and migrate our commercial features to OpenTelemetry. This was a time-consuming process but essential for decoupling our commercial code from the open-source code. +For the AI studio, they are using Java and Python hotel instrumentation for end-to-end observability. The next part is the cloud service management console. A lot of management console services have embraced open telemetry as well. For example, the elastic cloud computing service is using hotel Java instrumentation quite a lot and they extend and make some extensions to that. They want to do request-based traffic routing. They want to route the service to different machine groups. -#### Benefits of Migrating to OpenTelemetry +The video on demand service is also running a large amount of clusters written in Go and they have adopted the Golang compile time instrumentation mostly for runtime monitoring. They want to monitor the runtime metrics and also provide data. Another case is the institution detection service. There’s an agent running on the host written in Go. Before open telemetry, they had nothing to do with observability because they lacked techniques and tools to do that. Now they can use hotel Go compile time instrumentation for their runtime monitoring and profiling. -After migrating to OpenTelemetry, we experienced several benefits. Firstly, the OpenTelemetry semantic conventions became the foundation of our observability efforts. By adopting OpenTelemetry, users could independently access materials and documentation, which significantly reduced their learning burden. +As our own service, the cloud monitoring service also implements self-observability with Java instrumentation. We do a lot of things with Java instrumentation to do traces and profiling. This is a good example of dogfooding, right? -Secondly, the good architectural design and better scalability of OpenTelemetry instrumentation made it easier for us to promote our Java agent. Previously, if a user employed a framework that we did not support, they would create issues asking for support. Now, we can inform them that many extensions are available for them to use independently. +Yes. Besides our internal adoption, we are also trying to contribute these enhancements or extensions back to the open telemetry community. We have mentioned before that a lot of instrumentations, some of them have been contributed and some are also on the way to contributing back. We also helped promote open telemetry in the APAC area. We helped organize some meetings that are friendly to the Asia-Pacific areas like Java, Go, and Jingi meetings. We also engaged with conferences like KubeCon and the hotel community and we are also organizing events like KCD to help promote open telemetry in Asia-Pacific areas. -**Steve:** Thank you, Hushing. I will now introduce the remaining parts of our presentation. +In 2024, when we decided to donate this Golang compile time instrumentation to the open telemetry community and after a long discussion with the community, we also found that DataDog showed a very strong interest in this project. They actually submitted a proposal to donate their own framework, which is also a compile time instrumentation project, to open telemetry. So we decided that we do not want to depend on a single company's effort. We formed a special interest group (SIG) to build this project from scratch. This SIG is co-founded by Alibaba, DataDog, and Quesma. We have a common effort to implement this compile time instrumentation for Golang. This project is still in active development and we are working on the first release and the first MVP version of that this year. -### Compile-Time Instrumentation Approach +[00:30:24] If you are interested, please definitely check it out. Talking about future work, for Java instrumentations, we are planning to implement the proxy agent mode. That means that we want to use the Java agent as a dependency rather than booking upstream projects because we do not want to maintain such kind of project. We want to make sure all the extension points can be merged into the upstream so that we can extend this project very well. -In 2023, we sought a solution for Go applications that could simplify instrumentation. Initially, we explored the eBPF-based approach provided by the OpenTelemetry community, but encountered limitations. Eventually, we developed a compile-time instrumentation method, which hooks into the Go compile command, ensuring no modifications are required in the user's code. +For the Golang compile time instrumentation, we are working well with the MVP version and we're making sure we are working towards the first release. We will also support more signals including metrics and profiles. For the GI, we also want to contribute some of the instrumentations back to the community because the GI is still in active development. We put some of them in our repo and we want to contribute them back as well. -#### Highlights of Our Approach +Actually, we also made our own open telemetry distribution called Long Sweet. Long is the Chinese version of Dragon. It's kind of a distribution built on various language instrumentation as well as our collector project called the Long Collector. We want to make sure that all the enhancements we want to contribute back to the hotel project as much as possible. But for some reasons, the community may not accept our contributions; they will be in our own distributions. -1. **Asynchronous Context Propagation:** This feature automatically propagates context, even when users do not pass the object correctly, addressing common issues with incomplete traces. - -2. **Compatibility with Old SDK:** Users who have previously used manual instrumentation can continue to work seamlessly with our approach, allowing for a smooth transition. - -3. **Custom Extensions:** Users can instrument any line of code in their applications, enabling customizations for traffic management and security. +That is all for the presentation and thanks for everyone for listening. -### Adoption Cases +**Dan:** Thank you, Hushing and Steve. That was great. I think feel free to drop any questions in the chat and then, if you're watching live on YouTube or LinkedIn, I have one which is you mentioned the adoption of open telemetry within Alibaba. I wanted to know how easy was it for teams within Alibaba to migrate to open telemetry? Is there anything you can share in terms of facilitating a migration, maybe they use something like a common set of libraries or migration script or something else? -We've seen many adoption cases within Alibaba. For instance, our cloud service's data flow uses OpenTelemetry to ensure that all key services embrace the W3 tracing model by default. The AI Studio utilizes Java and Python instrumentation for end-to-end observability, while services like the Elastic Cloud Computing Service and Video on Demand services have adopted OpenTelemetry for runtime monitoring and profiling. +**Steve:** Yeah, I think it's not quite easy actually. It's not quite easy for them to migrate. We have done a lot of things to advocate for that. First, we wrote some internal articles to share some of our practices, like how a team adopted this technology and how they benefited from that. Then we put that into the internal article to help promote open telemetry. Secondly, we actually wanted to try to minimize the effort for the user to migrate because we have a legacy project called Ecoi and they already have some instrumentations there. If we are switching to this architecture to the auto instrumentations, they might encounter some conflicts or something like that. We actually have done a lot of things to prevent such kind of things from happening and we built that within our agent with our instrumentations as much as possible to make sure the users have minimized their effort to start up and to migrate. The idea is to try to put things to our team rather than put the efforts to the users to make sure that a seamless migration from the old architecture to the open telemetry based one. -### Contributions Back to the Community +**Dan:** Nice. So you dogfooded that to make sure that you adopted it internally as well. -Beyond our internal adoption, we strive to contribute enhancements and extensions back to the OpenTelemetry community. We've organized friendly meetings for the APAC region and engaged with conferences like KubeCon. In 2024, we decided to donate our Go compile-time instrumentation project to the community, forming a special interest group with collaborators from DataDog and Quesma to build this project from scratch. +**Steve:** Yeah. -### Future Work +**Dan:** Makes sense. Okay. I think we've got some comments in the chat but no questions so far as I can tell. I have another one as well from myself. You mentioned that you run on the hotel Java agent stuff. You run a fork or a distribution of that agent but in terms of API usage, do internal users in Alibaba use the open telemetry API directly when instrumenting applications or do you have some kind of abstraction on top or a helper library that they use instead of the open telemetry API? -Looking ahead, we plan to implement a proxy agent mode for our Java instrumentation to ensure all extension points can be merged into the upstream. We are also working towards the first release of our Go compile-time instrumentation and will support additional signals, including metrics and profiles. +**Hushing:** I think for most cases, we would have them use the auto instrumentations as the first choice because that is less effort for them to start their adoption. But for some special cases, actually, we can't satisfy. We will encourage them to use the SDK to add some custom instrumentations. For example, they want to add some custom spans or metrics. When that happens, we encourage them to use the official open telemetry SDK as much as possible because that is well documented and we actually have battle-tested it with the latest version of that SDK. So we do not want them to build another abstraction layer on top of that. But there are special cases that we introduced, like LLM application observability recently. Our AI studio is using the open telemetry SDK; they found that they have to write a lot of code in order to add their instrumentations because there are so many things they want to capture in a span. So they have to write a lot of instrumentations. So they asked us if we can provide an abstraction layer for them to simplify their work to do their instrumentation. So in that case, we actually worked with them to provide a higher level of that SDK. Their instrumentation effort has been reduced and they are very happy to do that. -### Conclusion +**Dan:** Yes. So that's one case I can share, and for most cases, I think it's using the official OT SDK. -Thank you for your attention, and we appreciate the opportunity to share our journey with OpenTelemetry at Alibaba! +**Hushing:** Yes. And in terms of the API as well because it's got that decoupling of the API and the implementation, like you can always swap the implementation underneath with the SDK, right? ---- +**Steve:** Yes. -**Dan:** Thank you, Hushing and Steve! That was an insightful presentation. If anyone has questions, feel free to drop them in the chat. +**Dan:** I want to share another point about this question. -**Audience Interaction** +**Hushing:** Sure. -One question I have is about the ease of migration within Alibaba. How did you facilitate the transition to OpenTelemetry? Did you use common libraries or migration scripts? +**Steve:** For our internal users, some of them, if we don't provide auto instrumentation such as C++ applications, we encourage them to use the API just like Hushing mentioned. Another point for Java applications, some users, as we know, if they use trace or span, sometimes that is a monitoring spot. If we just instrument some frameworks, some key methods and if the latency belongs to users' code, maybe they can't get the information why the latency is so long. During the traces, at this time, we provide out of box profiling ability. We correlate the traces with the profile and we can provide diagnosis for users to understand why the latency is so long and which code is responsible for the latency. So they don't need to use the API to instrument their own methods manually. -**Steve:** Migrating to OpenTelemetry wasn't easy. We wrote internal articles sharing our practices and benefits. We also aimed to minimize the effort for users by integrating our legacy project with the new architecture, ensuring a seamless transition. +**Dan:** Okay. Yeah, that makes sense. Last question from me. Again, if you're watching and want to drop a question, feel free. Both of you talked about context propagation as something that has definitely improved a lot thanks to open telemetry instrumentation. I know this from experience, especially when we're talking about async tasks. However, sometimes it's still challenging in certain scenarios to make sure that internally in a service, the context is propagated correctly. How do you encourage teams to ensure that context is propagated correctly within not just across services but also within their own services? And also, do you know if they measure that in some way if context is broken somewhere or leaking? -**Dan:** It sounds like you really focused on making it easy for your teams to adopt! +**Steve:** Yeah, that's a good question. Actually, I think for our internal users, it's very difficult for them. Even if we provide guidelines or encourage them to do so, they may not follow this rule. There is very much opportunity for them to miss that or fail to follow that. To solve this problem, it's our team's duty to make sure all the possible ways of asynchronous context propagations have been propagated correctly by our auto instrumentations. That is the way we solve this problem. We have popular frameworks in Java, for example, the thread pool or some kind of similar scenarios, and we do the test to make sure that the asynchronous context can be propagated properly. -**Steve:** Exactly! We aimed to take on the burden of migration rather than placing it on our users. +There are other frameworks that are very challenging for us to do. For example, Reactor Netty or such kinds of frameworks are very high frequency for them to switch context and asynchronous is common for them. For this kind of scenario, we might have something working not very well or as expected. For this case, we would like to try and work with the community to identify the issue and try to fix it. -**Dan:** Great approach! +**Dan:** Yeah, I used to remember the days where you had to pass a context object around in function calls and it wasn't pretty and it wasn't good either. I think auto instrumentation is the way to go and anything that can get done transparently is also always going to be covering more and resulting in better context propagation, right? -If there are no further questions, I want to thank Hushing and Steve again for joining us today. If you'd like to continue the conversation, please check out the End User SIG resources at OpenTelemetry.io. Thank you all for watching, and see you next time! +**Steve:** Yes. As long as the users stick to that automatic instrumentation approach, they will ask us if we satisfied this or make why did that happen? So they will come to us looking for help rather than doing it by themselves. + +**Dan:** Indeed. Okay. I think that's all we have time for today. Thank you so much for joining us both Hushing and Steve. It was great knowing more about Alibaba's approach to open telemetry. Thanks to all that joined live on LinkedIn and YouTube. Remember that if you want to continue the conversation, we've got the end user SEG resources as part of the opentelemetry.io website. Feel free to continue the conversation there and hope to see you there as well. Thank you again, both, and see you in the next one. + +**Hushing and Steve:** Thank you. Bye-bye. ## Raw YouTube Transcript diff --git a/video-transcripts/transcripts/2025-09-25T05:32:25Z-otel-in-practice-how-we-scaled-kafkalog-ingestion-for-otel-by-150.md b/video-transcripts/transcripts/2025-09-25T05:32:25Z-otel-in-practice-how-we-scaled-kafkalog-ingestion-for-otel-by-150.md index 554727d..06b7dc1 100644 --- a/video-transcripts/transcripts/2025-09-25T05:32:25Z-otel-in-practice-how-we-scaled-kafkalog-ingestion-for-otel-by-150.md +++ b/video-transcripts/transcripts/2025-09-25T05:32:25Z-otel-in-practice-how-we-scaled-kafkalog-ingestion-for-otel-by-150.md @@ -10,129 +10,152 @@ URL: https://www.youtube.com/watch?v=GrQyUPeCljo ## Summary -In this edition of "Hotel in Practice," host Henrik welcomed Dakota Passman from Bindplane to discuss scaling open telemetry logs with Kafka. Dakota, a software engineer at Bindplane, shared insights from a project where they helped a customer overcome performance bottlenecks while pulling events from Kafka. The customer initially faced issues with only 12,000 events per second per partition, which led to a growing backlog. Dakota highlighted key strategies that improved performance, including using a new Kafka client, optimizing log encoding, reconfiguring the data processing pipeline to prioritize batching, and switching the exporter protocol. After implementing these changes, the throughput increased significantly, demonstrating how careful configuration can enhance system performance. The session concluded with a Q&A segment where Dakota addressed audience questions related to performance tuning and configuration best practices. Viewers were encouraged to access the recording on YouTube and LinkedIn for further learning. +In this episode of "Hotel in Practice," host Dakota Passman from Bindplane discusses scaling open telemetry logs using Kafka. Dakota, a software engineer with over two years at Bindplane, presents a case study involving a customer who faced performance bottlenecks pulling events from Kafka, initially processing only 12,000 events per second per partition, while needing to reach 30,000. Key strategies to resolve this included switching to a more efficient Kafka client, optimizing configuration settings, adjusting log encoding from OTLP JSON to raw JSON, and repositioning the batching processor to improve throughput. The presentation also features a live demo showcasing the improvements in event processing capabilities after implementing these changes. Audience questions addressed the impacts of different exporters on performance, the configuration of batching, and resource usage considerations. The session concludes with reminders about upcoming events and opportunities for community engagement. ## Chapters -Here are the key moments from the livestream with timestamps: +00:00:00 Welcome and intro +00:01:40 Guest introduction: Dakota Passman +00:02:50 Overview of scaling issues +00:05:01 Key takeaways for performance +00:06:30 Customer performance metrics +00:09:00 Changes made to improve performance +00:12:00 Demo setup explanation +00:15:02 Live demo of performance changes +00:19:00 Q&A session begins +00:25:26 Closing remarks and housekeeping -00:00:00 Introductions and welcome to the stream -00:01:45 Introduction of speaker Dakota Passman -00:03:10 Overview of the topic: Scaling OpenTelemetry logs with Kafka -00:05:00 Discussing performance issues with the customer's Kafka integration -00:07:30 Key takeaways for improving Kafka receiver performance -00:09:20 Explanation of the customer's original configuration and bottleneck -00:13:00 Live demo setup and initial performance metrics -00:15:30 Implementation of changes to improve performance -00:20:00 Analysis of the changes made and their impact on throughput -00:25:00 Q&A session discussing exporters and batching strategies -00:30:00 Closing remarks and information about future events and community engagement +**Host:** Hello everyone and welcome to our latest edition of Hotel in Practice. We are so excited to have you join here today. And remember, tell your friends for anyone who wasn't able to make it today that we will have our recordings available after the fact. You can go to our YouTube channel at hotel-official, and you can also check out the hotel LinkedIn page. We should have the recording available on there too. -# Hotel in Practice: Scaling OpenTelemetry Logs with Kafka +We have a very special hotel in practice today. We have Dakota Passman from Bindplane talking about doing some gnarly scaling of open telemetry logs with Kafka. So without further ado, let's bring Dakota on. -Hello everyone, and welcome to our latest edition of Hotel in Practice! We are so excited to have you join us today. Remember to tell your friends that if they weren't able to make it, the recordings will be available afterward. You can check out our recordings on our YouTube channel at **hotel-official** and also on the Hotel LinkedIn page. +**Dakota:** Hello. -Today, we have a very special guest, Dakota Passman from Bindplane, who will talk about scaling OpenTelemetry logs with Kafka. Without further ado, let’s bring Dakota on! +**Host:** Welcome Dakota. ---- +**Dakota:** Hi. Thanks for having me. -### Introduction +**Host:** Super excited to have you here. Would you like to do a brief intro of yourself before starting? -**Dakota:** Hi! Thanks for having me. I'm Dakota, and I've been a software engineer at Bindplane for a little over two and a half years. I work on our platform, focusing a lot on the hotel project and making various contributions, especially regarding the supervisor. I'm happy to be here. +[00:01:40] **Dakota:** Yeah. Yeah. I'm Dakota. I've been a software engineer at Bindplane for a little over two and a half years now. Working on our platform, focusing a lot on the hotel project and making some different contributions. Focusing a lot these days on the supervisor. It's a pretty cool project going on there. But yeah, happy to be here. -**Host:** Excellent! If you're ready, let’s get started. For anyone interested, we will have time for questions after Dakota's presentation, so please feel free to post them in the chat. +**Host:** Excellent. If you're ready to go, then let's get started. By the way, for anyone who is interested, we will have some time after Dakota's presentation for questions. So please feel free to post them in the chat, and we will address them after the presentation. ---- +[00:02:50] **Dakota:** Yeah. Cool. All right. So yeah, today I want to talk about a situation we handled with a customer of ours using the Kafka receiver at scale. We had to scale their environment. They were having performance issues. They were running into a bottleneck trying to pull events from Kafka. When we started talking to them, they were pulling only 12,000 events per second per partition in their topic, which was just not enough to keep up. -### Presentation Overview +So we're going to talk about that bottleneck, how we got them to get up to a target EPS of 30,000. And then we're going to demo those changes live in a replica environment. To start, we're going to go over some of the takeaways just so that we can keep these in mind as we're going through this and see where they're coming up. -**Dakota:** Today, I want to discuss a situation we handled with a customer using the Kafka receiver at scale. They were experiencing performance issues and encountered a bottleneck while trying to pull events from Kafka. Initially, they were pulling only 12,000 events per second per partition, which was not enough to keep up. We're going to cover that bottleneck and how we helped them reach a target of 30,000 events per second (EPS). We’ll also demo those changes live in a replica environment. +For starters, don't trust defaults. The Kafka receiver by default uses a certain Kafka client, and these clients behave differently. They perform differently. It's a default. However, shortly before we had this issue with the customer, there was a new client implementation added that we were able to use, and using that client over the default client was a big improvement. -**Key Takeaways:** -1. **Don't Trust Defaults:** The Kafka receiver uses a default Kafka client that performs differently. A new client implementation added shortly before we encountered issues provided a significant improvement. - -2. **Configuration Matters:** Proper configuration of the collector can make a huge difference. This might seem obvious, but it’s easy to overlook if you’re unsure of the configuration options. - -3. **Pipeline Configuration:** The configuration of the pipeline is also crucial. It can greatly affect performance. - -4. **Understand Your Observability Goals:** This understanding helps tune the environment effectively to ensure your Kafka receiver performs well and meets your observability goals. +Secondly, configuration of the collector makes a huge difference. That might seem kind of obvious, but it's sometimes easy to overlook that, especially if you're not entirely sure of what configuration options you have and what they might exactly do. Something else that I think gets overlooked sometimes is configuration of the pipeline matters a lot too, especially in this situation. ---- +Finally, it's important to understand your observability goals so you can tune the environment to get it right and make sure that your Kafka receiver or really any receiver that you might be using is performing well in your environment and hitting the observability goals that you're looking for. -### The Customer's Situation +To kind of set the stage a bit, this customer of ours was pulling 192,000 events per second across all of their partitions. They had just a single topic with 16 partitions, again doing roughly 12,000 events per second per partition, which sums up to 192,000 events per second. However, they needed to really be at 480,000 events per second in order to keep up with the amount of data going into Kafka. At that point, they would be able to start cutting down the backlog that they had going. -The customer was pulling 192,000 events per second across all their partitions, with a single topic of 16 partitions. They needed to reach 480,000 events per second to keep up with the data going into Kafka and start cutting down their backlog. They were using the OpenTelemetry collector and had many default options set, leading to rapid backlog growth. Kafka was ingesting 30,000 events per second per partition, resulting in a backlog increasing by 288,000 events per second. +They were using the open telemetry collector using the Kafka receiver with a lot of default options set. One topic, 16 partitions, each one only doing 12,000 events per second. The backlog was growing rapidly. Kafka was ingesting 30,000 events per second per partition roughly, and so their backlog was growing by 288,000 events per second across all the partitions. They were quickly falling behind. ---- +[00:06:30] In this situation, it was security telemetry that they were collecting from Kafka and trying to send to a SIEM backend. Already, they were dealing with delayed telemetry, and at this poor performance, they were at risk of starting to drop some of this telemetry too. -### Changes Made +Some of the changes we made, this is kind of like an illustration of the configuration and the pipeline that they had going. You can see some of these changes as we moved through the pipeline. So first off, the Kafka receiver using the new FronGo client, which was the Kafka client I mentioned before, using that implementation, we saw a big jump. We started, we jumped up to about 30% just using that client instead of the default. -Here are some changes we implemented: +Another thing, the log encoding that we were pulling from Kafka made a big difference. The receiver was configured to use OTLP JSON, which didn't really make sense in their situation. There was a lot of unnecessary work being done to handle OTLP JSON. So instead, we switched it to just raw JSON or normal JSON, not OTLP specifically, and that was another big performance boost there. -1. **Client Upgrade:** We switched to the new FronGo client, which improved throughput by about 30%. - -2. **Log Encoding:** The receiver was initially set to use OTLP JSON, which was inefficient for their use case. Switching it to raw JSON provided a significant performance boost. +One other thing, batching. We moved our batch processor to the start of the processor pipeline, right after the receiver. The reason we did this, it allowed the receiver to pump out large quantities of events rather than being restricted by the data flow of the processor in front of it. It was reliably able to push out thousands of events in a batch at once rather than maybe a couple hundred and then jump up to a couple thousand back to a couple hundred. It was much more consistent performance batching right after the receiver. That's what we saw. As a result, we were able to boost our throughput doing that. -3. **Batching:** We moved the batch processor to the start of the pipeline, allowing the receiver to push out large quantities of events consistently. +[00:09:00] One final change we made, this is again specific to their environment and their pipeline. They were sending the SecOps SIEM backend, and we saw that changing the protocol used from gRPC to HTTP actually increased performance. In their specific case, this made a big impact. We were able to get increased throughput doing this as well. The idea here is similar to the batch processor we discussed. The exporter is just able to handle throughput better at HTTP than using gRPC. This affected the receiver upstream from the exporter, allowing it to push data out a lot faster. -4. **Protocol Change:** Changing the protocol from gRPC to HTTP for the exporter also improved performance significantly in their specific case. +In this demo, the customer had one topic, 16 partitions. They had a single collector per partition. In this demo, we're just going to be using a single collector, a single partition. Demoing with 16 partitions and managing collectors at that scale is difficult to do manually. We're just going to keep it simple here, just a single collector. ---- +We're going to start the environment, should be about 12,000 events per second that we're pulling. Then we're going to apply the changes to the environment, and then we'll see, you know, we'll discuss those changes more in depth and see how it's performing afterwards. -### Demo +So if I come over here and check this out. I started this up shortly before we started here. This green line up here, this is going to be the events per second that the Kafka topic in the partition is pulling in. We're stable at around 26,000 to 27,000 events per second. And then down here is our single receiver, a single copy receiver. This is using the default client. It's pulling in these events as OTLP JSON, and batching is being done afterwards. Here you can see that we're stable at 12,000 events. -In the demo, we will use a single collector and a single partition for simplicity. Initially, we were pulling around 12,000 events per second. After applying the changes, we expect to see significant improvements. +Now, I'm going to just quickly stop this collector, and then I'm going to start it up using the new config. Crucially here, I'm going to start it up using this feature gate to enable the FronGo client. I'm going to start that. I'm going to let that run in the background for a little bit, let these changes trickle through. I'm going to talk about the changes we made more in depth. -(At this point in the presentation, Dakota demonstrates the changes made to the configuration and shows the performance metrics.) +[00:12:00] On the left here, this is the first configuration that the collector was using. This is the configuration for the Kafka receiver. This is a pretty default config. The key difference here again is this OTLP JSON encoding versus on the right here in the new config. We're going to be using text. Using text is just going to be able to pull the data faster. It's not going to have to worry about transforming or handling that data as much. It can just push the events through faster. It's not going to get bogged down trying to transform it. ---- +I've got some collecting the metrics from this receiver. Here I've got some processors. These are defined the same across the two here. This is to generate or to simulate how moving the batch processor around in the pipeline affects the actual throughput. These processors are all the same, just pretty basic, adding some fields. -### Results +Next exporters, we've got no exporter. We're not going to show the effects of the SecOps exporter in this demo just because it's a little much. We're going to just keep it simple here using the OP exporter, you know, eliminate that lever from the situation. We're sending our internal telemetry. -We witnessed a jump from 12,000 events per second to around 25,000-26,000 events per second after implementing the changes. This improvement allowed the customer to catch up with Kafka's ingestion rate and start reducing their backlog. +The other important part here, this is the initial configuration. We can see here the order we define our processors. We've got our two transform processors and then the batch. Again, the improvement we saw was adding the batch processor at the start of this processors definition list. This is so that the receiver is sending directly to the batch processor. It can get consistent performance, consistently pushing out a high number of events rather than being bottlenecked by the other processors in front of it. ---- +The other crucial thing, again I pointed out when I restarted the collector, I used the new feature gate to use the FronGo client instead of the default. -### Conclusion +We'll come back here. We'll give this a refresh. This is going to keep going. But at the moment, we can see this new collector here. This is the old one turning off, and this is the new one coming back on. We briefly spike up, and I expect if we let this go for a little while, we'll see this stabilize and mirror the Kafka line. -In summary, we identified several key factors that contributed to the bottleneck: -- Back pressure from upstream components affecting the receiver. -- The performance differences between the FronGo client and the default client. -- Avoiding unnecessary encoding conversions based on telemetry goals. +[00:15:02] Let this go for a little bit. But I guess, just to kind of go back here and reiterate some of these points again, use the FronGo feature flag to use a FronGo client; it just performs a lot better than the default client in the Kafka receiver. Your log encoding, again, if you're handling the data unnecessarily, in this case, we're pulling data from Kafka that's not necessarily OTLP JSON, but because we have a receiver set up for that encoding, there's unnecessary work being done to transform it into that. We can just pull it in as normal JSON and handle it afterwards. That was a big source of improvement. -Thank you for attending this session! We have some links to share, including a blog post about this topic and a link to the CNCF Slack. +Again, batching early, the receiver is able to consistently push out a consistent number of events rather than having seen that fluctuate based on what's happening upstream of it. Again, in a similar vein, not shown in the demo but for SecOps or for this exporter, tuning and configuring that affects the receiver upstream of it. ---- +Those were again some of the changes we made. We'll see if we've got some better data here. It's still early, but you can see how this collector is starting to stabilize, kind of right around the throughput that the Kafka topic is getting. I expect this to get closer as it comes online and starts to stabilize. But again, you can see we jumped from doing 12,000 events per second, and then with all those changes we made, now we're closing in around 25,000 to 26,000 events per second. Drastically better improvement. This is how we were able to help this customer scale up their throughput so that they're able to keep up with Kafka. In their case, we were able to get past what Kafka was ingesting so that we could also start cutting down the backlog and catch up. -### Q&A Session +Got to refresh this a bit. Cool. You can see this is stabilizing. Why it worked? Back pressure. Back pressure is the term for this idea I've been talking about where what's happening upstream or downstream of the receiver is affecting the receiver. It's just not able to push events through as fast because it's got to wait on the components ahead of it to process their events. You don't notice it until, in this case, the receiver starts falling behind and is affected by it. -**Host:** Now, let’s move on to questions from the audience. +Again, very similar to the early batching. The FronGo client just performs a lot better than the default client in terms of pulling events from Kafka. Again, use that feature gate to enable it. Finally, avoid doing unnecessary encoding conversions. Again, this is understanding your telemetry goals in your environment, understanding what the data you're sending looks like so that you can make sure that your receivers and other components are configured properly to handle it. -1. **How do different exporters impact performance?** - - The impact will depend on the exporters used. For this customer, switching from gRPC to HTTP for the SecOps exporter resulted in better throughput. +I think that's all I have for a demo. We can keep checking out that graph to see how it's looking as it's coming online. Got a couple links here, one to the blog post that we made about this, and then also the link out to the CNCF Slack. I think that is it for my slides. -2. **What was the initial reason to put batching at the end of the pipeline?** - - This approach may have worked better in other environments, but it depends on the processors in your pipeline. +[00:19:00] **Host:** Cool. Sorry, muted mic. Looks like we have a question coming in from LinkedIn. How do different exporters impact performance? -3. **Would using a memory limiter bring extra performance?** - - I'm not familiar enough with that processor to give a definitive answer, but I’m open to looking into it. +**Dakota:** Yeah. I think that's definitely going to be very dependent on the exporters being used. It's going to be something you have to investigate per case and, again, fine-tune the configuration of them. In this case for the customer, the SecOps exporter using gRPC just wasn't able to keep up with the throughput we had, and in this situation, HTTP was able to perform a lot better. It's just a matter of trial and error to see what works best for your environment based on your telemetry and your goals. -4. **Is the FronGo feature flag only useful for the Kafka receiver?** - - No, it's a general implementation not specific to the receiver. Other Kafka components might utilize this client as well. +**Host:** Awesome. Looks like we have another question. Batch has been recommended to be the first processor of the pipeline. What was the initial reason to put it at the end of the first pipeline? -5. **Did you size the batching to improve performance?** - - Yes, we experimented with the size of batching to find what worked best for our environment. +**Dakota:** Yeah, so the reason for putting it at the end, you know, I'm not entirely sure of the decision to do that or what went into that decision. I think it's a perfectly valid approach. I know maybe in other environments that works better. I think it depends on the processors in your pipeline. I think some processors, it would be better to batch afterwards rather than before. Again, it's very dependent on what your pipeline looks like, what your goals are, and what you're trying to achieve. -6. **What about resource usage? Does it have any consequences?** - - The configuration changes didn’t significantly affect resource consumption, which remained reasonable. +**Host:** Awesome. Another follow-up question or another question. You did not use the memory limiter. Would that bring extra performance from using it if you used the memory limiter? ---- +**Dakota:** That's a great question. I can't say I'm too familiar with that processor, so I don't know if I have a good answer for that. I would certainly be willing to look into it and understand it and see if that could have applied here. -### Closing Remarks +**Host:** Fair enough. Another one that we have is, the FronGo feature flag, is it only useful for the Kafka receiver? -If there are no more questions, that wraps up our session. Thank you all for attending! Remember, the recording will be available on our LinkedIn page and YouTube channel. Also, for those interested in sharing their stories with the Hotel End User SIG, please reach out to us on CNCF Slack. +**Dakota:** It's a good question too. Yeah. No, the FronGo client implementation is just a general implementation not specific to the receiver. Some of the other Kafka components in the project, it's a matter of whether or not they utilize this client as well, and looking into seeing if there's a way to enable them to use this client or not. If not, I imagine that's something that will happen shortly, the ability to use the other Kafka components with that FronGo client. -We would love to hear from you, especially if you have experiences with OpenTelemetry to share. Thank you so much, and we hope to see you at CubeCon North America! +**Host:** All right. Awesome. Another question we have. Did you size the batching to get improvements, and did you adjust the exporter batch as well? + +**Dakota:** Yeah, that's another good question. We definitely did play around with the size of the batching. In this demo, we've got it configured like this. That's definitely something that you can play around with, toggle that, fine-tune it, see what works best for your environment and your throughput. + +**Host:** And then the final question that we have here, what about the resource usage? Does it have any consequences? + +**Dakota:** Yeah, that's a great question too. Definitely something you want to be aware of. With our customer, making these configuration changes, they didn't affect the resource consumption and usage like that. The amount of resources the collector was using was still pretty, not negligible, but it didn't affect the system. It wasn't like all of a sudden we jumped from using 20% memory to 80% memory or CPU. It was very reasonable, so it wasn't necessarily a concern. + +**Host:** Excellent. Do we have any other questions from the audience? Let me share these links in the chat as well. + +**Dakota:** Oh, perfect. Yeah. Stay tuned. + +**Host:** We need like a little drum roll. + +**Dakota:** Yeah. + +**Host:** Okay. Awesome. Henrik has shared the links. + +**Dakota:** Perfect. + +**Host:** Amazing. Well, I suppose then if we don't have any other questions, I guess that is a wrap. Short and sweet informative. That's a lot to cover in a short time and also lots of great questions, lots of great considerations from a performance side as well. So definitely appreciate the questions that we've gotten. Again, tell your friends for those who are on the stream that this recording will be available after the fact, both on our LinkedIn page, the open telemetry LinkedIn page, and on the open telemetry YouTube channel, which is hotel-official. + +[00:25:26] A couple of housekeeping notes for anyone who is interested. The CFPs are still open for CubeCon, and I think in the last week, the CFPs have also opened for the CubeCon collocated events. That's the CubeCon in Europe in Amsterdam. Not the upcoming one; the upcoming one is done. Those CFPs are closed. But if you are interested in applying to CubeCon EU and Amsterdam and/or to the collocated events, these are the links. Remember folks that the submission limit for CubeCon is now three proposals per person, and that includes whether you're the main submitter or a co-speaker. The submission limit for the collocated events is 10 across all collocated events, which is very cool. + +For anyone who is interested in sharing their stories with the hotel end user SIG, we love hearing from the community. Please reach out to us. You can find us in CNCF Slack. We have a lovely Slack channel, which is, I believe, SIG hotel end user. Henrik has been nice enough to put the link to our Slack channel on there as well. + +Oh, yeah, sorry, I had it backwards; it's hotel-end user. We would love to hear your stories, whether it is through hotel in practice, this type of presentation. If you have a presentation that you want to test out, this is a great proving ground. Or if it's a talk you've given somewhere, and you just want to share it with more folks, this is also another great way to get the topic out there. + +Dakota, you presented this at open source summit, right? The collo observability day or the, I forget what the hotel day was, is that correct? + +**Dakota:** I didn't present anything, but one of our coworkers at Bindplane did present a topic about the Kafka receiver. + +**Host:** Ah, okay. + +**Dakota:** His presentation was a deep dive into his understanding of it and some of the unique aspects of it he figured out. + +**Host:** Oh nice. Very nice. That's awesome. We would love to hear topics like that. We also love to hear anything where you've learned cool stuff about open telemetry. We would love to hear from you, so hit us up on our Slack channel. We also have hotel Q&A, so for anyone who wants to talk about their hotel journey, this one's an interview style format. Both of these are live streams. We would love to hear from you. + +We have recently had, I think our previous hotel in practice was with folks from Alibaba in the APAC region. We are looking for more folks in the APAC region as well who would love to share their story because, you know, open telemetry and CNCF, it's a global undertaking. We have lots of love from folks all across the globe. If you're in the APAC region and you have a story to share, we would schedule an APAC-friendly hotel in practice or hotel Q&A to cater to the time zone. + +That is it for us. If you're going to CubeCon North America, you'll probably see some of our crew at CubeCon North America. Excited to have folks connect in person on open telemetry. Thank you so much everyone for attending. + +**Dakota:** Thanks. ## Raw YouTube Transcript diff --git a/video-transcripts/transcripts/2025-10-22T04:08:27Z-whats-new-in-otel.md b/video-transcripts/transcripts/2025-10-22T04:08:27Z-whats-new-in-otel.md index 5da9ddf..58c26b7 100644 --- a/video-transcripts/transcripts/2025-10-22T04:08:27Z-whats-new-in-otel.md +++ b/video-transcripts/transcripts/2025-10-22T04:08:27Z-whats-new-in-otel.md @@ -10,318 +10,312 @@ URL: https://www.youtube.com/watch?v=NFtcp0LPtFA ## Summary -In this YouTube video, Bree Lee and Adriana Villa host a special edition of "Open Telemetry and User SIG Live," introducing the segment "What's New in Hotel." They discuss updates and developments in the OpenTelemetry community with guests Severin Newman and Marilia Gutierrez. Severin shares impressive statistics about contributor growth and community engagement in OpenTelemetry, highlighting the transition towards more robust configuration options via declarative configuration rather than just environment variables. Marilia provides insights into this new configuration system, explaining how it simplifies setups for various SDKs and is already being implemented in languages like Java and JavaScript. The session also features a Q&A, addressing audience questions about configuration precedence and the adoption of OpenTelemetry in various contexts. The hosts encourage community participation and outline plans for future events, including a live stream from KubeCon. +In this special edition of "Open Telemetry and User SIG Live," hosts Bree Lee and Adriana Vila introduced a new segment called "What's New in Hotel," aimed at sharing updates within the OpenTelemetry community. They engaged with guests Severin Newman and Marilia Gutierrez, who discussed recent developments in OpenTelemetry including community growth, updates on the configuration system, and upcoming events like KubeCon. Severin highlighted impressive contributor statistics, while Marilia detailed the transition from environment variables to a more structured declarative configuration system for SDKs, emphasizing its advantages for complex setups. The session concluded with a Q&A segment addressing various queries about OpenTelemetry functionalities and contributions, encouraging community engagement and participation. ## Chapters -00:00:00 Introductions and welcome -00:02:30 Overview of the new segment "What's New in Hotel" -00:03:50 Introducing the first guest, Severin Newman -00:05:30 Severin discusses OpenTelemetry community numbers and growth -00:12:00 Highlighting community updates and new translations on OpenTelemetry.io -00:18:00 Discussion about the upcoming governance committee elections -00:23:00 Severin shares insights on the technical updates in OpenTelemetry -00:33:00 Marilia Gutierrez introduces declarative configuration for OpenTelemetry -00:40:00 Marilia discusses the benefits of the new configuration system -00:47:00 Q&A session with audience questions about OpenTelemetry features and trends +00:00:00 Welcome and intro +00:01:10 Introduction of guests +00:03:56 Severin Newman introduction +00:06:00 Community growth statistics +00:12:00 Community updates +00:15:30 Governance committee election details +00:19:50 OpenTelemetry Unplugged announcement +00:23:20 Technical highlights overview +00:29:00 Marilia Gutierrez introduction +00:31:30 Declarative configuration discussion -# Open Telemetry and User SIG Live Transcript +**Bree:** Heat. Heat. Hello. -**Heat. Heat. Hello.** +**Adriana:** Hello everyone. Welcome. -**Bree:** Hello everyone. Welcome. If you're on Pacific time like me, it's early, 8:00 AM. Still dark. Thank you for joining if you're on Pacific time. +**Bree:** Welcome. If you're on Pacific time like me, it's early 8:00 AM. Still dark. Thank you for joining if you're on Pacific time. **Adriana:** You're welcome. -**Bree:** Well, hi. Welcome to this special edition of Open Telemetry and User SIG Live. We're very excited to bring you a brand new segment called "What's New in Hotel." I am Bree Lee, a senior developer relations engineer at New Relic. I get to work with the lovely Adriana Villa. Would you like to introduce yourself? +[00:01:10] **Bree:** Well, hi. Welcome to this special edition of Open Telemetry and User Sig Live. We're very excited to bring to you a brand new segment called What's New in Hotel. I am Bree Lee. I'm a senior developer relations engineer at New Relic, and I get to work with the lovely Adriana Villa. Would you like to introduce yourself? -**Adriana:** Yes. My name is Adriana Villa. I work with Bree in the Hotel and User SIG, and I am a principal developer advocate at Dynatrace. Like Bree said, this is our very first "What's New in Hotel." I think this is really exciting because I don’t know about you, but I can never keep up with all the stuff going on in Hotel. So, I kind of need someone to tell me. This is a little self-serving, and we're basically bringing folks in from the Hotel community to tell us some of the cool stuff that is going on. +**Adriana:** Yes. Yeah. My name is Adriana Vila. I work with Bree in the hotel and user SIG, and I am a principal developer advocate at Dynatrace. And like Bree said, this is our very first What's New in Hotel. And I think this is really exciting because I don't know about you, but for me, I can never keep up with all the stuff that's going on in hotel. So I kind of need someone to tell me. So this is a little self-serving, and we're basically bringing folks in from the hotel community to tell us some of the cool stuff that is going on. -**Bree:** Yes. And hello, Michael! +**Bree:** Yes. And hello, Michael. -**Adriana:** Hi! Yay, thanks for joining us. I know it's late-ish for you. It's 5:00 PM for you, so thank you for joining, Michael. +**Michael:** Hi. Yay. Thanks for joining us. I know it's lateish for you. -**Bree:** Yes. By the way, I am coming in from Portland, Oregon. If you would like to share where you're tuning in from in the chat of whichever app you're using, we would love to see where you're from because there's a pretty good variety of time zones on this call as well. For example, Adriana is coming from Toronto. +**Bree:** It's way 5:00 PM for you. So thank you for joining, Michael. -**Adriana:** That's right! Yeah. And you said it right; you said Toronto, not Toronto. +**Michael:** Yes. And by the way, I am coming in from Portland, Oregon. And if you would like to share where you're tuning in from in the chat of whichever app you're using, we would love to see where you're from because there's a pretty good variety of time zones on this call as well. For example, Adriana is coming from Toronto. -**Bree:** Amazing! Yes, I'm from Canada, representing. So, I guess this is a good time to bring in our first guest. +**Adriana:** That's right. Yeah. And you said it right. You said Toronto, not Toronto. -**Adriana:** Oh yeah, sorry! +**Bree:** Amazing. Yes, I'm from Canada, so representing. -**Bree:** Just to give you all a sneak peek into what our new segment is going to look like, we are going to speak with Severin Newman, then Marilia Gutierrez, and finally, we are going to bring on Lisa Jung, who will host our audience Q&A at the very end. So, if you have questions, put them in the chat, and we will get to them at the end of both of the first two interviews during the audience Q&A. Hopefully, that'll make sense. Yes, like Adriana said, let's bring on Severin. +**Adriana:** Yeah. -**Severin:** Hey! +**Bree:** So I guess this is a good time to bring in our first guest. -**Bree:** Happy to be here. +**Adriana:** Oh yeah, sorry. Sorry. Yes. -**Adriana:** Thanks for joining us! Where are you coming to us live from? +[00:03:56] **Bree:** Yes. So just to give you all a sneak peek into what our new segment is going to look like, we are going to speak with Severin Newman. And then we're going to speak with Marilia Gutierrez. And finally, we are going to bring on Lisa Jung to host our audience Q&A at the very end. So if you have questions, put them in the chat, and we will get to them at the end of both of the first two interviews during the audience Q&A. Hopefully, that'll make sense. And yes, like Adriana said, let's bring on Severin. Oh, Egypt is also watching. -**Severin:** I'm based in Germany, close to Nuremberg. So, central European time. +**Severin:** Hey, happy to be here. -**Bree:** So, it's evening for you. +**Bree:** Thanks for joining us. -**Severin:** Yes, it's 5:00 PM, so it's getting dark again. Summer is officially over, unfortunately. And next week, you guys get the time change of one hour back, and then North America gets the time change of one hour back this week. It's screwing up my whole calendar. Everything is now at 5:00 PM, which should be at 6:00 PM. +**Severin:** Yeah, sure. Where are you? Where are you coming to us live from? -**Bree:** Well, at least we have one more week of not having to worry about that. +**Severin:** Yeah, I'm based in Germany, so I'm close to Nuremberg. So yeah, Central European time. -**Severin:** Yeah. +**Bree:** So it's evening for you. -**Bree:** So, do you want to introduce yourself briefly? +**Severin:** Yeah, it's 5:00 PM. So it's getting dark again. So summer is officially over, unfortunately. -**Severin:** Yeah, sure. So, I am the head of community and developer relations at Cosley. I'm part of the Open Telemetry governance committee, and I'm also one of the co-maintainers of what we call the SIGCOMs. What we do is the website, the blog, social media channels, and everything around that. That's a very short version of what I do for the Open Telemetry community. I'm super excited to be here today to give you a first start into what's new in Open Telemetry. +**Bree:** And then next week you guys get the time change of 1 hour back, and then North America gets the time change of 1 hour back this week because it's like screwing up my whole calendar. So it's like, oh, everything is now at 5:00 PM, which should be at 6:00 PM and everything. So yeah, let's get over it. -**Bree:** Amazing! So, let's get started then. Do you have some goodies that you can share with us? +**Severin:** Yeah. Well, at least we have like one more week of not having to worry about that. -**Severin:** Yes, absolutely. I have prepared a little bit of slides—not to bore everybody with a lot of text, but to make a few of these things I wanted to share a little more visual. I have three topics to talk about: a little bit about Hotel and numbers, a little bit about the community, and then a little bit about the tech stuff. I think that's what most people are excited about. So, let me share my screen and get started with a little bit on the numbers. +**Bree:** Yeah. -So, those numbers are for all the time, right? Since Open Telemetry was created, what is it now, six years ago, we had almost 20,000 contributors from 120 countries and almost 2,000 organizations. I looked up the numbers for the last 12 months—still 6,000 contributors, over 70 countries, and over 600 organizations. So, you see there’s a lot of velocity in our project. This is growing every year, and we see it also with the PRs and the work happening on GitHub. +**Severin:** So do you want to introduce yourself briefly? -We have to add here that something like this session is not recorded by anything you see on CNCF DevStats. There’s a lot of work going on in our community that’s not tracked on those boards, unfortunately. So, a big shout-out to everybody doing that work outside of the repositories. But I still think those numbers are impressive. Since the foundation of the project, we had like 60,000 PRs reviewed, as you can see, most of them three to four times, and over half a million comments. This means there were over half a million times someone wrote something on a blog, issue, or pull request, which I think is really great. +**Severin:** Yeah, sure. So, sorry. Tell us all the things you do for hotel. -What’s exciting is that in the last year, 20% of those PRs have been done, meaning we are leveling up our reviews. We have more repositories where we say, "Hey, you're not done with one review; you maybe need two reviews." For example, in our Open Telemetry IO repository, we need a review from docs maintainers and from co-owners. I think this helps with the quality of the project. +**Severin:** Right. Thank you so much. I try my best. Yeah. So I'm Severin, however you pronounce it. I always also pronounce it differently when I speak English or German. I'm head of community and developer relations at Cosley. I'm part of the Open Telemetry governance committee, and I'm also one of the co-maintainers of what we call sigcoms. So what we do is the website, the blog, social media channels and like everything around that. So that's a very short version of what I do for the Open Telemetry community. Yeah, and I'm super excited to be here today to give you a first start into what's new in Open Telemetry. So yeah, that's me. -And who is using Open Telemetry? There are three groups we mention: first, the vendors or backends—companies that consume Open Telemetry data and do something with it. There are also open-source projects, including Jaeger or OpenSearch, and what we call integrations—projects that added Open Telemetry natively. This is probably a much bigger number than what we have on our website. +**Bree:** Amazing. So I guess let's get started then. Do you have some goodies that you can share with us? -Things like Kubernetes have their API server supporting Open Telemetry, or there's the PHP server called Roadrunner that has Open Telemetry. I always love mentioning that one because it’s written in Go and then does PHP. You can even do tracing from Go into PHP in this web server, which is really something I always love seeing. Oh, and also Docker! If you look at Docker BuildKit and another Docker project, you can see your containers being built with Open Telemetry. +[00:06:00] **Severin:** Yeah, absolutely. So I have prepared a little bit of slides, not to like bore everybody out with like a lot of text and more like to make a few of these things I wanted to share with you a little bit more visual. So I have three topics to talk about, right? A little bit hotel and numbers because I'm always excited about seeing how big this project has grown, and then a little bit about the community and then a little bit about the tech stuff. I think that's the stuff, the things that most people are excited about. So yeah, let me share my screen and get started with a little bit on the numbers. -I think that's also very fun. And then, of course, thousands of adopters. Every time I'm in a presentation or in a room, I ask people, "Hey, who of you knows about Open Telemetry?" Most of their hands go up. Then if I say, "Hey, and who of you is using Open Telemetry?" also like 80% of the people raise their hands. Unfortunately, we don’t have that many listed on our website, so here's the first call to action: if you are an Open Telemetry adopter, if you have maybe even written a blog from your company saying, "Here’s how we use Open Telemetry," just use that link, and you can reach out, and we can add you there. +**Severin:** Yeah. So those numbers are like for all the time, right? So since Open Telemetry was created, what is it now, 6 years ago, we had almost 20,000 contributors from 120 countries and like I call it organizations, but think about it's like companies and foundations, whatever, like almost 2,000 organizations. I also looked up the numbers for the last 12 months, which is still like 6,000 contributors, over 70 countries, over 600 organizations. So you see there's a lot of velocity in our project, right? So, and this is growing and growing every year, and we see it also with the PRs and the work that's happening on GitHub. I mean, we have to add here, right? Something like this session here is not recorded by anything that you see on CNCF Devstat, right? There's a lot of work going on in our community that's not tracked on those boards unfortunately. So also big shout out to everybody doing, let's say, the work outside of the repositories, but I still think those numbers are impressive, right? Since the foundation of the project, we had like those 60,000 PRs reviewed, as you can see, like most of them three to four times and then like over half a million comments, right? So that means like there were over half a million times someone wrote something on a below our issue or pull request or something like that, which I think is really great. And what's really exciting is like in the last year, I think 20% of those PRs have been just done in the last year, which is like huge growth, and especially like from the reviews, 30% of them have been done in the last year, meaning like we're leveling up our reviews, right? So we have more and more repositories where we say like, hey, you're not done with one review, you maybe need two reviews or for example, in our Open Telemetry IO repository, we need a review from docs maintainers and from like the co-owners sig, and I think this really helps with the quality of the project. -So, that’s a little bit on Open Telemetry numbers. I think that's just a good starting point. Now, let’s talk about the community. +**Severin:** And who's doing that, right? And this is some number I'm super happy to be able to share because we built a page for that specifically on the Open Telemetry website where we list all the triagers, all the approvers, and all the maintainers. And as you can see, like those are like people that stepped up and took over a role where they said like, hey, I help us triaging, I help us like managing the repositories, I help with running the sig meetings. And if you do the math, there's like almost 300 people just having those kinds of roles, which I think is really, really great to have. So yeah, a big shout out to everybody who's helping with that, and also a big shout out to all the people I'm not listing here because there's so many people in our community that are helping or that are just landing in our community and then wanting to step up and do much more things. -**Adriana:** Yeah, I think let's dive into the next topic. +**Severin:** Yeah, and then of course, who is using Open Telemetry, right? And there's always like three kinds of groups that we mention. There's first of all, of course, like what we call the vendors or like the backends. So this is of course companies that consume Open Telemetry data and do something with it. But also open source projects, right? This also includes Jaeger or OpenSearch. But then there's also what we call integrations, right? And that number is probably much, much bigger. So this is just what we have on our website. Those are projects that said like, hey, we add Open Telemetry to our project natively. So things like Kubernetes has their API server supporting Open Telemetry or there's the PHP server called Roadrunner that has Open Telemetry. I always love mentioning that one because like it's written in Go and then doing PHP. So you can even do tracing from Go into the PHP in this web server. This is really something I always love seeing. Oh, and also Docker, right? So if you, I think it's Docker Buildkit and another project of Docker, like you can see your containers being built with Open Telemetry. I think that's also very fun. -**Severin:** Awesome! As I said, I always love to start with those numbers to emphasize how big this community has grown. And that also means there's a lot of community updates. Especially, we all know KubeCon is coming, and for whatever reasons, September, October, and November are when all those community changes happen. +**Severin:** And then of course thousands of adopters, right? I mean every time I'm in a presentation or in a room and I ask people like, hey, who of you knows about Open Telemetry, most of their hands go up, and then if I say like, hey, and who of you is using Open Telemetry, also like 80% of the people are raising their hand. Unfortunately, this is something we don't have that many listed on our website. So here's the first call to action. If you are an Open Telemetry adopter, if you have maybe even written a blog from your company where you say like, hey, here's how we use Open Telemetry, here's how it's helping us, just use that link and you can reach out and we can have you add it there, right? So yeah, that's a little bit Open Telemetry numbers. I think that's just a good starting point, and let's talk about the community or is there anything you'd like to deep dive on that slide? I think it's just good to get started. -One I'd like to highlight, because I'm one of the maintainers of this project, is if you go to the OpenTelemetry.io website, there are now nine languages into which we have started to translate the OpenTelemetry website. That includes English, Bengali, Spanish, French, Japanese, Portuguese, Romanian, Ukrainian, and Chinese. I’ve written it down so I don't forget anybody. I think that’s amazing! We started with four languages a little more than a year ago, and it has grown to nine languages. This is really amazing for the community because I think it’s a great way to get started with OpenTelemetry—you can translate the things into your native language, learn about them, and then maybe move into different projects of the OpenTelemetry project with this additional understanding. +**Adriana:** Yeah, I think let's dive into the next topic. -Another big call-out: the technical committee has added two members. David Ashpole and Josh McDonald have joined the TC. David is a subject matter expert on metrics and Prometheus. He’s a Kubernetes contributor as well. Josh has stepped down from the TC for a while but is now coming back in. He’s involved in the sampling SIG and is one of the people behind OpenTelemetry Arrow and many other things. I’m probably not doing him justice by only calling out those two things, but big congratulations to both for joining the TC. We will see the impact they will have on the project very soon. +[00:12:00] **Severin:** Awesome. I think that's now more exciting. Right. As I said, I always love to start with those numbers to just emphasize how big this community has grown. And that also means like there's a lot of community updates, and especially like we all know KubeCon is coming, and for whatever reasons September, October, November is like all those community changes that are going on. One I'd like to highlight because like I'm one of the maintainers of this project is like if you go to the Open Telemetry.io website, there's now nine languages that we have started to translate the Open Telemetry website to. So that includes English just like the base language. But then we have Bengali, Spanish, French, Japanese, Portuguese, Romanian, Ukrainian, and Chinese. I have written it down so that I don't forget anybody. But I think that's amazing, right? I mean we have started with that a little bit more than a year ago with four languages, and it has grown to nine languages. And this is really amazing community because I think that's a really great way to get started with Open Telemetry, right? So you can translate the things into your native language, learn about them, and then maybe move into different projects of the Open Telemetry project with this additional understanding. -The governance committee election is coming up next week. We just announced the candidates. If you are a member of standing, as we call it in the OpenTelemetry community, you should have a notification about being allowed to elect. If you think you’re allowed to be in that election as a voter and haven’t received your access, reach out. There’s a link for that as well. +**Severin:** Yeah, then maybe a big call out, the technical committee has added two members. So David Ashpole and Josh McDonald have joined the TC. David is a subject matter expert on metrics Prometheus. He's a Kubernetes contributor as well. And Josh has stepped down from the TC a while ago, is now coming back in, and he's like in the sampling sig. He is one of the people behind the Open Telemetry Arrow and so many other things. So I'm probably not doing him justice only calling out those two things. But yeah, big congratulations to the two for joining the TC, and we will see the impact they will have on the project very soon. -For people new to the OpenTelemetry project, we have a new Slack channel called OpenTelemetry New Contributors. If you want to start contributing, go over there, and you can get started or ask your questions. Say, "Hey, I picked up that issue, and I need some help," or "Hey, I want to help with documentation. Can somebody guide me?" I think that's a really great opportunity to get started. +**Severin:** Then another one, the governance committee election is coming up next week. So we just announced the candidates. If you are a member of standing, how we call it in the Open Telemetry community, you should have a notification about you being allowed to elect. If you think that you're allowed to be in that election as a voter and have not gotten your access to that, reach out. There's a link for that also. You can find it from here to be added to that. For people that are new to the Open Telemetry project, a new Slack channel that we called Open Telemetry New Contributors. So if you want to start contributing, go over there, and you can get started or can ask your questions and say like, hey, I picked up that issue and I need some help with that or hey, I want to help with documentation, I want to help as the collector, can somebody guide me with that? So I think that's a really great opportunity to get started. -Another call to action: we have community awards! We started this last year, and we want to do this again. At KubeCon, we will call out a few members of the community who the community thinks deserve an award. Everybody is allowed to nominate someone, and this does not only include people contributing to the project. This also includes people promoting the project in any way. If you have a person in your company running around convincing everybody to use OpenTelemetry, you can nominate them! If you find four or five friends in your company and say, "Hey, let's nominate that person," that’s perfectly fine with us. We want to hear those stories of people promoting OpenTelemetry. +[00:15:30] **Severin:** Yeah, another call to action, I think I have a few more coming. So take down your paper and take some notes. We have community awards, right? So, we started with this last year, and we wanted to redo this once again. So at KubeCon, we will call out a few members of the community where the community thinks like they deserve an award, right? So everybody is allowed to nominate someone, and this also does not only include like people contributing to the project. This also includes people that are promoting the project in any way, right? So let's say you have a person in your company that's running around and convincing everybody to use Open Telemetry. You can nominate them, right? If you find four or five friends in your company and say like, "Hey, let's nominate that person. Let's add them to that list." Perfectly fine with us, right? We want to hear those stories. We want to hear about people that are promoting Open Telemetry. -Speaking of KubeCon, the observatory is back at KubeCon. If you want to speak with us at KubeCon, this will be my first time there, by the way, so if you want to catch up with us, you will find us there. We also have a schedule and some programs planned. You can probably tell more about that than I can. +**Severin:** Speaking about KubeCon, right? I mean, the observatory is back to KubeCon. So if you want to speak with us at KubeCon, I will be first time for me at KubeCon North America this year, by the way. So if you want to catch up with us, you will find us there. I think we also have some schedule and some program there planned. You can probably tell more than I can do on that, but yeah, I'm very excited about that. -**Adriana:** Yes, I wanted to interject quickly. I wanted to dig into the GC elections because that's pretty much open to anyone, right? I know you said it's happening next week, but anyone who's been involved in OpenTelemetry can nominate themselves for the GC elections, correct? +**Adriana:** Yeah. And if I may interject really quickly on a couple of things, I wanted to just dig in quickly to the GC elections because that's pretty much open to anyone, right? I know you said it's happening next week. But anyone who's been involved in Open Telemetry can nominate themselves for the GC elections, right? **Severin:** Sorry, can you now? I lost you for a minute. -**Adriana:** Oh, anyone can nominate themselves for the GC elections that has been working in OpenTelemetry. Is that correct? - -**Severin:** Yes, but I think the nomination is closed since like... - -**Adriana:** Okay, okay. +**Adriana:** Oh, can anyone nominate themselves for the GC elections that's been working in Open Telemetry? Is that correct? -**Severin:** Yeah, yeah. +**Severin:** Yeah, but I think the nomination is closed since like— -**Adriana:** So, that's why I didn't call it the nomination. But yeah, no, fair enough. +**Adriana:** Okay. Okay. Yeah. Yeah. So that's why I did not call that the nomination. But yeah, no, fair enough. -**Severin:** I think you don't have to be like— +**Severin:** I think even you don't have to be like— -**Adriana:** So, yeah, everybody can be nominated. I think you need two supporters, and then you're in. I said it's closed for this year, but next year, we will do this once again. Maybe it would be good for anyone interested in running next year to give them a heads-up about what the process might be like. +**Adriana:** So yeah, everybody can be nominated. I think you need two supporters, and then you're in. I said it's closed for this year, but next year, we will do this once again. So maybe it would be good for anyone who's interested in running next year to just give them a heads up of what the process might be like. -**Severin:** Yeah, sure. Normally, we send out a blog post, I think at the end of September or early October, where we say, "Hey, elections are coming up. If you want to run as a candidate, create a pull request against the community repository and have two people support you." If you want to do that for the first time, wait for the first people to submit their nominations or self-nominations. Let’s be honest; most people self-nominate, and there’s nothing wrong with that. I do that. Just use that as a template and think about if you would be elected in the governance committee of OpenTelemetry, which is responsible for managing the whole project, for the health of the project, and the community as a whole. What are the things you would do for the project? +**Severin:** Yeah, sure. So what we normally do is we will send out a blog post, I think, end of September, early October, where we say like, hey, elections are coming up. If you want to run as a candidate, create a pull request against the community repository and have two people supporting you or saying like, yeah, I think that person is a good candidate. I can recommend that if you want to do that for the first time, wait for the first people to submit their nomination or self-nomination. Let’s be honest, most people self-nominate, and there’s nothing wrong with that. I do that. And just use that as a template and then think about like if you would be elected in the governance committee of Open Telemetry, which is like responsible for managing the whole project, for the health of the project, for the community as a whole, what are the things that you would do for the project, right? So the GC, just to call this out, this is less about like, oh, I want the collector to do x, y, and z, or I want to have, I don’t know, added some specific language to the Open Telemetry project. Of course, from a GC perspective, let's say you can engage in those conversations with those communities, but the GC is much more around like how can we grow and make our community stronger and better and how can we manage the project as a whole? So that means like how can we get projects added to what we are doing? How can we make sure that they're sustainable? How can we make sure that our project is thriving, right? So yeah, next year around the same time window, if you want to run, that’s where you should take a look. -The GC is less about "I want the collector to do X, Y, and Z," or "I want to have added some specific language to the OpenTelemetry project." Of course, from a GC perspective, you can engage in those conversations with those communities, but the GC is much more around how we can grow and make our community stronger and better and how we can manage the project as a whole. That means how can we get projects added to what we are doing, how can we ensure they are sustainable, and how can we make sure our project is thriving? So yeah, next year, around the same time window, if you want to run, that’s where you should take a look. +[00:19:50] **Adriana:** There you go. And for anyone who's planning for next year, I guess keep an eye on the nominees this year. See what, like you said, see what they've written for their nominations to get an idea of the types of things that get you elected. You're on the GC, right? -**Adriana:** There you go. For anyone planning for next year, keep an eye on the nominees this year. See what they’ve written for their nominations to get an idea of the types of things that get you elected. You’re on the GC, right? - -**Severin:** Yes. +**Severin:** Yeah. -**Adriana:** So, what gets you elected? +**Adriana:** So yeah, I think what gets you elected is like that you show that interest in the community, right? I mean at the end of the day, it’s also like— -**Severin:** I think you show that interest in the community. At the end of the day, it’s also like... it’s very hard to say that since I’m a member of the GC myself. But before I was a member of the GC, I always looked at what those people had done before. Sure, putting in your nomination is one thing. Writing down, "Here are the things I want to do for the community" is one thing. But it's more about what have you done in the last few months to help people with that. +**Severin:** It’s very hard to say that since I'm a member of the GC myself. So what I look for, but before I was a member of the GC and before I was voting with this insight and perspective, I always looked at like what have those people done before that, right? Sure, putting on your nomination is one thing, like writing down, hey, here’s the things I want to do for the community is one thing. But yeah, it’s more about like what have you done in the last, I don’t know, few months to help people with that, right? -**Adriana:** From a voter perspective, it definitely helps if you’ve been visible in the community. It could be in a number of ways, right? Code contributions, talks, blog posts, being active in the channels, answering questions. +**Adriana:** Yeah, and I think from a voter perspective, it definitely helps if you've been visible in the community. And it could be in a number of ways, right? Code contributions, talks, blog posts, being active in the channels, answering questions. -**Severin:** That’s maybe a very important point, right? You don’t have to be part of the GC to do that. We need a lot of people to do a lot of work outside of writing all the code and documentation. There are so many great things you can contribute to this project. I say it very often to people that want to break into the OpenTelemetry community or any open-source community: you already have skills. You could be a good marketer, designer, or technical writer. Think about what you bring to the table. That helps you be appreciated in the community. Don’t think, "Oh, I have to be a good Go programmer to be valuable to the OpenTelemetry community." That’s just not true. +**Severin:** Yeah. And that’s maybe a very important point, right? You don’t have to be part of the GC to do that, right? I mean, we need a lot of people to do a lot of work outside of writing all the code, outside of writing all the documentation. There are so many great things you can contribute to this project, and I say it very often to people that want to either break into the Open Telemetry community or in any open-source community. I mean, you have skills already, right? You could be a good marketer person, or you could be a designer, or you could be a good technical writer, or you could be a good developer, right? And think about the things that you bring to the table, and that’s helping you to be appreciated in the community. Don’t think like, oh, I have to be a good Go programmer to be valuable to the Open Telemetry community. That’s just not true. -**Adriana:** That’s a great point! Before we transition over to Marilia, do you still have one more thing you wanted to show us quickly? +**Adriana:** That’s a great point. And I think before we transition over to Marilia, you still had one more thing that you wanted to show us really quickly. I think you're going to show hotel unplugged. -**Severin:** Yes, I have a slide and another one. I’m not sure how we’re doing on timing. I’m talking a little long now. I still have one more, but let’s see if we can get through that. I think people are more excited about the declarative configuration part. But people should be very excited about OpenTelemetry Unplugged. This is an unconference we’re planning for happening after FOSDEM next year, the Monday after FOSDEM. If you're still around, register for this event, and you can meet with the OpenTelemetry community. There's a blog post on that. +[00:23:20] **Severin:** Yeah, I have like the slide and another one. I'm not sure how we’re doing on timing. I'm talking a little bit long now. So I still have like one more. But let's see that we can get through that. I think people are more excited about the declarative configuration part, but people should be very excited about OpenTelemetry Unplugged. This is like an unconference we're planning for happening after FOSDEM next year. So Monday after FOSDEM, if you're still around, register for this event and then you can meet with the OpenTelemetry community. There's a blog post on that. I'm very excited about that one. And then last but not least, because people are asking about that, graduation is in progress. I can share a little bit more in the tech overview. So yeah, that’s on the community updates. Let me hopefully go into the technical highlights. -Last but not least, because people are asking about that, graduation is in progress. I can share a little bit more in the tech overview. So yeah, that’s on the community updates. Let me hopefully go into the technical highlights. +**Severin:** This was the text I wanted to show first, right? There are so many updates. I will not touch into anything and everything that the hotel project is doing right now, right? So, this is almost impossible. But I picked like the things that showed up recently, and since I saw this question coming up very early on when we started this session today, it’s like how can I stay updated? The OpenTelemetry blog is a really good place, and we try to motivate also SIGs to put things there. For example, the profiling SIG will come out very soon with like, hey, we are moving towards alpha. I think that's a really great thing that we're seeing in the community. And equally, we had a blog post just recently about sampling that they're going to adopt the W3C trace context level two. That’s probably worth another What's New in Hotel. I try to do like a two-sentence summary of that. It’s very difficult, but in a long story to make a long story short, it's about mainly about randomness and trace ID to ensure that there's enough randomness in the trace ID, and that allows you to do certain things and sampling. I can, like if we have time in the Q&A, I can try to dive into that for a little bit. -This was the text I wanted to show first, right? There are so many updates. I will not touch on everything that the Hotel project is doing right now; it’s almost impossible. But I picked the things that showed up recently. Since I saw this question coming up very early on when we started this session today: how can I stay updated? The OpenTelemetry blog is a really good place, and we try to motivate SIGs to put things there. For example, the profiling SIG will come out very soon with, "Hey, we are moving towards alpha." I think that’s a great thing that we’re seeing in the community. +**Severin:** The mainframe community has been doing a survey just recently, and I don't know if people know that we have a mainframe community with OpenTelemetry, and they just asked a bunch of questions to people on the mainframe like do you know about OpenTelemetry, how do you want to see it used? And they were talking about like, hey, we need good Python and Java support. We need to see that the OpenTelemetry collector is able to take metrics out of a mainframe, and of course the question about like what to do in COBOL. I have no more details into that, but it's a very exciting community that like, hey, even the mainframe, which is like a technology for how many years around, probably longer than I'm living, and they're looking into OpenTelemetry just to close this gap for end-to-end visibility. -Equally, we had a blog post just recently about sampling, and they're going to adopt the W3C trace context level two. That’s probably worth another "What’s New in Hotel." I’ll try to do a two-sentence summary of that. It’s very difficult, but in a long story to make a long story short, it’s mainly about randomness and trace ID to ensure enough randomness in the trace ID, allowing you to do certain things with sampling. If we have time in the Q&A, I can try to dive into that a little bit. +**Severin:** Declarative configuration, I just have it here for completeness, looking forward to hear about this more myself. And then we have next Android road to stable. So they're working towards a version 1.0. They have done a ton of work over the last few months. So also very excited about that one. There's even a demo app now in the hotel demo, so you can even play around with that. I think that's a really good starting point into that. And they're looking for feedback, right? So go to the OpenTelemetry website, go to the OpenTelemetry blog. In that blog, there's a link to like, hey, what do you want the hotel Android community doing for their next release? -The mainframe community has been doing a survey recently, and I don’t know if people know that we have a mainframe community with OpenTelemetry. They just asked a bunch of questions to people on the mainframe: Do you know about OpenTelemetry? How do you want to see it used? They were talking about needing good Python and Java support and seeing that the OpenTelemetry Collector is able to take metrics out of a mainframe. Of course, the question about what to do in COBOL is still open. I have no more details on that, but it’s a very exciting community, even for the mainframe, which is a technology that has been around for many years. +**Severin:** And then what I said about like we are working towards graduation. We met with the TOC of CNCF. There were a few adopter interviews we were discussing what are the things to go next. So there’s an issue on the CNCF TOC repository, it's number, I wrote it down 1739. You can look this up, and there's a summary of like what are the next steps, right? And one big thing, and this is maybe exciting because this is not new feedback for us, that people are looking for like how can we improve stability? How can we communicate stability better? And how can we look into ease of use? How can we make it easier for people to get started with OpenTelemetry, right? -We are looking forward to hearing more about the declarative configuration myself. We have the Android team working towards a stable version, and they have done a ton of work over the last few months. There’s even a demo app now in the OpenTelemetry demo with Android, so you can play around with that. I think that’s a really good starting point. They’re looking for feedback, so go to the OpenTelemetry website, check out the OpenTelemetry blog. In that blog, there’s a link to see what you want the OpenTelemetry Android community to do for their next release. +**Severin:** And I said probably this slide alone would have covered like another 20 to 25 minutes if I would have talked about browser phase one, the bail donation, which is now called OBI, the injector weaver, hotel arrow, they're doing a new phase, we have a call for contributors for Kotlin, a lot of good changes in sigcom. So yeah, I tried to start with that, and I really hope we do those kinds of sessions more regularly so people can stay on top of things. So yeah, this is how you say like scratching the surface, and then let's dig deeper next time. So yeah, hope it's helping. -As I said, we are working towards graduation. We met with the TOC of CNCF. There were a few adopter interviews where we discussed what the next steps are. There’s an issue in the CNCF TOC repository, issue number 1739, which you can look up. There’s a summary of what the next steps are. One big thing, and this is maybe exciting because this is not new feedback for us, is that people are looking for how we can improve stability, how we can communicate stability better, and how we can make it easier for people to get started with OpenTelemetry. +**Adriana:** This was amazing. -Probably this slide alone would have covered another 20 to 25 minutes if I had talked about browser phase one, the bail donation (now called OBI), the injector weaver, OpenTelemetry Arrow, and the call for contributors for Kotlin. There are a lot of good changes in SIGCOM, so I tried to start with that. I really hope we can do these kinds of sessions more regularly so people can stay on top of things. I hope this helps. +**Severin:** Oh, sorry. Go ahead. -**Bree:** This was amazing! +**Adriana:** No, I was going to say thank you so much. Like that was really helpful for me. -**Severin:** Oh, sorry. Go ahead! +**Severin:** Yeah, definitely. I was going to say this proves that we need more of these. -**Bree:** No, I was going to say thank you so much. That was really helpful for me. +**Adriana:** Yeah. Yeah. No, I mean I was learning a bunch of things while preparing. I was like, "Hey, let me ask around and let's talk with people like what should what's going on in your SIG? What’s happening?" Right? But I mean, our project is growing, and we are adding so many cool things to it. And yeah, I'm very excited about a lot of those things. Again, I only can recommend two things. Go to the OpenTelemetry blog from time to time. We have those kinds of announcements there. And if you're a maintainer or contributor, come to SIGCOMs and let us know about the latest and greatest of your SIG so we can share this with the wider world, right? -**Severin:** Yes, definitely. I was going to say this proves that we need more of these. +**Bree:** And again, if Severin covered a lot, if you have any questions, please put them in the chat of whichever platform you're tuning in from, and we will get to them after we speak with our next guest. Thank you so much, Severin. -**Bree:** Yes! I mean, I was learning a bunch while preparing. I was like, "Hey, let me ask around and talk with people about what’s going on in your SIG!" +**Severin:** Yeah, thank you. Speak to you in a minute. Thank you. -**Adriana:** Our project is growing, and we are adding so many cool things to it. I’m very excited about a lot of those things. And again, I only recommend two things: go to the OpenTelemetry blog from time to time. We have those kinds of announcements there. If you’re a maintainer or contributor, come to SIGCOMs and let us know about the latest and greatest of your SIG so we can share this with the wider world. +[00:29:00] **Bree:** And we are getting ready now to bring on Marilia Gutierrez to talk about what is she talking about? -If you have any questions, please put them in the chat of whichever platform you're tuning in from, and we will get to them after we speak with our next guest. Thank you so much, Severin. +**Adriana:** Amazing. Hi, Marilia. -**Severin:** Yeah, thank you! Speak to you in a minute. +**Marilia:** Hello everyone. Well, continuing on the theme of where you're tuning from, I'm also tuning from Toronto. So yeah, me and Adriana. -**Bree:** Thank you! Now we are getting ready to bring on Marilia Gutierrez to talk about... what is she talking about? Ree, remind me. +**Adriana:** Yeah, and we're both Brazilians tuning in from Toronto, which is a coincidence. -**Adriana:** Amazing! Hi, Marilia! +**Marilia:** Small subset. That's awesome. So Marilia does a lot in the community. Would you like to share some of the work that you do before you get into declarative configuration? -**Marilia:** Hello everyone! Continuing on the theme of where you're tuning from, I'm also tuning in from Toronto. So yeah, me and Adriana. +**Marilia:** Yes, of course. So yeah, that is one thing that I was even talking to colleagues yesterday that I keep finding more things in hotel, and I just trying to get involved, and I just like keep spreading the things that I work on. So for example, I am a maintainer for the contributor experience. So if you want to be a contributor and you're having some challenges, that would be the group that will help you with. I am an approver for the JavaScript SDK for Portuguese localization, database semantic conventions, and as of a couple of days ago, also for the communications SIG as well. -**Adriana:** And we’re both Brazilians tuning in from Toronto, which is a coincidence! +**Adriana:** Yay. Amazing. -**Marilia:** Small subset! That’s awesome. So, I do a lot in the community. Would you like to share some of the work that you do before we get into declarative configuration? +**Marilia:** Dang, that is so productive. -**Marilia:** Yes, of course! I was even talking with colleagues yesterday that I keep finding more things in Hotel, and I’m just trying to get involved and spread the things that I work on. For example, I am a maintainer for the contributor experience. If you want to be a contributor and you're having some challenges, that would be the group that will help you. I am an approver for the JavaScript SDK for Portuguese localization, database semantic conventions, and as of a couple of days ago, also for the communications SIG as well. +**Adriana:** No, you're an inspiration. -**Adriana:** Yay! Amazing! +**Marilia:** Thank you. -**Marilia:** Thank you! So, even for this one, we were like, "Okay, we need to pick one topic to go into deep." There are so many things, like, "Okay, I was working on this. I’m working on that." Let's pick one. I feel like the declarative config is something that is really cool and is getting to a very almost stable place. We’re going to see this implementation across several SDKs now, so I think it’s a really cool one to talk about. +**Adriana:** Seriously. -**Adriana:** Cool! So, let’s start. Before I start with the config, I feel like I should give a little context: what is this? Why do we need this? What are we talking about? +**Marilia:** Yeah. So, even like for this one that we were like, okay, we need to pick one topic to go into deep, like so many things like, okay, I was working on this, I'm working on that. Let's pick one. But I feel like the declarative config is something that is really cool that is getting to a very almost stable place, and we're going to see this implementation in several SDKs now. So I think it's a really cool one to talk about. -So, how would you set up your SDK? You’re using OpenTelemetry, and you want to set up some specific things for your case—things like my server's name or what is going to be my tracer exporter. How it was done up until now is basically using environment variables. It was very easy to do this because environment variables are universally available in all languages. It doesn’t matter the SDK; there was some way to do this. But then more complex things came up: "Oh, I need this." +[00:31:30] **Adriana:** Cool. So yeah, let's start. So before I start with the config, I feel like I should give a little context like what is this? Why do we need this? What are we talking about? So how you would set up your SDK, you’re like you're using hotel and you want to set up some specific things for your case, things like even like my server's name or like what is going to be my tracer exporter. So how it was done up until now is basically using environment variables, and it was very easy to do this because environment variables are just universally available on all languages. So it doesn't matter the SDK, there was like some way to do this. But then more complex things came up and like oh I need this, and we just published a blog post yesterday talking about the story of the declarative config. There was an issue created five years ago on the Java SIG like I just want to be able to filter out health checks traces. I don't want to keep sending this. There is no easy way to just like filter that specific one. Like I have my sampler, but how do I pass what is the thing that I want to filter? So that started to get very complex to do just with an environment variable, so we needed something more robust. So that is the idea that came for, let's work on with the declarative config. -We just published a blog post yesterday talking about the story of the declarative config. There was an issue created five years ago on the Java SIG: "I just want to be able to filter out health checks traces. I don’t want to keep sending this." There was no easy way to just filter that specific one. I have my sampler, but how do I specify what I want to filter? It started to get very complex to do just with an environment variable, so we needed something more robust. So that is the idea behind working on the declarative config. +**Marilia:** So at this point, we now disallowed environment variables just so we don't keep like adding more when we have to transition, and then we started with now declaring config. So this is pretty much a YAML file, and you can just put all the things that you care about on that one. It is a little more robust when just compared to environment variables. It's easier to expand because you can have like all the objects and types that you need there. And because this was a project, of course, I'm the one talking about it today, but there were several people that worked on this for several months, and now we're just on the phase of really implementing on several SDKs. But the process of actually creating the convention, there is this own repo specific for this one; it takes a lot of time to get feedback from people and see what's going to work with all the languages. -At this point, we have disallowed environment variables just so we don’t keep adding more when we have to transition. Then we started with declaring config, which is pretty much a YAML file where you can put all the things that you care about. It is a little more robust compared to environment variables and is easier to expand because you can have all the objects and types you need there. +**Marilia:** And also, people have concerns like I'm not going to use environment variables anymore, but I'm used to them. So you can still have environment variables on your config file. So I can put an example here like how do you use this? And like if I have this, use it; otherwise, use this other thing. And to just start using, which is available today in Java and also starting on JavaScript, I do recognize the irony that you have to set up the environment variable config file to use the config file. I do see the irony in that. Let's say like don't use my environment variables, use the file. But that's how you do it. And then from that means like the SDKs will be able to just use your config file instead of having to set up all the environments. So if you're curious, this is how a basic one looks like. -Of course, I’m the one talking about it today, but there were several people working on this for months. We are now in the phase of really implementing it across several SDKs. The process of creating the convention has its own repo. It takes a lot of time to get feedback from people and see what’s going to work with all languages. People have concerns about not using environment variables anymore, but you can still have environment variables in your config file. +**Marilia:** So here you see like the YAML. So we have like some resources. Here we have like an example that still points to environment variable if you care. And then I have like some examples like tracer provider. I only care about the endpoints. So this is all anything that I put in here. And then here, the example that I mentioned for like the issue that was created, like I just want to filter my health checks. This is an example how you do this. In Java, it's already working, and you can see here just like a sampler, and I say like my rule, I want to drop if I have these attributes with this pattern. So it's a lot easier than having to actually go into the code or your instrumentation, trying to like create your sampler or like trying to have this environment somehow. Now you just create this object, and that's it. You don't have to care about like how the SDK would do this. We are the ones that instrumented, like doing that for you, so you don't have to do it. -I have an example here of how to use this. If I have this, use it; otherwise, use this other thing. To just start using it, it is available today in Java and is starting in JavaScript. I do recognize the irony that you have to set up the environment variable config file to use the config file. I do see the irony in that: "Don’t use my environment variables; use the file." But that’s how you do it. +**Marilia:** And of course, there's a lot of things like I'm just trying to like put a few screenshots. This can be quite big. So you can see here like, oh, I have a processor, I have a batch with this schedule with this timeout, my exporter is this type, and it's a good one that we have a file called kitchen sync on that repo that every single spec that is created is added to this file, which is the one that I'm showing here. So if you want to see examples of everything you have here, and it has the comments of what it means, what are the values, what are like the default values if you don't put them. So it's a great one, and we also have one for migration. If you’re like I just want the basic because I'm using my environment variables and I don't want to have to like copy. So you just copy that file, and it's still going to use your environment variables for some basic stuff. -Here’s a basic example. In YAML format, we have some resources. Here we have an example that still points to environment variables if you care. I have an example of the tracer provider. I only care about the endpoints. This is anything I put in here. +**Marilia:** If you want to know what is available today, we have this matrix of compliance, which Java is fully compliant, PHP, JavaScript is partially, and we do have a lot of other languages that are also working on it. I think we just need to update this for a few of them, like for example Go and I believe Python are also have some implementation. And when I say like it's fully compliant, it does not mean that it's ended; it's like complete. That is not what it means. There are still features that still need to be added, but at least the basic of what you're seeing here is working for those languages. -The example I mentioned for the issue that was created is how to filter health checks. This is an example of how you do this. On Java, it’s already working. You can see here I have a sampler, and I say, "My rule is to drop if I have these attributes with this pattern." It’s a lot easier than having to go into the code or your instrumentation to create your sampler. You just create this object, and that’s it. You don’t have to care about how the SDK would do this. We are the ones instrumenting, doing that for you. +**Marilia:** And here I can probably copy this on the chat, and we can share, but here I put a few links for resources. So the blog post that I mentioned, which is in English and in Portuguese, that is like, see one of the cool things about localization that I already created like both of them. I have the file like the repo for the configuration and the examples that I mentioned. So the kitchen sync, the migration itself, and if you have questions about the config file in general, you can use the Slack channel. So the hotel config file, if you want to have specific for the SDK. So for example, how is this working on the Java one or the JavaScript one. So then you go to those specific channels, so like hotel Java, JavaScript, Go, and so on. And yeah, that's what I have for an overview for today. -Of course, there are a lot of things. I’m just trying to put a few screenshots here, as this can be quite big. You can see here, "Oh, I have a processor. I have a batch with this schedule, with this timeout. My exporter is this type." We have a file in that repo called "kitchen sink," where every single spec created is added to this file, which is the one I’m showing here. If you want to see examples of everything you have here, it has comments on what it means, what the values are, and what the default values are if you don’t put them. +**Adriana:** This is great. Now I had a follow-up question for you, Marilia. So these configurations, it sounds like they are language dependent. So you do them at the language level, not at the collector level. Is that correct? -We also have one for migration. If you want the basic because you’re using your environment variables and don’t want to copy, just copy that file, and it’s still going to use your environment variables for some basic stuff. +**Marilia:** So the, so one thing is supposed to be like completely agnostic. So you can use the same file; it doesn't matter which SDK you're using. But there are parts that are also for the collector. So for example, the collector is already using the declarative config for a couple of things. -If you want to know what is available today, we have this matrix of compliance. Java is fully compliant, PHP, and JavaScript are partially compliant. We have a lot of other languages that are also working on it. I think they need to update this for a few of them, like Go and Python, which also have some implementations. When I say it’s fully compliant, it doesn’t mean it’s complete. There are still features that need to be added, but at least the basics are working for those languages. +**Adriana:** Right. -Here, I can probably copy this in the chat, and we can share. I put a few links for resources: the blog post that I mentioned, which is in English and Portuguese, the file in the repo for the configuration, and the examples I mentioned. If you have questions about the config file in general, you can use the Slack channel for the hotel config file. For specifics about the SDK, you can go to the specific channels like hotel Java, JavaScript, Go, and so on. +**Marilia:** So it's pretty much like already added there. I don't have a list with me easily to just show what are the things that are working, but the idea is to have all components will be able to have the declarative config; it's just a matter of now we need to go and implement on all those places. -That’s what I have for an overview today. +**Adriana:** But does like how does the config file get loaded up? Does it get loaded like as part of, yeah, I guess I'm having trouble like understanding where in the flow it fits in. -**Adriana:** This is great! I have a follow-up question for you, Marilia. These configurations sound like they are language-dependent. You do them at the language level, not at the collector level. Is that correct? +**Marilia:** Are you talking specifically about the collector or in general? -**Marilia:** Yes, the config is supposed to be completely agnostic. You can use the same file regardless of which SDK you’re using. However, there are parts that are also for the collector. For example, the collector is already using the declarative config for a couple of things. +**Adriana:** Oh no, just in general, like the declarative config, like for the environment variables. -**Adriana:** Right. How does the config file get loaded? Does it get loaded as part of... I'm having trouble understanding where in the flow it fits in. +**Marilia:** So for example, I can talk about like the JavaScript, which is the one that I'm working on. So when you initialize the SDK, usually you do like for example new SDK, and sometimes you have to like pass parameters. This time, you’re not going to need to do this. So your file is going to exist on the same place that you're adding your instrumentation. -**Marilia:** Are you talking specifically about the collector or in general? +**Adriana:** So it picks it up from, does it pick it up like from as long as you have that file in a particular directory in your code, it'll pick it up? Is that the idea? -**Adriana:** Oh, just in general—the declarative config, like for the environment variables. +**Marilia:** Yeah, because the environment variable points to the file. So as long as you put the like this is the path to the file. So when you initialize the SDK, it's going to check like do you have set a config file? If you don't, it's gonna ignore and use whatever you were doing before. But if it has, it's gonna say, "Okay, let me check what you have on that file." And for some things, it's pretty much saying like if you have this value, I'm going to use it; otherwise, it has default values for a lot of things. So it's basically that is the moment when you initialize the SDK that it's going to read and do whatever it needs to do. -**Marilia:** For example, I can talk about the JavaScript, which is the one I’m working on. When you initialize the SDK, usually you do something like `new SDK` and sometimes you have to pass parameters. This time, you don't need to do this. Your file is going to exist in the same place that you’re adding your instrumentation. +**Adriana:** Okay. Okay. That makes a lot of sense. This is cool because I guess it also lets you, I guess if you're using like just the old way of like just the environment variables, that was very flat, right? And this kind of gives it a little bit more structure too. So it kind of lets you like lump configurations together, which is very cool. -**Adriana:** So, does it pick it up from a specific directory in your code? Is that the idea? +**Marilia:** Yeah, because we have a section for example this works for everybody. So it's supposed to be like very agnostic. So now my application is in Python, and I want to use the same. But of course, there are things that are very specific for languages, like this like agents only exist in Java. So, but oh, but I still want to configure this. So the config file has sections for languages as well. So if you have like this is specific for Java, do this. We have those types of things there as well. -**Marilia:** Yes, as long as you have that file in a particular directory, it’ll pick it up. The environment variable points to the file, so when you initialize the SDK, it checks if you have set a config file. If you don’t, it ignores it and uses whatever you were doing before. But if it has a config file, it checks what you have in there. For some things, it’s pretty much saying, "If you have this value, I’m going to use it; otherwise, it has default values for many things." This happens at the moment you initialize the SDK. +**Adriana:** That's awesome. Bree, do you have anything that you wanted to chime in on? -**Adriana:** Okay, that makes sense. This is cool because it also lets you... I guess if you’re using just the old way of environment variables, that was very flat, right? This kind of gives it a little more structure, which is cool. +**Bree:** I'm still processing. -**Marilia:** Yes, we have sections for everybody, so it’s supposed to be very agnostic. Now, my application is in Python; I can use the same config file. However, there are things that are very specific to languages. For example, agents only exist in Java. But I still want to configure this. The config file has sections for languages as well. +**Marilia:** It's so cool, right? Like my mind is blown. I'm like where were you like five years ago? -**Adriana:** That’s awesome! Ree, do you have anything you wanted to chime in on? +**Marilia:** Yeah. And the idea is even like for some of the things you don't have to like restart some of the components and can just like read as is. So that is also an advantage as well. But yeah. -**Ree:** I’m still processing! +**Bree:** Well, I think we can get right into some Q&A because I do have questions, and I think our audience has some questions. So Lisa, we would love to bring you on. -**Adriana:** It’s so cool, right? +**Lisa:** Hi, Lisa. -**Marilia:** Yes! +**Lisa:** All right, thank you so much. Hello everybody. I'm Lisa Jung, a staff developer advocate at Grafana Labs. I'm part of the communications and the end-user SIG. So yeah, now we are gathering questions from the audience and asking our lovely feature speakers here. So the first question is from Cecil. So he said if you have both options, environment variables or the configuration in a project, which one takes precedence? -**Adriana:** Well, I think we can get right into some Q&A because I have questions, and I think our audience has some questions. So, Lisa, we would love to bring you on. +**Marilia:** So if you do have the environment variable saying like this is my config file, that is the one that takes precedence, and it's going to use this, and it's not going to use the environment variable as a backup at all. It's going to ignore that unless you have your config file had the environment variable on that file. But that takes precedence. If you don't have that environment variable set up, then it uses, even if you have the file existing somewhere, it doesn't know. So it's going to ignore and use environment variables. -**Lisa:** Hi, everyone! I’m Lisa Jung, a staff developer advocate at Grafana Labs. I’m part of the communications and end-user SIG. Now we are gathering questions from the audience to ask our lovely featured speakers here. +**Lisa:** Gotcha. Thank you. And then we have a question from Kieran. Russ is still a four-letter word. Long way to go. Question mark. -The first question is from Cecil. He said, "If you have both options—environment variables or the configuration—in a project, which one takes precedence?" +**Marilia:** For specific for the declarative config, I believe this question came up during your— -**Marilia:** If you do have the environment variable saying, "This is my config file," that is the one that takes precedence. It’s going to use that one and not use the environment variable as a backup at all. It’s going to ignore that unless the config file itself has the environment variable. +**Lisa:** Okay. -**Lisa:** Gotcha. Thank you! We have a question from Kieran: "Rust is still a four-letter word. Long way to go?" +**Marilia:** So yeah, it is. So that is the thing because when people are creating the semantic convention and stuff, they try to get people from different languages so you know what's going on. And then you go to your own SIG and start implementing, but a few of the SIGs have very small groups, so it's hard sometimes for them to handle because they have to review PRs that come in and do releases, so it's hard sometimes to do and be able to keep up with everything going on. The same thing like database and conventions came out; a lot of SDKs still don't have implementation. So if you, for example, really care about REST, that is your chance to join the SIG and actually be a contributor. So people are really, all the materials are always happy to help and have people joining in. You can say like this is something that I'm interested in, and I really care about how do I start contributing? And it's a way for you to be able to have things faster that you want, and if your goal is to be like an approver or maintainer for small things, that is going to happen faster because your contribution is going to have a lot of weight if there are not a lot of people helping out. So that is something also to keep in mind. -**Marilia:** For the specific declarative config, I believe this question came up during your session. +**Lisa:** Gotcha. -**Severin:** Yes, it is a complicated topic. When people create the semantic conventions and stuff, they try to get people from different languages to know what’s going on. They go to their own SIG and start implementing. A few SIGs have very small groups, so it’s hard sometimes for them to handle because they have to review PRs and do releases. It’s hard to keep up with everything going on. If you really care about Rust, that is your chance to join the SIG and actually be a contributor. +**Lisa:** All right, we have a question for Severin. So what does W3C trace context L2 for tail sampling bring to the table? -**Lisa:** Gotcha! Another question for Severin: "What does W3C trace context L2 for tail sampling bring to the table?" +**Severin:** Yeah, I had to take some notes on that myself because it's a very, very complex topic, and I'm also not like a sampling expert, but I try my very best, right? So we have been doing sampling in OpenTelemetry for I think over four years now, and it has been evolving, and like we added a lot of good things. One of the things that we had is this trace ID ratio-based sampler, right? And this sampler had always like this is just me quoting straight from that blog post. It always had this to-do where we said like, hey, all you can do safely with that is sampling on root spans, right? So you had not a lot of good guarantees on probability downstream, right? Because what's happening is that like you use the trace ID as a random n-sized word, like you use some part of the trace ID, and then you compare it to like, let's say a boundary that’s like 2 to the ^ of n times the ratio. So you can think about it like if you take 50%, it’s like I don't know 128. If you have six, seven, six bits, something like that. And if the number is smaller or bigger, then it comes in or does not come in. But the big problem is that in the trace ID until level two, there were no guarantees on how many of those bits are truly random, right? Or are maybe encoding some additional information, right? And this is what has been changed in that new standard, and OpenTelemetry is adopting that, right? So you send an additional flag when you use that, and with that, you get 56 bits of sufficient randomness, and you can use that, right? So again, this is me just talking from a few notes that I took from the blog post. I can highly recommend reading into that. I think if we talk about sampling, we all know that like with the amount of data that people are consuming these days with OpenTelemetry, this is urgently needed, and they need a lot of good hands to try out those things. So give this a look, and yeah, try it out. -**Severin:** I had to take some notes on that myself because it’s a very complex topic, and I’m also not a sampling expert, but I’ll try my best. We have been doing sampling in OpenTelemetry for I think over four years now, and it has been evolving. We added a lot of good things. One of the things we had is this trace ID ratio-based sampler. This sampler could only sample on root spans. You had not a lot of good guarantees on probability downstream. +**Lisa:** Gotcha. And to piggyback on that, Henrik, what about consistent probabilistic sampling? -What’s happening is that you use the trace ID as a random n-sized word. You use part of the trace ID and compare it to a boundary that’s 2 to the power of n times the ratio. For example, if you take 50%, it’s like 128. If the number is smaller or bigger, then it comes in or does not come in. +**Severin:** I don't know. Very, very honest. I'm not exactly sure. So that's like maybe a good thing for the next round, right? Maybe invite someone from the sampling SIG and have them talk about that specifically, right? It looks like there's some interest in that. -The big problem was that until level two, there were no guarantees on how many of those bits are truly random or are encoding additional information. This is what has changed in that new standard, and OpenTelemetry is adopting that. You send an additional flag when using that, and with that, you get 56 bits of sufficient randomness. +**Lisa:** Yeah, absolutely. So Marilia, Fernando has a question for you. Can I have more than one config file, like a file with generated configs or another with only specific JavaScript configs? -I recommend reading into that. If we talk about sampling, we all know that with the amount of data that people are consuming these days with OpenTelemetry, this is urgently needed, and they need a lot of good hands to try these things out. So, give this a look! +**Marilia:** So the file itself is going to be one because otherwise, we don't know which one takes precedence. But for example, the part that I mentioned that are specifics to language is still part of the same file. So you have, for example, like oh, like tracer exporter, like resources, and then there is a section that we call like you should put at the bottom that like, okay, now it's specific, like some resources and stuff for like JavaScript for Java. So that would be the parts that you have that are specific for JavaScript. At the moment, there's nothing specific for JavaScript, so you wouldn't have that at the moment. But if you have things different for different applications, you need one file for each one. So this is something like, for example, that you can take advantage of some things environment-variable-related. Imagine that you have, oh, it’s exactly the same thing, but I want to have the service name different on those two. So what you do then, your config file points just to say like use the environment variable for this case, and then you have your two environments that use the different name. -**Lisa:** Gotcha! To piggyback on that, Henrik asks, "What about consistent probabilistic sampling?" +**Lisa:** Gotcha. Another question from Henrik for Marilia. Would the OTL operator provide a config CRD that could be deployed across Kubernetes objects? -**Severin:** I’m not exactly sure. That’s maybe a good topic for the next round. We can invite someone from the sampling SIG and have them talk specifically about that. +**Marilia:** Not sure because I don't see like for example a reason to add new config because the idea is to have all possible config that we want to exist on that file. It's just a matter of like does it exist already or something that needs to be added because it's a little more also about getting now feedback from the community just because we have something a little more stable and say like what is missing and what we can add there. So those are the points that we need feedback from people. I can check this one specifically, but from the top of my mind, I don't know the answer if we have something already for this one. -**Lisa:** Great! We have a question for Marilia from Fernando: "Can I have more than one config file, like a file with generated configs or another with only specific JavaScript configs?" +**Lisa:** Gotcha. -**Marilia:** The file itself is going to be one because otherwise, we don’t know which one takes precedence. For example, if you have things that are specific to language, they are still part of the same file. You have sections that are specific for Java, for example. At the moment, there’s nothing specific for JavaScript, but if you have things different for different applications, you need one file for each one. +**Lisa:** We have a question from Cecil. So please feel free to jump in, Severin or Marilia. Is there any guidance you can share around OpenTelemetry for long-running background processes? -If you have the same thing but want to have the service name different, you can say use the environment variable for this case. Then you have your two environments that use the different names. +**Severin:** Yeah, I can pick that. So this is actually like a very complicated topic, right? And I just looked it up. There's an issue on the OpenTelemetry specification from 2019. So since the project was founded by Armen around that topic, and the first guidance I can give is— -**Lisa:** Gotcha! Another question from Henrik for Marilia: "Would the OTL operator provide a config CRD that could be deployed across Kubernetes objects?" +**Marilia:** Yeah, no worries. -**Marilia:** I’m not sure because I don’t see a reason to add new config. The idea is to have all possible configs that exist in that file. It’s just a matter of getting feedback from the community about what’s missing and what we can add there. +**Severin:** Now, I lost track for a minute. Yeah, so the very, very first guidance I can give you is like go to that issue, go to that repository, and call out that you're interested in that and maybe even interested in helping with that. It's a very gnarly topic, right? So what can you do until then, right? I mean at the end it depends a little bit on what does long mean to you, right? I mean if we talk about if we talk about spans and traces taking a few minutes, then we are totally fine. You maybe should be just fine with tracing that you're doing today. But if you say like, hey, and this is the classic thing, right? You have a background job, and maybe this is doing some video encoding and maybe you want to do something around that. One thing you definitely can do today, right, is use logging and events for that and maybe announce like those kinds of things and say like, okay, now I'm starting, now I'm ending. It's not perfect, right? And let’s be super honest with that, and there’s probably still a lot of things you need to do on your back end of things. But yeah, you have to work around that and just let us know that we need to do more on that topic. So that's the guidance here. -**Lisa:** Gotcha! We have a question from Cecil: "Is there any guidance you can share around OpenTelemetry for long-running background processes?" +**Lisa:** Thanks, everyone. All right, I need to stop just reading straight. I need to process a little bit more before reading the questions out loud. So sorry about that. -**Severin:** Yes, this is actually a very complicated topic. I looked it up, and there’s an issue on the OpenTelemetry specification from 2019 on this topic. The first guidance I can give is to go to that issue in the repository and let them know you’re interested in that and maybe even interested in helping with that. +**Lisa:** Okay. So as we wrap up our Q&A, I'm going to ask one question to both Severin and Marilia. So, you've been contributing to hotel for a long time. So, I'm going to take you all the way to the beginning. How did you get your start in contributing to hotel? And do you have any advice for new contributors? Who goes first? You want to start or should I start? -What can you do until then? It depends on what "long" means to you. If we talk about spans and traces taking a few minutes, then you should be just fine with the tracing that you’re doing today. +**Marilia:** You can start. -But if you say you have a background job—maybe this is doing some video encoding—and you want to do something around that, one thing you can do today is use logging and events for that and announce those kinds of things. It’s not perfect, and let’s be super honest with that. There are probably still things you need to do on the backend. +**Severin:** Yeah. Yeah. Let me get started. So my reason to start with OpenTelemetry was a very selfish reason, right? And this is something I always tell people, right? If you want to get into open source, have your own good reason to start with open source, right? Have something where you say like I don't know, I want sampling to be better in OpenTelemetry because I have way too much telemetry, so let me get started with sampling and then make this thing work. So my situation back then was like I was in a consulting position, right? And I was talking with a lot of people about what we back then called APM, and we were still transitioning to call everything observability, and people had questions about OpenTelemetry, right? They told me like, hey, what is this thing? Should I use that or should I stick with this thing I have already? And this was like every time like something where it was like recognizing like, hey, I have to level up on that, right? So I went to the community and I said like okay let me get started on that. Let me learn myself what this OpenTelemetry thing is so that I can speak better to my users, to our customers, to people that want to get into our product. So I started to help with documentation, help us getting started. I started actually also in the JavaScript community, but then I stuck with documentation because again, this was the most helpful thing for myself to just explain OpenTelemetry better to other people. So yeah, for me, it was basically I have been working on observability in a few companies before, and I really liked this observability thing, and I really wanted to focus on that. So I was looking for a job that is basically my job. I only have to think about like the observability, and this is why I went to Grafana, that the team was like you have to contribute to hotel, so it was like oh exactly what I wanted. So this is kind of how I started contributing. So it was a mix of things that I found interesting, like oh I think this is important, I should contribute it, and a mix of what also makes sense for my own company. -**Lisa:** Thanks, everyone! As we wrap up our Q&A, I’m going to ask one question to both Severin and Marilia. You’ve been contributing to Hotel for a long time. How did you get your start in contributing to Hotel, and do you have any advice for new contributors? +**Marilia:** And I do actually have, I was going to say, we just created a post that I can share here. There are ideas on how to contribute because if you wanted again, you can have several reasons for it, or like you're just working on something because your company needs or something your own personal project needs, and you want to develop something that you just want to learn. You want to increase your network; people that you want to meet. So pick something that really interests you, like oh I already have knowledge, and I want to focus on that, or something like oh, I'm just curious about this. I don't know what this is, but I want to learn. You can have both ways and really dig deep into those ones. OpenTelemetry is a huge project, and there are so many components, so many languages, you will find. It's just a matter of finding what is your place there, and you can try it out. There is no thing like I picked this; now I have to go through this one until the end. Like no, join a couple of calls, just listen in. You don’t have to say anything. Like oh this thing seems interesting, like continue a little more; oh no, this is not for me, go to the next one until you find what is the thing that really excites you. -**Severin:** I can start. My reason to start with OpenTelemetry was very selfish. This is something I always tell people: have your own good reason to start with open source. Have something where you say, "I want sampling to be better in OpenTelemetry," for example. +**Lisa:** Nice. Thanks, Marilia. -My situation back then was that I was in a consulting position, and I was talking with a lot of people about APM. People had questions about OpenTelemetry. They told me, "Hey, what is this thing? Should I use it or stick with what I have?" I recognized I had to level up on that, so I went to the community to learn what OpenTelemetry is, helping with documentation to explain it better to others. +**Lisa:** So Severin, Marilia, Adriana, and Bree have been amazing guides and mentors when I was first getting started with OpenTelemetry. -**Marilia:** I can go next. My journey was a mix of things that I found interesting and what made sense for my own company. I wanted to focus on observability, so I was looking for a job where I only had to think about observability. This was why I went to Grafana. The team was like, "You have to contribute to Hotel." That was exactly what I wanted. +**Severin:** Now we— I lied; there's one more burning question for both of you. So the question is, what kind of interesting usage trends in terms of implementation, observability stack, docs page views, or anything have you seen over the past year? -**Severin:** I do actually have— we just created a post that I can share here. There are ideas on how to contribute. You can have several reasons: you're working on something because your company needs it, your project needs it, or you want to learn. Pick something that interests you, whether you have knowledge in it or are just curious. Our project is huge, with so many components and languages. You will find your place there. +**Marilia:** I think this is more of like whatever people did not think it was necessarily important before to have observability on, they're like, oh yeah, that would be good if I actually know what the hell was going on. So I think—and then just like time, people keep adding more and more. So it's a lot of old type of applications that people did not think about like oh yeah I should add it, but also new things. But I guess you're going to find a little of everything like people like really now until like mobile, like nobody was paying attention or like the browser one, that's being really active. So it's just like what we forgot to add it. Okay, let's work on all this thing because it's really important. So it's just like all over. So you're going to find from every single place and then even things that people do for fun like I want to monitor the quality of the air and humidity on my plants. There are people that do this to like oh no I'm actually working at NASA and want to know what's going on. So you're going to find so much spread. -**Marilia:** Yes, exactly! Join a couple of calls, listen in. You don’t have to say anything. If something seems interesting, continue, but if it’s not for you, go to the next one until you find what excites you. +**Severin:** Yeah, I can try to make it quick because there's so many things I could now answer that I was just thinking about it like yeah I had like five or ten things in my head. But let me pick a few. Like the one thing is like and this may be close to like hey let’s measure like the humidity in our room. I saw more and more also like databases being used, observability inside the database, right? I mean we saw like hey tracing going up down to the database, but I know that now people are thinking about like why not do tracing inside of the database, right? I mean that's something I found very funny. Similar to what I mentioned before, right? Your Docker build. So this is just this thing where we need to recognize like every software can be traced, logged, and metrics. Is there even a verb? I don't know. But I think you get the gist, right? So you can put OpenTelemetry into just everything and get out amazing data. And the other thing, and then I tried to make it quick so that we can get to an end is like I think people start to recognize that with OTLP, they can send their data everywhere, right? I mean, Adriana, you wrote about that. Don’t send your traces, logs, and metrics into different places. That's not what I'm talking about. It's more like hey, wait a minute. I can send a bunch of my data over into that back end, and I can send maybe another portion of that data into another specific tool. And people start to recognize like they have a lot of freedom with the data, right? They have a lot of freedom with the observability data beyond, let's say, what observability vendors offer today or what we do with trace explorers, alerting, and all the capabilities that we had before. This is actually a topic I'm very, very curious and excited about right now that we say like wait a minute now we have all this data, what else can we do with that, right? I think that's super, super interesting. -**Lisa:** There’s one more burning question for both of you: what kind of interesting usage trends in terms of implementation, observability stack, docs page views, or anything have you seen over the past year? +**Lisa:** Thank you. I—I—yeah, it has been really interesting. And thank you all so much for following along. If you have additional questions, please find us on the CNCF Slack instance at hotel-sig-end-user channel. There is a QR code on the screen for you to use. -**Marilia:** I think it’s people realizing that what they didn’t think was important before to have observability is important now. They’re adding more and more. It’s a lot of old applications that people didn’t think to add observability on. You’ll find a lot of everything—new things, and even things people do for fun, like monitoring the quality of the air for their plants! +**Lisa:** Also, huge shout out to Henrik Rexet, who is doing all this live streaming stuff for us on the back end. Thank you so much to Lisa for hosting our Q&A, and Adriana for being my co-host, and Severin and Marilia for taking the time to come on and keep us updated. Thank you so much. -**Severin:** I can try to make it quick! I had five or ten things in my head. One thing is that I saw more databases being used for observability. People are recognizing that every software can be traced, logged, and monitored. Also, people are now realizing that with OTLP, they can send data everywhere. They have more freedom with their observability data, which is super interesting! +**Severin:** Yeah, thank you for having us. -**Lisa:** Thank you all for following along! If you have additional questions, please find us on the CNCF Slack instance at hotel-sig-end-user channel. There’s a QR code on the screen for you to use. Huge shout-out to Henrik Rexet, who is doing all this live-streaming stuff for us on the back end. Thank you so much to Lisa for hosting our Q&A, and to Adriana for being my co-host, and to Severin and Marilia for taking the time to come on and keep us updated. +**Marilia:** Yeah, happy to be here. -**Severin:** Thank you for having us! +**Lisa:** We got a question asking when the next one is. So, this means that I guess we'll have to put on another one. We definitely plan on putting on another one, hoping for a monthly cadence. But we also have KubeCon coming up in November, so we'll see. There will be—speaking of KubeCon, as Severin mentioned, the Hotel Observatory is happening. If you haven't been to the Hotel Observatory booth, it's loads of fun. This is where all the cool hotel peeps come to hang out. And we're also doing as part of a thing that we've been doing the last little while for KubeCon, we're doing a live stream from KubeCon. And I guess it will be—we call it the Humans of Hotel live stream. I guess it will be to a certain extent a What's New in Hotel. So keep an eye out for that. I believe it's happening on Wednesday, November 12th from starting at 2:30 PM Eastern time. So keep an eye out for that. -**Marilia:** Happy to be here! +**Lisa:** And I think that's it. Do you have anything else, Bree, to add to that? -**Lisa:** We got a question asking when the next one is. This means we’ll have to put on another one. We definitely plan to do another. We’re hoping for a monthly cadence, but we also have KubeCon coming up in November, so we’ll see. +**Bree:** No, I thank you all for being here. -Speaking of KubeCon, as Severin mentioned, the Hotel Observatory is happening. If you haven’t been to the Hotel Observatory booth, it’s loads of fun. This is where all the cool OpenTelemetry people come to hang out. We’re also doing a live stream from KubeCon. I guess it’ll be something like a "Humans of Hotel" live stream, which will be to a certain extent a "What’s New in Hotel." +**Lisa:** Yeah, thank you. And also if you have a cool OpenTelemetry story to share, whether it's like how you use OpenTelemetry in your organization or you have like a presentation that you want to do about cool stuff you've done with OpenTelemetry, reach out to us in the hotel user SIG. Again, it’s hotel-sig-end-user on Slack. We would love to hear from you. -Keep an eye out for that! I believe it’s happening on Wednesday, November 12th, starting at 2:30 PM Eastern time. That’s it! Do you have anything else to add, Bree? +**Bree:** Yes. Thank you so much, everyone. We'll see you next time. -**Bree:** No! Thank you all for being here! +**All:** Thank you, everyone. -**Everyone:** Thank you! Bye! +**Marilia:** Bye. ## Raw YouTube Transcript diff --git a/video-transcripts/transcripts/2025-11-13T08:33:00Z-humans-of-otel-live-from-kubecon-na-2025.md b/video-transcripts/transcripts/2025-11-13T08:33:00Z-humans-of-otel-live-from-kubecon-na-2025.md index e8036c6..fb36108 100644 --- a/video-transcripts/transcripts/2025-11-13T08:33:00Z-humans-of-otel-live-from-kubecon-na-2025.md +++ b/video-transcripts/transcripts/2025-11-13T08:33:00Z-humans-of-otel-live-from-kubecon-na-2025.md @@ -10,249 +10,437 @@ URL: https://www.youtube.com/watch?v=NzbDui8hDdo ## Summary -In this video, Reese and co-host Sophia present a live discussion from KubeCon North America 2025 in Atlanta, featuring guests Jacob Marino and Diana. Jacob, a maintainer of OpenTelemetry, shares insights about his recent talk on Kubernetes and OpenTelemetry, highlighting the high attendance and diverse range of questions from beginners to experienced users. He emphasizes the importance of community feedback for improving the project and discusses ongoing initiatives like the new injector project using Zig programming language to enhance compatibility across environments. Diana, a developer experience engineer, discusses her involvement in localization efforts for OpenTelemetry documentation, aiming to make it accessible in multiple languages, including Romanian, and shares her passion for community engagement and the importance of empowering developers in open-source contributions. The conversation touches on the need for better onboarding practices, best practices for instrumentation, and the overall excitement surrounding developments in the OpenTelemetry community. +In this live segment from KubeCon North America 2025, hosts Reese and Sophia discuss various topics related to open telemetry with their guest, Jacob Marino, a maintainer in the open telemetry community. Jacob shares insights from his recent talk on introductory concepts of open telemetry and Kubernetes, highlighting the surprising engagement from the audience, with a wide range of questions from beginners to experienced users. The conversation also touches on the collaborative efforts in the open telemetry community, including the introduction of a new auto-instrumentation project using the Zig programming language, and the importance of user feedback for improving documentation and functionality. The hosts also welcome Diana, a developer experience engineer and recent Open Source Community Star award winner, who discusses her contributions to localization efforts within the open telemetry community, emphasizing the challenges and motivations in translating technical documentation for various languages. The session concludes with a focus on the community's commitment to improving user experiences and engagement through better practices and collaborative efforts. ## Chapters -00:00:00 Introductions and welcome from Atlanta, Georgia -00:01:30 Introduction of guest Jacob Marino and his background -00:03:15 Jacob shares his experience giving a talk at CubeCon -00:05:00 Discussion on audience engagement during Jacob's talk -00:07:45 Importance of user feedback in open source projects -00:10:00 Introduction to the OpenTelemetry injector project -00:13:30 Overview of the Zig programming language being used in the project -00:16:00 Discussion on stability and maturity of the OpenTelemetry project -00:19:45 Diana introduced as the next guest and her background in observability -00:22:15 Diana discusses her involvement in localization and translation efforts -00:25:00 Conversation about community engagement and upcoming trends in OpenTelemetry - -# CubeCon North America 2025 - Live Transcript +00:00:00 Welcome and intro +00:01:00 Guest introduction: Jacob +00:02:40 Jacob's talk experience +00:04:50 Audience Q&A insights +00:06:00 Community feedback challenges +00:08:40 Discussion on the injector project +00:10:50 Stability and project maturity +00:12:00 Graduation criteria explanation +00:14:50 Importance of documentation +00:27:20 Guest introduction: Diana +00:30:01 Localization efforts and challenges **Reese:** Just the light. Oh, action. **Sophia:** Oh, you did. You did. -**Reese:** Hello everybody! If you've been waiting around, we are so sorry for the delay, but we are very excited to be coming to you live from Atlanta, Georgia. Pretty much everybody was very surprised by a cold front that had us looking at 40°F (about 3°C) the past couple of days. Luckily, it's warming back up, so thank goodness! We are coming to you live from CubeCon North America 2025. I'm Reese, and I am joined here by my co-host, Sophia. +**Reese:** Hello everybody. If you've been waiting around, we are so sorry for the delay, but we are very excited to be coming to you live from Atlanta, Georgia. But pretty much everybody was very surprised by a cold front that had us looking at 40°, which I think is 3°C, the past couple days. Luckily, it's warming back up, so thank goodness. But yeah, we are coming to you live from CubeCon North America 2025. I'm Reese and I am joined here by my co-host Sophia. -**Sophia:** Hi! +**Sophia:** Hi. **Reese:** And our first guest, Jacob. -**Jacob:** Hello, my name is Jacob Marino. I'm a maintainer for OpenTelemetry operator, OpenTelemetry Helm charts, hotel injector, and I maintain a few random collection things. I contribute to OpAmp, and I'm missing one; I can't remember. I was wondering who had hands for all ten, honestly. +[00:01:00] **Jacob:** Hello, my name is Jacob Marino. I'm a maintainer for, let's see, OpenTelemetry operator, OpenTelemetry Helm charts, hotel injector, and I maintain a few random collection things and I contribute to OpAmp and, and I'm missing one. [laughter] I was wondering who had a hands for all 10 honestly. + +**Reese:** If Josh Sar were here, he's everywhere. + +**Jacob:** That's every single group. + +**Sophia:** That's wild and awesome at the same time. + +**Jacob:** No, it was very impressive talking with him. Learned so much. + +**Reese:** So, how I feel talking to you all? Um, now Jacob, you did a talk or was it two talks? + +**Jacob:** Uh, one talk. So I gave a talk about two hours ago, three hours ago. + +**Reese:** Cool. + +**Jacob:** Very fresh. + +**Sophia:** It was packed. + +**Jacob:** It was like I was worried that—I've given this is my second time giving a talk. First talk that I gave was in Chicago two years ago. + +**Reese:** Oh, nice. + +[00:02:40] **Jacob:** And the room we got a very low slot, like it was like towards the end of the day on Thursday, I think. So it was a tough tougher day to give a talk. We had a solid audience, and for this one I was worried that with the content we were doing, which is like introductory to hotel and Kubernetes, I always worry with these introduction ones, you know, it's like is this going to be enough, is it going to be too much? So my fear is that nobody shows up and I'm speaking to an empty room. This was the reverse where it was a full room and people were standing in the room and then people got turned away from that door. + +**Sophia:** Oh my gosh. + +**Jacob:** Which I was happily surprised by. Then [laughter] I also like we only reserved five minutes for questions at the end because I was like, you know, I don't know what questions people will have. Usually people aren't asking a ton of questions like after they'll come up to you later, but it's like usually like a Q&A session does not last very long. So oh five minutes for questions, like that's fine. We do questions; there's a line of like 15 people waiting to ask questions. Like they just kept growing and growing. We stayed after for another 30 minutes just answering questions. + +**Reese:** Interesting. What level of questions? Like was it more, um, you know, people who are newer to the topic or who are asking like more like stage two kind of questions? + +**Jacob:** Huge range. So we had some people who were, you know, very introductory level, just like I'm getting my bearings here. How does this part of it work? You know, what's the deal with Prometheus and hotel and what do you use for a backend? You know, that's like a solid starter question. And then other people came in they were, "I've been using this chart for the past year and there's this one thing that I want to be able to do and when are you going to like do the work to like enable that and how can we like architect this?" Big range. + +**Sophia:** Very big and each one was like a different part of the area. + +**Jacob:** Okay, that's so cool. + +**Reese:** Yeah, very, very cool and very surprising. I'm always happy to because, you know, I think as a maintainer it's like we get people who come to us with [laughter] issues on GitHub and it's like you'll get someone complaining on a GitHub issue or like, "Hey, this thing broke. The most recent release broke me, um, you know, what are we doing about that?" + +**Sophia:** Broke! [laughter] + +**Jacob:** But no one's broke their spirit. You know, I've had releases break my spirit. + +**Sophia:** [laughter] + +[00:04:50] **Jacob:** But the real thing is we don't get positive feedback. We don't get a lot of questions because realistically most people are asking track or going on Stack Overflow, right? It's like they're not bringing the questions to us. So we don't really hear or understand what usage is. I mean, it's kind of the benefit [laughter] and the curse of open source is that like we don't necessarily get that user feedback directly. It's why like I love what you all do because getting the user feedback is one of the most valuable things to the community. Like in order for us to push forward, we need to know what isn't working right now where people just don't see the effort. + +**Reese:** Um, we actually—I hosted a meetup in New York last month, two months ago, something like that. + +**Sophia:** Yeah. + +[00:06:00] **Jacob:** And that was eye-opening because it was like wow, people are really using these things and things that I feel like I've talked about so many times are still not known, you know? And it's like I'll mention like how many people know what like the target allocator is in a show of hands and nobody raised their hand in a room of like 40, you know, senior SREs. I'm like, okay, uh, would anybody like it if you could do distributed, sharded, Prometheus scraping? And everybody raises their hand and I'm like, well we have that, like it's done, it's there, you know? You can use it, it's been a thing for a few years. You know, so I guess all to say the community just is still—there are still things for people to learn about the vast ecosystem that we have here. + +**Sophia:** Yeah, I think, you know, I think a common theme just in general is people not being aware of, you know, different abilities and features and yeah on the end you see too, you know, it's a bit of a challenge because like we feel like we're talking about it, um, you know, at our work and like in the Ozone community and then, yeah, like you said, we have all these conscious people who don't know and we're trying to figure out like where are they going for like how can we make them aware of, you know, all these cool things that are happening. + +**Jacob:** Um yeah, so, you know, one of the things we started doing is um the what's the hotel segment, which we just had our first one um a month ago or so. And so we're hoping to, you know, I don't know, have something we can point people to. + +**Reese:** Of course. + +**Sophia:** But you mentioned the injector. + +**Jacob:** I did. Tell us more about please do. So I am the, uh, I'm a maintainer for it but I'm mostly just doing reviewing because this is actually a joint effort between uh Splunk and D-Zero. Both of them have already written their own uh injectors for auto instrumentation. + +**Reese:** Oh, right. + +**Jacob:** And they're essentially donating a merge of both of those projects to the tot alpha version of this for their customers. Um, so we're just testing out that everything is working the way we need to. [clears throat] There's a lot of really random edges here. Um, interestingly enough, this does introduce a new language to hotel, which is exciting, which is it's very rare for us to not use a language in a project that is meant to instrument languages. + +**Sophia:** Right. + +**Jacob:** And we are now using Zig in the project. + +**Sophia:** Zig. + +**Jacob:** Zig um is a fun language that is sort of a merge between like Go, Rust, C++, and C. + +**Reese:** Oh, super fun. [laughter] + +[00:08:40] **Jacob:** But so the real value of it is it doesn't require uh a lot of like core Linux dependencies. So like GCC is one that we really think about a lot. Uh, if you're on AWS, something that I've been bitten by is not all of the nodes have GCC. And so when you're trying to do injection, like say you're trying to run a load balancer like to or Envoy, um, is similarly injected like we inject auto instrumentation and that will just not work. It used to not work. I think they've changed this but it used to not work uh on certain types of AWS nodes. So we have that same issue because we're trying to inject to all these different environments. So, Zig is pre-built with a bunch of these C libraries, but from scratch, part of its standard library and not based on anything existing. So it has all of its own dependencies. So, the benefit of this injector project is that we're going to be able to really meet users where they're at without needing to know a ton about their environment. And that's the real problem today. + +**Sophia:** Yeah. + +**Jacob:** Um, so it's an exciting project. I think we're a few months away from it being alpha beta. Other people are on the release timeline. Ted Young is thinking a lot about the release timeline there. + +**Sophia:** Um, but all TC. + +**Jacob:** TD, that's so cool. + +**Reese:** Yeah. What are the parts of, you know, whether it's something working on or not working on directly that you are excited about? + +[00:10:50] **Jacob:** Good question. I think that the biggest thing for the project right now that everybody cares about is stability, right? I think that uh we're trying to get to the place where users can reliably get, you know, a new update without them needing to do any configuration changes, without needing to worry about uh components being deprecated, things like that, right? I talked to, at that meetup in New York, I talked to a developer at a major bank who was, you know, they've developed all of their own uh custom components for the collector and so every time that we've been doing these reworkings, the collector's architecture, they need to then refactor a bunch of their code as well. And you know, after a few months you lose track of all these updates. So it becomes like a weeks-long project to keep up to date. When you have to do that, their release cycle is every quarter, right? It gets a lot to manage and a lot to update. So for the users, I think what people really want to see is that level of maturity and stability. We can get to that graduated status with the CCF would be huge. That would be massive for the project to have. Uh, I think it would really—I mean our adoption already has been really high. I think that's when we would come to the next level. + +**Sophia:** Great. Can you explain a little bit for those of us like CCS stages like what going from incubated, incubating projects to graduated means? + +**Jacob:** Sure. + +**Sophia:** Yeah. + +**Jacob:** I'm not the best. I can tell you what I know. I'm definitely not the person in charge of this. There are people who are far more confident in the stability part of this governance than I. + +**Sophia:** Yeah. + +**Jacob:** So I will speak on only what I know. + +**Sophia:** That's all we're asking for. [laughter] + +[00:12:00] **Jacob:** So right now we're in, as you said, the project is relatively lenient in terms of its standards. There are standards, but graduation is really strict, I would say for good reason, right? Like you want to be sure that when you are considered a graduated project you have a healthy community, which [laughter] we definitely do. You have stability guarantees, which is the main thing we're working towards, and lastly, adoption. So adoption has to be at a certain amount of companies; you have to have testimonials [laughter] from those companies about how the project's been helpful. That's another thing that we have plenty of. So now it really just comes down to stability and stability. You can think of Kubernetes as the example here. Kubernetes is, you know, the first graduated project. I imagine it's the first one. Maybe there maybe someone beat them to it, but you don't [laughter] know. But essentially, every time that Kubernetes improves their security and reliability, uh, you know, release stability, anything like that, it raises the bar for everybody else to continue to improve. So as we are getting towards stability, Kubernetes is chugging along, you know, really positively and improving their posture constantly. So it's up to us to not only meet where we started for our graduation criteria, but also to continue building towards that. So that means that you're going to see more security features, much more security guarantees, which is going to be very helpful for enterprises. M um, you're going to hear more about quarterly releases rather than uh the sort of disparate release calendar that we have today. Uh, you'll hear more about documentation which is also going to be huge. So uh later when that comes on, I assume that we'll be talking about some mobilization efforts and documentation. + +**Reese:** Ah yes, uh that is super, super valuable, right? + +**Jacob:** Um, all of that is going to get us to that graduation phase. Uh, it's just time, it's time and effort and uh we have a dedicated group of people who want to see this happen and are spending, you know, all of our time at all of these large companies trying to make that happen. Very exciting. + +**Sophia:** Awesome. + +**Reese:** That is very exciting. And if you are yourself an open end user and um your company is using in fashion, um, and you want to share your story, we would love to have your story as part of um our legacy [laughter] our catalog. + +**Jacob:** Yeah. + +**Reese:** Like to help, you know, help graduate the project and help take us out to us. Um, we'll have it in the show notes um after this. But yeah, it's pretty easy. As long as you remember me Slack, you can find us at hotel-dash-dash-and-dash. + +**Jacob:** I think that's all the dashes. + +**Reese:** I thought we were going to dash zero. [laughter] + +**Jacob:** Oh, so we're like dash. + +[00:14:50] **Reese:** And similarly, if anybody wants to help out with documentation, documentation is one of the best ways to contribute to the project. It is so valuable. We had somebody for the operator group contribute a whole uh bit of documentation for the target allocator and it was so well done. Adriana did this as well, super helpful, and it has been such a resource for me to be able to point users that come into our Slack, come into our S meetings to just say, "Hey, you have a question about this? We have a full document to answer everything you might need to know about it." That has been such a way to offer. Um, it's lovely. So, great way to contribute and get started. Doesn't require writing code. Requires asking questions and writing a bunch of stuff down. + +**Sophia:** Yeah. And I can confirm that everyone that I met in the local community has been super helpful. Um, anything you know you can take on and be responsible for, like documentation, that's something that you're taking off their plate. So I know they're having to help you. And it's also cool to see your work on an official website. + +**Jacob:** Yeah. Like, uh, definitely join the communication SIG for that. I'm currently in it and we're working on a lot of refactoring of the documentation, like just like the early stages since I'm more of a beginner with the OpenTelemetry and everything. I've been working on kind of refactoring those beginner level documentations. So like it's been really helpful getting people like acquainted with that and localization efforts too. Like I know Lisa is working on a lot of localization efforts. + +**Reese:** So yeah, it'll help. + +**Jacob:** Super helpful. And the beginner stuff especially. I mean, again from the talk today I think it's just so—it was so useful to learn where people are at in their implementation journeys, right? Uh we do have people that really need reference architectures, a ton of code examples, right? These things help project adoption that also make it so that users actually are delighted by the experience, right? Ultimately, that's what I think we all want to see and that's what users want. Like you want to be able to install hotel wherever you need it and not have to scour the internet to figure out how to make it work, right? Different companies have so many different needs and so getting started is always going to be the most important thing for us to really nail. + +**Sophia:** Uh, the next thing that I heard from, uh, when I did the event in New York is reference architectures and showing how we can do hotel in all of these various environments, so that people can have some code samples to copy from, uh, some architectures to, you know, design around. I was just talking with an engineer at [company name] and they are really interested in understanding how to do various trace sampling architectures, right? Where we can really have like a whole, as you said, catalog these architectures to show people. And we all of our companies, like you know, when I was at [another company name], we had all these architectures internally that we would post blog posts about to share. + +**Jacob:** We have a lot of companies that are using hotel as part of the ingestion path as well, which involves a lot of this architectural work. Similarly, a lot of companies that are contributors to hotel also are obviously using the tooling themselves for their own company's observability. We should reach out to the people who are doing that and try to publish what are the reference architectures that hotel maintainers use ourselves, right? + +**Sophia:** I think that would be really useful for people who are trying to implement this right now. + +**Reese:** Absolutely. + +**Sophia:** And so are we planning to host them like in a repository or like on sites directly? + +**Jacob:** Oh, I think would be the best. I think, you know, we have documentation places, the operator group, and I think the idea of everything being the doc is going to be the way to go. + +**Reese:** Right. + +**Jacob:** Like centralizing it and making sure that it's well organized, which is a lot of the work that you're doing, super valuable. + +**Sophia:** Of course. + +**Jacob:** But I think it just—it needs to be there right now. I think it exists on a few different company blogs, a few different companies— + +**Sophia:** Head scattered. + +**Jacob:** —bringing that together I think would be really, really valuable. + +**Reese:** One reason let's do it. + +**Jacob:** Right? There's so many companies like contributing. So there's definitely a bunch of references that people in our community could use. + +**Sophia:** Yes. + +**Jacob:** Yeah. + +**Reese:** Absolutely. And so many people are, people are so friendly, right? Like back to that, like going to uh the hotel booth. We've had so many people who have never come up to us before and are just asking great questions, and we have so many people who just, you know, we hang out there because we love to talk to people about this. It's rare that you get to have this level of engagement with end users, right? Uh, that direct type of engagement is so valuable and so people coming by has been one of my favorite experiences of it always. I think the reason that I love to come to these is just getting to talk to people, hearing people's problems, digging in because then it helps us plan what I like what I need to do for the next two years, right? I can take that feedback on it. + +**Sophia:** What, um, so in during your time here at CubeCon, like from the audience questions that you had at your talk and people who come through the OpenTelemetry observatory, do you have—are you seeing any like general trends as to like what people are leaning toward or needing more help with? + +**Jacob:** Yeah. So I say the main thing that is of concern to me is around OpAmp, which has been getting a lot of—there are a lot of talks in OpAmp this CubeCon. I don't know if that if we've noticed that there were at least four that mentioned it, which is a lot. + +**Sophia:** Could you explain that a little bit more for us? + +**Jacob:** Nothing. [laughter] Sorry. + +**Sophia:** We're slowing down a little bit. + +**Jacob:** So, OpAmp is the uh Open Agent Management Protocol, uh, part of the hotel project but is also designed to be in the same way that hotel is very vendor-neutral, it's neutral to agents as well. So, it's not just for the collector necessarily to be for a lot of different uh agents out there. The goal of the protocol is to enable things like uh component status reporting, health reporting, and remote configuration. And remote configuration is definitely the top of the town is what I would say. + +**Sophia:** Yeah. Okay. Very cool. + +**Jacob:** Um, and so in that vein, what it sounds like users really want is a way to dynamically update what the collector can do, what the SDKs can do, uh, in real time. And that's challenging. The way that the collector was designed was never really with remote configuration in mind. There are ways to reload the process. Like we can actually go in and restart the process from the collector binary. Like you just do that, right? + +**Sophia:** Yeah. + +**Jacob:** But it's different than a remote. What users really want is a way to push a rule to a collector and the collector accepts it and continues going. + +**Sophia:** Yeah. -**Reese:** If Josh Sar were here, he's in every single group. That's wild and awesome at the same time. +**Jacob:** So that's what a company like Bindplane has already built. Uh, and so what we want to do is take a lot of the lessons from what Bindplane has been successful with and obviously we're working with them as well. They are big contributors to uh, the idea is that we work all together to improve the provider posture here and improve the SDKs as well so that we can actually realize this vision that people can dynamically update their configurations wherever they might be. [laughter] Uh, but it's challenging. I mean that that's like a really tough topic. -**Jacob:** No, it was very impressive talking with him. I learned so much. +**Sophia:** Yeah. -**Reese:** So, how do you feel talking to us all? Jacob, you did a talk or was it two talks? +**Jacob:** There's a lot of companies in hotel have done this work beyond Bindplane before. Elastic comes to mind. They have their own protocol for doing road configuration already. -**Jacob:** One talk. I gave a talk about two hours ago, three hours ago. +**Sophia:** Okay. Okay. -**Reese:** Cool. Very fresh! +**Jacob:** We've talked with a few of their containers and there's definitely some like room for some good collaboration that we've talked about. -**Jacob:** It was packed. I was worried because this is my second time giving a talk. The first talk I gave was in Chicago two years ago. We got a very low slot, like towards the end of the day on Thursday. It was a tougher day to give a talk. We had a solid audience, and for this one, I was worried that with the content we were doing, which is like introductory to hotel and Kubernetes, I always worry with these introduction ones. You know, is this going to be enough? Is it going to be too much? My fear is that nobody shows up and I'm speaking to an empty room. This was the reverse, where it was a full room, and people were standing in the room, and then people got turned away from that door. +**Sophia:** Awesome. -**Sophia:** Oh my gosh! +**Jacob:** I think it's all to be seen, but this is definitely the thing that we hear a lot about. The thing that I hear a lot about on the operator side is in Kubernetes, how can I do uh, call it not just remote configuration but layered configuration. So what this means is how can I make it so that if I'm running in an enterprise company that I'm in a multi-tenant environment, I want like a base collector. I want to overlay. I want my teams to be able to overlay different pieces of config on top of that collector for their needs. -**Jacob:** I was happily surprised by that. We only reserved five minutes for questions at the end because I was like, you know, I don't know what questions people will have. Usually, people aren't asking a ton of questions after; they'll come up to you later. But it's like usually a Q&A session does not last very long. So I thought five minutes for questions is fine. We do questions, and there's a line of like 15 people waiting to ask questions. It just kept growing and growing. We stayed after for another 30 minutes just answering questions. +**Reese:** Right. -**Sophia:** Interesting! What level of questions? Was it more, you know, people who are newer to the topic, or who were asking more like stage two kind of questions? +**Jacob:** It's a really complicated topic. -**Jacob:** Huge range! We had some people who were very introductory level, just like, "I'm getting my bearings here. How does this part of it work? What's the deal with Prometheus and hotel? What do you use for a backend?" That's like a solid starter question. Then other people came in, saying, "I've been using this chart for the past year and there's this one thing that I want to be able to do. When are you going to do the work to enable that? How can we architect this?" Very big and each one was from a different part of the area. +**Sophia:** Yeah, it's really— -**Reese:** Okay, that's so cool! +**Jacob:** Sounds like it. And the thing that I sort of struggle with is how do we make this extensible and configurable but also simple to understand. The problem with this, that it starts getting into a lot of moving parts, and as soon as you start to introduce more moving parts is when complexity, and usually reliability, both complexity increases and reliability decreases, right? You need consistency and more moving parts, uh, will reduce that. It's sort of that—there's a great quote in reliability that I love. It's u—the most reliable system is the one that does—that never—that you, right? We know this quote. I'm totally botching it right now. [laughter] -**Jacob:** Yeah, very, very cool and very surprising. I'm always happy because, as a maintainer, we get people who come to us with issues on GitHub, and it's like you'll get someone complaining, "Hey, this thing broke. The most recent release broke me." But we don't get a lot of positive feedback. We don't hear what usage is. It's kind of the benefit and the curse of open source—we don't necessarily get that user feedback directly. It's why I love what you all do, because getting user feedback is one of the most valuable things to the community. In order for us to push forward, we need to know what isn't working right now. +**Sophia:** No, I hear it. I hear it. -**Sophia:** Right! It's a bit of a challenge because we feel like we're talking about it at our work and in the OpenTelemetry community, and then, like you said, we have all these conscious people who don't know. +**Jacob:** The idea is that like [laughter] the most reliable system is the one that like doesn't exist, right? -**Jacob:** Exactly! We're trying to figure out how we can make them aware of all these cool things that are happening. One of the things we started doing is the "What's New in Hotel" segment, which we just had our first one about a month ago or so. We're hoping to have something we can point people to. +**Sophia:** Right. -**Sophia:** Of course! +**Jacob:** Because it's always achieving its goal of non-existing. And so if you want to really improve the reliability of the service, you delete it because then you don't have to care. Then it's at 100% because it's always giving you the same answers, which is nothing. -**Jacob:** You mentioned the injector. +**Sophia:** Yeah. [laughter] -**Reese:** I did! Tell us more about it, please. +**Jacob:** [laughter] Sorry. -**Jacob:** I’m a maintainer for it, but I'm mostly doing reviewing because this is actually a joint effort between Splunk and D-Zero. Both of them have written their own injectors for auto instrumentation, and they are essentially donating a merge of both of those projects to the alpha version of this for their customers. We are just testing out that everything is working the way we need to. It's a very exciting project because it introduces a new language to Hotel, which is rare for us. We're now using Zig in the project. +**Sophia:** No, I'm here for it. I'm here for it. -**Sophia:** Zig! +**Jacob:** Yeah. -**Jacob:** Zig is a fun language that is sort of a merge between Go, Rust, C++, and C. The real value of it is it doesn't require a lot of core Linux dependencies. For example, on AWS, not all of the nodes have GCC. When you're trying to do injection, like say you're trying to run a load balancer like Envoy, it used to not work on certain types of AWS nodes. Zig is pre-built with a bunch of these C libraries and has its own dependencies. So the benefit of this injector project is that we'll be able to meet users where they're at without needing to know a ton about their environment. +**Reese:** Someone—I’ll say that to, I don’t know, someone at the Ozone booth and they'll be like, "You totally messed up." -**Reese:** That's exciting! +**Sophia:** You know, [laughter] we'll have the correct throw in the shout out. -**Jacob:** I think we're a few months away from it being in alpha or beta. Ted Young is thinking a lot about the release timeline. +**Reese:** Yeah. Thank you so much, Jacob. Um, we are going to bring on our next guest, Diana, in a moment. Um, and in the meantime, I really want to show off this baseball jersey. -**Sophia:** That’s so cool! What parts of the project are you excited about? +**Sophia:** Looks really good. -**Jacob:** Good question! I think the biggest thing for the project right now that everybody cares about is stability. We're trying to get to a place where users can reliably get a new update without needing to do any configuration changes, without needing to worry about components being deprecated. I talked to a developer at a major bank who has developed all of their own custom components for the collector, and every time we do these reworkings, they need to refactor a bunch of their code as well. It becomes a weeks-long project to keep up to date. +**Reese:** But, um, our friends at Honeycomb have been kind enough to um, sponsor for the hotel community. Um, thank you, Austin Parker, for the design. Um, I would stand and show the back. It says uh the number 11 ACTUALLY I DON'T KNOW WHY IT SAYS 11 like [laughter] there. But yeah, it says maintainer on the back. Um, we'll have some pictures hopefully to show you. But yeah, you've got the logo over here and it says open too across the front. Very, very snazzy, very sharp. [laughter] Um, yeah. And it also comes in black, which I had the option, but I was like, the client looks sharp. Yeah. Um, and also, you can see the blue and the yellow very clearly. Yeah. Just wanted to show you a little bit of a conference fashion. Um, and then when we get um our second guest in a setup, I also want to show you her nails because they're amazing. -**Reese:** Wow, that sounds challenging. +**Sophia:** Oh yes. -**Jacob:** Yes! Users want to see that level of maturity and stability. If we can get to that graduated status with the CCF, it would be huge. Our adoption has already been really high, but that would take us to the next level. +**Reese:** I don't know how close I can get. -**Sophia:** Can you explain a bit about the CCF stages? What does it mean to go from incubated to graduated? +**Sophia:** Yeah. -**Jacob:** Sure! Right now, the project has relatively lenient standards. There are standards, but graduation is really strict for good reason. You want to be sure that when you are considered a graduated project, you have a healthy community, which we definitely do, and stability guarantees, which is the main thing we're working towards. Lastly, adoption has to be at a certain amount of companies, and you have to have testimonials from those companies about how the project has been helpful. +**Reese:** And like I said, we have a group right here. I'm very excited that it's for today. So that's why we're able to, you know, have lunch on right now. But yeah, I brought my winter puppy out. Um, it was intense, but we made it through and we are now day two of the main event with one more day to go and I hope everyone is doing well. -**Reese:** That makes sense! +**Sophia:** Staying hydrated. -**Jacob:** Stability can be thought of as the main goal. You can think of Kubernetes as the example here. Kubernetes is, you know, the first graduated project. Every time Kubernetes improves their security and reliability, it raises the bar for everyone else to continue to improve. We are getting towards stability, and as we do, we will be introducing more security features and guarantees, which will be very helpful for enterprises. +**Reese:** Staying hydrated is so large [laughter] and important. And yeah, I think we're ready to speak to our next guest, Diana. -**Sophia:** That is super valuable! +**Sophia:** Yes. Awesome. -**Jacob:** Yes! Later on, we will be talking about mobilization efforts and documentation as well. +[00:27:20] **Reese:** Diana, welcome. And thank you guys. Congratulations on uh winning the Open Salt Community Star 2025. She was one of several winners. Um, the only one I want to point out, but that's why it's so important. Um, Diana, can you please introduce yourself? And also, the earrings. -**Reese:** If you are an OpenTelemetry user and your company is using it, we would love to hear your story! It will help us graduate the project and take us further. +**Diana:** Yeah. Oh my gosh. Yes. Oh, those are gorgeous. Oh my goodness. Okay. Sorry. Continue. [laughter] -**Jacob:** Absolutely! Documentation is one of the best ways to contribute to the project. It is so valuable. We had someone for the operator group contribute a whole bit of documentation for the target allocator, and it was so well done. +**Reese:** Like, hold up your treasury. -**Sophia:** That’s wonderful! +**Diana:** Yeah. Yeah, I'm Diana. Uh, I'm a developer experience engineer at Parametric and I work all my life in engineering uh in observability the last couple of, like, five years. Uh, and I discovered like I really like going and explaining things to the community. So I had a chance like years back to be a speaker on one of the first conferences I've ever been in my life. Uh, I was super nervous but I liked it a lot and I saw like so many women on stage giving talks that I felt like I needed to do more. -**Jacob:** It's been a fantastic resource for me to point users who come into our Slack or our meetings to just say, "Hey, you have a question about this? We have a full document to answer everything you might need to know about it." +**Sophia:** Of course. -**Reese:** And the local community has been super helpful! Anything you can take on, like documentation, is something that you're taking off their plate. +[00:30:01] **Diana:** Uh, so I kind of did that for two years. Uh, and actually last year, like I was hearing so much about OpenTelemetry, I said, "Oh, I want to get involved. I want to do something." Uh, and I got lucky because in the same moment when I thought about it, the foundation was like, "Oh, we are looking for observability people, we are looking for uh, let's get involved, we want to create the first open and learn fundamentals." Uh, I got in, so it was like first start with the documentation, reading questions, questions about and that's like simple months into certain words. I knew it was finally polished. So that was like for me the kick into like sense and slowly I kind of like I was like, okay, I'm sorry things, how about I'm going to do my company? That was like uh for my current company actually. Um, so uh it was like really interesting because I wanted like from explaining OpenTelemetry to the entire company, getting software engineers involved like instrumenting some things in like SDKs, uh, testing everything. -**Sophia:** Yes! Join the communication SIG for that. I'm currently in it, and we're working on a lot of refactoring of the documentation. +**Reese:** Um, so yeah, production, so that was like a decision. -**Jacob:** Exactly! The beginner stuff is especially important. From the talk today, it was so useful to learn where people are at in their implementation journeys. +**Diana:** Uh, but that was like a really cool experience that I've been talking about since one of my talks and explaining how, you know, companies, some particular companies that to be honest they're not super big times to build [laughter] in takes a very important. So I need to be involved in the community. So for me that was like a very, very experience also like understand [laughter] yeah, I thought about myself. Okay, but if right now I'm all alone and I still want to contribute, how can I do it? And I saw translating documents. -**Reese:** Definitely! Getting started is always going to be the most important thing for us to nail. +**Sophia:** Oh, nice. That's interesting. -**Jacob:** Yes! I heard from the event in New York that reference architectures and showing how we can do hotel in various environments are also needed. +**Diana:** I can do my own time. Start something, translations and yeah, can you um— -**Sophia:** That would be really useful! +**Reese:** So global organization, the projects or localization just refers to translating the documentation to different languages. -**Reese:** Are we planning to host them in a repository or on sites directly? +**Diana:** Exactly. Yeah. So the idea that—okay, there may be a bit more um, other terms around localization. So it's not necessarily only the technical terms or the tech writing part, but there are like also like um how you're going to automate some CI, going to automate specific pipelines documentation. -**Jacob:** I think that would be the best. Centralizing it and making sure that it's well organized is a lot of the work that you're doing. +**Reese:** Right. Right. -**Sophia:** Absolutely! +**Diana:** Can also be that and also how are you communicating a specific message uh in a language, right? Interpretation of concepts and terms and technology in other languages. So I think that's how I see things. -**Jacob:** So, we should bring all that together. +**Sophia:** Well, you mentioned Spanish. Um, are you involved with other languages? Um, and can you also tell us like what other languages are being are for the project right now? -**Reese:** Yes! There's a ton of companies contributing, so there are definitely references that our community could use. +**Diana:** Yeah. Yeah. Yeah. Um, so when um back when I got involved, I didn't think of this year, uh, Spanish was available, obviously English, French. -**Jacob:** Exactly! People are so friendly, and the engagement with end users is so valuable. +**Reese:** Ah yes. -**Sophia:** Do you have any general trends that you are seeing during your time here at CubeCon? +**Diana:** And that was right. So from the beginning of the year up to now, I think three more languages emerged. So three more communities. Uh, I know for sure Bengali. -**Jacob:** Yes! The main concern seems to be around OpAmp, which has been getting a lot of attention. There are at least four talks that mentioned it, which is a lot. OpAmp is the Open Agent Management Protocol, part of the OpenTelemetry project, designed to be vendor-neutral. The goal of the protocol is to enable things like component status reporting, health reporting, and remote configuration. +**Reese:** Oh nice. -**Reese:** That sounds interesting! +**Diana:** And that's like so they really got together and they started to contribute more and more like their only technology. -**Jacob:** Yes! What users really want is a way to dynamically update what the collector can do, what the SDKs can do in real-time, but that’s challenging. The way the collector was designed was never really with remote configuration in mind. +**Sophia:** Yeah. -**Sophia:** That does sound tough! +**Diana:** Um, Ukrainian, that's locked up recently. So I know for sure they need more contributors. Uh, so somebody came with the idea and they started to do that. Um, and I started, uh, for my own language Romanian. I'm native to Romanian. Uh, I started like a couple of months back and I said, "Look, yeah, I think it's an amazing opportunity for my language to represent and for developers back home to understand a bit more language. It's going to really perspective like it is a really nice community." -**Jacob:** Yes! But we want to realize this vision that people can dynamically update their configurations. +**Reese:** Yeah, that's really beautiful. -**Reese:** That’s exciting! +**Diana:** So like what is the hardest thing about localization that you find when you do that kind of work? -**Jacob:** Thank you so much! +**Diana:** Yeah, I think um definitely translating specific terms into your own language or in that particular language, but giving it the flavor or giving it like [laughter] that really gets the essence so it doesn't have to sound like a very dry terminology and that word captures the essence of the meaning. So for example, I don't know about—if you translate really like [laughter] literally a word from English or another language, it might sound a bit weird. So you have to like, oh, does it sound really like this in that language? I have to like, you know, modify it a bit. I have to give it like a nice ring to it so like you can get it, you know, it's kind of like, uh, people like it immediately and they want to use it, right? Um, so that's I think that's the difficult part and also the difficult part is that there are many, many languages that without technical terms directly in English, right? -**Reese:** We will bring on our next guest, Diana, in a moment. I want to show off this baseball jersey. It looks really good! +**Sophia:** Right. -**Sophia:** It does! +**Diana:** And when you want to translate it in that language quite like the appropriate meaning is like, okay, I know you know uses that word in my language and they always something in English, right? Oh, that get immediately, you know, it doesn't sound strange, so those are like words basically. -**Reese:** Our friends at Honeycomb have been kind enough to sponsor us for the OpenTelemetry community. Thank you, Austin Parker, for the design! +**Reese:** That's so wonderful. -**Sophia:** It looks sharp! +**Diana:** Yeah, it's really cool. -**Reese:** Yes! And when we get our second guest set up, I also want to show you her nails because they are amazing! +**Sophia:** Is there um like a way or um to kind of see, you know, the rate of adoption or involvement from the communities that now have these translations available? -**Sophia:** Yes! +**Diana:** Um, in terms of metrics or in terms of adoption, I think that we are uh kind of primitive. Um, and I think that you need to correlate it a bit with a ground meeting event or with some type of local community that you could uh speak about it on site, right? -**Reese:** I brought my winter puppy out. It was intense, but we made it through day two of the main event. I hope everyone is doing well and staying hydrated! +**Reese:** Or this makes some noise back home for example. -**Sophia:** Staying hydrated is so important! +**Diana:** Uh, and say look, we are starting this, we are doing this. We need more contributors. Uh, we need, you know, like the people from the local communities. So like that for each one of us is very much needed whether online or on site. Uh, so for me, for example, I just started this with Romania. I started going like to people like I contributed before or knew from like events and I'm also Romanian. Around there, like can you put it, can you promote it? Like we are like actually entitling our language to be heard in the open community like really jumped into it and yeah, for sure, let's do it. -**Reese:** I think we're ready to speak to our next guest, Diana. +**Reese:** Like a lot of people started. It's really nice. -**Diana:** Thank you! Congratulations on winning the Open Source Community Star 2025. +**Diana:** Yeah, I think as you know, native English speakers we get so used to everything being— -**Reese:** Thank you! Can you please introduce yourself? +**Reese:** To English. -**Diana:** Yes! I'm Diana, a Developer Experience Engineer at Parametric. I have worked all my life in engineering and observability for the last five years. I discovered that I really enjoy explaining things to the community. I had a chance years back to be a speaker at one of the first conferences I ever attended. I was super nervous, but I liked it a lot, and I saw so many women on stage giving talks that I felt I needed to do more. +**Diana:** And honestly, you know, a lot of um non-native English speakers, you know, they usually speak multiple languages. Um, and so I think it's really impressive um and awesome the work that you all are doing. -**Sophia:** That's amazing! +**Diana:** Yeah, I appreciate it. And also I think down the line people get motivated not only because of modernization but they start thinking, okay, uh, do I like overall open source? Uh, should I do something else maybe in another project? Uh, does it help me in like my work projects, whatever I'm doing? Uh, and I think this is how I practically, you know, promoted it to my community. I said if you guys like me, uh to say like I'm doing an open source project as your contributor like if your employer needs you to have this experience or maybe you want to go to a conference and talk about it. So all of these are very, very nice to have, you know, on your screen and you know, different age groups, you know, there, university or later on their career and I think it's really important to keep motivation because it can be sometimes really hard to start in your free time or remind yourself that you know [laughter] you have an objective online. It's kind of like a gateway project into contributing to open source a little bit. [laughter] -**Diana:** So, I did that for two years. Last year, I heard so much about OpenTelemetry and wanted to get involved. I got lucky because the foundation was looking for observability people. We wanted to create the first OpenTelemetry Learn Fundamentals, and I got in. It was a kickstart for me to delve into the documentation and engage with the community. +**Reese:** I always keep telling them that I didn't necessarily start with the organization and I share my new stuff here. I'll definitely go for example—I being an engineer, I still like doing engineering work. -**Reese:** That’s wonderful! +**Diana:** Oh yeah. -**Diana:** I wanted to explain OpenTelemetry to my entire company, getting software engineers involved in instrumenting things in SDKs and testing everything in production. It was a cool experience, and I’ve been talking about it since in my talks. +**Reese:** And I want to get involved in the codes part as well because I've been keeping an eye on everything that's happening talking like there are many, many things that for sure my company or other people will definitely find. -**Sophia:** That's great! +**Diana:** Um, so yeah, you know, one project or one S is not enough. -**Diana:** I also thought about how, if I'm all alone and still want to contribute, I could translate documents. +**Reese:** We're going to jump [laughter] into something else at some point and then the community especially is really, really and they keep talking about the same issues over and over again. -**Reese:** Nice! What does localization refer to in this context? +**Diana:** Yeah. So just recently talking about I heard about the same kind of issues in Kubernetes. So I'm kind of like went back and forth because I'm kind of like struggling with similar issues. So I think it's very good to keep an open mind open and keep going. For me at least personally this is what motivates us going forward. You can diversify your contributions, not stop, and we always find something. -**Diana:** Localization not only refers to translating documentation into different languages but also includes how to automate specific pipelines and how you communicate a message in a language. +**Reese:** Thank you. -**Sophia:** That's interesting! +**Diana:** Yeah, that's awesome. And then kind of pivoting a little bit. Um, I would love to know what you're looking forward to in the coming year for hotel. Like what are you excited about? What kind of trends you're seeing like implementation? -**Diana:** Back when I got involved, Spanish and French were available. Since then, three more languages have emerged: Bengali, Ukrainian, and I started translating into Romanian. +**Diana:** Anything that has— -**Reese:** That's beautiful! +**Jacob:** So um, just like the last couple of weeks been to like specific events. I saw a lot of interesting talks about yeah obviously projector and uh I think these are two things that are really interesting at the moment. -**Diana:** The hardest part about localization is translating specific terms into your own language while giving them the right flavor and essence. +**Sophia:** So instrumentation, particular end users need a lot more best practices so people don't see users and say, "Oh, but we don't know how to do all this instrumentation or we don't know like which to add." I don't know like what's the easy stuff you can implement? Like can we can you add some like best practices? Can you, right? -**Sophia:** Yes, I can imagine! +**Diana:** You know, kind of like tell us, you know, like what would be like best starting point onboarding, let's say in this case. -**Diana:** It’s challenging to find the right words that capture the meaning without sounding dry. +**Jacob:** Uh, and I've been hearing like uh community and for OpenTelemetry. -**Reese:** That’s a great point! +**Reese:** Yeah. -**Diana:** Yes! There are many languages that lack technical terms directly in English, and translating them appropriately can be tough. +**Diana:** You know, uh, like get them in. -**Sophia:** That’s wonderful that you’re addressing that! +**Sophia:** Yeah. [laughter] -**Reese:** Is there a way to see the rate of adoption or involvement from the communities that have these translations available? +**Reese:** I know that you're complaining, BUT TELL US more about your use. -**Diana:** Currently, we are kind of primitive in terms of metrics. It’s important to correlate it with ground meetings or local communities that can promote the translations. +**Diana:** Right? -**Sophia:** That makes sense! +**Sophia:** Yeah, that's so cool. -**Diana:** For example, I started with Romania and reached out to people, encouraging them to get involved. +**Jacob:** Yeah. -**Reese:** That's fantastic! +**Reese:** Um, I just wanted to know, you said that you wanted to get involved in more of the engineering side, the coding side. Is there somewhere specifically that you're looking forward to or some somewhere specifically that you would like to work in? Like anything exciting to you specifically? -**Diana:** It’s important to keep motivation high, especially when starting in free time. +**Diana:** Uh, for instance, I'm very interested in the river and the river. -**Sophia:** For sure! +**Sophia:** Can you explain it a little bit to us? -**Diana:** It's a gateway project into contributing to open source. +**Diana:** Uh, yeah. Well, I—I don't know. -**Reese:** Absolutely! What are you looking forward to in the coming year for OpenTelemetry? +**Reese:** No, right. Right. -**Diana:** I’ve seen a lot of interesting talks about instrumentation lately. Users need more best practices to understand how to implement instrumentation easily. +**Diana:** You're getting [laughter] -**Reese:** That’s a great point! +**Reese:** I think people just last [laughter] maintainers and people that got involved and all the idea about this convention around it. -**Diana:** Yes! They want to know the best starting points and onboarding processes. +**Diana:** Uh, and although I—I’m not sure like how they are implementing it like practically, something that I want to do is stop from going to so many events like [laughter] my laptop stuff like this. -**Sophia:** That's super important. +**Reese:** So, um, I think it needs a bit of time and time and you know, with my team and my brother. -**Diana:** I’m also interested in contributing more to the engineering side, particularly around the collector and how to implement various configurations. +**Sophia:** That's awesome. I love it. -**Reese:** That sounds exciting! +**Reese:** Really cool. Awesome. Well, thank you so much, Diana. Congratulations again on your movie star award. It is so cute. And thank you so much, Sophia, for being my co-host. -**Diana:** Thank you so much for having me, and congratulations again on your award! +**Sophia:** Yes. Thank you so much, Reese, for being the host host. -**Reese:** Thank you, Diana! Thank you, Sophia, for being my co-host. +**Reese:** You're amazing. Our co-co-host [laughter] co-host. -**Sophia:** Yes! Thank you so much, Reese, for being our amazing host! +**Sophia:** Um, yeah, we hope to see you at a future soon. Um, Amsterdam in EU and soon for our next year. Um, and we'll have some show up. So check us, check it out there and talk to us if you like please. [laughter] -**Reese:** We hope to see you again soon, maybe in Amsterdam or at next year's event. +**Reese:** See you everybody. Bye everybody. -**Sophia:** Yes! Check us out, and talk to us if you like! +[music] -**Reese:** See you, everybody! Bye! +[music] ## Raw YouTube Transcript -Just the light. Oh, action. >> Oh, you did. You did. >> Hello everybody. If you've been waiting around, we are so sorry for the delay, but we are very excited to be coming to you by from Atlanta, Georgia, but pretty much everybody was um very surprised by a cold front that had us looking at 40° uh which I think is a 3° C um the past couple days. Luckily, it's warming back up, so thank goodness. But yeah, we are coming to you live uh from CubeCon North America 2025. I'm Reese and I am joined here by my co-host Sophia. >> Hi. >> And our first guest, Jacob. >> Hello, my name is Jacob Marino. I'm a maintainer for let's see open telemetry operator, open telemetry helm charts, hotel injector, and I maintain a few random collection things and I contribute to opamp and and I'm missing one. I can't remember. I was wondering who had a hands for all 10 honestly. >> If Josh Sar were here, he's he's in he he's everywhere. >> That's every single group. >> That's wild and awesome at the same time. >> No, it was very impressive talking with him. Learned so much. >> So, how I feel talking to you all? Um, now Jacob, you did a talk or was it two talks? >> Uh, one talk. So I gave a talk uh about two hours ago, three hours ago. >> Cool. >> Very fresh. >> It was packed. >> It was like I was worried that uh I've given this is my second time giving a talk. First talk that I gave was in Chicago 2 years ago. >> Oh, nice. >> And the room we got a very low slot like it was like towards the end of the day on Thursday, I think. So it was a tough tougher day to give a talk. uh we had a solid audience uh and for this one I was worried that with the content we were doing which is like introductory to hotel and kubernetes I always worry with these introduction ones you know it's like is this going to be enough is it going to be too much so my fear is that nobody shows up and I'm speaking to an empty room this was the reverse where it was a full room and people were standing in the room and then people got turned away from that door >> oh my gosh >> which I was happily like surprised by. Then I also like we only reserve 5 minutes for questions at the end cuz I was I was like you know I don't know what questions people will have. Usually people aren't asking the ton of questions like after they'll come up to you later but it's like usually like a Q&A session does not last very long. So oh 5 minutes for questions like that's fine. I we do questions there's a line of like 15 people waiting to ask questions. like they just keep it kept growing and growing. We stayed after for another 30 minutes just answering questions. >> Interesting. What level of questions like was it more um you know people who are newer to the topic or who are asking like more like stage two kind of questions >> huge range so >> we had some people who were you know very introductory level just like I'm getting my bearings here how does this part of it work you know what's the deal with Prometheus and hotel and what do you use for a back end you know that that's like a solid solid starter question and then other people came in they were I've been using this chart for the past year and there's this one thing that I want to be able to do and uh when are you going to like do the work to like enable that and how can we like architect this big range >> very big and each one was like a different part of the of the area. >> Okay, that's so cool. >> Yeah, very very cool and very surprising. I'm always happy to cuz you know I think as a maintainer it's like we get people who come to us with issues on GitHub and it's like you'll get someone complaining on a GitHub issue or like hey this thing broke the most recent release broke me um you know what are we doing about that broke >> but no one's broke their spirit you know I've had releases break my spirit >> but the uh the real thing is we don't get positive feedback. We don't get a lot of questions cuz realistically most people are asking track or going on Stack Overflow, >> right? It's like they're not bringing the questions to us. So, we don't really hear or understand what usage is. I mean, it's kind of the benefit and the curse of open source is that like we don't necessarily get that user feedback directly. It's why like I love what you all do cuz getting the user feedback is one of the most valuable things to the community. like in order for us to push forward, we need to know what isn't working right now where people just don't see the effort. Um we actually I hosted a meetup in New York uh last month, 2 months ago, something like that. >> Yeah. Uh, and that was eye opening because it was like wow, people are really using these things and things that I I feel like I've talked about so many times are still not known, you know, and it's like I'll mention like how many people know what like the target allocator is in a show of hands and nobody raised their hand in a room of like 40, you know, senior SRRES. I'm like, okay, uh, would anybody like it if you could do distributed, sharded, retheus scraping? and everybody raises their hand and I'm like well we have that like it's done it's there you know you can use it it's been a thing for a few years you know so I guess all to say the community just is still there's still things for people to learn about the vast ecosystem that we have here >> yeah I think you know I think a common theme just in general is people not being aware of you know different abilities and features and yeah on the end you see too you know it's it is a bit of a challenge cuz like we feel like we're talking about it um you know at our work and like in the ozone community and then yeah like you said we have all these conscious people who don't know and it's we're trying to figure out like where are they going for like how can we >> make them aware >> of you know all these cool things that are happening. Um yeah, so you know, one of the things we we started doing is um the what's the hotel segment, which we just had our first one um a month ago or so. And so we're hoping to, you know, I don't know, have something we can point people to. >> Of course. >> But you mentioned the injector. >> I did. Tell us more about please do. So I am the uh I'm a maintainer for it but I'm mostly just doing reviewing because this is actually a joint effort between uh Splunk and D-Zero. Both of them have already written their own uh injectors for auto instrumentation. Oh, right. >> And they're essentially donating a merge of both of those projects to the tot alpha version of this for their customers. Um, so we're just testing out that everything is working the way we need to. There's a lot of really random edges here. Um, interestingly enough, this does introduce a new language to hotel, which is exciting, which is it's very rare for us to not use a language in a project that is meant to instrumental languages. Right. >> Right. >> And we are now using Zig in the project. >> Zig. >> Zig um is a fun language that is sort of a merged between like Go, Rust, C++, and C. Oh, super fun. >> But so the real value of it is it doesn't require uh a lot of like core Linux dependencies. So like GIC C is one that we really think about a lot. Uh if you're on AWS, something that I've been bitten by is not all of the nodes have g. And so when you're trying to do injection like say you're trying to run a load balancer liketo or envoy >> um is similarly injected like we inject auto instrumentation and that will just not work. It used to not work. I think they've changed this but it used to not work uh on certain types of AWS nodes. So we have that same issue because we're trying to inject to all these different environments. So, Zigg is pre-built with a bunch of these C libraries, but from scratch, part of it standard library and not based on anything existing. So, it has all of its own dependencies. So, the benefit of this injector project is that we're going to be able to really meet users where they're at without needing to know a ton about their environment. And that's the real problem today. >> Yeah. >> Um, so it's an exciting project. I think we're a few months away from it being alpha beta. Other people are on the release timeline. Ted Young is thinking a lot about the release timeline there. Um, but all TC >> TD, that's so cool. >> Yeah. What are the parts of, you know, whether it's something working on or not working on directly that you are excited about? >> Good question. I think that the biggest thing for project right now that everybody cares about is stability, right? I think that uh we're trying to get to the place where users can reliably get you know a new update without them needing to do any configuration changes without needing to worry about uh components being deprecated things like that right I talked to at that meet up in New York I talked to a developer at a major bank who was you know they've developed all of their own uh custom components for the collector and so every time that we've been doing these reworkings the collector's architecture. They need to then refactor a bunch of their code as well. And you know after a few months you lose track of all these updates. So it becomes like a weeks long project to keep up to date. When you have to do that their release cycle is every quarter, right? It gets a lot to manage and a lot to update. So for the users, I think what people really want to see is that level of maturity and stability. We can get to that graduated status with the CCF would be huge. That would be massive for the project to have. Uh I think it would really I mean our adoption already has been really high. I think that's when we would come to the next level. Great. >> Can you explain a little bit for those of us like CCS stages like what going from incubated incubating projects to graduated means? >> Sure. >> Yeah. I'm not the best. I can tell you what I know. I'm definitely not the person in charge of this. There are people who are far more confident in the stability part of this governance than I. >> Yeah. >> So I I will speak on only >> that's all we're asking for. >> So right now we're in as you said uh project is relatively lenient in terms of its standards. There are standards but graduation is is really strict I would say for good reason right like you want to be sure that when you are considered a graduated project you have a healthy community which we definitely do uh you have stability guarantees which is the main thing we're working towards and lastly adoption so adoption has to be at certain amount of companies you have to have testimonials from those companies about how the project's been helpful uh that's another thing that we have plenty of so now it really just comes down to stability and stability. You can think of Kubernetes as the example here. Kubernetes is, you know, the first graduated project. I imagine it's the first one. Maybe there maybe someone beat them to it, but you don't know. But essentially, every time that Kubernetes improves their security and reliability, uh, you know, release stability, anything like that, it raises the bar for everybody else to continue to improve. So as we are getting towards stability, Kubernetes is is chugging along, you know, really positively and improving their posture constantly. So it's up to us to not only meet where we started for our uh graduation criteria, but also to continue building towards that. So that means that you're going to see more uh security features, much more uh security guarantees, which is going to be very helpful for enterprises. M um you're going to hear more about quarterly releases rather than uh the sort of disperate release calendar that we have today. >> Uh you'll hear more about documentation which is also going to be huge. So uh later when that comes on I assume that we'll be talking about some mobilization efforts and documentation. >> Ah yes, uh that is super super valuable right um all of that is going to get us to that graduation phase. Uh it's just time it's time and effort and uh we have a dedicated group of people who want to see this happen and are spending you know all of our time at all of these large companies trying to make that happen. Very exciting. >> Awesome. >> That is very exciting. And if you are yourself an open end user and um your company is using in fashion um and you want to share your story, we would love to have your story as part of um our uh legacy our catalog. >> Yeah. like to help, you know, help graduate the project and help take us out to us. Um, we'll have it in the show notes um after this. But yeah, it's pretty easy. As long as you remember me Slack, you can find us at hotel dash dash and dash. >> I think that's all the dashes. >> I thought we were going to dash zero. Oh, so we're like dash. >> And similarly, if anybody wants to help out with documentation, documentation is >> one of the best ways to contribute to the project. It is so valuable. We had somebody for the operator group contribute a whole uh bit of documentation for the target alligator and it it was so well done. Adriana did this as well. super helpful >> and it has been such a resource for me to be able to point users that come into our Slack, come into our S meetings to just say, "Hey, you have a question about this? We have a full document to answer everything you might need to know about it." That has been such a way to offer. Um, it's lovely. So, great way to contribute and get started. Doesn't require writing code. Requires asking questions and writing a bunch of stuff down. >> Yeah. And I can confirm that everyone that I met in the local community has been super helpful. Um, anything you know you can take on and be responsible for like documentation that's something that you're taking off their plate. So I know they're having to help you. And it's also cool to see your work on an official website. >> Yeah. Like uh definitely join the communication sig for that. I'm currently in it and we're working on a lot of refactoring of the documentation like just like the early stages since I'm more of a beginner with the open telemetry and everything. I've been working on kind of refactoring those beginner level documentations. So like it's been really helpful getting people like acquainted with that and localization efforts too. Like I know Lisa is working on a lot of localization efforts. So yeah, it'll help. >> Super helpful and the the beginner stuff especially. I mean >> again from the talk today I think it's just so it was so useful to learn where people are at in their implementation journeys >> right >> uh we do have people that really need reference architectures a ton of code examples right >> these things help project adoption that also make it so that users actually are delighted by the experience right that and ultimately that that's what I think we all want to see and that's what users want like you want to be able to install hotel wherever you need it and not have to scour the internet to figure out how to how to make it work, right? >> Different companies have so many different needs and so getting started is always going to be the most important thing for us to really nail. Uh the next thing that I heard from uh when I did the event in New York is reference architectures and showing how we can do hotel in all of these various environments. so that people can have some code samples to copy from uh some architectures to you know design around. I was just talking with an engineer at and they >> are really interested in understanding how to do various trace sampling architectures, right? where we can really have like a whole as you said catalog these architectures to show people and we all of our companies like you know when I was at our flight we had all these architectures internally that we would post blog posts about to share we have a lot of companies that are using hotel as part of the ingestion path as well which involves a lot of this uh architectural work similarly a lot of companies are like that are contributors to hotel also are obviously using the tooling themselves for their own company's observability. We should reach out to the people who are doing that and try to publish what are the reference architectures that hotel maintainers use ourselves, right? >> I think that would be really useful uh for people who are trying to implement this right now. >> Absolutely. >> For sure. Um are we planning to host them like in a repository or like on sites directly? >> Oh, I think would be the best. I think you know we have documentation places the operator group and I think the idea of everything being the doc is going to be the way to go >> right >> like centralizing it and making sure that it's well organized which is a lot of the work that you're doing super valuable >> of course >> uh but I think it just it needs to be there right now I think it exists on a few different company blogs a few different companies >> head scattered >> bringing that together I think would really really valuable. >> One reason let's do it, >> right? There's so many companies like contributing. So there's definitely a bunch of references that people in our community could use. >> Yes. >> Yeah. >> Absolutely. And so many people are people are so friendly, right? like back to that like going to uh the hotel booth. We've had so many people who have never come up to us before and are just asking great questions and we have so many people who just, you know, we hang out there cuz we love to talk to people about this. It's rare that you get to have this level of engagement with end users, right? Uh that direct type of engagement is so valuable and so people coming by has been one of my favorite experiences of it always. I think the reason that I love to come to these is just getting to talk to people, hearing people's problems, digging in because then it helps us plan what I like what I need to do for the next 2 years, right? I can take that feedback on it. What um so in during your time here at coupon like from the audience questions that you had at your talk and people who come through the open tree observatory do you have are you seeing any like general trends as to like what people are leaning toward or needing more help with or >> Yeah. So I say the the main thing that is of concern to me is around amp which has been getting a lot of there are a lot of talks in opamp this coupon. I don't know if that if we've noticed that there were at least four that mentioned it which is a lot. >> Could you explain that a little bit more for us? Nothing. Sorry. >> We're slowing down a little bit. So, opamp is the uh open agent management protocol uh part of the hotel project but is also designed to be in in the same way the hotel is very vendor neutral it's neutral to agents as well. So, it's not just for the collector necessarily to be for a lot of different uh agents out there. The goal of the protocol is to enable things like uh component status reporting, health reporting, and remote configuration. And remote configuration is definitely the top of the town is what I would say. >> Yeah. Okay. Very cool. >> Um and so in that vein, what it sounds like users really want is a way to dynamically update what the collector can do, what the SDKs can do uh in real time. And that's challenging. The way that the collector was designed was never really with remote configuration in mind. There are ways to reload the process. Like we can actually go in and restart the process from the collector binary. Like you just do that, right? >> But it's different than a remote. What users really want is a way to push a rule to a collector and the collector accepts it and continues going. >> Yeah. So that's what a company like bind plane has already built. Uh and so what we want to do is take a lot of the lessons from what BL has been successful with and obviously we're working with them as well. They are big contributors to uh the idea is that we work all together to improve the provider posture here and improve the SDKs as well so that we can actually realize this vision that people can dynamically update their configurations wherever they might be. Uh but it's challenging. I mean that that's like a really tough topic. >> Yeah. >> There's a lot of a lot of companies in hotel have done this work beyond my plan before. Elastic comes to mind. They have their own >> I work here. >> Oh yeah. >> Elastic has uh their own protocol for doing road configuration already. >> Okay. Okay. >> We've talked with a few of their containers and there's definitely some like room for some good collaboration that we've talked about. >> Awesome. >> I think it's all to be seen but this is definitely the thing that we hear a lot about. The thing that I hear a lot about on the operator side is in Kubernetes, how can I do uh call it not just remote configuration but layered configuration. So what this means is how can I make it so that if I'm running in an enterprise company that I'm in a multi-tenant environment, I want like a base collector. I want to overlay. I want my teams to be able to overlay different pieces of config on top of that collector for their needs. >> Right. >> It's a really complicated topic. We >> Yeah, it's really >> sounds like it. And the thing that I sort of struggle with is how do we make this extensible and configurable but also simple to understand. The problem with this that it starts getting into a lot of moving parts and as soon as you start to introduce more moving parts is when complexity and usually reliability both complexity increases and reliability decreases right you need consistency and more moving parts uh will reduce that it's sort of that there's a great quote in reliability that I love it's u the most reliable system is the one that does that never that you right we know this quote. I'm totally botching it right now. >> No, I hear it. I hear it. >> The idea is that like the most reliable system is the one that like doesn't exist, >> right? >> Right. Cuz it's always achieving its goal of non-existing. >> And so if you want to really improve the reliability of the service, you delete it >> because then you don't have to care then it's at 100%. Cuz it's always it's always giving you the same answers which is nothing. Yeah. >> Sorry. >> No, I'm here for it. I'm here for it. Yeah. >> Someone I I'll say that to I don't know, someone at the Ozel booth and they'll be like, "You totally messed up." >> You know, we'll have the correct throw in the shout out. >> Yeah. Thank you so much, Jacob. Um, we are going to bring on our next guest, Diana, in a moment. Um, and in the meantime, I really want to show off this baseball jersey. >> Looks really good. >> But, um, our friends at Honeycomb have been kind enough to um, sponsor for the hotel community. Um, thank you, Austin Parker, for the design. Um, I would stand and show the back. It says uh the number 11 actually I don't know why it says 11 like there. But yeah, it says maintainer on the back. Um, we'll have some pictures hopefully to show you. But yeah, you've got the logo over here and it says open too across the front. Very, very snazzy, very sharp. Um, yeah. And it also comes in black, which I had the option, but I was like, the client looks sharp. Yeah. Um, and also, you can see the blue and the yellow very clearly. Yeah. Just wanted to show you a little bit of a conference fashion. Um, and then when we get um our second guest in a setup, I also want to show you her nails because they're amazing. >> Oh, yes. >> I don't know how close I can get. >> Yeah. And like I said, we have a group right here. I'm very excited that it's for today. So that's why we're able to, you know, have lunch on right now. But yeah, I brought my winter puppy out. Um it was it was intense, but we made it through and we are now day two of the main event with one more day to go and I hope everyone is doing well. >> Staying hydrated. staying hydrated is so large and important. And yeah, I think we're ready to speak to our next guest, Diana. >> Yes. Awesome. >> Diana, welcome. And thank you guys. Congratulations on uh winning the Open Salt Community Star 2025. She was one of several winners. Um, the only one I want to point out, but that's why it's so important. Um, Diana, can you please introduce yourself? And also, the earrings. >> Yeah. Oh my gosh. Yes. Oh, those are gorgeous. Oh my goodness. Okay. Sorry. Continue. >> Like, hold up your treasury. >> Yeah. Yeah. I'm Diana. Uh I'm a developer experience engineer parametric and I work all my life in engineering uh in observability the last couple of like 5 years. Uh and I discovered like I really like going and explaining things to the community. So I had a chance like years back to be a speaker on one of the first conferences I've ever been in my life. Uh I was super nervous but I liked it a lot and I saw like so many women on stage giving talks that I felt like I needed to know to do more. >> Of course. >> Uh so I kind of did that for 2 years. Uh and actually last year like I was hearing so much about open I said like oh I want to get involved. I want to do something. uh and I got lucky because in the same moment when I thought about it foundation was like oh we are looking for observability people we are looking for uh let's get involved we want to create the first open and learn fundamentals uh I got in so it was like first start with the documentation reading questions questions about and that's like simple months into certain words. I knew it was finally polished. So that was like for me the kick into like sens and slowly I kind of like I was like okay I'm sorry things how about I'm going to do my company. that was like uh for my current company actually. Um so uh it was like really interesting because I I wanted like from explaining open telemetry to the entire company, getting software engineers involved like instrumenting some things in like SDKs uh testing everything. Um so yeah production so that was like decision. Uh but that was like a really cool experience that I've been talking about since one of my talks and explaining how you know companies some particular companies that to be honest they're not super big times to build in takes a very important. So I need to be involved in the community. So for me that was like a very very experience also like understand Yeah, I thought about myself. Okay, but if right now I'm all alone and I still want to contribute, how can I do it? And I saw translating documents. >> Oh, nice. That's interesting. I can do my own time. start something translations and yeah, can you um so global organization the projects or localization just refers to translating the documentation to different languages. >> Exactly. Yeah. So the idea that okay there may be a bit more um other terms around localization. So it's not necessarily only the the technical terms or the tech writing part but there are like also like um how you're going to automate some CI going to automate specific u pipelines documentation. Right. Right. >> Can also be that and also how are you communicating a specific message uh in a language right interpretation of of concepts and terms and technology in other languages. So I think that's that's how I see things. >> Well, you mentioned Spanish. Um are you involved with other languages? Um and can you also tell us like what other languages are being are for the project right now? >> Yeah. Yeah. Yeah. Um so when um back when I got involved I didn't think of this year uh Spanish was available obviously English, French. >> Ah yes >> and that was right. So from the beginning of the year up to now I think three more languages emerged. So three more communities. Uh I know for sure Bengali. >> Oh nice. >> And that's like so they really got together and and they started to contribute more and more like their only technology. >> Yeah. >> Um Ukrainian that's locked up recently. So I know for sure they need more contributors. Uh so somebody came with the idea and they started to do that. Um, and I started uh for my own language Romanian. I'm native to Romanian. Uh, I started like a couple of months back and I said, "Look, yeah, I think it's an amazing opportunity for for my language to represent and for developers back home to understand a bit more language. It's going to really perspective like it is a really nice nice community." Yeah, that's really beautiful. So like what is the hardest thing about localization that you find when you do that kind of work? Yeah, I think um definitely translating specific terms into your own language or in that particular language, but giving it the flavor or giving it like that that really gets the essence so it doesn't have to sound like a very dry terminology and that word captures the essence of of the meaning. So for example, I don't know about If you translate really like like literally a word from English or another language, it might sound a bit weird. So you have to like al oh does it sound really like this in that language? I have to like you know modify it a bit. I have to give it like a nice ring to it. So like you can get it you know it's kind of like >> uh people like like it immediately and they want to use it right. Um so that's I think that's the difficult part and also the difficult part is that there are many many languages that without technical terms directly in English right >> and when you want to translate it in that language quite like the appropriate meaning is like okay I know you know uses that word in my language and they always something in English right oh that get immediately you know it doesn't sound strange so those are like words basically That's so wonderful. >> Yeah, it's really cool. >> Is there um like a way or um to kind of see, you know, the rate of adoption or involvement from the communities that now have these translations available. >> Um in terms of metrics or in terms of adoption, I think that we are uh kind of primitive. Um, and I think that you need to correlate it a bit with a ground meeting event or with some type of local community that you could uh speak about it on site, right? >> Or this makes some noise back home for example. Uh, and say look, we are starting this, we are doing this. We need more contributors. Uh, we need, you know, like the people from the local communities. So like that for each one of is is very much needed whether online or on site. Uh so for me for example I just started this with Romania. I started going like to people like I contributed before or new from like events and I'm also Romania. around there like can you put it can you promote it like we are like actually entitling our language to to be heard in the open community like really jumped into it and yeah for sure let's do it like a lot of people started it's really nice >> I think as you know native English speakers we get so used to everything being >> to English And honestly, you know, a lot of um non-native English speakers, you know, they usually speak multiple languages. Um and so I think it's really impressive um and awesome the work that you all are doing. >> Yeah, I appreciate it. And also I think down the line people get motivated not only because of modernization but they start thinking okay uh do I like overall open source uh should I do something else maybe in another project uh does it help me in like my work projects whatever I'm doing uh and I think this is how I practically you know promoted it to to my community I said if you guys like me uh to say like I'm doing a open source project as your contributor like if your employer needs you to have this experience or maybe you want to go to a conference and talk about it. So all of these are very very nice to have you know on your screen and you know different age groups you know there university or later on their career and I think it's really important to keep motivation because it can be sometimes really hard to start in your free time or remind yourself that you know you have an objective online. It's kind of like a gateway project into contributing to open source a little bit. >> I always keep telling them that I I didn't necessarily start with the organization and I share my new stuff here. I'll definitely going for example I being an engineer I still like doing engineering work. >> Oh yeah. And I want to get involved in the codes part as well because I've been keeping an eye on everything that's happening talking like there are many many things that for sure my company or other people will definitely find. Um, so yeah, you know, one project or one S is is not enough. >> We're going to jump into something else at some point and then the community especially is really really and they keep talking about the same issues over and over again. >> Yeah. So just recently talking about I heard about the same kind of issues in Kubernetes. So I'm kind of like went back and forth because I'm kind of like struggling with similar issues. So I think it's very good to keep an open mind open and keep going. For me at least personally this is what motivates us going forward you can diversify your contributions not stop and we always find something. >> Thank you. >> Yeah that's awesome. And then kind of pivoting a little bit. Um I would love to know what you're looking forward to in the coming year for hotel. Like what are you excited about? What kind of trends you're seeing like implementation? Anything that has so um just like the last couple of weeks been to like specific events. I saw a lot of interesting talks about yeah obviously projector and uh I think these are two things that are really interesting at the moment so instrumentation particular end users need a lot more best practices so people don't see users and say oh but we don't know how to do all this instrumentation or we don't know like which to add. I don't know like like what's the easy stuff you can implement like can we can you add some like best practices can you right >> you know kind of like tell tell us you know like what would be like best starting point on boarding let's say in this case uh and I've been hearing like uh community and for open tele Yeah. >> You know, >> like get them in. Yeah. >> I know that you're complaining, but tell us more about your use, >> right? >> Yeah, that's so cool. Yeah. Um, I just wanted to know, you said that you wanted to get involved in more of the engineering side, the coding side. Is there somewhere specifically that you're looking forward to or or some somewhere specifically that you would like to work in? Like anything exciting to you specifically? >> Uh, for instance, I'm very interested in the river and the river. >> Can you explain it a little bit to us? >> Uh, yeah. Well, I I don't know. >> No, right. Right. You're getting >> I think people just last maintainers and people that got involved and all the idea about this convention around it uh and although I I I'm not sure like how they are implementing it like practically something that I want to do is stop from going to so many events like my laptop stuff like this. So, um I think it needs a bit of time and time and you know with my team and my brother. >> That's awesome. I love it. Really cool. Awesome. Well, thank you so much, Diana. Congratulations again on your movie star award. It is so cute. And thank you so much, Sophia, for being my co-host. >> Yes. Thank you so much, Reese, for being the host host. >> You're amazing. Our co- co-host co-host. >> Um, yeah, we hope to see you at a future soon. um6 Amsterdam in EU and soon for our next year. Um and we'll have some show up. So check us check it out there and talk to us if you like please. >> See you everybody. Bye everybody. +Just the light. Oh, action. >> Oh, you did. You did. >> Hello everybody. If you've been waiting around, we are so sorry for the delay, but we are very excited to be coming to you by from Atlanta, GEORGIA, BUT PRETTY MUCH EVERYBODY WAS um very surprised by a cold front that had us looking at 40° uh which I think is a 3° C um the past couple days. Luckily, it's warming back up, so thank goodness. But yeah, we are coming to you live uh from CubeCon North America 2025. I'm Reese and I am joined here by my co-host Sophia. >> Hi. >> And our first guest, Jacob. >> Hello, my name is Jacob Marino. I'm a maintainer for let's see open telemetry operator, open telemetry helm charts, hotel injector, and I maintain a few random collection things and I contribute to opamp and and I'm missing one. I can't remember. [laughter] I was wondering who had a hands for all 10 honestly. >> If Josh Sar were here, he's he's in he he's everywhere. >> That's every single group. >> That's wild and awesome at the same time. >> No, it was very impressive talking with him. Learned so much. >> So, how I feel talking to you all? Um, now Jacob, YOU DID A TALK OR WAS IT TWO TALKS? >> UH, one talk. So I gave a talk uh about two hours ago, three hours ago. >> Cool. >> Very fresh. >> It was packed. >> It was like I was worried that uh I've given this is my second time giving a talk. First talk that I gave was in Chicago 2 years ago. >> Oh, nice. >> And the room we got a very low slot like it was like towards the end of the day on Thursday, I think. So it was a tough tougher day to give a talk. uh we had a solid audience uh and for this one I was worried that with the content we were doing which is like introductory to hotel and kubernetes I always worry with these introduction ones you know it's like is this going to be enough is it going to be too much so my fear is that nobody shows up and I'm speaking to an empty room this was the reverse where it was a full room and people were standing in the room and then people got turned away from that door >> oh my gosh >> which I was happily like surprised by. Then [laughter] I also like we only reserve 5 minutes for questions at the end cuz I was I was like you know I don't know what questions people will have. Usually people aren't asking the ton of questions like after they'll come up to you later but it's like usually like a Q&A session does not last very long. So oh 5 minutes for questions like that's fine. I we do questions there's a line of like 15 people waiting to ask questions. like they just keep it kept growing and growing. We stayed after for another 30 minutes just answering questions. >> Interesting. What level of questions like was it more um you know people who are newer to the topic or who are asking like more like stage two kind of questions >> huge range so >> we had some people who were you know very introductory level just like I'm getting my bearings here how does this part of it work you know what's the deal with Prometheus and hotel and what do you use for a back end you know that that's like a solid solid starter question and then other people came in they were I've been using this chart for the past year and there's this one thing that I want to be able to do and uh when are you going to like do the work to like enable that and how can we like architect this big range >> very big and each one was like a different part of the of the area. >> Okay, that's so cool. >> Yeah, very very cool and very surprising. I'm always happy to cuz you know I think as a maintainer it's like we get people who come to us with [laughter] issues on GitHub and it's like you'll get someone complaining on a GitHub issue or like hey this thing broke the most recent release broke me um you know what are we doing about that broke [laughter] >> but no one's broke their spirit you [laughter] know I've had releases break my spirit >> but the uh the real thing is we don't get positive feedback. We don't get a lot of questions cuz realistically most people are asking track or going on Stack Overflow, >> right? It's like they're not bringing the questions to us. So, we don't really hear or understand what usage is. I mean, it's kind of the benefit [laughter] and the curse of open source is that like we don't necessarily get that user feedback directly. It's why like I love what you all do cuz getting the user feedback is one of the most valuable things to the community. like in order for us to push forward, we need to know what isn't working right now where people just don't see the effort. Um we actually I hosted a meetup in New York uh last month, 2 months ago, something like that. >> Yeah. Uh, and that was eye opening because it was like wow, people are really using these things and things that I I feel like I've talked about so many times are still not known, you know, and it's like I'll mention like how many people know what like the target allocator is in a show of hands and nobody raised their hand in a room of like 40, you know, senior SRRES. I'm like, okay, uh, would anybody like it if you could do distributed, sharded, retheus scraping? and everybody raises their hand and I'm like well we have that like it's done it's there you know you can use it it's been a thing for a few years you know so I guess all to say the community just is still there's still things for people to learn about the vast ecosystem that we have here >> yeah I think you know I think a common theme just in general is people not being aware of you know different abilities and features and yeah on the end you see too you know it's it is a bit of a challenge cuz like we feel like we're talking about it um you know at our work and like in the ozone community and then yeah like you said we have all these conscious people who don't know and it's we're trying to figure out like where are they going for like how can we >> make them aware >> of you know all these cool things that are happening. Um yeah, so you know, one of the things we we started doing is um the what's the hotel segment, which we just had our first one um a month ago or so. And so we're hoping to, you know, I don't know, have something we can point people to. >> Of course. >> But you mentioned the injector. >> I did. Tell us more about please do. So I am the uh I'm a maintainer for it but I'm mostly just doing reviewing because this is actually a joint effort between uh Splunk and D-Zero. Both of them have already written their own uh injectors for auto instrumentation. Oh, right. >> And they're essentially donating a merge of both of those projects to the tot alpha version of this for their customers. Um, so we're just testing out that everything is working the way we need to. [clears throat] There's a lot of really random edges here. Um, interestingly enough, this does introduce a new language to hotel, which is exciting, which is it's very rare for us to not use a language in a project that is meant to instrumental languages. Right. >> Right. >> And we are now using Zig in the project. >> Zig. >> Zig um is a fun language that is sort of a merged between like Go, Rust, C++, and C. Oh, super fun. [laughter] >> But so the real value of it is it doesn't require uh a lot of like core Linux dependencies. So like GIC C is one that we really think about a lot. Uh if you're on AWS, something that I've been bitten by is not all of the nodes have g. And so when you're trying to do injection like say you're trying to run a load balancer liketo or envoy >> um is similarly injected like we inject auto instrumentation and that will just not work. It used to not work. I think they've changed this but it used to not work uh on certain types of AWS nodes. So we have that same issue because we're trying to inject to all these different environments. So, Zigg is pre-built with a bunch of these C libraries, but from scratch, part of it standard library and not based on anything existing. So, it has all of its own dependencies. So, the benefit of this injector project is that we're going to be able to really meet users where they're at without needing to know a ton about their environment. And that's the real problem today. >> Yeah. >> Um, so it's an exciting project. I think we're a few months away from it being alpha beta. Other people are on the release timeline. Ted Young is thinking a lot about the release timeline there. Um, but all TC >> TD, that's so cool. >> Yeah. What are the parts of, you know, whether it's something working on or not working on directly that you are excited about? >> Good question. I think that the biggest thing for project right now that everybody cares about is stability, right? I think that uh we're trying to get to the place where users can reliably get you know a new update without them needing to do any configuration changes without needing to worry about uh components being deprecated things like that right I talked to at that meet up in New York I talked to a developer at a major bank who was you know they've developed all of their own uh custom components for the collector and so every time that we've been doing these reworkings the collector's architecture. They need to then refactor a bunch of their code as well. And you know after a few months you lose track of all these updates. So it becomes like a weeks long project to keep up to date. When you have to do that their release cycle is every quarter, right? It gets a lot to manage and a lot to update. So for the users, I think what people really want to see is that level of maturity and stability. We can get to that graduated status with the CCF would be huge. That would be massive for the project to have. Uh I think it would really I mean our adoption already has been really high. I think that's when we would come to the next level. Great. >> Can you explain a little bit for those of us like CCS stages like what going from incubated incubating projects to graduated means? >> Sure. >> Yeah. I'm not the best. I can tell you what I know. I'm definitely not the person in charge of this. There are people who are far more confident in the stability part of this governance than I. >> Yeah. >> So I I will speak on only >> that's all we're asking for. [laughter] >> So right now we're in as you said uh project is relatively lenient in terms of its standards. There are standards but graduation is is really strict I would say for good reason right like you want to be sure that when you are considered a graduated project you have a healthy community which [laughter] we definitely do uh you have stability guarantees which is the main thing we're working towards and lastly adoption so adoption has to be at certain amount of companies you have to have testimonials [laughter] from those companies about how the project's been helpful uh that's another thing that we have plenty of so now it really just comes down to stability and stability. You can think of Kubernetes as the example here. Kubernetes is, you know, the first graduated project. I imagine it's the first one. Maybe there maybe someone beat them to it, but you don't [laughter] know. But essentially, every time that Kubernetes improves their security and reliability, uh, you know, release stability, anything like that, it raises the bar for everybody else to continue to improve. So as we are getting towards stability, Kubernetes is is chugging along, you know, really positively and improving their posture constantly. So it's up to us to not only meet where we started for our uh graduation criteria, but also to continue building towards that. So that means that you're going to see more uh security features, much more uh security guarantees, which is going to be very helpful for enterprises. M um you're going to hear more about quarterly releases rather than uh the sort of disperate release calendar that we have today. >> Uh you'll hear more about documentation which is also going to be huge. So uh later when that comes on I assume that we'll be talking about some mobilization efforts and documentation. >> Ah yes, uh that is super super valuable right um all of that is going to get us to that graduation phase. Uh it's just time it's time and effort and uh we have a dedicated group of people who want to see this happen and are spending you know all of our time at all of these large companies trying to make that happen. Very exciting. >> Awesome. >> That is very exciting. And if you are yourself an open end user and um your company is using in fashion um and you want to share your story, we would love to have your story as part of um our uh legacy [laughter] our catalog. >> Yeah. like to help, you know, help graduate the project and help take us out to us. Um, we'll have it in the show notes um after this. But yeah, it's pretty easy. As long as you remember me Slack, you can find us at hotel dash dash and dash. >> I think that's all the dashes. >> I thought we were going to dash zero. [laughter] Oh, so we're like dash. >> And similarly, if anybody wants to help out with documentation, documentation is >> one of the best ways to contribute to the project. It is so valuable. We had somebody for the operator group contribute a whole uh bit of documentation for the target alligator and it it was so well done. Adriana did this as well. super helpful >> and it has been such a resource for me to be able to point users that come into our Slack, come into our S meetings to just say, "Hey, you have a question about this? We have a full document to answer everything you might need to know about it." That has been such a way to offer. Um, it's lovely. So, great way to contribute and get started. Doesn't require writing code. Requires asking questions and writing a bunch of stuff down. >> Yeah. And I can confirm that everyone that I met in the local community has been super helpful. Um, anything you know you can take on and be responsible for like documentation that's something that you're taking off their plate. So I know they're having to help you. And it's also cool to see your work on an official website. >> Yeah. Like uh definitely join the communication sig for that. I'm currently in it and we're working on a lot of refactoring of the documentation like just like the early stages since I'm more of a beginner with the open telemetry and everything. I've been working on kind of refactoring those beginner level documentations. So like it's been really helpful getting people like acquainted with that and localization efforts too. Like I know Lisa is working on a lot of localization efforts. So yeah, it'll help. >> Super helpful and the the beginner stuff especially. I mean >> again from the talk today I think it's just so it was so useful to learn where people are at in their implementation journeys >> right >> uh we do have people that really need reference architectures a ton of code examples right >> these things help project adoption that also make it so that users actually are delighted by the experience right that and ultimately that that's what I think we all want to see and that's what users want like you want to be able to install hotel wherever you need it and not have to scour the internet to figure out how to how to make it work, right? >> Different companies have so many different needs and so getting started is always going to be the most important thing for us to really nail. Uh the next thing that I heard from uh when I did the event in New York is reference architectures and showing how we can do hotel in all of these various environments. so that people can have some code samples to copy from uh some architectures to you know design around. I was just talking with an engineer at and they >> are really interested in understanding how to do various trace sampling architectures, right? where we can really have like a whole as you said catalog these architectures to show people and we all of our companies like you know when I was at our flight we had all these architectures internally that we would post blog posts about to share we have a lot of companies that are using hotel as part of the ingestion path as well which involves a lot of this uh architectural work similarly a lot of companies are like that are contributors to hotel also are obviously using the tooling themselves for their own company's observability. We should reach out to the people who are doing that and try to publish what are the reference architectures that hotel maintainers use ourselves, right? >> I think that would be really useful uh for people who are trying to implement this right now. >> Absolutely. >> For sure. Um are we planning to host them like in a repository or like on sites directly? >> Oh, I think would be the best. I think you know we have documentation places the operator group and I think the idea of everything being the doc is going to be the way to go >> right >> like centralizing it and making sure that it's well organized which is a lot of the work that you're doing super valuable >> of course >> uh but I think it just it needs to be there right now I think it exists on a few different company blogs a few different companies >> head scattered >> bringing that together I think would really really valuable. >> One reason let's do it, >> right? There's so many companies like contributing. So there's definitely a bunch of references that people in our community could use. >> Yes. >> Yeah. >> Absolutely. And so many people are people are so friendly, right? like back to that like going to uh the hotel booth. We've had so many people who have never come up to us before and are just asking great questions and we have so many people who just, you know, we hang out there cuz we love to talk to people about this. It's rare that you get to have this level of engagement with end users, right? Uh that direct type of engagement is so valuable and so people coming by has been one of my favorite experiences of it always. I think the reason that I love to come to these is just getting to talk to people, hearing people's problems, digging in because then it helps us plan what I like what I need to do for the next 2 years, right? I can take that feedback on it. What um so in during your time here at coupon like from the audience questions that you had at your talk and people who come through the open tree observatory do you have are you seeing any like general trends as to like what people are leaning toward or needing more help with or >> Yeah. So I say the the main thing that is of concern to me is around amp which has been getting a lot of there are a lot of talks in opamp this coupon. I don't know if that if we've noticed that there were at least four that mentioned it which is a lot. >> Could you explain that a little bit more for us? Nothing. [laughter] Sorry. >> We're slowing down a little bit. So, opamp is the uh open agent management protocol uh part of the hotel project but is also designed to be in in the same way the hotel is very vendor neutral it's neutral to agents as well. So, it's not just for the collector necessarily to be for a lot of different uh agents out there. The goal of the protocol is to enable things like uh component status reporting, health reporting, and remote configuration. And remote configuration is definitely the top of the town is what I would say. >> Yeah. Okay. Very cool. >> Um and so in that vein, what it sounds like users really want is a way to dynamically update what the collector can do, what the SDKs can do uh in real time. And that's challenging. The way that the collector was designed was never really with remote configuration in mind. There are ways to reload the process. Like we can actually go in and restart the process from the collector binary. Like you just do that, right? >> But it's different than a remote. What users really want is a way to push a rule to a collector and the collector accepts it and continues going. >> Yeah. So that's what a company like bind plane has already built. Uh and so what we want to do is take a lot of the lessons from what BL has been successful with and obviously we're working with them as well. They are big contributors to uh the idea is that we work all together to improve the provider posture here and improve the SDKs as well so that we can actually realize this vision that people can dynamically update their configurations wherever they might be. [laughter] Uh but it's challenging. I mean that that's like a really tough topic. >> Yeah. >> There's a lot of a lot of companies in hotel have done this work beyond my plan before. Elastic comes to mind. They have their own >> I work here. >> Oh yeah. [laughter] >> Elastic has uh their own protocol for doing road configuration already. >> Okay. Okay. >> We've talked with a few of their containers and there's definitely some like room for some good collaboration that we've talked about. >> Awesome. >> I think it's all to be seen but this is definitely the thing that we hear a lot about. The thing that I hear a lot about on the operator side is in Kubernetes, how can I do uh call it not just remote configuration but layered configuration. So what this means is how can I make it so that if I'm running in an enterprise company that I'm in a multi-tenant environment, I want like a base collector. I want to overlay. I want my teams to be able to overlay different pieces of config on top of that collector for their needs. >> Right. >> It's a really complicated topic. We >> Yeah, it's really >> sounds like it. And the thing that I sort of struggle with is how do we make this extensible and configurable but also simple to understand. The problem with this that it starts getting into a lot of moving parts and as soon as you start to introduce more moving parts is when complexity and usually reliability both complexity increases and reliability decreases right you need consistency and more moving parts uh will reduce that it's sort of that there's a great quote in reliability that I love it's u the most reliable system is the one that does that never that you right we know this quote. I'm totally botching it right now. [laughter] >> No, I hear it. I hear it. >> The idea is that like [laughter] the most reliable system is the one that like doesn't exist, >> right? >> Right. Cuz it's always achieving its goal of non-existing. >> And so if you want to really improve the reliability of the service, you delete it >> because then you don't have to care then it's at 100%. Cuz it's always it's always giving you the same answers which is nothing. Yeah. [laughter] [laughter] >> Sorry. >> No, I'm here for it. I'm here for it. Yeah. >> Someone I I'll say that to I don't know, someone at the Ozel booth and they'll be like, "You totally messed up." >> You know, [laughter] we'll have the correct throw in the shout out. >> Yeah. Thank you so much, Jacob. Um, we are going to bring on our next guest, Diana, in a moment. Um, and in the meantime, I really want to show off this baseball jersey. >> Looks really good. >> But, um, our friends at Honeycomb have been kind enough to um, sponsor for the hotel community. Um, thank you, Austin Parker, for the design. Um, I would stand and show the back. It says uh the number 11 ACTUALLY I DON'T KNOW WHY IT SAYS 11 like [laughter] there. But yeah, it says maintainer on the back. Um, we'll have some pictures hopefully to show you. But yeah, you've got the logo over here and it says open too across the front. Very, very snazzy, very sharp. [laughter] Um, yeah. And it also comes in black, which I had the option, but I was like, the client looks sharp. Yeah. Um, and also, you can see the blue and the yellow very clearly. Yeah. Just wanted to show you a little bit of a conference fashion. Um, and then when we get um our second guest in a setup, I also want to show you her nails because they're amazing. >> Oh, yes. >> I don't know how close I can get. >> Yeah. And like I said, we have a group right here. I'm very excited that it's for today. So that's why we're able to, you know, have lunch on right now. But yeah, I brought my winter puppy out. Um it was it was intense, but we made it through and we are now day two of the main event with one more day to go and I hope everyone is doing well. >> Staying hydrated. staying hydrated is so large [laughter] and important. And yeah, I think we're ready to speak to our next guest, Diana. >> Yes. Awesome. >> Diana, welcome. And thank you guys. Congratulations on uh winning the Open Salt Community Star 2025. She was one of several winners. Um, the only one I want to point out, but that's why it's so important. Um, Diana, can you please introduce yourself? And also, the earrings. >> Yeah. Oh my gosh. Yes. Oh, those are gorgeous. Oh my goodness. Okay. Sorry. Continue. [laughter] >> Like, hold up your treasury. [laughter] >> Yeah. Yeah. I'm Diana. Uh I'm a developer experience engineer parametric and I work all my life in engineering uh in observability the last couple of like 5 years. Uh and I discovered like I really like going and explaining things to the community. So I had a chance like years back to be a speaker on one of the first conferences I've ever been in my life. Uh I was super nervous but I liked it a lot and I saw like so many women on stage giving talks that I felt like I needed to know to do more. >> Of course. >> Uh so I kind of did that for 2 years. Uh and actually last year like I was hearing so much about open I said like oh I want to get involved. I want to do something. uh and I got lucky because in the same moment when I thought about it foundation was like oh we are looking for observability people we are looking for uh let's get involved we want to create the first open and learn fundamentals uh I got in so it was like first start with the documentation reading questions questions about and that's like simple months into certain words. I knew it was finally polished. So that was like for me the kick into like sens and slowly I kind of like I was like okay I'm sorry things how about I'm going to do my company. that was like uh for my current company actually. Um so uh it was like really interesting because I I wanted like from explaining open telemetry to the entire company, getting software engineers involved like instrumenting some things in like SDKs uh testing everything. Um so yeah production so that was like decision. Uh but that was like a really cool experience that I've been talking about since one of my talks and explaining how you know companies some particular companies that to be honest they're not super big times to build [laughter] in takes a very important. So I need to be involved in the community. So for me that was like a very very experience also like understand [laughter] Yeah, I thought about myself. Okay, but if right now I'm all alone and I still want to contribute, how can I do it? And I saw translating documents. >> Oh, nice. That's interesting. I can do my own time. start something translations and yeah, can you um so global organization the projects or localization just refers to translating the documentation to different languages. >> Exactly. Yeah. So the idea that okay there may be a bit more um other terms around localization. So it's not necessarily only the the technical terms or the tech writing part but there are like also like um how you're going to automate some CI going to automate specific u pipelines documentation. Right. Right. >> Can also be that and also how are you communicating a specific message uh in a language right interpretation of of concepts and terms and technology in other languages. So I think that's that's how I see things. >> Well, you mentioned Spanish. Um are you involved with other languages? Um and can you also tell us like what other languages are being are for the project right now? >> Yeah. Yeah. Yeah. Um so when um back when I got involved I didn't think of this year uh Spanish was available obviously English, French. >> Ah yes >> and that was right. So from the beginning of the year up to now I think three more languages emerged. So three more communities. Uh I know for sure Bengali. >> Oh nice. >> And that's like so they really got together and and they started to contribute more and more like their only technology. >> Yeah. >> Um Ukrainian that's locked up recently. So I know for sure they need more contributors. Uh so somebody came with the idea and they started to do that. Um, and I started uh for my own language Romanian. I'm native to Romanian. Uh, I started like a couple of months back and I said, "Look, yeah, I think it's an amazing opportunity for for my language to represent and for developers back home to understand a bit more language. It's going to really perspective like it is a really nice nice community." Yeah, that's really beautiful. So like what is the hardest thing about localization that you find when you do that kind of work? Yeah, I think um definitely translating specific terms into your own language or in that particular language, but giving it the flavor or giving it like [laughter] that that really gets the essence so it doesn't have to sound like a very dry terminology and that word captures the essence of of the meaning. So for example, I don't know about If you translate really like like [laughter] literally a word from English or another language, it might sound a bit weird. So you have to like al oh does it sound really like this in that language? I have to like you know modify it a bit. I have to give it like a nice ring to it. So like you can get it you know it's kind of like >> uh people like like it immediately and they want to use it right. Um so that's I think that's the difficult part and also the difficult part is that there are many many languages that without technical terms directly in English right >> and when you want to translate it in that language quite like the appropriate meaning is like okay I know you know uses that word in my language and they always something in English right oh that get immediately you know it doesn't sound strange so those are like words basically That's so wonderful. >> Yeah, it's really cool. >> Is there um like a way or um to kind of see, you know, the rate of adoption or involvement from the communities that now have these translations available. >> Um in terms of metrics or in terms of adoption, I think that we are uh kind of primitive. Um, and I think that you need to correlate it a bit with a ground meeting event or with some type of local community that you could uh speak about it on site, right? >> Or this makes some noise back home for example. Uh, and say look, we are starting this, we are doing this. We need more contributors. Uh, we need, you know, like the people from the local communities. So like that for each one of is is very much needed whether online or on site. Uh so for me for example I just started this with Romania. I started going like to people like I contributed before or new from like events and I'm also Romania. around there like can you put it can you promote it like we are like actually entitling our language to to be heard in the open community like really jumped into it and yeah for sure let's do it like a lot of people started it's really nice >> I think as you know native English speakers we get so used to everything being >> to English And honestly, you know, a lot of um non-native English speakers, you know, they usually speak multiple languages. Um and so I think it's really impressive um and awesome the work that you all are doing. >> Yeah, I appreciate it. And also I think down the line people get motivated not only because of modernization but they start thinking okay uh do I like overall open source uh should I do something else maybe in another project uh does it help me in like my work projects whatever I'm doing uh and I think this is how I practically you know promoted it to to my community I said if you guys like me uh to say like I'm doing a open source project as your contributor like if your employer needs you to have this experience or maybe you want to go to a conference and talk about it. So all of these are very very nice to have you know on your screen and you know different age groups you know there university or later on their career and I think it's really important to keep motivation because it can be sometimes really hard to start in your free time or remind yourself that you know [laughter] you have an objective online. It's kind of like a gateway project into contributing to open source a little bit. [laughter] >> I always keep telling them that I I didn't necessarily start with the organization and I share my new stuff here. I'll definitely going for example I being an engineer I still like doing engineering work. >> Oh yeah. And I want to get involved in the codes part as well because I've been keeping an eye on everything that's happening talking like there are many many things that for sure my company or other people will definitely find. Um, so yeah, you know, one project or one S is is not enough. >> We're going to jump [laughter] into something else at some point and then the community especially is really really and they keep talking about the same issues over and over again. >> Yeah. So just recently talking about I heard about the same kind of issues in Kubernetes. So I'm kind of like went back and forth because I'm kind of like struggling with similar issues. So I think it's very good to keep an open mind open and keep going. For me at least personally this is what motivates us going forward you can diversify your contributions not stop and we always find something. >> Thank you. >> Yeah that's awesome. And then kind of pivoting a little bit. Um I would love to know what you're looking forward to in the coming year for hotel. Like what are you excited about? What kind of trends you're seeing like implementation? Anything that has so um just like the last couple of weeks been to like specific events. I saw a lot of interesting talks about yeah obviously projector and uh I think these are two things that are really interesting at the moment so instrumentation particular end users need a lot more best practices so people don't see users and say oh but we don't know how to do all this instrumentation or we don't know like which to add. I don't know like like what's the easy stuff you can implement like can we can you add some like best practices can you right >> you know kind of like tell tell us you know like what would be like best starting point on boarding let's say in this case uh and I've been hearing like uh community and for open tele Yeah. >> You know, >> like get them in. Yeah. [laughter] >> I know that you're complaining, BUT TELL US more about your use, >> right? >> Yeah, that's so cool. Yeah. Um, I just wanted to know, you said that you wanted to get involved in more of the engineering side, the coding side. Is there somewhere specifically that you're looking forward to or or some somewhere specifically that you would like to work in? Like anything exciting to you specifically? >> Uh, for instance, I'm very interested in the river and the river. >> Can you explain it a little bit to us? >> Uh, yeah. Well, I I don't know. >> No, right. Right. You're getting >> I think people just last [laughter] maintainers and people that got involved and all the idea about this convention around it uh and although I I I'm not sure like how they are implementing it like practically something that I want to do is stop from going to so many events like [laughter] my laptop stuff like this. So, um I think it needs a bit of time and time and you know with my team and my brother. >> That's awesome. I love it. Really cool. Awesome. Well, thank you so much, Diana. Congratulations again on your movie star award. It is so cute. And thank you so much, Sophia, for being my co-host. >> Yes. Thank you so much, Reese, for being the host host. >> You're amazing. Our co- co-host [laughter] co-host. >> Um, yeah, we hope to see you at a future soon. um6 Amsterdam in EU and soon for our next year. Um and we'll have some show up. So check us check it out there and talk to us if you like please. [laughter] >> See you everybody. Bye everybody. [music] [music] From 389d9bfed46485f5a5713bd3a6036b673871d0fc Mon Sep 17 00:00:00 2001 From: Nicole van der Hoeven Date: Tue, 25 Nov 2025 11:01:20 +0100 Subject: [PATCH 14/28] Simplify timestamp window creation in find_correct_timestamp - Replace segment-based windowing with simpler 10-second intervals - Remove complex window_key logic and seen_times tracking - Create windows directly at 10-second intervals within search range - Makes timestamp finding more predictable and easier to debug --- video-transcripts/transcripts.py | 127 ++++++------------------------- 1 file changed, 25 insertions(+), 102 deletions(-) diff --git a/video-transcripts/transcripts.py b/video-transcripts/transcripts.py index 699daeb..b1e6968 100644 --- a/video-transcripts/transcripts.py +++ b/video-transcripts/transcripts.py @@ -778,31 +778,20 @@ def find_correct_timestamp(description, all_segments, time_to_text, client, orig if next_chapter_seconds is not None: max_time = min(max_time, next_chapter_seconds - 5) # Stay 5 seconds before next chapter - # Use actual timestamps from the transcript within the search range - # This preserves second-level precision instead of rounding to 10-second intervals + # Create text windows every 10 seconds within the search range windows = [] - seen_times = set() + current_time = min_time - for segment_time, segment_text in all_segments: - if min_time <= segment_time <= max_time: - # Group by 5-second windows to avoid too many entries - window_key = (segment_time // 5) * 5 - - if window_key not in seen_times: - seen_times.add(window_key) - - # Collect text from a small window around this timestamp - window_texts = [] - for time_sec in range(segment_time - 2, segment_time + 3): - if time_sec in time_to_text: - window_texts.extend(time_to_text[time_sec]) - - if window_texts: - # Use the actual segment time, not the rounded window_key - windows.append((segment_time, ' '.join(window_texts[:30]))) - - # Sort windows by time - windows.sort(key=lambda x: x[0]) + while current_time <= max_time: + window_texts = [] + for time_sec in range(current_time, min(current_time + 10, max_time + 1)): + if time_sec in time_to_text: + window_texts.extend(time_to_text[time_sec]) + + if window_texts: + windows.append((current_time, ' '.join(window_texts[:30]))) # Limit text per window + + current_time += 10 if not windows: return None @@ -816,13 +805,12 @@ def find_correct_timestamp(description, all_segments, time_to_text, client, orig Topic: "{description}" Original timestamp: {format_duration(original_seconds)} - Here is text from within ±60 seconds of that timestamp (with ACTUAL precise timestamps): + Here is text from within ±60 seconds of that timestamp: {windows_text} Which timestamp is the BEST match for when "{description}" begins? - Use the EXACT timestamp shown in brackets where the topic starts. - Do NOT round or approximate - use the precise timestamp from the list above. + The timestamp should be close to {format_duration(original_seconds)}. Respond with ONLY the timestamp in HH:MM:SS format (e.g., 00:05:17). If the original timestamp is already good, respond with: {format_duration(original_seconds)} @@ -859,53 +847,6 @@ def find_correct_timestamp(description, all_segments, time_to_text, client, orig print(f" Error searching for timestamp: {str(e)}") return None -def build_timeline_from_transcript(raw_transcript, sample_interval=10): - """ - Build a timeline showing what's being said at regular intervals. - @param raw_transcript: Raw transcript data with timing information - @param sample_interval: Sample interval in seconds (default: 10) - @return: String representation of the timeline - """ - if not raw_transcript: - return "" - - # Build a map of timestamp to text - time_to_text = {} - max_time = 0 - - for entry in raw_transcript: - if isinstance(entry, dict) and 'text' in entry and 'start' in entry: - text = entry['text'].strip() - if text and text not in ['[Music]', '[Applause]', '[Laughter]']: - start_time = int(entry['start']) - if start_time not in time_to_text: - time_to_text[start_time] = [] - time_to_text[start_time].append(text) - if 'duration' in entry: - end_time = entry['start'] + entry['duration'] - max_time = max(max_time, end_time) - - # Sample at regular intervals - timeline_lines = [] - current_time = 0 - - while current_time <= max_time: - # Collect text from a window around this time (±5 seconds) - window_texts = [] - for time_sec in range(max(0, current_time - 5), current_time + 6): - if time_sec in time_to_text: - window_texts.extend(time_to_text[time_sec]) - - if window_texts: - # Take first 100 characters of the combined text to keep it manageable - combined = ' '.join(window_texts)[:150] - timestamp_str = format_duration(current_time) - timeline_lines.append(f"[{timestamp_str}] {combined}...") - - current_time += sample_interval - - return '\n'.join(timeline_lines) - def create_chapters(transcript, video_id, raw_transcript=None): """ Create timestamps and chapters for a YouTube video transcript. @@ -927,42 +868,24 @@ def create_chapters(transcript, video_id, raw_transcript=None): duration_str = format_duration(duration_seconds) duration_constraint = f"\n\nIMPORTANT: This video is {duration_str} long. DO NOT generate any timestamps beyond {duration_str}. All timestamps must be less than or equal to {duration_str}." - # Build timeline from raw transcript if available - timeline = "" - if raw_transcript: - print(f"Building timeline from raw transcript for video {video_id}") - timeline = build_timeline_from_transcript(raw_transcript, sample_interval=10) - if timeline: - timeline = f"\n\n=== TIMELINE WITH ACTUAL TIMESTAMPS ===\n{timeline}\n=== END TIMELINE ===\n" - chapters_prompt = f""" - This is a transcript of a YouTube livestream. Identify key moments in the stream and provide timestamps in the format for YouTube like this: - 00:00:00 Welcome and intro - 00:01:47 Structured metadata - 00:03:22 Metrics discussion + This is a transcript of a YouTube livestream. Could you please identify up to 10 key moments in the stream and give me the timestamps in the format for YouTube like this?: + 00:00:00 Introductions + 00:01:47 What is structured metadata? + 00:03:22 Discussion about metrics CRITICAL INSTRUCTIONS: - - You MUST provide a MAXIMUM of 10 chapters - NO MORE than 10 - Always start with 00:00:00 - - Use ONLY timestamps that appear in the TIMELINE below - these are the ACTUAL timestamps from the video - - Pick timestamps where topic changes or new segments begin - - DO NOT make up timestamps - only use timestamps that appear in the timeline - - Look at what's being said at each timestamp in the timeline to determine good chapter points + - Use the ACTUAL timestamps from where topics begin in the transcript + - DO NOT round timestamps to neat intervals like :00 or :30 + - Use precise timestamps like 00:05:17, 00:12:43, 00:08:09, etc. + - Look at the actual flow of conversation to determine when topics change - The chapter description MUST accurately describe what is being said RIGHT AT that timestamp - Do NOT describe what happens later - only describe what is happening at the exact moment of the timestamp - - If a guest is introduced at a specific timestamp, use that exact timestamp + - Read the text carefully around each timestamp to ensure your description matches what's actually being discussed + - If a guest is introduced at 00:05:30, don't put "Guest introduction" at 00:20:00 - Be precise and honest about what's happening at each moment - - CHAPTER DESCRIPTION FORMAT: - - Use SHORT PHRASES, not full sentences - - Keep descriptions 2-5 words when possible - - Avoid articles (a, an, the) at the start - - Avoid question format - use noun phrases instead - - Make descriptions easy to search for - - Examples of GOOD descriptions: "OpenTelemetry basics", "Demo walkthrough", "Q&A session", "Kubernetes integration" - - Examples of BAD descriptions: "What is OpenTelemetry?", "A discussion about metrics", "The team talks about features" - - REMEMBER: Generate NO MORE than 10 chapters total. Select the most important moments only.{duration_constraint}{timeline} + - Use simple but descriptive language that makes it easier for users to understand what is being spoken about{duration_constraint} """ print(f"Creating chapters for video {video_id}") From 3d777da83475ba9d4ecf2f5520c2f48cf327d730 Mon Sep 17 00:00:00 2001 From: Nicole van der Hoeven Date: Tue, 25 Nov 2025 11:02:30 +0100 Subject: [PATCH 15/28] Add unit tests for transcript processing - Test timeline building with various intervals and edge cases - Test filtering of [Music] and [Applause] markers - Test timestamp parsing and formatting utilities - Test roundtrip conversions and precision handling --- video-transcripts/test_transcripts.py | 128 ++++++++++++++++++++++++++ 1 file changed, 128 insertions(+) create mode 100644 video-transcripts/test_transcripts.py diff --git a/video-transcripts/test_transcripts.py b/video-transcripts/test_transcripts.py new file mode 100644 index 0000000..2f0b700 --- /dev/null +++ b/video-transcripts/test_transcripts.py @@ -0,0 +1,128 @@ +import unittest +from transcripts import build_timeline_from_transcript, parse_timestamp_to_seconds, format_duration + + +class TestTimelineBuilder(unittest.TestCase): + """Test the timeline building functionality for chapter generation.""" + + def test_build_timeline_basic(self): + """Test that timeline is built correctly from raw transcript data.""" + raw_transcript = [ + {'text': 'Hello everyone', 'start': 0, 'duration': 2}, + {'text': 'Welcome to the show', 'start': 2, 'duration': 3}, + {'text': 'Today we are talking about OpenTelemetry', 'start': 35, 'duration': 4}, + {'text': 'It is very exciting', 'start': 39, 'duration': 2}, + ] + + timeline = build_timeline_from_transcript(raw_transcript, sample_interval=30) + + # Should have entries at 00:00:00 and 00:00:30 + self.assertIn('[00:00:00]', timeline) + self.assertIn('[00:00:30]', timeline) + self.assertIn('Hello everyone', timeline) + self.assertIn('OpenTelemetry', timeline) + + def test_build_timeline_filters_music(self): + """Test that [Music] and similar markers are filtered out.""" + raw_transcript = [ + {'text': 'Hello everyone', 'start': 0, 'duration': 2}, + {'text': '[Music]', 'start': 2, 'duration': 5}, + {'text': '[Applause]', 'start': 7, 'duration': 3}, + {'text': 'Welcome back', 'start': 28, 'duration': 2}, # Within ±5 window of 30s + ] + + timeline = build_timeline_from_transcript(raw_transcript, sample_interval=30) + + # Should not contain filtered markers + self.assertNotIn('[Music]', timeline) + self.assertNotIn('[Applause]', timeline) + # Should contain actual speech + self.assertIn('Hello everyone', timeline) + self.assertIn('Welcome back', timeline) + + def test_build_timeline_empty_transcript(self): + """Test that empty transcript returns empty timeline.""" + timeline = build_timeline_from_transcript([], sample_interval=30) + self.assertEqual(timeline, "") + + timeline = build_timeline_from_transcript(None, sample_interval=30) + self.assertEqual(timeline, "") + + def test_build_timeline_custom_interval(self): + """Test that custom sample intervals work correctly.""" + raw_transcript = [ + {'text': 'Start', 'start': 0, 'duration': 1}, + {'text': 'At 10 seconds', 'start': 10, 'duration': 2}, + {'text': 'At 20 seconds', 'start': 20, 'duration': 2}, + {'text': 'At 30 seconds', 'start': 30, 'duration': 2}, + ] + + # Sample every 10 seconds + timeline = build_timeline_from_transcript(raw_transcript, sample_interval=10) + + # Should have entries at 0, 10, 20, 30 + self.assertIn('[00:00:00]', timeline) + self.assertIn('[00:00:10]', timeline) + self.assertIn('[00:00:20]', timeline) + self.assertIn('[00:00:30]', timeline) + + + def test_build_timeline_preserves_precision(self): + """Test that timeline captures precise timestamps, not just rounded intervals.""" + raw_transcript = [ + {'text': 'Starting at 3 seconds', 'start': 3, 'duration': 2}, + {'text': 'Now at 32 seconds', 'start': 32, 'duration': 2}, + {'text': 'And at 91 seconds', 'start': 91, 'duration': 2}, + ] + + # Sample every 30 seconds with ±5 second window + # 0s sample (window 0-5) captures text at 3s + # 30s sample (window 25-35) captures text at 32s + # 90s sample (window 85-95) captures text at 91s + timeline = build_timeline_from_transcript(raw_transcript, sample_interval=30) + + # Should capture text from the sampling points + self.assertIn('[00:00:00]', timeline) + self.assertIn('[00:00:30]', timeline) + self.assertIn('[00:01:30]', timeline) + # Verify text is captured at each sample point + self.assertIn('3 seconds', timeline) + self.assertIn('32 seconds', timeline) + self.assertIn('91 seconds', timeline) + + +class TestTimestampParsing(unittest.TestCase): + """Test timestamp parsing and formatting functions.""" + + def test_parse_timestamp_to_seconds(self): + """Test converting timestamp strings to seconds.""" + self.assertEqual(parse_timestamp_to_seconds('00:00:00'), 0) + self.assertEqual(parse_timestamp_to_seconds('00:01:00'), 60) + self.assertEqual(parse_timestamp_to_seconds('00:05:30'), 330) + self.assertEqual(parse_timestamp_to_seconds('01:00:00'), 3600) + self.assertEqual(parse_timestamp_to_seconds('01:23:45'), 5025) + + # Test MM:SS format + self.assertEqual(parse_timestamp_to_seconds('05:30'), 330) + self.assertEqual(parse_timestamp_to_seconds('1:00'), 60) + + def test_format_duration(self): + """Test formatting seconds into HH:MM:SS.""" + self.assertEqual(format_duration(0), '00:00:00') + self.assertEqual(format_duration(60), '00:01:00') + self.assertEqual(format_duration(330), '00:05:30') + self.assertEqual(format_duration(3600), '01:00:00') + self.assertEqual(format_duration(5025), '01:23:45') + + def test_timestamp_roundtrip(self): + """Test that parsing and formatting are inverse operations.""" + timestamps = ['00:00:00', '00:05:30', '01:23:45', '02:00:00'] + for ts in timestamps: + seconds = parse_timestamp_to_seconds(ts) + formatted = format_duration(seconds) + self.assertEqual(formatted, ts) + + +if __name__ == '__main__': + unittest.main() + From 550cfaaf3ffa7d4b3209ffd65e7a0555b3a40b08 Mon Sep 17 00:00:00 2001 From: Nicole van der Hoeven Date: Tue, 25 Nov 2025 11:03:41 +0100 Subject: [PATCH 16/28] Add Python virtual environment entries to .gitignore - Add venv/, .venv/, env/, ENV/ directories - Add .env file (for API keys/secrets) - Prevents accidentally committing large virtual environments --- .gitignore | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/.gitignore b/.gitignore index 9756d97..bba8c50 100644 --- a/.gitignore +++ b/.gitignore @@ -306,6 +306,15 @@ paket-files/ __pycache__/ *.pyc +# Python virtual environments +venv/ +.venv/ +env/ +.env +ENV/ +env.bak/ +venv.bak/ + # Cake - Uncomment if you are using it # tools/** # !tools/packages.config From 4589139439de7e0e65bc81c7cff9b325dc4b15cf Mon Sep 17 00:00:00 2001 From: Nicole van der Hoeven Date: Fri, 28 Nov 2025 11:26:51 +0100 Subject: [PATCH 17/28] Add ## Transcript heading before cleaned transcript section --- video-transcripts/transcripts.py | 37 ++++++++++++++++++-------------- 1 file changed, 21 insertions(+), 16 deletions(-) diff --git a/video-transcripts/transcripts.py b/video-transcripts/transcripts.py index b1e6968..9b262d4 100644 --- a/video-transcripts/transcripts.py +++ b/video-transcripts/transcripts.py @@ -524,7 +524,7 @@ def insert_timestamps_in_transcript(cleaned_transcript, chapters, raw_transcript lines = cleaned_transcript.split('\n') # For each chapter, find the best line to insert it at - line_to_timestamp = {} # Maps line index to timestamp to insert + line_to_chapter = {} # Maps line index to (timestamp_str, title) to insert for seconds, timestamp_str, title in chapters: # Skip 00:00:00 as it's at the beginning @@ -567,8 +567,8 @@ def insert_timestamps_in_transcript(cleaned_transcript, chapters, raw_transcript best_score = 0 for idx, line in enumerate(lines): - # Skip empty lines and lines that already have timestamps - if not line.strip() or line.strip().startswith('['): + # Skip empty lines and lines that already have chapter markers + if not line.strip() or line.strip().startswith('###'): continue line_lower = line.lower() @@ -585,8 +585,8 @@ def insert_timestamps_in_transcript(cleaned_transcript, chapters, raw_transcript # Prefer lines that appear after already-placed timestamps (chronological order) position_bonus = 0 - if line_to_timestamp: - last_placed = max(line_to_timestamp.keys()) + if line_to_chapter: + last_placed = max(line_to_chapter.keys()) if idx > last_placed: position_bonus = 0.5 @@ -596,19 +596,21 @@ def insert_timestamps_in_transcript(cleaned_transcript, chapters, raw_transcript best_score = score best_line_idx = idx - # Insert timestamp at the best matching line + # Insert chapter marker at the best matching line if best_line_idx is not None and best_score > 1: # Require at least some confidence - # Only insert if we don't already have a timestamp for this line - if best_line_idx not in line_to_timestamp: - line_to_timestamp[best_line_idx] = timestamp_str + # Only insert if we don't already have a chapter marker for this line + if best_line_idx not in line_to_chapter: + line_to_chapter[best_line_idx] = (timestamp_str, title) - # Build the result by inserting timestamps at marked lines + # Build the result by inserting chapter headings before marked lines result_lines = [] for idx, line in enumerate(lines): - if idx in line_to_timestamp: - # Insert timestamp at the beginning of this line - timestamp = line_to_timestamp[idx] - result_lines.append(f"[{timestamp}] {line}") + if idx in line_to_chapter: + # Insert H3 heading before this line with timestamp and chapter title + timestamp, title = line_to_chapter[idx] + result_lines.append(f"### [{timestamp}] {title}") + result_lines.append("") # Add blank line after heading + result_lines.append(line) else: result_lines.append(line) @@ -873,6 +875,7 @@ def create_chapters(transcript, video_id, raw_transcript=None): 00:00:00 Introductions 00:01:47 What is structured metadata? 00:03:22 Discussion about metrics + 00:05:30 Guest introduction: Diana CRITICAL INSTRUCTIONS: - Always start with 00:00:00 @@ -885,7 +888,9 @@ def create_chapters(transcript, video_id, raw_transcript=None): - Read the text carefully around each timestamp to ensure your description matches what's actually being discussed - If a guest is introduced at 00:05:30, don't put "Guest introduction" at 00:20:00 - Be precise and honest about what's happening at each moment - - Use simple but descriptive language that makes it easier for users to understand what is being spoken about{duration_constraint} + - KEEP DESCRIPTIONS CONCISE: Use 2-6 words maximum, not full sentences + - Descriptions should be SHORT PHRASES like "Guest introduction", "Discussion about X", "Demo of Y feature" + - DO NOT write full sentences or lengthy explanations in the chapter titles{duration_constraint} """ print(f"Creating chapters for video {video_id}") @@ -950,7 +955,7 @@ def write_markdown(args, video): file.write(f"{chapters}\n\n") if cleaned_up: - # Don't write a heading because OpenAI was instructed to output markdown. + file.write("## Transcript\n\n") file.write(f"{cleaned_up}\n\n") # TODO: if we're happy with OpenAI output, this is extraneous and can go. From d3ab651d7c90f0ae8f960ec5f2dd7fb66fc5c732 Mon Sep 17 00:00:00 2001 From: Nicole van der Hoeven Date: Fri, 28 Nov 2025 11:49:48 +0100 Subject: [PATCH 18/28] Improve chapter timestamp accuracy by using raw transcript timeline - Add timeline skeleton to chapter generation prompt showing actual timestamps every 30s - Remove skip logic for 00:00:00 chapter heading in transcript insertion - Reuse existing time-to-text mapping logic for consistency - Fixes issue where chapter timestamps could be off by several minutes --- video-transcripts/transcripts.py | 59 ++++++++++++++++++++++++++++---- 1 file changed, 52 insertions(+), 7 deletions(-) diff --git a/video-transcripts/transcripts.py b/video-transcripts/transcripts.py index 9b262d4..f7d08ec 100644 --- a/video-transcripts/transcripts.py +++ b/video-transcripts/transcripts.py @@ -527,10 +527,6 @@ def insert_timestamps_in_transcript(cleaned_transcript, chapters, raw_transcript line_to_chapter = {} # Maps line index to (timestamp_str, title) to insert for seconds, timestamp_str, title in chapters: - # Skip 00:00:00 as it's at the beginning - if seconds == 0: - continue - # Get text from a window around the timestamp (±5 seconds) window_texts = [] for time_sec in range(max(0, seconds - 5), seconds + 6): @@ -870,6 +866,55 @@ def create_chapters(transcript, video_id, raw_transcript=None): duration_str = format_duration(duration_seconds) duration_constraint = f"\n\nIMPORTANT: This video is {duration_str} long. DO NOT generate any timestamps beyond {duration_str}. All timestamps must be less than or equal to {duration_str}." + # Build a timeline skeleton from raw transcript if available + timeline_context = "" + if raw_transcript: + print(f"Building timeline skeleton from raw transcript for video {video_id}") + + # Build mapping of time to text (reusing existing logic) + time_to_texts = {} + for entry in raw_transcript: + if isinstance(entry, dict) and 'text' in entry and 'start' in entry: + text = entry['text'].strip() + if text and text not in ['[Music]', '[Applause]', '[Laughter]']: + start_time = int(entry['start']) + if start_time not in time_to_texts: + time_to_texts[start_time] = [] + time_to_texts[start_time].append(text) + + # Create timeline samples every 30 seconds + timeline_samples = [] + sample_interval = 30 # seconds + max_samples = 40 # Limit to avoid token overflow + + current_time = 0 + sample_count = 0 + + while current_time <= (duration_seconds or 3600) and sample_count < max_samples: + # Get text from a small window around this time + window_texts = [] + for time_sec in range(current_time, min(current_time + 10, (duration_seconds or 3600) + 1)): + if time_sec in time_to_texts: + window_texts.extend(time_to_texts[time_sec]) + + if window_texts: + # Take first few words as a sample + sample_text = ' '.join(window_texts)[:150] # Limit length + timestamp_str = format_duration(current_time) + timeline_samples.append(f"[{timestamp_str}]: {sample_text}") + sample_count += 1 + + current_time += sample_interval + + if timeline_samples: + timeline_context = f""" + +TIMELINE REFERENCE - These are actual timestamps from the video showing what is being said at different times: + +{chr(10).join(timeline_samples)} + +USE THESE ACTUAL TIMESTAMPS to determine when topics change. Your chapter timestamps MUST come from the times shown above or nearby times. DO NOT guess or make up timestamps.""" + chapters_prompt = f""" This is a transcript of a YouTube livestream. Could you please identify up to 10 key moments in the stream and give me the timestamps in the format for YouTube like this?: 00:00:00 Introductions @@ -879,18 +924,18 @@ def create_chapters(transcript, video_id, raw_transcript=None): CRITICAL INSTRUCTIONS: - Always start with 00:00:00 - - Use the ACTUAL timestamps from where topics begin in the transcript + - Use the ACTUAL timestamps from the timeline reference below - DO NOT round timestamps to neat intervals like :00 or :30 - Use precise timestamps like 00:05:17, 00:12:43, 00:08:09, etc. - Look at the actual flow of conversation to determine when topics change - The chapter description MUST accurately describe what is being said RIGHT AT that timestamp - Do NOT describe what happens later - only describe what is happening at the exact moment of the timestamp - - Read the text carefully around each timestamp to ensure your description matches what's actually being discussed + - Read the timeline reference carefully to see what's actually being said at each time - If a guest is introduced at 00:05:30, don't put "Guest introduction" at 00:20:00 - Be precise and honest about what's happening at each moment - KEEP DESCRIPTIONS CONCISE: Use 2-6 words maximum, not full sentences - Descriptions should be SHORT PHRASES like "Guest introduction", "Discussion about X", "Demo of Y feature" - - DO NOT write full sentences or lengthy explanations in the chapter titles{duration_constraint} + - DO NOT write full sentences or lengthy explanations in the chapter titles{duration_constraint}{timeline_context} """ print(f"Creating chapters for video {video_id}") From c1cff879f40f17e8d638fd940072c2014e12624b Mon Sep 17 00:00:00 2001 From: Nicole van der Hoeven Date: Fri, 28 Nov 2025 12:16:33 +0100 Subject: [PATCH 19/28] Improve chapter selection using summary and full video coverage - Pass video summary to chapter generation to guide topic selection - Add explicit guidance to skip small talk not mentioned in summary - Fix timeline sampling to cover entire video duration dynamically * Was limited to 40 samples (20 minutes) regardless of video length * Now uses ~55 samples distributed across full video duration * Sample interval adjusts based on video length (min 15s) - Update prompt to emphasize reviewing ENTIRE video before selecting chapters - Add better logging showing sample count, interval, and video duration - Fixes issues with: * Chapters for unimportant small talk (e.g., weather discussion) * Missing chapters in second half of longer videos * AI only seeing beginning of video --- video-transcripts/transcripts.py | 75 ++++++++++++++++++++++++++------ 1 file changed, 61 insertions(+), 14 deletions(-) diff --git a/video-transcripts/transcripts.py b/video-transcripts/transcripts.py index f7d08ec..1385358 100644 --- a/video-transcripts/transcripts.py +++ b/video-transcripts/transcripts.py @@ -845,12 +845,13 @@ def find_correct_timestamp(description, all_segments, time_to_text, client, orig print(f" Error searching for timestamp: {str(e)}") return None -def create_chapters(transcript, video_id, raw_transcript=None): +def create_chapters(transcript, video_id, raw_transcript=None, summary=None): """ Create timestamps and chapters for a YouTube video transcript. @param transcript: The raw transcript of a YouTube video @param video_id: The YouTube video ID @param raw_transcript: Raw transcript data with timing information (optional) + @param summary: Summary of the video content (optional) @return: A string containing formatted timestamps and chapters """ if openai_key == "": @@ -869,7 +870,7 @@ def create_chapters(transcript, video_id, raw_transcript=None): # Build a timeline skeleton from raw transcript if available timeline_context = "" if raw_transcript: - print(f"Building timeline skeleton from raw transcript for video {video_id}") + print(f"Building timeline skeleton from raw transcript for video {video_id} (duration: {format_duration(duration_seconds) if duration_seconds else 'unknown'})") # Build mapping of time to text (reusing existing logic) time_to_texts = {} @@ -882,18 +883,22 @@ def create_chapters(transcript, video_id, raw_transcript=None): time_to_texts[start_time] = [] time_to_texts[start_time].append(text) - # Create timeline samples every 30 seconds + # Create timeline samples dynamically based on video duration timeline_samples = [] - sample_interval = 30 # seconds - max_samples = 40 # Limit to avoid token overflow + video_duration = duration_seconds or 3600 + + # Dynamically adjust sample interval based on video length + # Target ~50-60 samples total to cover the entire video while staying within token limits + target_samples = 55 + sample_interval = max(15, video_duration // target_samples) # At least 15 seconds between samples current_time = 0 sample_count = 0 - while current_time <= (duration_seconds or 3600) and sample_count < max_samples: + while current_time <= video_duration and sample_count < target_samples: # Get text from a small window around this time window_texts = [] - for time_sec in range(current_time, min(current_time + 10, (duration_seconds or 3600) + 1)): + for time_sec in range(current_time, min(current_time + 10, video_duration + 1)): if time_sec in time_to_texts: window_texts.extend(time_to_texts[time_sec]) @@ -907,27 +912,69 @@ def create_chapters(transcript, video_id, raw_transcript=None): current_time += sample_interval if timeline_samples: + print(f" Created {len(timeline_samples)} timeline samples (interval: ~{sample_interval}s) covering entire video") timeline_context = f""" -TIMELINE REFERENCE - These are actual timestamps from the video showing what is being said at different times: +TIMELINE REFERENCE - These are actual timestamps from the video showing what is being said at different times (covering the ENTIRE {format_duration(video_duration)} video): {chr(10).join(timeline_samples)} -USE THESE ACTUAL TIMESTAMPS to determine when topics change. Your chapter timestamps MUST come from the times shown above or nearby times. DO NOT guess or make up timestamps.""" +USE THESE ACTUAL TIMESTAMPS to determine when topics change. Review the ENTIRE timeline above before selecting chapters. Your chapter timestamps MUST come from the times shown above or nearby times. DO NOT guess or make up timestamps.""" + + # Include summary context if available + summary_context = "" + if summary: + summary_context = f""" + +VIDEO SUMMARY - Use this to understand what topics and people are actually important in this video: + +{summary} + +IMPORTANT: Create chapters ONLY for the topics, people, and discussions mentioned in the summary above. If something isn't in the summary (like casual small talk about weather), it's not important enough for a chapter.""" chapters_prompt = f""" - This is a transcript of a YouTube livestream. Could you please identify up to 10 key moments in the stream and give me the timestamps in the format for YouTube like this?: + This is a transcript of a YouTube livestream. + + TASK: Read through the ENTIRE video (see timeline reference below) and identify the 10 MOST IMPORTANT moments to create chapters for, in this format: 00:00:00 Introductions 00:01:47 What is structured metadata? 00:03:22 Discussion about metrics 00:05:30 Guest introduction: Diana + + IMPORTANT: Review the timeline reference showing the ENTIRE video before deciding on chapters. Distribute chapters across the full video length, not just the beginning. + + WHAT MAKES A GOOD CHAPTER: + - Topics and people mentioned in the VIDEO SUMMARY above + - Guest introductions (when new people join the conversation) + - Major topic changes (switching from one main subject to another) + - Substantive discussions, demos, or explanations that align with the summary + - Q&A sessions or audience interactions + - Significant announcements or reveals + + DO NOT CREATE CHAPTERS FOR: + - Anything NOT mentioned in the VIDEO SUMMARY (if summary is provided) + - Brief small talk, pleasantries, or casual mentions (weather, travel, casual greetings) + - Filler conversation or off-topic tangents + - Single sentences or passing comments about unrelated topics + - Technical difficulties, pauses, or transitions + - Topics that last less than 30 seconds + + DECISION PROCESS: + 1. Read through the ENTIRE timeline reference to understand the full video + 2. Identify all significant moments that match topics in the VIDEO SUMMARY + 3. Select the 10 MOST IMPORTANT moments distributed across the entire video + 4. For each moment: check if the topic/person is mentioned in the VIDEO SUMMARY + 5. If YES and it's a significant moment → include as a chapter + 6. If NO or it's just small talk → skip it + + FOCUS ON: What would viewers actually want to jump to? What are the main topics identified in the summary? Make sure to cover the ENTIRE video, not just the beginning. CRITICAL INSTRUCTIONS: - - Always start with 00:00:00 + - Always start with 00:00:00 (typically "Introductions" or "Welcome") - Use the ACTUAL timestamps from the timeline reference below - DO NOT round timestamps to neat intervals like :00 or :30 - Use precise timestamps like 00:05:17, 00:12:43, 00:08:09, etc. - - Look at the actual flow of conversation to determine when topics change + - Look at the actual flow of conversation to determine when MAJOR topics change - The chapter description MUST accurately describe what is being said RIGHT AT that timestamp - Do NOT describe what happens later - only describe what is happening at the exact moment of the timestamp - Read the timeline reference carefully to see what's actually being said at each time @@ -935,7 +982,7 @@ def create_chapters(transcript, video_id, raw_transcript=None): - Be precise and honest about what's happening at each moment - KEEP DESCRIPTIONS CONCISE: Use 2-6 words maximum, not full sentences - Descriptions should be SHORT PHRASES like "Guest introduction", "Discussion about X", "Demo of Y feature" - - DO NOT write full sentences or lengthy explanations in the chapter titles{duration_constraint}{timeline_context} + - DO NOT write full sentences or lengthy explanations in the chapter titles{duration_constraint}{summary_context}{timeline_context} """ print(f"Creating chapters for video {video_id}") @@ -980,7 +1027,7 @@ def write_markdown(args, video): filename = file_for_video(args, video) [summary, cleaned_up] = openai_cleanup(transcript, video_id) - chapters = create_chapters(transcript, video_id, raw_transcript) + chapters = create_chapters(transcript, video_id, raw_transcript, summary) # Parse chapters and insert timestamps into cleaned transcript parsed_chapters = parse_chapters(chapters) From ffbda60822b980240213540f695e4fe2639f2a14 Mon Sep 17 00:00:00 2001 From: Nicole van der Hoeven Date: Fri, 28 Nov 2025 12:34:59 +0100 Subject: [PATCH 20/28] Remove unused TooManyRequests import and add exception tests - Remove unused TooManyRequests import and defensive fallback - Add TestYouTubeAPIErrorHandling class to validate exception API contract - Tests ensure YouTube API exceptions are importable and catchable - Will catch breaking changes if youtube-transcript-api is upgraded --- video-transcripts/test_transcripts.py | 35 +++++++++++++++++++++++++++ video-transcripts/transcripts.py | 5 ---- 2 files changed, 35 insertions(+), 5 deletions(-) diff --git a/video-transcripts/test_transcripts.py b/video-transcripts/test_transcripts.py index 2f0b700..477e48b 100644 --- a/video-transcripts/test_transcripts.py +++ b/video-transcripts/test_transcripts.py @@ -91,6 +91,41 @@ def test_build_timeline_preserves_precision(self): self.assertIn('91 seconds', timeline) +class TestYouTubeAPIErrorHandling(unittest.TestCase): + """Test that YouTube API exceptions work as expected with our pinned version.""" + + def test_youtube_api_exceptions_importable(self): + """Test that all YouTube API exceptions we depend on can be imported. + + This will fail if the API changes in a way that breaks our imports. + """ + # Import all exceptions we use + from youtube_transcript_api import TranscriptsDisabled, NoTranscriptFound, VideoUnavailable + from youtube_transcript_api._errors import YouTubeRequestFailed + + # Verify they're actual exception classes + self.assertTrue(issubclass(TranscriptsDisabled, Exception)) + self.assertTrue(issubclass(NoTranscriptFound, Exception)) + self.assertTrue(issubclass(VideoUnavailable, Exception)) + self.assertTrue(issubclass(YouTubeRequestFailed, Exception)) + + def test_youtube_api_exceptions_catchable(self): + """Test that YouTube API exceptions can be caught. + + This validates that exceptions raised by the library can be caught + and their error messages can be inspected (as our code does). + """ + from youtube_transcript_api import TranscriptsDisabled, NoTranscriptFound + + # Test we can catch TranscriptsDisabled + with self.assertRaises(TranscriptsDisabled): + raise TranscriptsDisabled("video_id") + + # Test we can catch NoTranscriptFound + with self.assertRaises(NoTranscriptFound): + raise NoTranscriptFound("video_id", [], "en") + + class TestTimestampParsing(unittest.TestCase): """Test timestamp parsing and formatting functions.""" diff --git a/video-transcripts/transcripts.py b/video-transcripts/transcripts.py index 1385358..8add321 100644 --- a/video-transcripts/transcripts.py +++ b/video-transcripts/transcripts.py @@ -5,11 +5,6 @@ from youtube_transcript_api import YouTubeTranscriptApi from youtube_transcript_api import TranscriptsDisabled, NoTranscriptFound, VideoUnavailable from youtube_transcript_api._errors import YouTubeRequestFailed -try: - from youtube_transcript_api._errors import TooManyRequests -except ImportError: - # Fallback if TooManyRequests doesn't exist in this version - TooManyRequests = YouTubeRequestFailed from dotenv import load_dotenv from slugify import slugify import json From 3119c3b7a6b7c8451e995d7583f539a69cf3ac79 Mon Sep 17 00:00:00 2001 From: Nicole van der Hoeven Date: Fri, 28 Nov 2025 12:39:20 +0100 Subject: [PATCH 21/28] refactor: replace while True with explicit loop conditions and safeguards - Add max_pages parameter (default 100) to prevent infinite loops - Add max_retries (3) for quota exceeded errors - Track page_count and retry_count for better control flow - Make loop condition explicit: while page_count < max_pages - Reset retry_count on successful requests - Raise RuntimeError when max retries exceeded This addresses reviewer feedback about the risks of using while True loops. --- video-transcripts/transcripts.py | 19 +++++++++++++++---- 1 file changed, 15 insertions(+), 4 deletions(-) diff --git a/video-transcripts/transcripts.py b/video-transcripts/transcripts.py index 8add321..95ba960 100644 --- a/video-transcripts/transcripts.py +++ b/video-transcripts/transcripts.py @@ -21,15 +21,20 @@ # Second one is optional, but if it's missing, we'll skip the summary and cleanup. openai_key = os.environ.get("OPENAI_API_KEY","") -def get_playlist_videos(playlist_id): +def get_playlist_videos(playlist_id, max_pages=100): youtube = build('youtube', 'v3', developerKey=youtube_key) video_ids = [] next_page_token = None + page_count = 0 + max_retries = 3 + retry_count = 0 - # This returns a list of youtube:playlistItem with rate limiting - while True: + # Paginate through playlist items with rate limiting and safeguards + while page_count < max_pages: try: + page_count += 1 + # Build request parameters request_params = { 'part': 'snippet', @@ -50,6 +55,9 @@ def get_playlist_videos(playlist_id): next_page_token = response.get('nextPageToken') if not next_page_token: break + + # Reset retry count on successful request + retry_count = 0 # Small delay between pagination requests delay = random.uniform(1, 3) # Random delay between 1-3 seconds @@ -58,7 +66,10 @@ def get_playlist_videos(playlist_id): except HttpError as e: if e.resp.status == 403 and 'quota' in str(e).lower(): - print("Quota exceeded while fetching playlist items. Waiting before retry...") + retry_count += 1 + if retry_count >= max_retries: + raise RuntimeError(f"Max retries ({max_retries}) exceeded due to quota limits") + print(f"Quota exceeded while fetching playlist items (attempt {retry_count}/{max_retries}). Waiting before retry...") time.sleep(60) # Wait 1 minute continue else: From c382b4d3b05902415dc3968e0550421b2e046455 Mon Sep 17 00:00:00 2001 From: Nicole van der Hoeven Date: Fri, 28 Nov 2025 13:15:44 +0100 Subject: [PATCH 22/28] Refactor get_playlist_videos into smaller, more readable functions - Extract video ID fetching logic into _fetch_playlist_video_ids - Extract video details fetching logic into _fetch_video_details_batch - Simplify main get_playlist_videos function to coordinate the two helpers - Each function now has a single, clear responsibility - Improve code readability and maintainability --- video-transcripts/transcripts.py | 41 ++++++++++++++++++++++++++++---- 1 file changed, 36 insertions(+), 5 deletions(-) diff --git a/video-transcripts/transcripts.py b/video-transcripts/transcripts.py index 95ba960..24c825a 100644 --- a/video-transcripts/transcripts.py +++ b/video-transcripts/transcripts.py @@ -21,16 +21,20 @@ # Second one is optional, but if it's missing, we'll skip the summary and cleanup. openai_key = os.environ.get("OPENAI_API_KEY","") -def get_playlist_videos(playlist_id, max_pages=100): - youtube = build('youtube', 'v3', developerKey=youtube_key) - +def _fetch_playlist_video_ids(youtube, playlist_id, max_pages=100): + """ + Fetch all video IDs from a playlist with pagination and retry logic. + @param youtube: YouTube API client instance + @param playlist_id: YouTube playlist ID + @param max_pages: Maximum number of pages to fetch + @return: List of video IDs + """ video_ids = [] next_page_token = None page_count = 0 max_retries = 3 retry_count = 0 - # Paginate through playlist items with rate limiting and safeguards while page_count < max_pages: try: page_count += 1 @@ -76,7 +80,16 @@ def get_playlist_videos(playlist_id, max_pages=100): print(f"Error fetching playlist items: {e}") raise - # Fetch video details in batches with rate limiting + return video_ids + + +def _fetch_video_details_batch(youtube, video_ids): + """ + Fetch detailed video information for a list of video IDs in batches. + @param youtube: YouTube API client instance + @param video_ids: List of YouTube video IDs + @return: List of video objects with detailed information + """ videos = [] batch_size = 50 # YouTube API allows up to 50 IDs per request @@ -108,6 +121,24 @@ def get_playlist_videos(playlist_id, max_pages=100): return videos + +def get_playlist_videos(playlist_id, max_pages=100): + """ + Fetch all videos from a YouTube playlist with their detailed information. + @param playlist_id: YouTube playlist ID + @param max_pages: Maximum number of pages to fetch (default 100) + @return: List of video objects with detailed information + """ + youtube = build('youtube', 'v3', developerKey=youtube_key) + + # Step 1: Get all video IDs from the playlist + video_ids = _fetch_playlist_video_ids(youtube, playlist_id, max_pages) + + # Step 2: Fetch detailed information for each video + videos = _fetch_video_details_batch(youtube, video_ids) + + return videos + def get_channel_videos(channel_id, start_date, end_date): youtube = build('youtube', 'v3', developerKey=youtube_key) From e543eedebebc664fc9331f6c9a355c2bc556a521 Mon Sep 17 00:00:00 2001 From: Nicole van der Hoeven Date: Fri, 28 Nov 2025 13:28:43 +0100 Subject: [PATCH 23/28] Remove unnecessary YouTube API delays and fix retry bug - Removed 200ms delay in _fetch_video_details_batch - Removed 1-3 second delays in _fetch_playlist_video_ids and get_channel_videos - Removed 2-4 second delay between playlist and channel fetches in main - Fixed retry bug in _fetch_video_details_batch where continue would skip to next batch instead of retrying the failed one - Added proper retry loop with max_retries tracking per batch - 403 quota handlers now properly catch and retry after 60 second wait --- video-transcripts/transcripts.py | 65 ++++++++++++++------------------ 1 file changed, 28 insertions(+), 37 deletions(-) diff --git a/video-transcripts/transcripts.py b/video-transcripts/transcripts.py index 24c825a..a695b3d 100644 --- a/video-transcripts/transcripts.py +++ b/video-transcripts/transcripts.py @@ -62,11 +62,6 @@ def _fetch_playlist_video_ids(youtube, playlist_id, max_pages=100): # Reset retry count on successful request retry_count = 0 - - # Small delay between pagination requests - delay = random.uniform(1, 3) # Random delay between 1-3 seconds - print(f"Waiting {delay:.1f} seconds before next page...") - time.sleep(delay) except HttpError as e: if e.resp.status == 403 and 'quota' in str(e).lower(): @@ -92,32 +87,38 @@ def _fetch_video_details_batch(youtube, video_ids): """ videos = [] batch_size = 50 # YouTube API allows up to 50 IDs per request + max_retries = 3 for i in range(0, len(video_ids), batch_size): batch_ids = video_ids[i:i + batch_size] + retry_count = 0 - try: - request = youtube.videos().list( - part='snippet', - id=','.join(batch_ids) - ) - response = request.execute() - - for item in response['items']: - print(f"Found video: {item['snippet']['title']}") - videos.append(item) + while retry_count < max_retries: + try: + request = youtube.videos().list( + part='snippet', + id=','.join(batch_ids) + ) + response = request.execute() - # Rate limiting between batches - time.sleep(0.2) # 200ms delay between batches - - except HttpError as e: - if e.resp.status == 403 and 'quota' in str(e).lower(): - print(f"Quota exceeded on video details batch {i//batch_size + 1}. Waiting before retry...") - time.sleep(60) - continue - else: - print(f"Error fetching video details batch {i//batch_size + 1}: {e}") - continue + for item in response['items']: + print(f"Found video: {item['snippet']['title']}") + videos.append(item) + + # Success - break out of retry loop and move to next batch + break + + except HttpError as e: + if e.resp.status == 403 and 'quota' in str(e).lower(): + retry_count += 1 + if retry_count >= max_retries: + print(f"Max retries ({max_retries}) exceeded on video details batch {i//batch_size + 1}") + break + print(f"Quota exceeded on video details batch {i//batch_size + 1} (attempt {retry_count}/{max_retries}). Waiting before retry...") + time.sleep(60) + else: + print(f"Error fetching video details batch {i//batch_size + 1}: {e}") + break # Don't retry on non-quota errors return videos @@ -173,11 +174,7 @@ def get_channel_videos(channel_id, start_date, end_date): next_page_token = response.get('nextPageToken') if not next_page_token: break - - # Small delay between pagination requests - delay = random.uniform(1, 3) # Random delay between 1-3 seconds - print(f"Waiting {delay:.1f} seconds before next page...") - time.sleep(delay) + page_count += 1 except HttpError as e: @@ -1105,12 +1102,6 @@ def main(args): playlist_videos = get_playlist_videos(args.playlist) videos_to_transcribe = videos_to_transcribe + playlist_videos print(f"Found {len(playlist_videos)} videos in playlist %s" % args.playlist) - - # Small delay between different API calls - if args.channel: - delay = random.uniform(2, 4) - print(f"Waiting {delay:.1f} seconds before fetching channel videos...") - time.sleep(delay) if args.channel: print("Fetching channel videos") From aa70804a87f068271be28247a03e3c556b4117a7 Mon Sep 17 00:00:00 2001 From: Nicole van der Hoeven Date: Fri, 28 Nov 2025 13:54:04 +0100 Subject: [PATCH 24/28] docs: clarify rate limiting delay between transcript fetches - Add explanatory comment in transcripts.py explaining that the 2-5s delay prevents YouTube rate limiting (429 errors) - Update README to accurately reflect actual delays used (2-5s, not 10-30s) - Clarify distinction between proactive delays and reactive retry waits --- video-transcripts/README.md | 8 ++++---- video-transcripts/transcripts.py | 4 +++- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/video-transcripts/README.md b/video-transcripts/README.md index bfbb23c..de5a795 100644 --- a/video-transcripts/README.md +++ b/video-transcripts/README.md @@ -83,10 +83,10 @@ The generated markdown includes: ### Rate Limiting The script includes comprehensive rate limiting to avoid YouTube API quota issues: -- Random delays between API requests (3-8 seconds for pagination) -- Longer delays between video processing (10-30 seconds) -- Initial startup delay to "cool down" the API -- Automatic 60-second wait and retry when quota limits are hit +- Small delays between video transcript fetches (2-5 seconds) to prevent rate limiting +- Automatic 60-120 second wait and retry when rate limits (429 errors) are hit +- Exponential backoff for transient errors +- Detects and reports IP blocking vs. temporary rate limiting ### Error Handling - Exponential backoff with up to 3 retry attempts for transcript fetching diff --git a/video-transcripts/transcripts.py b/video-transcripts/transcripts.py index a695b3d..d7e08f1 100644 --- a/video-transcripts/transcripts.py +++ b/video-transcripts/transcripts.py @@ -228,7 +228,9 @@ def fetch_transcripts(args, videos): else: print(f"Skipping video {video_id} due to transcript issues.") - # Small delay between transcript fetches + # Rate limiting prevention: Small delay between transcript fetches to avoid + # triggering YouTube's rate limits (429 errors). Better to wait 2-5 seconds + # proactively than 60-120 seconds reactively if rate limited. if i < len(videos) - 1: # Don't sleep after the last video delay = random.uniform(2, 5) # Random delay between 2-5 seconds print(f"Waiting {delay:.1f} seconds before next video...") From 4b1f380f343ee1df16730b1641257da119a9bd51 Mon Sep 17 00:00:00 2001 From: Nicole van der Hoeven Date: Fri, 28 Nov 2025 14:10:07 +0100 Subject: [PATCH 25/28] Simplify verbose error print statements - Condense multiline IP blocking error messages to 2 lines - Shorten rate limiting error messages - Simplify XML parse error messages - All changes reference README for detailed troubleshooting --- video-transcripts/transcripts.py | 24 ++++++------------------ 1 file changed, 6 insertions(+), 18 deletions(-) diff --git a/video-transcripts/transcripts.py b/video-transcripts/transcripts.py index d7e08f1..d1128bf 100644 --- a/video-transcripts/transcripts.py +++ b/video-transcripts/transcripts.py @@ -311,27 +311,18 @@ def get_video_transcript_with_retry(video_id, max_retries=5): print(f"Video {video_id} is unavailable") return None, None except YouTubeRequestFailed as e: - # Check if this is an IP block error error_str = str(e) if 'blocking requests from your IP' in error_str or 'IP has been blocked' in error_str: - print(f"❌ YouTube has blocked your IP address for video {video_id}") - print(f" This is not a temporary rate limit - your IP is blocked.") - print(f"\n Workarounds:") - print(f" 1. Wait 24-48 hours for the block to clear") - print(f" 2. Use a different network/WiFi connection") - print(f" 3. Set up cookie-based authentication (see README)") - print(f" 4. Use a residential proxy or VPN (not cloud-based)") + print(f"❌ YouTube IP block detected for video {video_id}") + print(f" See README 'YouTube IP Blocking' section for solutions") return None, None elif '429' in error_str or 'Too Many Requests' in error_str: if attempt < max_retries - 1: - # Use much longer wait time for rate limiting (60-120 seconds) wait_time = random.uniform(60, 120) - print(f"⚠️ YouTube rate limit (429) detected for video {video_id}") - print(f" Waiting {wait_time:.0f} seconds before retry {attempt + 2}/{max_retries}...") + print(f"⚠️ Rate limit detected (429) - waiting {wait_time:.0f}s (retry {attempt + 2}/{max_retries})") time.sleep(wait_time) else: - print(f"❌ YouTube rate limit persists after {max_retries} attempts for video {video_id}") - print(f" Consider waiting 10-15 minutes before running the script again.") + print(f"❌ Rate limit persists after {max_retries} attempts - wait 10-15 minutes and retry") return None, None else: # Other YouTube API errors @@ -347,14 +338,11 @@ def get_video_transcript_with_retry(video_id, max_retries=5): # Check if this is likely a rate limit error disguised as XML parse error if 'no element found' in error_str or 'line 1, column 0' in error_str: if attempt < max_retries - 1: - # Use longer wait time as this is likely a rate limit issue wait_time = random.uniform(60, 120) - print(f"⚠️ Possible YouTube rate limit detected for video {video_id} (XML parse error)") - print(f" Waiting {wait_time:.0f} seconds before retry {attempt + 2}/{max_retries}...") + print(f"⚠️ Possible rate limit (XML parse error) - waiting {wait_time:.0f}s (retry {attempt + 2}/{max_retries})") time.sleep(wait_time) else: - print(f"❌ Persistent error for video {video_id} after {max_retries} attempts") - print(f" This may be due to YouTube rate limiting. Wait 10-15 minutes and try again.") + print(f"❌ Persistent error after {max_retries} attempts - see README troubleshooting") return None, None elif attempt < max_retries - 1: # Standard exponential backoff for other errors From d2e3f8ac1a92f2cec8c4be3c77aa50227454abff Mon Sep 17 00:00:00 2001 From: Nicole van der Hoeven Date: Fri, 28 Nov 2025 14:16:43 +0100 Subject: [PATCH 26/28] Add explicit output format instructions to chapters prompt - Instruct AI to output only the chapter list without preamble text - Prevent conversational phrases like 'Here are the chapters' or 'Sure, I'll help' - Addresses reviewer feedback to handle this in the prompt rather than post-processing --- video-transcripts/transcripts.py | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/video-transcripts/transcripts.py b/video-transcripts/transcripts.py index d1128bf..25f8d8e 100644 --- a/video-transcripts/transcripts.py +++ b/video-transcripts/transcripts.py @@ -315,7 +315,7 @@ def get_video_transcript_with_retry(video_id, max_retries=5): if 'blocking requests from your IP' in error_str or 'IP has been blocked' in error_str: print(f"❌ YouTube IP block detected for video {video_id}") print(f" See README 'YouTube IP Blocking' section for solutions") - return None, None + return None, None´ elif '429' in error_str or 'Too Many Requests' in error_str: if attempt < max_retries - 1: wait_time = random.uniform(60, 120) @@ -1006,7 +1006,15 @@ def create_chapters(transcript, video_id, raw_transcript=None, summary=None): - Be precise and honest about what's happening at each moment - KEEP DESCRIPTIONS CONCISE: Use 2-6 words maximum, not full sentences - Descriptions should be SHORT PHRASES like "Guest introduction", "Discussion about X", "Demo of Y feature" - - DO NOT write full sentences or lengthy explanations in the chapter titles{duration_constraint}{summary_context}{timeline_context} + - DO NOT write full sentences or lengthy explanations in the chapter titles + + OUTPUT FORMAT REQUIREMENTS: + - Output ONLY the chapter list, nothing else + - Start directly with the first timestamp line: "00:00:00 Introductions" + - Do NOT include any preamble, introduction, or explanatory text before the chapters + - Do NOT say things like "Here are the chapters", "Sure, I'll help", or "Here are 10 key moments" + - Do NOT add any text after the last chapter + - Just provide the raw chapter list in the specified format{duration_constraint}{summary_context}{timeline_context} """ print(f"Creating chapters for video {video_id}") From 17eed88cb02f527917bbdfa8edcc0f6f7753649d Mon Sep 17 00:00:00 2001 From: Nicole van der Hoeven Date: Fri, 28 Nov 2025 14:22:44 +0100 Subject: [PATCH 27/28] refactor: break insert_timestamps_in_transcript into smaller functions - Extract _build_time_to_text_mapping() for building time-to-text mapping - Extract _get_window_texts() for getting text from time windows - Extract _extract_key_words() for keyword extraction with filtering - Extract _calculate_line_score() for scoring line matches - Extract _find_best_insertion_line() for finding best insertion points - Extract _build_transcript_with_chapters() for final transcript assembly - Main function now acts as orchestrator (36 lines, down from 113) - Fix syntax error on line 318 (invalid character in return statement) Benefits: improved testability, readability, and maintainability --- video-transcripts/transcripts.py | 237 ++++++++++++++++++++----------- 1 file changed, 157 insertions(+), 80 deletions(-) diff --git a/video-transcripts/transcripts.py b/video-transcripts/transcripts.py index 25f8d8e..82253c2 100644 --- a/video-transcripts/transcripts.py +++ b/video-transcripts/transcripts.py @@ -315,7 +315,7 @@ def get_video_transcript_with_retry(video_id, max_retries=5): if 'blocking requests from your IP' in error_str or 'IP has been blocked' in error_str: print(f"❌ YouTube IP block detected for video {video_id}") print(f" See README 'YouTube IP Blocking' section for solutions") - return None, None´ + return None, None elif '429' in error_str or 'Too Many Requests' in error_str: if attempt < max_retries - 1: wait_time = random.uniform(60, 120) @@ -521,19 +521,12 @@ def parse_chapters(chapters_text): chapters.append((seconds, timestamp_str, title)) return chapters -def insert_timestamps_in_transcript(cleaned_transcript, chapters, raw_transcript): +def _build_time_to_text_mapping(raw_transcript): """ - Insert timestamp markers into the cleaned transcript at appropriate positions. - @param cleaned_transcript: The cleaned transcript text from OpenAI - @param chapters: List of (seconds, timestamp_str, title) tuples + Build a mapping of time (in seconds) to text at that time from raw transcript. @param raw_transcript: Raw transcript data from YouTube with timing info - @return: Transcript with timestamps inserted + @return: Dictionary mapping time in seconds to list of text snippets """ - if not cleaned_transcript or not chapters or not raw_transcript: - return cleaned_transcript - - # Build a mapping of time (in seconds) to text at that time - # Store multiple entries per time window for better matching time_to_texts = {} for entry in raw_transcript: if isinstance(entry, dict) and 'text' in entry and 'start' in entry: @@ -543,6 +536,146 @@ def insert_timestamps_in_transcript(cleaned_transcript, chapters, raw_transcript if start_time not in time_to_texts: time_to_texts[start_time] = [] time_to_texts[start_time].append(text) + return time_to_texts + + +def _get_window_texts(time_to_texts, seconds, window_size=5): + """ + Get text snippets from a time window around the target timestamp. + @param time_to_texts: Dictionary mapping time to text snippets + @param seconds: Target timestamp in seconds + @param window_size: Size of window in seconds (±window_size) + @return: List of text snippets from the time window + """ + window_texts = [] + for time_sec in range(max(0, seconds - window_size), seconds + window_size + 1): + if time_sec in time_to_texts: + window_texts.extend(time_to_texts[time_sec]) + + if not window_texts: + # Fallback to closest time if window is empty + closest_time = min(time_to_texts.keys(), + key=lambda t: abs(t - seconds), + default=None) + if closest_time: + window_texts = time_to_texts[closest_time] + + return window_texts + + +def _extract_key_words(text, min_length=3, excluded_words=None): + """ + Extract meaningful key words from text, filtering out common words. + @param text: Text to extract words from + @param min_length: Minimum word length to include + @param excluded_words: Set of words to exclude + @return: List of lowercase key words + """ + if excluded_words is None: + excluded_words = set() + + words = [w.lower() for w in text.split() + if len(w) > min_length and w.lower() not in excluded_words] + + # Fallback to shorter words if we got nothing + if not words: + words = [w.lower() for w in text.split() if len(w) > 2][:10] + + return words + + +def _calculate_line_score(line, sample_words, title_words, idx, line_to_chapter): + """ + Calculate how well a line matches the chapter based on multiple factors. + @param line: Line of text to score + @param sample_words: Key words from the timestamp window + @param title_words: Key words from the chapter title + @param idx: Index of the line in the transcript + @param line_to_chapter: Dictionary of already-placed chapters + @return: Score (higher is better match) + """ + # Skip empty lines and lines that already have chapter markers + if not line.strip() or line.strip().startswith('###'): + return 0 + + line_lower = line.lower() + + # Count matching words from the window + text_matches = sum(1 for word in sample_words if word in line_lower) + + # Count matching words from the chapter title (weighted more heavily) + title_matches = sum(1 for word in title_words if word in line_lower) * 2 + + # Bonus for lines that start with speaker markers + is_speaker_line = '**' in line and ':' in line + speaker_bonus = 1 if is_speaker_line else 0 + + # Prefer lines that appear after already-placed timestamps (chronological order) + position_bonus = 0 + if line_to_chapter: + last_placed = max(line_to_chapter.keys()) + if idx > last_placed: + position_bonus = 0.5 + + return text_matches + title_matches + speaker_bonus + position_bonus + + +def _find_best_insertion_line(lines, sample_words, title_words, line_to_chapter): + """ + Find the best line index to insert a chapter marker. + @param lines: List of transcript lines + @param sample_words: Key words from the timestamp window + @param title_words: Key words from the chapter title + @param line_to_chapter: Dictionary of already-placed chapters + @return: Tuple of (best_line_idx, best_score) or (None, 0) if no match + """ + best_line_idx = None + best_score = 0 + + for idx, line in enumerate(lines): + score = _calculate_line_score(line, sample_words, title_words, idx, line_to_chapter) + + if score > best_score: + best_score = score + best_line_idx = idx + + return best_line_idx, best_score + + +def _build_transcript_with_chapters(lines, line_to_chapter): + """ + Build final transcript with chapter headings inserted at marked positions. + @param lines: List of transcript lines + @param line_to_chapter: Dictionary mapping line index to (timestamp, title) + @return: Transcript string with chapter headings inserted + """ + result_lines = [] + for idx, line in enumerate(lines): + if idx in line_to_chapter: + # Insert H3 heading before this line with timestamp and chapter title + timestamp, title = line_to_chapter[idx] + result_lines.append(f"### [{timestamp}] {title}") + result_lines.append("") # Add blank line after heading + result_lines.append(line) + else: + result_lines.append(line) + + return '\n'.join(result_lines) + + +def insert_timestamps_in_transcript(cleaned_transcript, chapters, raw_transcript): + """ + Insert timestamp markers into the cleaned transcript at appropriate positions. + @param cleaned_transcript: The cleaned transcript text from OpenAI + @param chapters: List of (seconds, timestamp_str, title) tuples + @param raw_transcript: Raw transcript data from YouTube with timing info + @return: Transcript with timestamps inserted + """ + if not cleaned_transcript or not chapters or not raw_transcript: + return cleaned_transcript + + # Build time-to-text mapping from raw transcript + time_to_texts = _build_time_to_text_mapping(raw_transcript) # Split transcript into lines lines = cleaned_transcript.split('\n') @@ -550,71 +683,26 @@ def insert_timestamps_in_transcript(cleaned_transcript, chapters, raw_transcript # For each chapter, find the best line to insert it at line_to_chapter = {} # Maps line index to (timestamp_str, title) to insert + # Common words to exclude when extracting key words + common_words = {'that', 'this', 'with', 'from', 'have', 'were', 'been', 'they', 'them'} + title_common_words = {'the', 'and', 'for', 'with', 'about', 'discussion', 'introduction'} + for seconds, timestamp_str, title in chapters: - # Get text from a window around the timestamp (±5 seconds) - window_texts = [] - for time_sec in range(max(0, seconds - 5), seconds + 6): - if time_sec in time_to_texts: - window_texts.extend(time_to_texts[time_sec]) - - if not window_texts: - # Fallback to closest time if window is empty - closest_time = min(time_to_texts.keys(), - key=lambda t: abs(t - seconds), - default=None) - if closest_time: - window_texts = time_to_texts[closest_time] + # Get text from a window around the timestamp + window_texts = _get_window_texts(time_to_texts, seconds, window_size=5) if not window_texts: continue - # Combine all texts in the window + # Extract key words from the window text and chapter title combined_text = ' '.join(window_texts) - - # Extract key words from the window text - sample_words = [w.lower() for w in combined_text.split() - if len(w) > 3 and w.lower() not in ['that', 'this', 'with', 'from', 'have', 'were', 'been', 'they', 'them']] - - if not sample_words: - sample_words = [w.lower() for w in combined_text.split() if len(w) > 2][:10] - - # Also use words from the chapter title itself - title_words = [w.lower() for w in title.split() - if len(w) > 3 and w.lower() not in ['the', 'and', 'for', 'with', 'about', 'discussion', 'introduction']] + sample_words = _extract_key_words(combined_text, min_length=3, excluded_words=common_words) + title_words = _extract_key_words(title, min_length=3, excluded_words=title_common_words) # Find the best matching line in cleaned transcript - best_line_idx = None - best_score = 0 - - for idx, line in enumerate(lines): - # Skip empty lines and lines that already have chapter markers - if not line.strip() or line.strip().startswith('###'): - continue - - line_lower = line.lower() - - # Count matching words from the window - text_matches = sum(1 for word in sample_words if word in line_lower) - - # Count matching words from the chapter title - title_matches = sum(1 for word in title_words if word in line_lower) * 2 # Weight title matches more - - # Bonus for lines that start with speaker markers - is_speaker_line = '**' in line and ':' in line - speaker_bonus = 1 if is_speaker_line else 0 - - # Prefer lines that appear after already-placed timestamps (chronological order) - position_bonus = 0 - if line_to_chapter: - last_placed = max(line_to_chapter.keys()) - if idx > last_placed: - position_bonus = 0.5 - - score = text_matches + title_matches + speaker_bonus + position_bonus - - if score > best_score: - best_score = score - best_line_idx = idx + best_line_idx, best_score = _find_best_insertion_line( + lines, sample_words, title_words, line_to_chapter + ) # Insert chapter marker at the best matching line if best_line_idx is not None and best_score > 1: # Require at least some confidence @@ -623,18 +711,7 @@ def insert_timestamps_in_transcript(cleaned_transcript, chapters, raw_transcript line_to_chapter[best_line_idx] = (timestamp_str, title) # Build the result by inserting chapter headings before marked lines - result_lines = [] - for idx, line in enumerate(lines): - if idx in line_to_chapter: - # Insert H3 heading before this line with timestamp and chapter title - timestamp, title = line_to_chapter[idx] - result_lines.append(f"### [{timestamp}] {title}") - result_lines.append("") # Add blank line after heading - result_lines.append(line) - else: - result_lines.append(line) - - return '\n'.join(result_lines) + return _build_transcript_with_chapters(lines, line_to_chapter) def format_duration(seconds): """ From db4af169618849eb6389507fdcbbbf4227973892 Mon Sep 17 00:00:00 2001 From: Nicole van der Hoeven Date: Fri, 28 Nov 2025 19:04:32 +0100 Subject: [PATCH 28/28] Regenerated all transcripts --- ...etry-q-a-feat-iris-dyrmishi-of-farfetch.md | 300 +++++------- ...-whats-an-observability-engineer-anyway.md | 20 +- ...8Z-teaser-observability-is-a-team-sport.md | 24 +- ...o-you-foster-a-culture-of-observability.md | 13 +- ...-a-team-sport-with-iris-dyrmishi-teaser.md | 17 +- ...eam-sport-with-iris-dyrmishi-full-video.md | 274 +++++------ ...ob-aronoff-of-lightstep-from-servicenow.md | 403 ++++++++-------- ...10Z-opentelemetry-q-a-feat-hazel-weakly.md | 335 +++++++------ ...-threading-the-needle-with-hazel-weakly.md | 278 +++++------ ...n-distributed-tracing-with-doug-ramirez.md | 225 +++++---- ...in-practice-fireside-chat-december-2022.md | 378 ++++++++------- ...-end-user-discussions-amer-january-2023.md | 282 ++++++++--- ...otel-migration-story-with-jacob-aronoff.md | 292 ++++++----- ...he-evolution-of-observability-practices.md | 332 +++++++------ ...ng-parsing-data-with-the-otel-collector.md | 298 ++++++------ ...T23:34:44Z-otel-q-a-feat-jennifer-moore.md | 202 ++++---- ...Z-otel-q-a-feat-luiz-aoqui-of-hashicorp.md | 272 +++++------ ...:51Z-the-humans-of-otel-kubecon-na-2023.md | 130 ++--- ...ty-majors-amy-tobey-and-adriana-villela.md | 356 ++++++-------- ...ow-to-train-your-teams-on-observability.md | 296 ++++++------ ...:24Z-otel-collector-user-feedback-panel.md | 174 ++++--- ...us-interoperability-user-feedback-panel.md | 179 ++++--- ...:59Z-the-humans-of-otel-kubecon-eu-2024.md | 166 +++---- ...aniel-dias-and-oscar-reyes-of-tracetest.md | 146 +++--- ...ile-apps-with-hanson-ho-and-eliab-sisay.md | 255 ++++++---- ...liab-sisay-and-austin-emmons-of-embrace.md | 191 +++++--- ...4T15:54:54Z-otel-q-a-with-steven-swartz.md | 285 +++++------ ...T21:03:07Z-otel-q-a-with-dan-ravenstone.md | 360 ++++---------- ...umans-of-otel-live-from-kubecon-na-2024.md | 397 ++++++++------- ...2:14:29Z-humans-of-otel-kubecon-na-2024.md | 135 +++--- ...d-user-conversation-with-ariel-valentin.md | 282 +++++------ ...31T05:58:14Z-cfp-writing-q-a-livestream.md | 454 +++++++++++++----- ...el-for-beginners-the-javascript-journey.md | 66 +-- ...-otel-me-with-jerome-johnson-cal-loomis.md | 170 ++++--- ...on-with-austin-parker-marylia-gutierrez.md | 454 ++++++++++++------ ...6:00:52Z-otel-me-with-eromosele-akhigbe.md | 218 ++++----- ...humans-of-opentelemetry-kubecon-eu-2025.md | 128 ++--- ...025-05-spring-starter-for-opentelemetry.md | 355 +++++++++----- ...05-leveraging-ai-for-opentelemetry-data.md | 286 +++++++---- ...h-oluwatomisin-taiwo-and-andrei-morozov.md | 252 ++++------ ...practice-alibabas-opentelemetry-journey.md | 168 ++++--- ...aled-kafkalog-ingestion-for-otel-by-150.md | 128 ++--- .../2025-10-22T04:08:27Z-whats-new-in-otel.md | 326 +++++++------ ...umans-of-otel-live-from-kubecon-na-2025.md | 383 ++++++++------- 44 files changed, 5859 insertions(+), 4826 deletions(-) diff --git a/video-transcripts/transcripts/2023-06-01T17:22:03Z-opentelemetry-q-a-feat-iris-dyrmishi-of-farfetch.md b/video-transcripts/transcripts/2023-06-01T17:22:03Z-opentelemetry-q-a-feat-iris-dyrmishi-of-farfetch.md index 615df39..8e2df8f 100644 --- a/video-transcripts/transcripts/2023-06-01T17:22:03Z-opentelemetry-q-a-feat-iris-dyrmishi-of-farfetch.md +++ b/video-transcripts/transcripts/2023-06-01T17:22:03Z-opentelemetry-q-a-feat-iris-dyrmishi-of-farfetch.md @@ -10,300 +10,232 @@ URL: https://www.youtube.com/watch?v=9iaGG-YZw5I ## Summary -In this YouTube video, a discussion is held about OpenTelemetry and its implementation at Farfetch, featuring Iris, a platform engineer at the company. Iris shares her journey from a software engineer to her current role in observability, emphasizing the importance of OpenTelemetry in managing complex architectures with around 3,000 engineers. The conversation highlights the tools used for observability, including Grafana, Prometheus, and Tempo, as well as the challenges faced during the implementation of OpenTelemetry, particularly in integrating tracing and metrics. Iris notes the positive culture around observability at Farfetch, where the team has embraced OpenTelemetry without significant resistance, and expresses pride in contributing to the OpenTelemetry community. The video also touches on the significance of collaboration among engineering teams and the continuous evolution of observability practices at Farfetch. Overall, the discussion serves as an insightful look into the practical applications and benefits of OpenTelemetry in a large organization. +In this YouTube video, Iris from Farfetch leads a Q&A session featuring Edith, a platform engineer on the observability team at Farfetch, who shares her passion for OpenTelemetry and the company's journey in implementing observability practices. Edith discusses her background, the complex architecture at Farfetch involving various platforms, and how OpenTelemetry has been embraced within the organization to improve service monitoring. The conversation covers the tools and technologies being utilized, such as Prometheus, Grafana, and Tempo, in conjunction with OpenTelemetry for metrics, traces, and logs. Edith highlights the cultural acceptance of observability within Farfetch, the challenges faced during implementation, and the importance of collaboration among engineering teams. She also emphasizes the role of documentation and community support in their OpenTelemetry journey. Overall, the session provides insights into the practical applications of OpenTelemetry and the collaborative spirit surrounding observability efforts at Farfetch. ## Chapters -00:00:00 Welcome and intro -00:01:00 Iris introduction -00:02:10 Edis background and journey -00:04:30 OpenTelemetry architecture discussion -00:06:00 CI/CD pipeline overview -00:07:10 Observability tooling at Farfetch -00:08:10 OpenTelemetry journey at Farfetch -00:10:00 Team enabling observability practices -00:12:30 OpenTelemetry implementation process -00:14:40 OpenTelemetry collector usage +00:00:00 Introductions +00:01:58 Iris's background and role +00:05:24 Discussion on Farfetch architecture +00:09:54 Importance of observability +00:12:36 OpenTelemetry journey at Farfetch +00:19:48 Observability tooling overview +00:27:20 Challenges in implementing OpenTelemetry +00:31:30 Discussion on traces and metrics +00:38:42 Contributions to OpenTelemetry +00:46:48 Future plans for OpenTelemetry at Farfetch -**Iris:** Thank you. I guess we can get started. It's a cozy group today, which is cool. I like cozy groups. It means we get to have a more intimate conversation. But like I said, there will be ways of consuming this information as well because what Edis has to say is awesome. She's very passionate about OpenTelemetry, so I'm very excited to have her join us. Edith works at Farfetch. +## Transcript -Do you want to do like a brief little intro for everybody? +### [00:00:00] Introductions -[00:01:00] **Edis:** For sure. Hello everyone, again. My name is Edis. I work as a platform engineer. Yeah, I know this title changes every time, so in my LinkedIn it could be something different. You know how it is. I'm a platform engineer, part of the observability team currently in Farfetch. I belong to the central team that provides tools for all the engineering teams across Farfetch to monitor their services, including traces, metrics, logs, and alerting. Again, I'm very excited to be here. +**Iris:** Thank you. I guess we can get started. It's a cozy group today, which is cool. I like cozy groups. It means we get to have a more intimate conversation. But like I said, there will be ways of consuming this information as well because what Edis has to say is awesome. She's very passionate about open Telemetry, so I'm very excited to have her join us. Edith works at Farfetch. Do you want to do like a brief little intro for everybody? -So we're just gonna do this like just a regular Q&A. And because we have such a small audience, also if we have time at the end, y'all are more than welcome to ask questions. You know, the purpose of this is to understand what Edis is doing at Farfetch around OpenTelemetry observability to help the rest of the community share use cases across the community so that we can all learn from each other, right? +**Edis:** For sure! Hello everyone. Again, my name is Edis. I work as a platform engineer. Yeah, I know this title changes every time, so in my LinkedIn it could be something different; you know how it is. I'm a platform engineer, part of the observability team currently at Farfetch. I belong to the central team that provides tools for all the engineering teams across Farfetch to monitor their services, including traces, metrics, logs, and alerting. Again, I'm very excited to be here. -That is that. So first things first, how did you come about to your current role at Farfetch? +We're just gonna do this like a regular Q&A. And because we have such a small audience, if we have time at the end, you all are more than welcome to ask questions. The purpose of this is to understand what Edis is doing at Farfetch around open Telemetry observability to help the rest of the community share use cases across the community so that we can all learn from each other, right? -**Edis:** So observability has been a part of a build-up for me. When I started my career, I actually started as a software engineer, a back-end developer, and because it was a type of company that offered service to different clients, I became a DevOps engineer. I was like, okay, I was put there. So I started working very small scale with monitoring, mostly in AWS and Azure with CloudWatch, a little bit with insights. And then it started becoming more of a passion for me, the more I learned about it. +### [00:01:58] Iris's background and role -Then I changed the position that I was in, and I started working in a more, let's say, well-equipped observability platform. And that's when I was like, okay, I really, really like this. I heard about OpenTelemetry for the first time and had my chance to touch it a little bit. Prometheus, Grafana, so I saw there was a lot of potential there. +So, first things first, how did you come about your current role at Farfetch? -And then where I currently am, of course, my previous experience really helped to come now. It's been one year and a little bit of learning and continuously evolving in observability. So now I think I've become pretty good at it. I started from zero. Yay, that's the best! +**Edis:** Observability has been a part of my build-up. When I started my career, I actually started as a software engineer, a back-end developer. Because it was a type of company that offered services to different clients, I became a DevOps engineer. I was like, okay, I was put there. So, I started working on a very small scale with monitoring, mostly in AWS and Azure, with CloudWatch and a little bit with Insights. It started becoming more of a passion for me the more I learned about it. Then I changed the position that I was in and I started working in a more, let's say, well-equipped observability platform. That's when I was like, okay, I really, really like this. -But how did you hear about OpenTelemetry specifically? Is it like one of those things where someone mentioned it to you, you saw it on the interweb somewhere? What's your story? +I heard about open Telemetry for the first time and had my chance to touch it a little bit, along with Prometheus and Grafana. I saw there was a lot of potential there. My previous experience really helped me come now, and it's been one year and a little bit of learning and continuously evolving in observability. Now, I think I've become pretty good at it. I started from zero. Yay! -**Edis:** I think it was LinkedIn somewhere. I know that I was working—we were working with no traces at the time. I know, a blasphemy. But we were looking into tracing solutions and somewhere like that I saw OpenTelemetry. So I was like, okay, I'm gonna give this a try and made a small POC for my manager. It never went more than that to the POC. It was almost more than one year ago. But that's how I heard about it. I really liked it. I had it at the back of my mind. +**Iris:** That's the best! How did you hear about open Telemetry specifically? Was it one of those things where someone mentioned it to you, or you saw it on the interweb somewhere? What's your story? -So now that there was an opportunity here in Farfetch, OpenTelemetry came up again. I was like, okay, I like that. I'm gonna go for it. +**Edis:** I think it was LinkedIn somewhere. I know that while we were working with no traces at the time, I know, a blasphemy, but we were looking into tracing solutions. Somewhere like that, I saw open Telemetry. I was like, okay, I'm gonna give this a try and made a small POC for my manager. It never went more than that to the POC; it was almost more than one year ago, but that's how I heard about it. I really liked it and had it at the back of my mind. So, now that there was an opportunity here at Farfetch, open Telemetry came up again, and I was like, okay, I like that. I'm gonna go for it. -So at Farfetch, what can you tell us a little bit about the architecture of the system that you're working with and why observability and OpenTelemetry are so important to that? +At Farfetch, can you tell us a little bit about the architecture of the system that you're working with and why observability and open Telemetry are so important to that? -[00:04:30] **Edis:** So I guess we'll start with like, let's give us a sense of what the architecture is. We are around 3,000 engineers currently in Farfetch, only in tech. We have an extremely complex architecture because we have different types of different sides of the business. So we have cloud-native, we have Kubernetes, and we have virtual machines, Ubuntu Linux, the three types of different cloud providers. +**Edis:** Sure! We are around 3,000 engineers currently at Farfetch, only in tech, and we have an extremely complex architecture because we have different types and different sides of the business. We have cloud-native, we have Kubernetes, and we have virtual machines—Ubuntu, Linux—the three types of different cloud providers. Every team, of course, we have uniformity and we have guidelines that need to be followed, but it's a lot of years in the process. Some things are still like in the past; every team decides to do things how they think is best. -Every team, of course, we have uniformity and we have guidelines that need to be followed, but it's a lot of years in the process and some things are still like in the past. Every team decides to do things the way they think is best. So it is a lot of information coming from everywhere, and it's not uniform. +### [00:05:24] Discussion on Farfetch architecture -For example, we were relying heavily on Prometheus to collect metrics, but some of our applications, some of our engineers, Prometheus was just not a good idea. So we were like, okay, what could be better? Here's OpenTelemetry. The same for the traces. And of course, this will help us not only collect these telemetry signals from places where we couldn't before and from services that were not possible, but also it's helping us put everything in a uniform way, which is amazing. +It is a lot of information coming from everywhere, and it's not uniform. For example, we were relying heavily on Prometheus to collect metrics, but some of our applications, some of our engineers thought Prometheus was just not a good idea. So, we were like, okay, what could be better? Here's open Telemetry. The same for the traces. Of course, this will help us not only collect these Telemetry signals from places where we couldn't before and from services that were not possible, but also it's helping us put everything in a uniform way, which is amazing. -[00:06:00] Very cool. And now what about your deployment process? What do you mean by building in the organization? What's your CI/CD pipeline like? Is it something that you're involved in to any extent or not really involved? +**Iris:** Very cool! What about your deployment process? What do you mean by building in the organization? What's your CI/CD pipeline like? Is it something that you're involved in to any extent or not really involved? -**Edis:** More in the sense that I use it a lot, and I'm in close communication. But yeah, we currently use Jenkins in Farfetch, and we have a separate team taking care of that, which tells again how complex everything is. We have teams and everything is segregated in my position, which is strictly observability—basically everything with observability, all the tools, deployments, maintenance, and releasing new features on top of what we have. +**Edis:** Not really involved, more in the sense that I use it a lot and I'm in class communication. But yeah, we currently use Jenkins at Farfetch, and we have a separate team taking care of that, which again tells how complex everything is. We have teams and everything is segregated. My position is strictly observability, basically everything with the durability— all the tools, deployments, maintenance, and releasing new features on top of what we have. -Okay, cool. So what observability tooling are you using? And you know, if you're using an observability vendor that you'd rather not divulge, that's totally okay. But just give us a sense of maybe your OpenTelemetry stack, how it's set up, that kind of thing. +**Iris:** Okay, cool! What observability tooling are you using? If you're using an observability vendor that you'd rather not divulge, that's totally okay. But just give us a sense of maybe your open Telemetry stack and how it's set up. -[00:02:10] **Edis:** Yeah, currently we're mostly open source and some things that have been created in-house by us, but mostly open source. We use Grafana for dashboards, we use Tempo as a data source, which is also part of Grafana, mostly for as a tracing back-end. That was something new. Sorry for the baby cats in the background. +**Edis:** Yeah, currently we're mostly open source with some tools that have been created in-house by us, but mostly open source. We use Grafana for dashboards, we use Tempo as a data source, which is also part of Grafana, mostly as a tracing back-end. That was something new. Sorry for the baby cats in the background! We use Prometheus and Thanos for metrics, and now we have added open Telemetry there as well. We used to have Jaeger, and we still do to a certain degree because we still haven't completely moved to open Telemetry when it comes to the frameworks that some of the teams are using. They are working hand in hand with Jaeger, and that's pretty much it. That's all that I can think of right now, but these are the main ones. -We use Prometheus and Thanos for metrics, and now we have added OpenTelemetry there as well. We used to have Jaeger, and we still do to a certain degree because we still haven't completely moved up on OpenTelemetry when it comes to the frameworks that some of the teams are using. So they're working hand in hand, Jaeger, and that's pretty much it. That's all that I can think of right now, but these are the main ones. +**Iris:** Cool! What about your organization's open Telemetry journey? We hear it's always a mixed bag, right? Some organizations are like giddy-up, more open Telemetry observability, and others are like... What was it like at Farfetch? -[00:08:10] Cool. And what about in terms of your organization's OpenTelemetry journey? We hear it's always a mixed bag, right? Some organizations are like, giddy up, more OpenTelemetry observability, and others are like, what was it like at Farfetch? +**Edis:** Well, I'm very proud to say actually that we have a very good observability culture at Farfetch. It was actually one of the ideas that was welcomed immediately. Okay, open Telemetry, let's do it, let's go for it! It took us a bit more time to make some space on our yearly plan for it, but the moment that it was mentioned, everyone was okay, let's jump to it. We only see positives there. We couldn't see any negatives other than the time spent, which is a necessity. It was very well accepted. I was surprised. I'm very happy, obviously. -**Edis:** Well, I'm very proud to say actually that we have a very good observability culture in Farfetch. It was actually one of the ideas that was welcomed immediately. Okay, OpenTelemetry, let's do it, let's go for it. It took us a bit more time to make some space on our yearly plan for it, but the moment that it was mentioned, everyone was like, okay, let's jump to it. We only see positives there. We couldn't see any negatives other than the time spent, which is a necessity. And yeah, it was very well accepted. I was surprised. I'm very happy, obviously. +**Iris:** That's amazing! It sounds like it came from the top then? -That's amazing! Yeah, so it sounds like it came from the top then. +**Edis:** Yeah, exactly. Of course, I don't want to mention names, but some of our senior leaders are very involved in the community, and they're always seeing new technologies and saying, "Hey, what do you think about this? Do you like it?" Of course, me, I'm extremely ambitious and I'm always on the internet, on LinkedIn, seeing new things to implement. It's a great combination there. -**Edis:** Yeah, exactly. Of course, I don't want to mention names, but some of our senior leaders are very involved in the community, and they're always seeing new technologies and saying, "Hey, what do you think about this? Do you like it?" And of course, me, I'm extremely ambitious, and I'm always on the internet on LinkedIn seeing for new things to implement. It's a great combination there. +**Iris:** Awesome! Now, in terms of your team starting to enable observability practices and open Telemetry, what did you and your team have to go through in order to make that happen? -Oh, awesome! Now in terms of your team starting to enable observability practices and OpenTelemetry, what did you and your team have to go through in order to make that happen? +### [00:09:54] Importance of observability -**Edis:** Well, the moment that I joined the team, at a point in time, I think the biggest struggle had already passed because observability became a very new thing three years ago in Farfetch. I think the people that struggled the most were the engineers that worked in the team back then. Of course, it was a new thing. Observability, why do we need it? Why is it so important? +**Edis:** Well, at the moment that I joined the team, I think the biggest struggle had already passed because observability became a very new thing three years ago at Farfetch. I think the people that struggled the most were the engineers that worked in the team back then. Of course, it was a new thing: observability. Why do we need it? Why is it so important? -[00:10:00] But I think at the point that I actually joined, everyone had already embraced how important observability is for everyone. Of course, that's an overestimation, but most of the engineers had already embraced it. So for us saying that, hey, we are implementing this amazing new thing, OpenTelemetry, that everyone had heard about, it was very popular right now and it was embraced immediately. And of course, knowing the benefits that they were getting from it—more metrics, more traces, more uniform way of collecting everything—it was a blast immediately. +But I think at the point that I actually joined, everyone had already embraced how important observability is for everyone. Of course, that's an overestimation, but most of the engineers had already embraced it. So, for us saying, "Hey, we are implementing this amazing new thing, open Telemetry," that everyone had heard about—it was very popular right now—it was embraced immediately. Knowing the benefits they were getting for it—more metrics, more traces, more uniform ways of collecting everything—it was a blast immediately. -And how did your team skill up in terms of being able to start implementing OpenTelemetry? Because it sounds like some people were kind of familiar with it. Maybe, I'm assuming it might have been newer to other folks. You said actually when the directive came that your team was gonna start implementing observability principles, OpenTelemetry, that was something that you know there was like work involved. So what was the kind of work that was involved in order to enable that for your team? +**Iris:** How did your team skill up in terms of being able to implement open Telemetry? Because it sounds like some people were kind of familiar with it; maybe it was newer to other folks. You said, actually, when the directive came that your team was gonna start implementing observability principles with open Telemetry, there was work involved. What was the kind of work that was involved in enabling that for your team? -**Edis:** Well, I think the biggest challenge—what we thought at the time was a challenge—was how to release this in patches and in parts so we could... we didn't do any damage or that the engineering teams that use observability every day didn't feel the change for the bad, of course. We would like them to see the improvement. +**Edis:** Well, I think the biggest challenge—what we thought at the time was a challenge—was how to release this in patches and in parts so we could do no damage and that the engineering teams that use observability every day didn't feel the change for the bad, of course. We would like them to see the improvement. -So we thought that that was going to be a challenge. Of course, we were going to move from one technology to the other. But actually, we always have someone who is a driver for the project, and in this case, in my team, it was me. So I did a thorough investigation, and every time I was reading something, I was like, wow, okay, this is cool. Wow, okay, this is amazing because everything managed to fit together very well for us. +We thought that was going to be a challenge. Of course, we were going to move from one technology to another. But actually, we always have someone who is a driver for the project, and in this case, in my team, it was me. I did a thorough investigation, and every time I was reading something, I was like, "Wow, okay, this is cool! Wow, okay, this is amazing!" Because everything managed to fit together very well for us. -Of course, the fact that we use open source really helps, with OpenTelemetry having compatibility with Prometheus, with Jaeger, and with everything that we were working with, it became easier over time to just put everything out there. +Of course, the fact that we use open source really helps with open Telemetry. It has compatibility with Prometheus, with Jaeger, with everything that we were working with. It became easier over time to just put everything out there. -So it sounds like you were the primary driver for getting people leveled up on OpenTelemetry. +**Iris:** It sounds like you were the primary driver for getting people leveled up on open Telemetry. -[00:12:30] **Edis:** Yeah, I'm proud of that. Together with the ones who were the biggest pushers and supporters of OpenTelemetry, and now we're in production. So I think we were pretty successful. +**Edis:** Yeah, I'm proud of that! Together with the ones who were the biggest pushers and supporters of open Telemetry, and now we're in production. I think we were pretty successful. -That is super cool! I don't know that we hear too many stories of organizations using OpenTelemetry in production, so I think that's really awesome. And how long did it take you guys to get to the point where you've got OpenTelemetry in production? +### [00:12:36] OpenTelemetry journey at Farfetch -**Edis:** So let me see. We planned to start implementing OpenTelemetry around January, and we started the first investigation gathering information. I think by mid-March, we were already ready in production. But again, we are still not there, not 100%. We're still relying on a lot of things with Prometheus and Jaeger. We're just using OpenTelemetry to transport. We still need to do instrumentation with OpenTelemetry. Some parts of our engineering teams are still not using it, but currently, OpenTelemetry is our main transport of our telemetry signals, basically. +**Iris:** That is super cool! I don't know that we hear too many stories of organizations using open Telemetry in production, so I think that's really awesome. How long did it take you guys to get to the point where you've got open Telemetry in production? -Are you relying heavily on the OpenTelemetry collector then to do that for you? +**Edis:** Let me see. We planned to start implementing open Telemetry around January, and we started the first investigation, gathering information. I think by mid-March, we were already ready in production. But again, we are still not 100% there. We're still relying on a lot of things with Prometheus and Jaeger. We're just using open Telemetry to transport. We still need to do instrumentation with open Telemetry. Some parts of our engineering teams are still not using it, but currently, open Telemetry is our main transport for our technology signals, basically. -**Edis:** Yeah, especially with traces. Traces were one of the neglected parts of the observability stack, you know. It's typical because traces are a bit, let's say, more modern and it takes more time to implement and to actually understand how good they are. Now traces are becoming the eat of observability. So it really helped us. OpenTelemetry really helped us get the best of the tracing in Farfetch. +**Iris:** Are you relying heavily on the open Telemetry collector then to do that for you? -We were actually doing some numbers today with our architect, and we were moving around 1,000 spans per second in the past, and now we have 40,000 that flawlessly without even needing to lift a finger. And we're not there yet. We still have a lot to do there. +**Edis:** Yeah, especially with traces. Traces were one of the neglected parts of the observability stack. It's typical because traces are a bit more modern and it takes more time to implement and to actually understand how good they are. Now traces are becoming the heart of observability, so it really helped us. Open Telemetry really helped us get the best of tracing at Farfetch. We were actually doing some numbers today with our architect, and we were moving around 1,000 spans per second in the past, and now we have 40,000 that flawlessly without even needing to lift a finger. We're not there yet; we still have a lot to do there. -And so how are you collecting your traces right now? Is it like manual instrumentation, auto instrumentation, a combination of both? Where are y'all at with that? +**Iris:** How are you collecting your traces right now? Is it like manual instrumentation, auto instrumentation, a combination of both? Where are you all at with that? -[00:14:40] **Edis:** It's a combination of both. So we're currently doing... We have another team working with us that helps provide the instrumentation of the frameworks because we have a huge variety of languages. So we still have the OpenTracing framework that teams are still using. We also have some teams using manual instrumentation, and we're also implementing the OpenTelemetry operator with auto instrumentation, mostly for .NET and Java currently, but looking forward for Go because we have a lot of applications running in Go. +**Edis:** It's a combination of both. We currently have another team working with us that helps provide the instrumentation of the frameworks because we have a huge variety of languages. We still have the open tracing framework that teams are still using. We also have some teams using manual instrumentation. We're also implementing the open Telemetry operator with auto instrumentation, mostly for .NET and Java currently, but looking forward for Go because we have a lot of applications running in Go. Our main goal is that very soon we want to have all applications using open Telemetry instrumentation, but it's going to be a process, as slow and as fast depending on the team's pace. -So our main goal is that very soon we want to have all applications using OpenTelemetry instrumentation, but it's going to be a process, as slow and as fast depending on the team's pace. +**Iris:** For the teams who are using Go—which to my understanding there is no auto instrumentation—is there any kind of support that your team provides around that in terms of helping these teams instrument? -So for the teams who are using Go, which to my understanding there is no auto instrumentation, is there any kind of support that your team provides around that in terms of helping these teams instrument? +**Edis:** Yeah, of course! One of the main teams that this is Go, especially because of the open source, is my team. We provide documentation, we provide guidelines, and there is a lot of very good documentation in the open Telemetry as well about manual instrumentation. So basically, we have sessions with engineers—not to train them because I think they can do it better than we can because it's their code—but just to show them the best practices and to introduce them to that and to tell them that our tools are here for you, and they take it from there. It's a team sport, let's say. -**Edis:** Yeah, of course. One of the main teams that this is for Go, especially because of the open source, it's my team. But yeah, of course, we provide documentation, we provide guidelines, and there is a lot of very good documentation in the OpenTelemetry as well about manual instrumentation. So basically, we have sessions with engineers—not to train them because I think they can do it better than we can because it's their code—but just to show them the best practices and to introduce them to that and to tell them that, hey, our tools are here for you, and they take it from there. It's a team sport, let's say. +**Iris:** Awesome! I love that because I think what you said is really important: that you're not instrumenting their code for them because it's their code, but that you provide the guidance and guidelines on the instrumentation, which I think is super important. Also, I want to call out, Ubuntu just shared a link in the chat that indicates that there is auto instrumentation for Go, so yay! -Awesome! I love that. Because yeah, I think what you said is really important that you're not instrumenting their code for them because it's their code, but that you provide the guidance on the instrumentation, which I think is super important. +**Edis:** Something to look at! -Also, I want to call out, Ubuntu just shared a link in the chat that indicates that there is auto instrumentation for Go. So yay! Something to look at. +**Iris:** Yeah, it's a more recent thing, and it's still a work in progress, but I came across this last week only. Since you mentioned Go, I was like, okay, go take a look if there's something interesting. -**Edis:** Yeah, it's a more recent thing, and it's still a work in progress, but I came across this last week only. Since you mentioned Go, I was like, okay, go take a look if there's something interesting. +**Edis:** No, I've been following it as well actually because it's my stack, and the things that I usually want to go. I'm like, every day, is there news? Is there news? It's become like a passion now! -I've been following it as well, actually, because it's my stack and the things that I'm—it's very close to me that I usually want to use Go. So I'm like, every day, is there news? Is there news? +**Iris:** Very cool! I want to ask about the auto instrumentation because, you know, keeping on that thread, I think you mentioned you're leveraging auto instrumentation for Java through the open Telemetry operator. Can you share a little bit about your experience around using the open Telemetry operator? I came across it like maybe a couple months ago, and for me, I was like, this thing is amazing! -**Iris:** That's very cool! Now I want to ask on the auto instrumentation because, you know, keeping on that thread, I think you mentioned you're leveraging auto instrumentation for Java through the OpenTelemetry operator. Can you share a little bit of your experience around using the OpenTelemetry operator? +**Edis:** Yeah, it's cool to hear someone who's actually using it. Is that in production right now for you? -I came across it like maybe a couple of months ago, and for me, I was like, this thing is amazing! So yeah, like, it's cool to hear someone who's actually using it. Is that in production right now for you? +**Edis:** Yes, it is partly in production, but it's very isolated. It's not available for everyone. The operator is amazing. We loved it when we just discovered it; we were like amazed. The thing is that it is a bit getting used to. We had some challenges when we started—well, we still do—but especially in the beginning because the operator and the instrumentation object and the collector, well, it was my bad obviously. -**Edis:** Yes, it is partly in production, but it's very isolated. It's not available for everyone. And yeah, the operator is amazing. We loved it when we just discovered it. We were like amazed. +We were having some certificate issues and trying to rotate certificates, and I was trying to delete something, and they just deleted and created. It was like a big mess! I would just leave my computer and come back because of how coupled the operator and the collector and its orientation are. But yeah, now we've gotten the hang of it, and it is amazing. -The thing is that it is a bit getting used to. We had some challenges when we started—well, we still do—but especially in the beginning because the operator and the instrumentation object and the collector, well, it was my bad, obviously. We were having some certificate issues and trying to rotate certificates. I was trying to delete something, and I just deleted and created—it was like a big mess. I would just like leave my computer and come back because of how coupled the operator and the collector and its orientation is. +One other challenge that I didn't think of—that's the beauty of it—we were having a discussion with another engineer, and I was complaining that Prometheus cannot target the operator. The collector and the operator, for some reason, and we cannot create alerts from it. He told me, "Well, have you tried using open Telemetry to send everything to Prometheus?" I'm like, hmm! It's the beauty of it; it's compatible! -But yeah, now we've gotten the hang of it, and it is amazing. One other challenge that I didn't think of—that's the beauty of it—we were having a discussion with another engineer, and I was complaining that Prometheus cannot target the operator, the collector, for some reason, and we cannot create alerts from it. +I guess there's challenges every day. One day it's time! -He told me, "Well, have you tried using OpenTelemetry and then sending everything to Prometheus?" And I'm like, hmm. It's the beauty of it. You know, it's compatible. +### [00:19:48] Observability tooling overview -Yeah, I guess there's challenges every day. One day it's time. +**Iris:** That's very cool! Speaking of, you mentioned you're ingesting traces. I know the log signal is a newer player in the land of open Telemetry. Have you and your team or anyone at Farfetch started playing around with open Telemetry logging as well? -That's very cool! And speaking of like, you mentioned you're ingesting traces. I know the log signal is like a newer player in the land of OpenTelemetry. Have you and your team or anyone at Farfetch started playing around with OpenTelemetry logging as well? +### [00:46:48] Future plans for OpenTelemetry at Farfetch -**Edis:** Very, very little. Mostly consuming from a Kafka topic and see how that goes. It's pretty good, but we know that this is not there yet, and it's not stable. So we don't expect it to go into production or to have it part of the day-to-day because currently we have a huge volume of logs going through Farfetch, more than places, so a lot more. +**Edis:** Very little, mostly consuming from a Kafka topic and seeing how that goes. It's pretty good, but we know that this is not there yet, and it's not stable. So we don't expect it to go into production or to have it part of the day-to-day because currently we have a huge volume of logs going through Farfetch—more than places. So yeah, it's not worth risking, but we're definitely experimenting, and we expect in one year from now open Telemetry will be the only receiver that we're going to use for everything. -So yeah, it's not worth risking, but we're definitely experimenting. We expect in one year from now OpenTelemetry will be the only receiver that we're going to use for everything. +**Iris:** Cool! For the little experiment that you've done in using open Telemetry logs, have you taken advantage of the log-to-trace correlation, or is that just the logs in isolation? How's that been going? -Oh cool! For the little experiment that you've done in using OpenTelemetry logs, have you taken advantage of the log-to-trace correlation, or is that just the logs in isolation? How's that been going? +**Edis:** Actually, no, that's a very good suggestion because I've been focusing mostly just on getting things across and mostly the processing and the obfuscation of the data. That's something that we would like to use open Telemetry for, which is amazing. But no, that's something that it's definitely worth for me to investigate into next. -**Edis:** Actually, no. That's a very good suggestion because I've been focusing mostly just getting things across and mostly the processing of the obfuscation of the data because that's something that we would like to use OpenTelemetry, which is amazing for. But no, that's something that it's definitely worth for me to investigate into the next. +**Iris:** Awesome! What about the metric signal? How are you ingesting the metrics? Is Prometheus passing the metrics over to you? Are you using the Prometheus receiver? I know from personal experience when I started playing around with the Prometheus receiver, it was barely a thing. It was so unstable; there was a disclaimer. I'm just wondering how you're using the Prometheus receiver and if so, what's been your experience around that? -Awesome! What about the metric signal? How are you ingesting the metrics? Is Prometheus passing the metrics over to you, and are you using the Prometheus receiver? I know like from personal experience when I started playing around with the Prometheus receiver, it was like barely a thing. It was like so unstable. There was a disclaimer. I'm just wondering how you use the Prometheus receiver and if so, what's been your experience around that? +### [00:38:42] Contributions to OpenTelemetry -**Edis:** Yes, actually, we use the Prometheus receiver. So throughout the instrumentation, just to get this out of the way, we do get some OTLP metrics as well, but the majority of our metrics is Prometheus. So the Prometheus receiver works very good for us because we already had an observability system in place, and we use Console for target control. So it was extremely easy for us to use the receiver, and it can handle a huge amount of data. +**Edis:** Yes, actually, we use the Prometheus receiver. So throughout the instrumentation, just to get this out of the way, we do get some OTLP metrics as well, but the majority of our metrics is from Prometheus. The Prometheus receiver works very well for us because we already had an observability system in place, and we use Console for target control. So it was extremely easy for us to use the receiver, and it can handle a huge amount of data. The scrape configs are the same as in Prometheus, so technically nothing changes. You're just using another tool to scrape all these metrics. It is very straightforward for us, and currently, we're using both open Telemetry for some scenarios and Prometheus. But again, in the future, the goal is to use only open Telemetry. It's just that it's a process. -The scrape configs are the same as in Prometheus, so technically nothing changes. You're just using another tool to scrape all these metrics. It is very straightforward for us, and currently, we're using both OpenTelemetry for some scenarios and Prometheus. But again, in the future, the goal is to use only OpenTelemetry; it's just that it's a process. +**Iris:** Do you think then the Prometheus receiver will be the way to ingest all of your metrics, and so you'll be able to scrap Prometheus altogether as your end goal? -Do you think then the Prometheus receiver will be the way to ingest all of your metrics and so you'll be able to scrape Prometheus altogether as your end goal? +**Edis:** I would say so, but it would be maybe the majority because we have a lot of our services running on virtual machines as well, and we have Prometheus exporters there. It's best that those, let's say, stand remain in touch, and it's going to be more difficult to adapt them. But when it comes to our Kubernetes, I think we're going to go more with open Telemetry because everything is Prometheus, but we have the Kubernetes SD configs for the Prometheus receiver, which is amazing. We also have the target control target allocator through the operator, which I have been testing, and I really liked it. So I think we're going to go completely, especially on Kubernetes and in the cloud, with open Telemetry. -**Edis:** I would say so, but it would be maybe the majority because we have a lot of our services running on virtual machines as well, and we have Prometheus exporters there. So it's best that those, let's say, remain in touch, and it's going to be more difficult to adapt them. But when it comes to our Kubernetes, I think we're going to go more with OpenTelemetry because everything is Prometheus, but we have the Kubernetes SD configs for the Prometheus receiver, which is amazing. +**Iris:** Going back to Kubernetes, how many clusters are you typically involved with that you're observing? -We also have the target control target allocator through the operator, which I have been testing, and I really liked it. I think we're going to go completely, especially on Kubernetes and in the cloud. +**Edis:** I would say maybe a hundred in total for different data centers. Yes, and thousands of virtual machines. We have a huge stack, so your team is responsible then for ensuring that if there's an issue with the operator, you and your team are managing the open Telemetry operator across all these Kubernetes clusters? -Cool! And going back to Kubernetes, how many clusters are you typically involved that you're observing? +**Edis:** Exactly! That's why we're also trained in Kubernetes—all of us that are part of the team—because it's a very important part of our job to actually maintain everything. -**Edis:** I would say maybe a hundred in total for different data centers. Yes, and thousands of virtual machines. We have a huge stack. - -So your team is responsible then for ensuring that if there's an issue with the operator, you and your team are managing the OpenTelemetry operator across all these Kubernetes clusters? - -**Edis:** Exactly. That's why we're also trained in Kubernetes, all of us that are part of the team, because it's a very important part of our job to actually maintain everything. - -**Iris:** Oh nice! Nice! Okay, so you wear multiple hats. So you maintain the clusters as well, is what it sounds like? - -**Edis:** Or I don't know. - -Okay, cool! And I seem to recall—I want to say like version 1.26 of Kubernetes—they enabled some OpenTelemetry capabilities as an experimental feature. Is that something that you've ever dabbled with or heard of? Just curious. +**Iris:** Nice! I seem to recall, I want to say like version 1.26 of Kubernetes enabled some open Telemetry capabilities as an experimental feature. Is that something that you've ever dabbled with or heard of? Just curious. **Edis:** Not yet, actually. No, I haven't come across it, but something that I can note and test. There's always a lot to learn. -**Iris:** Yeah, I'll see if I can find a link around that. I feel like it was a very niche thing that was not often talked about, but I'd love to hear if anyone's played around with that. - -Now, for me, one of the things that I always love to hear is how teams are structuring their collectors. Do you have one collector, multiple collectors? If so, how are you deploying them? - -**Edis:** So different configurations. Yeah, we currently have an agent and a central collector type of organization. Well, as I mentioned, we have especially when it comes to Kubernetes clusters, we have a huge number of them. So having just one point of interest is going to be overwhelming. - -So we are deploying—well, currently we have one OpenTelemetry agent in each of the clusters, and we are starting to substitute it with OpenTelemetry operator, which has the collector and auto instrumentation. So that's the end goal. That's where we're getting it. So everything is sent to a central collector where we do the observability of data, the sampling, and everything, and all that is sent. Currently, we're using Tempo, but in the future, we might use a vendor. - -Basically, there's going to be a central collector collecting all that per data center. - -Okay, so you basically like each Kubernetes cluster has its own collector, and then they feed into your central collector. Where does your central collector reside? Does it reside on a VM? Does it reside on another Kubernetes cluster? - -**Edis:** It's a Kubernetes cluster, and it's... yeah, we call it the central collector because most of the stock on that cluster is dedicated to us. We have a lot of data, so we have a lot of requirements for memory. So most of the applications running in this cluster are for observability and very few for the platform. - -So yeah, that's where everything is. - -Okay, cool! And then how do you ensure, you know, if everything's being sent to that central collector, that becomes a single point of failure. So how do you ensure that, you know, if that goes down, what's your backup plan? - -**Edis:** Well, we have fallback clusters with fallback collectors. So if this one fails, it goes to the other one immediately. We have also implemented—well, it's not currently running, but we also have implemented the—you know that the OpenTelemetry collectors can send to as many exporters as possible? So we are currently using a fallback cluster to send, for example, if one fails, send to the other one, and we can immediately enable it without an issue. - -It's like a background. Yeah, and we have equipped our collectors with very—we well, we use auto-scaling a lot. It's based on our metrics, so we will make sure that our collectors have a lot of memory and CPU liberty so their queues will be big as well. If there is a small downtime, everything will be saved into the queue and then sent to the central collector. - -Awesome! And on that central collector—well, speaking like continuing on the collector thread—what were some of the challenges that you experienced initially when you started implementing the collector? Because I would imagine, you know, you said that you want to make sure you have enough memory allocated. I'm assuming that might have been perhaps an initial challenge. Are there any others, or can you talk more about that? - -**Edis:** Well, yeah, I think knowing the collector and how it works is a new technology introduced to us was the biggest challenge. We're very fortunate in Farfetch because we rely heavily on auto-scaling. So the first thing that we did was to enable auto-scaling. But yeah, we had to do some tests to see how it was working with a small amount of data because again, it was completely new, and it took us a while to know the memory and CPU requirements. - -So at the same time, we're not wasting a huge amount of money just to have this available, but at the same time, we need to have this available because it is so crucial. So yeah, definitely the resources, memory, and CPU are very important. Everything else, I would say the Helm charts in the community are so good. We just needed to use them, enable it, and of course modify everything, the configuration basically, the exporters and receivers, and that's it. But yeah, it was pretty straightforward when it comes to that. - -Okay, and in terms of configuration, are you using any processors to do any data masking or to add attributes, remove attributes? Do you have custom processors? What's your processor story like on the collector? - -**Edis:** Well, we're currently experimenting with processors. I think we only have the batch processor or whatever it is enabled by default on the charts. But we are playing a lot with the data masking processors because especially for the logging part in tracing and in metrics, we do not have any data that could be sensitive, but for logging, that's something that we are relying on heavily, and that's what we're testing the most. - -But currently, we haven't implemented anything special for the telemetry data that we're currently passing. - -Cool, cool! And I was curious because, you know, you mentioned like you're using traces, you're using metrics, playing around with logs. Within traces, are you aware of any teams using span events, for example? - -**Edis:** No, not yet. And that's something that we are really, really planning to introduce currently because of the limitations that we had with our previous tracing system. We had a very low sampling; it was 0.1 percent. The interest in the teams was not big. They didn't really care much about tracing. It was very rare to find an engineer that relied on tracing. - -So now that we implemented Tempo and OpenTelemetry, we are gradually increasing the sampling size and allowing more information to come through. I think we are getting better at it, but still not there. That's part of our package of making traces first-class citizens. - -That's what I love to hear! And I don't think this is the thing yet now, but my understanding in talking to a few of the OpenTelemetry folks is that the idea is that the logs are going to replace the span events because, I mean, span events are basically like logs embedded in your traces anyway. But with the idea that, obviously, you continue having that correlation, but I believe my understanding is also you get access to the fact that the logs specification is a lot richer than the span events specification, so you get to take advantage of having more information potentially at your disposal. - -So that's kind of a thing that I'm looking forward to personally. I noticed that Sebastian just posted a link in the chat regarding the traces for Kubernetes cluster in version 1.27. So for anyone, I'm not sure if that was the one that you were speaking of. I was just trying to find a reference. - -**Edis:** Yeah, yeah, I believe that is the one. I believe that is the one. - -Yeah, that's super cool! I do seem to remember that for enabling it, you have to go into deep, deep in the bowels of Kubernetes configuration to be able to enable that feature. So I think if you were using a cloud provider, do so at your own risk kind of thing. I think it was a lot easier to use if you're doing Kubernetes locally on your machine. But anyway, still a cool feature nonetheless. - -But I definitely think it's something that is worth exploring as it matures. Just going back to our discussion, I had a question also with regards to have you encountered any folks who were resistant to this whole OpenTelemetry thing? Like on development teams or even within your own team? What was the vibe around that? And if so, what did you do to help alleviate their stress? - -**Edis:** To be honest, not really. It's curious, actually. I haven't really met anyone that opposes it. Yeah, I've met plenty of people that simply do not care for it. Like they're like, okay, we have it, it's okay, we do not have it. - -In this situation, it's mostly, for example, someone asks for something related to observability and traces, and I'm like, hey, look at this cool thing that we did with OpenTelemetry. Now it is available for you, and they're like, okay. I just keep sending things that I consider that are so cool and it's good for them to use, and that's pretty much it. I know that they're going to use it because it's a but never had someone that was against it or like, no, we don't need it. - -It's interesting! - -**Iris:** Awesome! I think we need more companies like yours where people are like, yeah, if it's a lemon tree. - -**Edis:** Yeah, we have a very good observability culture. I'm actually really, really proud of that. I mean, I'm proud because I'm helping continue and build it, but the previous team where that worked for that could also... +**Iris:** I'll see if I can find a link around that. I feel like it was a very niche thing that was not often talked about, but I'd love to hear if anyone's played around with that. Now, one of the things that I always love to hear is how teams are structuring their collectors. Do you have one collector, multiple collectors? If so, how are you deploying them? -**Iris:** That's awesome! Yeah, and I think like that's where we really see success in OpenTelemetry is always having like a group of people who are like amazingly enthusiastic about it and are just like out there and believe in it and want to make sure that it happens. +**Edis:** We currently have an agent and central collector type of organization. Well, as I mentioned, we have especially a huge number of Kubernetes clusters, so having just one point of interest is going to be overwhelming. We are deploying... Well, currently, we have one open Telemetry agent in each of the clusters, and we are starting to substitute it with open Telemetry operator, which has the collector and auto instrumentation. So that's the end goal; that's where we're getting it. Everything is sent to a central collector where we do the obfuscation of data and the sampling and everything. All that is sent—currently we're using Tempo, but in the future, we might use a vendor. Basically, there's going to be a central collector collecting all that per data center. -So, you know, and I know like Shubanchu, who's also on the call, like he's done a lot of evangelism around OpenTelemetry in his organization, which is super awesome. So, you know, hats off to y'all who do that in your organizations because I think it's gonna keep helping make OpenTelemetry awesome. +**Iris:** So you basically have each Kubernetes cluster has its own collector, and then they feed into your central collector. Where does your central collector reside? Does it reside on a VM, or does it reside on another Kubernetes cluster? -Now, is your organization at the point now where you've started to like make contributions to OpenTelemetry, or is that still something where you're like not there yet? +**Edis:** It's a Kubernetes cluster. We call it the central collector because most of the stock on that cluster is dedicated to us. We have a lot of data, so we have a lot of requirements, so memory-wise, most of the applications running in this cluster are for observability and very few for the platform. -**Edis:** Yes, actually we made the contribution recently to the operator because of the certification. I think the certification was relying on search manager and not with custom certificates, and that didn't work for us. So there was a feature request and then a request to fix that, and that's why we are working so heavily now with operators so we can use our own certificates basically until search manager is available for us as well. +### [00:27:20] Challenges in implementing OpenTelemetry -That is so cool! How did it feel like making the contribution? +**Iris:** Cool! How do you ensure that if everything's being sent to that central collector, that becomes like a single point of failure? What’s your backup plan? -**Edis:** It was great! Well, actually, it was a joint effort. Our architect was the main figure, let's say, after this. But yeah, it feels great, and the community is super welcoming, and like it was approved so fast because when he submitted the feature request, we were thinking that it was going to take like months or like, okay, yeah, we will have to wait. And then a week later he's like, hey, guess what? It's more! Let's go forward and test it. It's amazing! +**Edis:** Well, we have fallback clusters with fallback collectors. If this one fails, it goes to the other one immediately. We have also implemented—well, it's not currently running—but we also have implemented the fact that the open Telemetry collectors can send to as many exporters as possible. So we are currently using a fallback cluster to send, for example, if one fails, send to the other one, and we can immediately enable it without an issue. It's like a background. -That is so cool! I'm always like a huge fan of that, you know, like don't wait around for the feature request, just do it yourself. So it's really cool that you and your team were able to achieve that. I think that's like, you know, I think it's a great accomplishment because like putting yourself out there for open source is like you have to be vulnerable, and you have to be okay with people saying, like, well, that's not really correct. +We have equipped our collectors with auto-scaling a lot; it's based on our metrics. We will make sure that our collectors have a lot of memory and CPU liberty so their queues will be big as well. If there is a small downtime, everything will be saved into the queue and then sent to the central collector. -It's scary, right? So yay, congrats! That's super amazing! I hope the team continues to make OpenTelemetry contributions. One thing I wanted to pivot to is, you know, like I am so happy that you have awesome things to say about OpenTelemetry. Is there anything that you think you and your team have encountered where OpenTelemetry could improve? Because that's, you know, that feedback is also super important so that we can continue to improve as a whole. +**Iris:** Awesome! Speaking of the central collector, what were some of the challenges you experienced initially when you started implementing the collector? I would imagine, you said that you want to make sure you have enough memory allocated. I'm assuming that might have been perhaps an initial challenge. Are there any others, or can you talk more about that? -**Edis:** Well, to be honest, I've had a super positive experience with it. Working with it at every step of the way has been super easy, and there's been a lot of support from the community. The only thing that I would say maybe is that it is a bit lacking in documentation in some parts. For example, when I was implementing the Prometheus receiver first, it was very confusing to use. I was using the console, the console SD config, and it was a mess. I couldn't find information anywhere on how to implement it or some good documentation. +**Edis:** Yeah, I think knowing the collector and how it works was a new technology introduced to us. We were very fortunate at Farfetch because we rely heavily on auto-scaling. The first thing that we did was to enable auto-scaling. But yeah, we had to do some tests to see how it was working with a small amount of data because again, it was completely new, and it took us a while to know the memory and CPU requirements. -Of course, I'm planning to contribute to that, but that's all that I would have to say. It takes a bit longer to figure it out, or you have to dig very, very deep to find some documentation for certain features, exporters, or receivers. But other than that, it's been an amazing experience. +At the same time, we were not wasting a huge amount of money just to have this available, but at the same time, we need to have this available because it is so crucial. So yeah, definitely the resources—memory and CPU—are very important. Everything else I would say, the Helm charts in the community are so good. We just needed to use them, enable it, and of course modify everything—the configuration, basically the exporters and receivers—and that's it. But yeah, it was pretty straightforward when it comes to that. -Cool, cool! Awesome! Yeah, I have to agree with you; sometimes it is a bit of an archaeological dig. So anything that can be done to help bring that up to the surface is most welcome. We definitely look forward to a contribution to the docs. +**Iris:** In terms of configuration, are you using any processors to do any data masking or to add attributes or remove attributes? Do you have custom processors? What's your processor story like on the collector? -Also, for anyone on here who's played around with OpenTelemetry, you can always submit a pull request to the OpenTelemetry.io repo if you want to write a blog post about anything OpenTelemetry-related. Because I know the comms folks are always looking for contributions. +**Edis:** Well, we're currently experimenting with processors. I think we only have the batch processor or whatever it is enabled by default on the charts, but we are playing a lot with the data masking processors. Especially for the logging part, in tracing and in metrics, we do not have any data that could be sensitive, but for logging, that's something that we are relying on heavily, and that's what we're testing the most. But currently, we haven't implemented anything special for the Telemetry data that we're currently passing. -And so also if you're looking for a first contribution on OpenTelemetry, that is a great place to start. +**Iris:** I was curious because you mentioned you're using traces, you're using metrics, and playing around with logs. Within traces, are you aware of any teams using span events, for example? -Now, you know, let's turn the tables around to our lovely audience here today. Does anybody have any questions for Edis? +### [00:31:30] Discussion on traces and metrics -**Audience Member:** No burning questions? We all good? +**Edis:** No, not yet. That's something that we are really, really planning to introduce currently. Because of the limitations that we had with our previous tracing system, we had a very low sampling; it was 0.1 percent. The interest in the teams was not big; they didn't really care much about tracing. It was very rare finding an engineer that relied on tracing. So now that we implemented Tempo with open Telemetry, we are gradually increasing the sampling size and allowing more information to come through. I think we are getting better at it, but still not there. That's part of our package of making traces first-class citizens. -**Iris:** I haven't even wanted to say thank you for doing this. I appreciate it. +**Iris:** That's what I love to hear! I don't think this is the thing yet now, but my understanding in talking to a few of the open Telemetry folks is that the idea is that the logs are going to replace the span events because span events are basically logs embedded in your traces anyway, but with the idea that you continue having that correlation. But I believe my understanding is also you get access to the fact that the logs specification is a lot richer than the span events specification, so you get to take advantage of having more information potentially at your disposal. So that's kind of a thing that I'm looking forward to personally. -**Audience Member:** Yeah, I will second that to both of you. I think you covered a lot, Adriana and Edis. You gave pretty good insight into her company. It's amazing to see how little objections there are to change, not just observability, but change in general. So it's nice to see. Almost wish like, can I have some of it? +I noticed that Sebastian just posted a link in the chat regarding the traces for Kubernetes clusters in version 1.27. For anyone who is not sure, I believe that is the one I was referring to. -I've been grinding for the last year basically talking observability and OpenTelemetry non-stop, and I finally made some inroads, I think. But yeah, it's nice to see that there are easier companies or easier paths. Not everybody has struggles the same. It's good to know. +**Edis:** Yeah, I believe that is the one! That's super cool! I do seem to remember that for enabling it, you have to go deep into the bowels of Kubernetes configuration to be able to enable that feature. So, I think if you were using a cloud provider, do so at your own risk kind of thing. I think it was a lot easier to use locally on your machine, but anyway, still a cool feature nonetheless. I definitely think it's something that is worth exploring as it matures. -**Edis:** Yeah, that's so true! I think my experience in the past has also been like it's the uphill battle to OpenTelemetry. So it is so refreshing to hear like such a lovely positive story that there is light at the end of the tunnel. There are people who get it. +Going back to our discussion, I had a question regarding whether you encountered any folks who were resistant to this whole open Telemetry thing, like on development teams or even within your own team. What was the vibe around that? If so, what did you do to help alleviate their stress? -**Audience Member:** There isn't it, but now it's becoming—I think it is the second most contributed project and one of the most well-known. So more people are getting a smell of it, so it's getting more and more accepted, I'd say, than it was one year ago. +**Edis:** To be honest, not really. It's curious actually. I haven't really met anyone that opposes it. I've met plenty of people that simply do not care for it—like they're like, "Okay, we have it, it's okay, we do not have it." In this situation, it’s mostly, for example, someone asks for something related to observability and traces, and I'm like, "Hey, look at this cool thing that we did with open Telemetry. Now it is available for you," and they're like, "Okay." I just keep sending things that I consider are so cool, and it's good for them to use, and that's pretty much it. I know that they're going to use it because it's a good thing. I've never had someone that was against it, or like, "No, we don't need it." -**Edis:** Yeah, so true! Like when I started dabbling around in OpenTelemetry, the traces specification wasn't even finalized. I was pushing the organization I was at the time to be like OpenTelemetry is going to be the big thing, y'all! And they're like, uh-huh. So it was an uphill battle. I feel like now at least like a lot of the specs are finalized, so it makes for a very compelling narrative, and you get more and more user stories of people actually using this in production, even if it's like, you know, kind of like where Farfetch is at—where you're not fully productionalized, but you've got some stuff running in production. +**Iris:** That's interesting! Awesome! I think we need more companies like yours where people are like, "Yeah, if it's open Telemetry." -And I think that's a compelling story to tell as well, right? Which is like get it out there, start using it! +**Edis:** Yeah, we have a very good observability culture! I'm actually really proud of that. I'm proud because I'm helping to continue building it, but the previous team where I worked helped with that too! -I think that's a really great message to share with folks. Do you have any parting thoughts? +**Iris:** That's awesome. I think that's where we really see success in open Telemetry—having a group of people who are amazingly enthusiastic about it and just believe in it and want to make sure that it happens. -**Edis:** I was gonna say something as well. We're, yeah, it's not fully there, but we're passing a crazy amount of data after those collectors, and they're tough. Trust me, they're tough. It's worth it. It might be difficult to convince your peers to start implementing it, but it's worth it. They are durable collectors! You can pass thousands and millions of data per second, per minute. +Do you have any parting thoughts? -Yeah, if you do a good job with auto-scaling, Eden's gonna manage how much memory and CPU are they using while doing that kind of load. Do you know offhand? +**Edis:** I was going to say something as well. We're not fully there, but we're passing a crazy amount of data after those collectors, and they're tough! Trust me, they're tough! It's worth it. It might be difficult to convince your peers to start implementing it, but it's worth it. They are durable collectors; you can pass thousands and millions of data per second per minute. If you do a good job with auto-scaling, you're going to manage how much memory and CPU they are using while doing that kind of load. -**Edis:** When we were tracing around 30,000 spans per second, I think we were like at around eight gigabytes of memory, and I don't know how many instances of the collector we were running, but maybe like four. But around eight gigabytes. +**Iris:** Thank you! -Cool, thank you! +**Sebastian:** One question from your earlier conversation about using open Telemetry metrics and open Telemetry tracing: are you guys using correlation in any way in terms of the metric data, trace data, or log data at all? Or is that still something you are exploring? -**Iris:** Any other questions or comments for Edis? +**Edis:** Something that we're still exploring. Our first traces-metrics correlation that we're doing, we're investigating through open Telemetry, but it was easier for us to implement it through Tempo. They have a metrics generator, and that's what we were doing—generating some metrics from traces. But we're planning to move that to open Telemetry because it's best. It was just easier for us that way. -Well, if you like what you heard today, you just will be back for our OpenTelemetry in practice session on June the 8th, and it's gonna be, I believe, at the same time. So keep an eye out for an invite then. +**Sebastian:** Thank you! -Edis, why don't you tell folks what you'll be presenting about? +**Iris:** Do you have any other questions or comments for Edis? -**Edis:** Well, my topic will be observability as a team sport. It's something that I'm extremely passionate about, and I see that in many companies, observability teams are the ones that are making the guidelines, instrumenting the code, creating the alerts, and responding to them. I am completely against that. I think that observability, everyone should do their part. Engineers know their code and their product better. We, as observability engineers, are there to help them and empower them and provide the tools, but they should be the ones taking charge when it comes to this part. So that's what I'm going to be talking about. +**Audience Member:** Well, if you like what you heard today, Edis will be back for our open Telemetry in practice session on June the 8th, and it's gonna be, I believe, at the same time. So keep an eye out for an invite then. -**Iris:** Awesome! I cannot wait to hear this talk! So that'll be on June 8th at one o'clock Eastern, which is 10 o'clock Pacific and plus six in Central European Time. I think it's going to be awesome, so I hope you all can join and tell your friends. We would love to have more people here to hear what Edis has to say because I think it's awesome. +**Iris:** Why don't you tell folks what you'll be presenting about? -I think you're a great champion of OpenTelemetry, so keep on keeping on! This was great! Thanks so much for joining us here today. +**Edis:** My topic will be observability as a team sport. It's something that I'm extremely passionate about, and I see that in many companies, observability teams are the ones that are making the guidelines, instrumenting the code, creating the alerts, and responding to them. I am completely against that. I think that observability—everyone should do their part. Engineers know their code and their product better. Us as observability engineers are there to help them and empower them and provide the tools, but they should be the ones taking charge of this part. That's what I'm going to be talking about. -And, you know, if you have a friend who—or if you yourself are interested in participating in one of these Q&As or even OpenTelemetry in practice, please reach out to either Therese or Rin or me on the OpenTelemetry end-users Slack. We are more than happy to hear your stories and help folks share them with the world. +**Iris:** Awesome! I cannot wait to hear this talk! That'll be on June 8th at one o'clock Eastern, which is ten o'clock Pacific and plus six in Central European Time. I think it's going to be awesome, so I hope you all can join and tell your friends. We would love to have more people here to hear what Edis has to say because I think it's awesome. -We will be providing a blog post summary of today's Q&A for all to see. Thank you so much, everyone! +I think you're a great champion of open Telemetry, so keep on keeping on! This was great! Thanks so much for joining us here today. If you have a friend who or if you yourself are interested in participating in one of these Q&As or even open Telemetry in practice, please reach out to either Therese or Rin or me on the open Telemetry end users Slack. We are more than happy to hear your stories and help folks share them with the world. We will be providing a blog post summary of today's Q&A for all to see. -**Edis:** Thank you so much, Adriana! +Thank you so much, everyone! -**Iris:** Thank you, everyone! +**Edis:** Thank you so much, Adriana! Thank you, everyone! ## Raw YouTube Transcript diff --git a/video-transcripts/transcripts/2023-06-01T17:22:03Z-whats-an-observability-engineer-anyway.md b/video-transcripts/transcripts/2023-06-01T17:22:03Z-whats-an-observability-engineer-anyway.md index 6be0ced..11ac340 100644 --- a/video-transcripts/transcripts/2023-06-01T17:22:03Z-whats-an-observability-engineer-anyway.md +++ b/video-transcripts/transcripts/2023-06-01T17:22:03Z-whats-an-observability-engineer-anyway.md @@ -10,22 +10,24 @@ URL: https://www.youtube.com/watch?v=KRjYXPS3_so ## Summary -In this video, the speaker discusses the importance of providing documentation and guidelines for engineers regarding manual instrumentation in OpenTelemetry. They emphasize that while they are not training engineers, they facilitate sessions to showcase best practices and introduce the tools available to them. The speaker highlights the collaborative nature of the process, suggesting that it is a "team sport" where engineers can effectively utilize the tools and resources to enhance their work. +In this video, the speaker discusses the role of OpenTelemetry in providing documentation and guidelines for manual instrumentation. They emphasize the importance of collaboration with engineers, stating that while the engineers are skilled in their own code, the sessions aim to share best practices and introduce tools available for their use. The speaker highlights that the process is a team effort, encouraging engineers to utilize the resources provided. ## Chapters -00:00:00 Welcome and intro -00:00:10 Documentation and guidelines -00:00:20 Manual instrumentation overview +00:00:00 Introductions +00:00:15 OpenTelemetry documentation 00:00:30 Best practices introduction -00:00:40 Tools and teamwork -00:00:53 Closing remarks +00:00:40 Tools for engineers -[00:00:10] **Speaker:** Thank you. We provide documentation. We provide guidelines, and there is a lot of very good documentation in the OpenTelemetry as well about manual instrumentation. +## Transcript -[00:00:30] So basically, we have sessions with engineers, not to train them because I think they can do it better than we can because it's their code, but just to show the best practices and to introduce them to that and to tell them that, hey, our tools are here for you. +### [00:00:00] Introductions -[00:00:40] They take it from there. It's a team sport, let's say. Thank you. +**Speaker:** Thank you. We provide documentation. We provide guidelines, and there is a lot of very good documentation in the OpenTelemetry as well about manual instrumentation. + +### [00:00:30] Best practices introduction + +Basically, we have sessions with engineers, not to train them because I think they can do it better than we can because it's their code, but just to show the best practices and to introduce them to that. We tell them that, hey, our tools are here for you, and they take it from there. It's a team sport, let's say. Thank you. ## Raw YouTube Transcript diff --git a/video-transcripts/transcripts/2023-06-05T15:14:28Z-teaser-observability-is-a-team-sport.md b/video-transcripts/transcripts/2023-06-05T15:14:28Z-teaser-observability-is-a-team-sport.md index 22d6b75..238f0de 100644 --- a/video-transcripts/transcripts/2023-06-05T15:14:28Z-teaser-observability-is-a-team-sport.md +++ b/video-transcripts/transcripts/2023-06-05T15:14:28Z-teaser-observability-is-a-team-sport.md @@ -10,22 +10,26 @@ URL: https://www.youtube.com/watch?v=HLJe2RLPlWY ## Summary -In this video, the speaker discusses the concept of observability as a collaborative effort within teams, emphasizing that it should not be solely the responsibility of dedicated observability teams. The speaker, who is passionate about this topic, argues that engineers should take ownership of observability in their code and products, while observability engineers should act as facilitators, providing tools and support. The main points include the importance of empowering engineers to handle observability and the need for a collective approach rather than a top-down model. +In this video, the speaker passionately discusses the concept of observability as a collaborative effort within software engineering teams. They express their belief that observability should not be solely the responsibility of dedicated observability teams, who typically create guidelines, instrument code, and manage alerts. Instead, the speaker advocates for a more inclusive approach where all engineers take an active role in observability, leveraging their deep understanding of the code and product. The speaker emphasizes the importance of empowering engineers with the necessary tools, positioning observability as a shared responsibility among the entire team. ## Chapters -00:00:00 Welcome and intro -00:00:10 Observability as a team sport -00:00:20 Role of observability teams -00:00:30 Empowering engineers -00:00:40 Engineers taking charge -00:00:55 Conclusion and wrap-up +00:00:00 Introductions +00:00:15 Observability as a team sport +00:00:30 Engineers' role in observability +00:00:45 Empowering engineers with tools -[00:00:10] **Speaker:** Foreign topic will be observability as a team sport. It's something that I'm extremely passionate about. +## Transcript -[00:00:20] I see that in many companies, observability teams are the ones that are making the guidelines, instrumenting the code, creating the alerts, and responding to them. I am completely against that. I think that observability, everyone should do their part. Engineers know their code and their product better. +### [00:00:00] Introductions -[00:00:40] Us as observability engineers are there to help them and empower them and provide the tools, but they should be the ones taking charge when it comes to this part. So that's what I'm going to be talking about. +**Speaker:** Foreign topic will be observability as a team sport. It's something that I'm extremely passionate about, and I see that in many companies, observability teams are the ones that are making the guidelines, instrumenting the code, creating the alerts, and responding to them. + +### [00:00:30] Engineers' role in observability + +**Speaker:** I am completely against that. I think that observability, everyone should do their part. Engineers know their code and their product better. Us as observability engineers are there to help them and empower them and provide the tools, but they should be the one taking charge when it comes to this part. + +**Speaker:** So that's what I'm going to be talking about. ## Raw YouTube Transcript diff --git a/video-transcripts/transcripts/2023-06-08T02:24:12Z-teaser-how-do-you-foster-a-culture-of-observability.md b/video-transcripts/transcripts/2023-06-08T02:24:12Z-teaser-how-do-you-foster-a-culture-of-observability.md index fa19803..679e44a 100644 --- a/video-transcripts/transcripts/2023-06-08T02:24:12Z-teaser-how-do-you-foster-a-culture-of-observability.md +++ b/video-transcripts/transcripts/2023-06-08T02:24:12Z-teaser-how-do-you-foster-a-culture-of-observability.md @@ -10,15 +10,18 @@ URL: https://www.youtube.com/watch?v=RxP76ahb29s ## Summary -In this video, the speaker expresses pride in fostering a strong observability culture within their team or organization. They discuss the importance of maintaining and developing this culture, emphasizing its role in improving operational efficiency and transparency. The speaker highlights their contributions to this initiative and reflects on the positive impact that a robust observability culture has on the team's overall performance and collaboration. +In this YouTube video, the speaker expresses pride in fostering a strong observability culture within their organization. They emphasize the importance of this culture in enhancing performance and accountability. The discussion highlights key aspects of building and maintaining an effective observability framework, focusing on collaboration and continuous improvement. The speaker reflects on their role in contributing to this positive environment, underscoring the value of observability in driving success. ## Chapters -00:00:00 Welcome and intro -00:00:10 Observability culture discussion -00:00:20 Building observability culture +00:00:00 Introductions +00:00:15 Pride in observability culture -[00:00:10] **Speaker:** Thank you. I have a very good observability culture. I'm actually really, really proud of that. I mean, I'm proud because I'm helping continue and build it. +## Transcript + +### [00:00:00] Introductions + +**Speaker:** Thank you. I have a very good observability culture. I'm actually really, really proud of that. I mean, I'm proud because I'm helping continue and build it. ## Raw YouTube Transcript diff --git a/video-transcripts/transcripts/2023-06-12T23:45:47Z-teaser-observability-is-a-team-sport-with-iris-dyrmishi-teaser.md b/video-transcripts/transcripts/2023-06-12T23:45:47Z-teaser-observability-is-a-team-sport-with-iris-dyrmishi-teaser.md index e13ec4c..e981b9d 100644 --- a/video-transcripts/transcripts/2023-06-12T23:45:47Z-teaser-observability-is-a-team-sport-with-iris-dyrmishi-teaser.md +++ b/video-transcripts/transcripts/2023-06-12T23:45:47Z-teaser-observability-is-a-team-sport-with-iris-dyrmishi-teaser.md @@ -10,17 +10,20 @@ URL: https://www.youtube.com/watch?v=IHeqL36AlK0 ## Summary -In this video, the speaker discusses the importance of internal teams being involved in building their own observability dashboards and alert systems. They argue that if external teams are responsible for these tasks, the alerts and dashboards will lack specificity and relevance, leading to delayed responses to incidents. The speaker emphasizes that an internal team’s familiarity with their products and code allows them to create more tailored and effective monitoring tools. Overall, the video highlights the necessity of team involvement in incident response systems to ensure timely and accurate reactions to potential issues. +In this video, the speaker discusses the importance of internal team involvement in creating dashboards and alerts for observability in software development. The speaker argues that if external resources (ERS) handle these tasks, the alerts and dashboards may be too generic and not tailored to the specific products or code of the team, potentially leading to delayed incident responses. The emphasis is on the need for team members to build their own tools to ensure they are relevant and effective. ## Chapters -00:00:00 Introduction to dashboard building -00:00:10 Discussion on alerting systems -00:00:20 Observability data challenges -00:00:30 Generic alerting thresholds -00:00:40 Importance of team involvement +00:00:00 Introductions +00:00:15 Importance of team involvement +00:00:30 Risks of generic alerts +00:00:45 Need for tailored dashboards -[00:00:10] **Speaker:** foreign ERS were not involved in the part of building their own dashboard, building their own alert, knowing what data they're sending for their observability reasons. They will not be able to respond to an incident in time. Simple as that. If I was the one making the alerting for them, it would be something extremely generic with some thresholds that I just thought would be right because I don't know their products or their code. Obviously, the dashboards would be completely generic, but if those were done by a person inside the team itself, it would be so easy for them. +## Transcript + +### [00:00:00] Introductions + +**Speaker:** Foreign ERS were not involved in the part of building their own dashboard, building their own alert, knowing what data they're sending for their observability reasons. They will not be able to respond to an incident in time. Simple as that. If I was the one making the alerting for them, it would be something extremely generic with some thresholds that I just thought would be right because I don't know their products or their code. Obviously, the dashboards would be completely generic, but if those were done by a person inside the team itself, it would be so easy for them. ## Raw YouTube Transcript diff --git a/video-transcripts/transcripts/2023-06-12T23:45:52Z-otel-in-practice-presents-observability-is-a-team-sport-with-iris-dyrmishi-full-video.md b/video-transcripts/transcripts/2023-06-12T23:45:52Z-otel-in-practice-presents-observability-is-a-team-sport-with-iris-dyrmishi-full-video.md index 8dba675..874c533 100644 --- a/video-transcripts/transcripts/2023-06-12T23:45:52Z-otel-in-practice-presents-observability-is-a-team-sport-with-iris-dyrmishi-full-video.md +++ b/video-transcripts/transcripts/2023-06-12T23:45:52Z-otel-in-practice-presents-observability-is-a-team-sport-with-iris-dyrmishi-full-video.md @@ -10,274 +10,262 @@ URL: https://www.youtube.com/watch?v=U1yLXnMONkc ## Summary -In this YouTube video, Iris, a platform engineer at Farfetch with a strong focus on observability, discusses the importance of fostering an observability culture within engineering teams. She emphasizes that observability is a collective responsibility rather than the duty of a single team, advocating for active participation from all engineers in creating and maintaining observability systems. Key points include defining a perfect observability system, the significance of team ownership in monitoring and alerting, the challenges of managing telemetry data, and the need for a collaborative approach to optimize observability practices. Iris also highlights the role of observability engineers in guiding teams without taking over responsibilities, and she shares insights into the tools and frameworks used at Farfetch, such as OpenTelemetry and Grafana. The discussion touches on various topics, including incident response, auto-scaling, and the need for structured data to avoid excessive costs associated with observability. The video concludes with an invitation for further dialogue and collaboration in the observability community. +In this YouTube video, Iris, a platform engineer with a focus on observability at Farfetch, leads a discussion about the importance of observability culture in engineering teams. She emphasizes that observability should be a shared responsibility among all engineers, not just a centralized team, and discusses the characteristics of a perfect observability system. Key points include the need for a centralized view of telemetry data, optimized data usage to control costs, and the crucial role of each engineering team in setting up their own monitoring tools. Iris highlights the necessity for effective incident response and detection, the importance of collaboration in creating custom alerts, and the need for engineers to take ownership of their observability practices. Throughout the conversation, she encourages open dialogue and exchange of experiences among attendees, reinforcing that a robust observability culture is essential for operational success. The video features questions and interactions from participants, including discussions on practical tools and strategies for improving observability within organizations. ## Chapters -00:00:00 Welcome and introduction -00:01:10 Observability as a team sport -00:03:05 Perfect observability system overview -00:05:30 Importance of observability engineers -00:08:42 Incident response and team involvement -00:10:40 Custom dashboards and alerts -00:12:30 Challenges of observability data -00:15:00 Observability culture and team ownership -00:20:00 Changing observability culture -00:24:00 Q&A session +00:00:00 Introductions +00:01:04 Observability as a team sport +00:03:28 Characteristics of a perfect observability system +00:05:32 Importance of observability engineers +00:06:56 Tools for observability: OpenTelemetry, Grafana, Prometheus +00:08:40 Incident response and detection +00:10:24 Custom alerts and dashboards +00:12:08 Importance of team involvement in observability +00:20:48 Observability culture at Farfetch +00:39:52 Anomaly detection and machine learning in observability -**Iris:** Thank you for joining us again. We had her for the Q&A last month, a couple of weeks ago I guess, and she did a bang-up job, so we asked her to come back and I'll let you take it away. +## Transcript -Hello everyone, thank you so much for joining today. My name is Iris, I'm a platform engineer with a focus on observability working at Firefits currently, and I'm a super passionate person when it comes to observability. So yeah, I like to talk about it as much as I can, and I like to share the observability culture all over, not just in my company. +**Iris:** Thank you for joining us again. We had her for the Q&A last month, a couple of weeks ago, I guess, and she did a bang-up job. So we asked her to come back, and I'll let you take it away. -[00:01:10] So today I wanted to have a talk, and I would like to have it as an open discussion. I would love to hear about the observability culture in your companies as well. I believe that observability is a team sport, and it is not something that is being done just by one central team in a company, no matter how big or small it is. I think everyone should put their hands and be involved in the observability. +### [00:01:04] Observability as a team sport -But of course, let's say observability has been there forever, but now it's becoming this huge thing, and now we have this amazing culture that is spreading, but we're still not there. I feel like this conversation and these discussions are so important to put out there for us to discuss and then to take them to our companies and to learn obviously from each other. - -So yeah, this was just a small intro. Please interrupt me because I do not have a good visibility, so if you have a question, please don't hesitate to unmute yourself and just speak. If you have any opinions about something that I'm saying, I just want to have a spoiler alert: these are mostly things that I've learned from my job and things that I believe in. So of course, if you have a completely different opinion, please let me know. Let's discuss it; it would be amazing to have a conversation about this here. - -Okay, so the first question that I ask all engineers that are being interviewed for a job in my team is: whose responsibility is observability? Because, let's be honest, observability is being implemented differently in many different companies. I've seen so many ways everywhere that I've interviewed for; it's different. So of course, observability is everyone's responsibility. It's simple. If your engineer says that answer to me, then I'm like, "Okay, okay, we're on board, we're getting somewhere." +**Iris:** Well, hello everyone. Thank you so much for joining today. My name is Iris. I'm a platform engineer with a focus on observability working at Farfetch currently, and I'm a super passionate person when it comes to observability. I like to talk about it as much as I can, and I like to share the observability culture all over, not just in my company. So today I wanted to have a talk, and I would like to have it as an open discussion. I would love to hear about the observability culture in your companies as well. I believe that observability is a team sport, and it is not something that is being done just by one central team in a company, no matter how big or small it is. I think everyone should put their hands in and be involved in the observability. -[00:03:05] And why do I believe this? First of all, in order to get deeper into why I believe that everyone is responsible for the observability in their company, I want to give an overview of what I think a perfect observability system is. And that's perfect—let's take it with a grain of salt, of course. It's perfect for me; my company works in another person's company, doesn't work. +But of course, let's say observability has been there forever, but now it's becoming this huge thing, and now we have this amazing culture that is spreading, but we're still not there. I feel like this conversation and these discussions are so important to put out there for us to discuss and then to take them to our companies and to learn obviously from each other. -So if you have any more ideas here, this was just what came to the top of my mind. For me, a perfect observability system has a centralized view of systems observability. Data engineers can go, and they can access their traces, metrics, logs, dashboards, alerts, everywhere. They don't have to memorize a lot of URLs or have to go to many places to find the information. +So yeah, this was just a small intro. Please interrupt me because I do not have good visibility. If you have a question, please don't hesitate to unmute yourself and just speak, or if you have any opinions about something that I'm saying, I just want to have a spoiler alert. These are mostly things that I've learned from my job and things that I believe in. So of course, if you have a completely different opinion, please let me know. Let's discuss it. It would be amazing to have a conversation about this here. -So that's the first thing. The second one is optimized data will optimize costs because doing observability doesn't necessarily mean that you will send everything that you can and rack up huge bills. But it needs to be optimized data and some good old data that is actually going to help other teams troubleshoot and monitor their systems. Obviously, reliable alerting, reliable and custom dashboards, rich traces, correlation between the different telemetry signals—so this is something that makes a perfect observability system for us. +Okay, so the first question that I ask all engineers that are being interviewed for a job in my team is, "Whose responsibility is observability?" Because let's be honest, observability is being implemented differently in many different companies. I've seen so many ways everywhere that I've interviewed for. It’s different. So of course, observability is everyone's responsibility. It's simple. If your engineer says that answer to me, then I'm like, "Okay, okay, we're on board, we're getting somewhere." -And my job as an observability engineer, especially in perfect, is to provide that for the engineers. But I cannot do it alone because no matter how much I try, many—let's say most of these characteristics—are a teamwork and something that other people also have to contribute. +### [00:03:28] Characteristics of a perfect observability system -This means the engineers that are part of, I would say, our customers. I don't know if anyone has any other ideas about what a perfect observability system is for them. I would love to hear more about it or if you completely agree with my goals of having this system. But at least I'm with you all throughout me any moment. +And why do I believe this? First of all, in order to get deeper into why I believe that everyone is responsible for the durability in their company, I want to give an overview of what I think a perfect observability system is. And that's perfect, let's take it with a grain of salt. Of course, it's perfect for me; my company works, and in another person's company it doesn't work. So if you have any more ideas here, this was just what came to the top of my mind. -**Derek:** Thank you! I definitely agree with a lot of the things that you said. Like, it's just, yeah, spot on. Sorry, I'm not able to see the comments here, but if there is something interesting, do let me know, please. +For me, a perfect observability system has a centralized view of systems observability. Data engineers can go and they can access their traces, metrics, logs, dashboards, alerts everywhere. They don’t have to memorize a lot of URLs or have to go to many places to find the information. So that's the first thing. The second one is optimized data will optimize costs because doing observability doesn’t necessarily mean that you will send everything that you can and rack up huge bills, but it needs to be optimized data and some good old data that is actually going to help other teams troubleshoot and monitor their systems. -[00:05:30] **Iris:** Okay, so the question is: okay, you have this vision of a perfect observability system. How do we get to this observability system? Well, observability engineers like myself, my team with the knowledge to shape observability systems into the correct path, I'm not saying that observability is something that a simple engineer that is working can't get into. They can be participants and active participants in it, but of course, it is important that there are engineers who know observability better than everyone, who know what guidelines to implement, what tools to use, or how to optimize data. It's very important that these people exist in the company. +Obviously, reliable alerting, reliable and custom dashboards, rich traces, correlation between the different telemetry signals—so this is something that makes a perfect observability system for us. My job as an observability engineer, especially in perfect, is to provide that for the engineers. But I cannot do it alone because no matter how much I try, most of these characteristics are teamwork and something that other people also have to contribute, which means the engineers that are part of, I would say, our customers. I don’t know if anyone has any other ideas about what a perfect observability system is for them. I would love to hear more about it, or if you completely agree with my goals of having this system, but at least I'm with yourself and with them throughout me any moment. -Because as I said before, observability is one of those fields that is moving so fast that it is impossible for a person that is working, let's say, with databases or in Java and their full-time job is called getting back-end, whatever else, to keep up with this. So it is important for people like us to be in the company and to preach, let's say, these values and to bring all these good tools and good things that are being developed and shared in the community. +**Audience Member:** Thank you! I definitely agree with a lot of the things that you said; like it just, yeah, spot on. -Of course, as I mentioned, observability engineers also bring the data to collect and process telemetry data. But my fourth favorite so far, of course, is open telemetry, which I'm a huge fan of. We have fans, Grafana, Prometheus; those are some of the tools that are open source that are being used so massively right now by observability engineers. +**Iris:** Okay, sorry, I'm not able to see the comments here, but if there is something interesting, do let me know, please. -And now I'm getting to what the rest of the presentation will be about. It's not only observability engineers that are needed when it seems to actively participate in optimizing the data that they're sending and actively building their monitoring. Five people, 10 people, 20, depending on the size of the observability team, cannot do the work of two thousand, three thousand, depending on the work of so many engineers. Everyone needs to have control of their own code. +### [00:05:32] Importance of observability engineers -So some of the things why observability is so important to be done as a team is because of how crucial it is in a company. I don't—in some, let's say that in some companies, it's still not understood how important it is to have observability. But that is, well, what I believe is because of the small scale. But the more that the companies grow, they understand how important it is to have highly available and reliable systems. It is crucial to have optimized performance because you could have—you could allocate a lot of resources, and you are not saving on cost. Basically, you're spending too much money or you're cutting back on resources, and you are basically not giving a good performance or a good experience to your users, depending on the type of product it is. +Okay, so the question is, okay, you have this vision of a perfect observability system—how do we get to this observability system? Well, observability engineers like myself, my team, with the knowledge to shape observability systems into the correct path. I'm not saying that observability is something that a simple engineer that is working can't get into; they can be participants and active participants in it. But of course, it is important that there are engineers who know observability better than everyone, who know what guidelines to implement, what tools to use, or how to optimize data. It's very important that these people exist in the company because, as I said before, observability is one of those fields that is moving so fast that it is impossible for a person that is working, let’s say, with databases or in Java, and their full-time job is called getting back-end whatever else to keep up with this. -[00:08:42] Also, it is crucial for incident response. This is something that is very important—in my company, incident response is very important, considering that we have a lot of customers that rely on our product, very expensive products at times. So having all the teams involved in the crucial incident response is extremely important. We have around 2000 engineers. I actually thought that we were 3000; we were actually 2000 that are working in completely different areas, some of them I don't even know, except for what they are. So when I hear about them, I'm like, "Wow, okay," because it's so big and so diverse in our company. +### [00:06:56] Tools for observability: OpenTelemetry, Grafana, Prometheus -And if these engineers were not involved in the part of building their own dashboard, building their own alert, knowing what data they're sending for their observability reasons, they will not be able to respond to an incident in time, simple as that. If I was the one making the alerting for them, it would be something extremely generic with some thresholds that I just thought would be right because I don't know their products or their code, obviously. +So it is important for people like us to be in the company and to preach, let's say, these values and to bring all these good tools and good things that are being developed and are being shared in the community. Of course, as I mentioned, observability engineers also bring the data to collect and process telemetry data. But my fourth favorite so far, of course, is OpenTelemetry, which I'm a huge fan of. That's why we're here after all. I'm a huge fan. We have Grafana, Prometheus—that are some of the tools that are open source that are being used so massively right now by observability engineers. -The dashboards would be completely generic, but if those were done by a person inside the team itself, it would be so easy for them because they know it inside out, their work, what are some of the issues that they might have. So if an incident happens, you reduce the response from hours that it would be if it was for me dealing with their incident to seconds or minutes if someone from their engineering team is involved. +And now I'm getting to what the rest of the presentation will be about. It's not only observability engineers that are needed; teams need to actively participate in optimizing the data that they're sending and things, actively building their monitoring. Five people, ten people, twenty, depending on the size of the observability team, cannot do the work of two thousand, three thousand, depending on the work of so many engineers. Everyone needs to have control of their own code. -[00:10:40] Also, it is crucial for incident detection, as I said before. Us as engineers, the rebuilt engineers, we can provide a set of alerting dashboards that are very generic that each team can have as a baseline to implement their monitoring system, but I don't feel comfortable in a way to go and implement custom alerts, custom dashboards for these teams. Again, it goes to this: even if I wanted to, there are thousands and thousands of engineers, technologies, hundreds of teams. I would never be able to do that properly. +Some of the reasons why observability is so important to be done as a team is because of how crucial it is in a company. In some companies, it’s still not understood how important it is to have observability. But that is, well, what I believe is because of the small scale. The more that the companies grow, they understand how important it is to have a highly available and reliable system. It is crucial to have optimized performance because you could allocate a lot of resources and you are not saving on costs. Basically, you're spending too much money or you're cutting back on resources, and you are basically not giving a good performance or a good experience to your users, depending on the type of product. -That's why it's so important that each team is involved and does it themselves. If they need it, they change it; they don't need our permission; they are owners of that solution. It's extremely important and also critical for auto-scaling. I say this because in perfect, we rely heavily on telemetry data to auto-scale, because you never know the amount of traffic that you're going to have in your application. +### [00:08:40] Incident response and detection -But for example, let's say for me or for another person in the observability, if we were to set some auto-scaling rules for you to be based on CPU or on memory, it would be something that is completely generic. But an engineer that knows their system so well, they would go and say, "Let's do a custom auto-scaling depending on the number of throughput," or whatever. +Also, it is crucial for incident response. This is something that is very important. In my company, incident response is very important considering that we have a lot of customers that rely on our product—very expensive products at times. So having all the teams involved in the crucial incident response is extremely important. We have around two thousand engineers. I actually thought that we were three thousand; we were actually two thousand that are working in completely different areas, some of them I don’t even know except for what they are. So when I hear about them, I'm like, "Wow, okay," because it's so big and so diverse in our company. -Basically, my bottom goal here would be to say that all this requires people from the teams that are monitoring the system to take part in. If it is done by people that are outside—I am an infrastructure engineer; I work with Kubernetes, I work with VMs, with Azure, with cloud, and I'm not very good at back-end coding, let's say. So if I was the one doing this, I wouldn't do it properly, no matter how much I tried and how much effort I put. +If these engineers were not involved in the part of building their own dashboards, building their own alerts, knowing what data they're sending for their observability reasons, they would not be able to respond to an incident in time. Simple as that. If I was the one making the alerting for them, it would be something extremely generic with some thresholds that I just thought would be right because I don’t know their products or their code, obviously. The dashboards would be completely generic. But if those were done by a person inside the team itself, it would be so easy for them because they know it inside out, their work, what some of the issues that they might have. -[00:12:30] So let's not do observability for the sake of it. Yeah, let's have some generic alerts just for having them. No, let's do it properly. Let's have the people that are working day-to-day with the product implement them, make it their baby as well. It's very important to show it to the engineers. And I am very proud to say that in my company, we have a pretty good observability culture that started a few years back, so now it is widely accepted. +### [00:10:24] Custom alerts and dashboards -But I've seen it happen that teams rely heavily on other people to set up alerting for them, and that is something that scares me. If something happens, if an incident happens, and I wouldn't be comfortable working in a team like that if I wasn't the one setting up my own monitoring for the system. Also, observability can be so messy depending on the scale of the company. +So if an incident happens, you reduce the response from hours that it would be if it was for me dealing with their incident to seconds or minutes if someone from their engineering team is involved. Also, it is crucial for incident detection. As I said before, us as observability engineers can provide a set of alerting dashboards that are very generic, that each team can have as a baseline to implement their monitoring system, but I don’t feel comfortable in a way to go and implement custom alerts, custom dashboards for these teams. -And so many people that come and go, everyone wants to add the custom metric, the custom log. It can get extremely messy. It can get too many metrics, too many logs, especially—for example, living debugging logs flowing all day. You know, because it can—yeah, you just sense, "I'm an engineer; I want to have full observability of my application." It's not affecting my performance, so I'm sending everything: debugging info, warning metrics. I create like a bunch of them on top of each other, or the same metric with different names. +Again, it goes to this: even if I wanted to, there are thousands and thousands of engineers, technologies, hundreds of teams. I would never be able to do that properly. That's why it's so important that each team is involved and does it themselves. If they need it, they change it; they don’t need our permission. They are owners of that solution. It’s extremely important and also critical for auto-scaling. I say this because in Farfetch, we rely heavily on telemetry data to auto-scale because you never know the amount of traffic that you're going to have in your application. -And yeah, what can I, as an observability engineer, do about it alone? I detect it; I see it, and I say, "Okay, this is completely wrong." But of course, I'm going to need the collaboration of the engineers that are working with our product to actually change it. Otherwise, it would never be optimized, and it should never be, let's say, good observability. +### [00:12:08] Importance of team involvement in observability -Too many traces—well, I'm a firm believer that there is no such thing as too many traces. I'm a big advocate of that. But yeah, it's possible that we are sending 100% of traces with a mess and completely no information there. And what can I do, considering that I'm not the one doing the day-to-day developing? I have to go and talk to the teams that they need to be active participants on that. Without it, for me, there is no durability, or it is something very generic just for the sake of it. That happens so much, and to be honest, it rates me. +But for example, let’s say for me or for another person in observability, if we were to set some auto-scaling rules for you, it would be based on CPU or memory. It would be something completely generic, but an engineer that knows their system so well would go and say, "Let’s do a custom auto-scaling depending on the number of throughput," or whatever. Basically, my bottom goal here would be to say that all this requires people from the teams that are monitoring the system to take part in. If it is done by people that are outside—I am an infrastructure engineer. I work with Kubernetes, I work with VMs, with Azure, with cloud, and I'm not very good at back-end coding, let's say. So if I was the one doing this, I wouldn’t do it properly, no matter how much I tried and how much effort I put. -[00:15:00] But there are also too many dashboards, too many alerts. If someone else creates the alerts or they're just created automatically, you have them ringing back and forth, and nobody actually looks at them because they know that it is spam. So it is very important for people to go and for engineers to go there and say, "Hey, okay, I don't need this alert; I need this. I need to create this." And the same for the dashboards. +So let’s not do observability for the sake of it. Yeah, let’s have some generic alerts just for having them. No, let’s do it properly. Let's have the people that are working day-to-day with the product implement them, make it their baby as well. It's very important to show it to the engineers. I am very proud to say that in my company we have a pretty good observability culture that has started a few years back. Now it is widely accepted, but I've seen it happen that teams rely heavily on other people to set up alerting for them, and that is something that scares me. If something happens, if an incident happens, I wouldn't be comfortable working in a team like that if I wasn't the one setting up my own monitoring for the system. -It can be thousands and thousands of dashboards, and when you actually have an incident, that's at midnight, 3 a.m. in the morning, you wake up, your eyes cannot open, and you have to scroll through thousands of those or try to find a person that might know what this is. It's very important to be about as simple as that. +Also, observability can be so messy depending on the scale of the company. So many people come and go. Everyone wants to add the custom metric, the custom log. It can get extremely messy. It can get too many metrics, too many logs, especially, for example, living debugging logs flowing all day. You know, because it can—yeah, you just sense it. I’m an engineer; I want to have full observability of my application. It's not affecting my performance, so I'm sending everything: debugging info, warning metrics. I create a bunch of them on top of each other, or the same metric with different names. -Also, it is not easy; it makes it easy. I think they go hand in hand. It's, let's say, mostly the same thing with too many alerts and dashboards, and you just cannot get the hang of it, and you just give up. So basically, you do not rely on this to troubleshoot them. On what do you rely? Maybe the application logs or it takes you hours to solve an incident, which is not great either. +What can I, as an observability engineer, do about it alone? I detect it. I see it, and I say, "Okay, this is completely wrong." But of course, I'm going to need the collaboration of the engineers that are working with our product to actually change it. Otherwise, it would never be optimized, and it should never be, let's say, good observability. Too many traces—well, I'm a firm believer that there is no such thing as too many traces. I'm a big advocate of that. But yeah, it's possible that we are sending 100% of traces with a mess and completely no information there. -Too few or too many telemetry signals—I talked about too many, but there is also one thing: it is possible that you don't know how metrics are, how logs are generated, how traces are generated. In this case, that same place is considering right now, it’s this huge movement to make them first-class citizens. Many engineers, let's say, do not know how important tracing is and not how the great capabilities that they can achieve to that. +What can I do considering that I'm not the one doing the day-to-day developing? I have to go and talk to the teams that they need to be active participants in that. Without it, for me, there is no durability, or it is something very generic just for the sake of it. That happens so much, and to be honest, it irks me. But there are also too many dashboards, too many alerts. If someone else creates the alerts or they're just created automatically, you have them ringing back and forth, and nobody actually looks at them because they know that it is spam. -And my job as an engineer would be to advise them and to help them with it, but not to do it for them because I don't know the type of information they might need. It's simple. And it is not cheap. Absolutely, it is not cheap. Depending on the scale of the information that we are having, it could rack up to millions. +So it is very important for people to go, and for engineers to go there and say, "Hey, okay, I don't need this alert. I need this. I need to create this." And the same for the dashboards. It can be thousands and thousands of dashboards, and when you actually have an incident at midnight, 3 a.m. in the morning, you wake up, your eyes cannot open, and you have to scroll through thousands of those or try to find a person that might know what this is. -I was reading some articles recently about some companies that were paying a huge bill in observability, and the question in those articles was, "Who wants to blame?" And the stack of different billing is a team sport. It is also the blame is to share because the engineers of observability are clearly doing a great job at educating, or maybe they're not even there; that could be one of the cases, or the teams are not participating. +It's very important to be about as simple as that. Yeah, also, it is not easy. It makes it easy. I think they go hand in hand. It’s, let’s say, mostly the same thing with too many alerts and dashboards, and you just cannot get the hang of it, and you just give up. So basically, you do not rely on this to troubleshoot. What do you rely on? Maybe the application logs? Or it takes you hours to solve an incident, which is not great either. -Basically, you have this framework that just sends 100% of logging, tracing, metrics, and that is all sent somewhere. So yeah, observability is something that is a very delicate topic when it comes to costs because so many engineers consider it as not a very important thing until they actually need it. For example, when something happens and they don't receive an alert, "Oh my God, it is actually important!" Or "Oh my God, the information on that dashboard now would be so nice!" +Too few or too many telemetry signals—I talked about too many, but there is also one thing: it is possible that you don't know how metrics are generated, how logs are generated, how traces are generated. In this case, that same place is considering right now. It's this huge movement to make them first-class citizens. Many engineers, let’s say, do not know how important tracing is and the great capabilities that they can achieve with that. My job as an engineer would be to advise them and to help them with it, but not to do it for them because I don't know the type of information they might need. It's simple. -But at the same time, we have to know what to collect, how to, and to also advise our team to do it and to empower them to do it themselves. It's not an easy task, especially when it's so comfortable for them to have all that information. Let's say, for example, logging. I have a vision of talking in general, but nothing against it; it's used so much. They say, "Okay, I just will have the debug logs, and even if something happens, I will have it there." +And it is not cheap. Absolutely it is not cheap. Depending on the scale of the information that we are having, it could rack up to millions. I was reading some articles recently about some companies that were paying a huge bill in observability, and the question in those articles was, "Who wants to blame?" The stack of different billing is a team sport. The blame is shared because the engineers of observability are clearly doing a great job at educating, or maybe they're not even there. That could be one of the cases. Or the teams are not participating. Basically, you have this framework that just sends 100% logging, tracing, metrics, and that is all sent somewhere. -Yeah, it's our job to convince them to show them and to actually make a plan and follow it through together. I would say our mission as observability engineers is not just to teach people and to show them because sometimes that sounds like, I don't know if it is the right word, but like stack ups. "Oh yeah, you do it." It's not that; it's just trying to do it in a collaborative manner to consider it as a teamwork. +So yeah, observability is something that is a very delicate topic when it comes to costs because so many engineers consider it as not a very important thing until they actually need it. For example, when something happens and they don't receive an alert, "Oh my God, it is actually important!" Or, "Oh my God, the information on that dashboard now would be so nice." But at the same time, we have to know what to collect, how to do it, and to also advise our team to do it and to empower them to do it themselves. -That's a real team. It's not an external team that just works completely away from you, but it's, let's say, part of your team. And at the same time, it's not—it serves as an advisory. +It's not an easy task, especially when it's so comfortable for them to have all that information. Let's say, for example, logging. I have a vision of talking in general, but nothing against it, but it's used so much. They say, "Okay, I just will have the debug logs, and even if something happens, I will have it there." Yeah, it's our job to convince them, to show them, and to actually make a plan and follow it through together. -So how do we change to this observability culture? It is very, very difficult. Because of my passion for observability, I talk a lot about it, and it is very, very difficult to convince an engineer with a higher grade than you to follow a certain observability culture if sometimes it's not even the engineers themselves. +I would say our mission as observability engineers is not just to teach people and to show them because sometimes that sounds like, I don't know if it is the right word, but like stack-ups, "Oh yeah, you do it." It's not that. It's just trying to do it in a collaborative manner to consider it as a teamwork. It's a team; it's not an external team that just works completely away from you, but it's, let's say, part of your team. And at the same time, it's not—it serves as an advisory. -[00:20:00] It could be senior leadership; it could be higher-ups. They just do not believe it, and some companies—this is very prevalent and it's very difficult to crack in there to actually implement it. But I think that our job is to make a change. We need to be ambassadors in our companies, and I think what we're doing today and all these sessions are a great way for us to talk and to share opinions and to know how to approach our teams better, how to be better at what we're doing, and to actually show the importance of our work. +So how do we change to this observability culture? It is very, very difficult. Because of my passion for observability, I talk a lot about it, and it is very, very difficult to convince an engineer with a higher grading as you to follow a certain observability culture if sometimes it's not even the engineers themselves. It could be senior leadership, it could be higher-ups. I just do not believe it in some companies, and this is very prevalent, and it's very difficult to crack in there to actually implement it. -We need to show the importance of the observability of the data. In fact, you can just go and say, "Hey, observability, a rainbow is amazing, beautiful." We have to show data, and that's very important to collect and very easy as well, considering the amount of incidents that are happening. Or you just show how easy it is to get information about one aspect of the application that they never thought about. +But I think that our job is to make a change. We need to be ambassadors in our companies, and I think what we're doing today and all these sessions are a great way for us to talk and to share opinions and to know how to approach our teams better, how to be better at what we're doing, and to actually show the importance of our work. We need to show the importance of the observability of the data. In fact, you can just go and say, "Hey, observability, a rainbow is amazing, beautiful!" We have to show data, and that's very important to collect and very easy as well considering the amount of incidents that are happening. -It's important to provide up-to-date tools. Our job is amazing because we can work with someone in modern technologies that are moving so fast. At the same time, it is very difficult because you need to provide tools that aren't mature enough. You have to provide tools that the engineers need and are looking for, and sometimes this can get overwhelming. But it is part of the challenge, and actually, this is my favorite aspect of the job: how fast it moves and how fast you have to adapt. It's part of being an observability engineer. +### [00:20:48] Observability culture at Farfetch -Yeah, we have to provide guidelines. Not everyone knows—I mentioned this before, but I'm a firm believer of not judging. It might sound like a childish thing to say or kindergarten issue, but I know that tech people judge each other. "Oh, you don't know that much about Kubernetes parts of durability." It's very important for us to offer help and to avoid judgment of that sort because observability is not difficult, but at the same time, it's not easy, as I said before. +Or you just show how easy it is to get information about one aspect of the application that they never thought about. It's important to provide up-to-date tools. Our job is amazing because we can work with some modern technologies that are moving so fast. At the same time, it is very difficult because you need to provide tools that aren’t mature enough. You have to provide tools that the engineers need and are looking for, and sometimes this can get overwhelming. But it is part of the challenge, and actually, this is my favorite aspect of the job: how fast it moves and how fast you have to adapt. It's part of being an observability engineer. -It's very fast; it's moving; it's modern. You need to adapt very fast, and the person that is completely detached from that will not be able to do it. So there is no place for judgment here, and I think that if there is judgment, that doesn't make you a good observability engineer. I think we're like the saints of the engineers. +We have to provide guidelines. Not everyone knows; I mentioned this before, but I'm a firm believer of not judging. It might sound like a childish thing to say or a kindergarten issue, but I know that tech people judge each other. "Oh, you don't know that much about Kubernetes parts of durability." It's very important for us to offer help and to avoid judgment of that sort because observability is not difficult, but at the same time, it's not easy. -And also, we need to give space and time to adapt. In some companies where the durability cultures exist, it's easy, obviously. If you have a majority of engineers that follow these practices, let's say, it becomes very easy for it to spread. But when you're starting from zero, it needs time. People that have been working in other companies that have completely different working mentality, it's going to be difficult for them to just land in the company and be like, "Ah, okay, now I accept observability as this amazing thing that it is." +As I said before, it's very fast, it's moving, it's modern. You need to read; you need to adapt very fast, and the person that is completely detached from that will not be able to do it. So there is no place for judgment here, and I think that if there is judgment, that doesn't make you a good observability engineer. I think we’re like the saints of the engineers. -No, we have to give them space and time to adapt, show them, be patient, be there to help, to implement. Even an engineer that is excellent at his job but has no idea how observability works, we have to be there for them. I think the human aspect of this job is more important than the technical one because it is good; it's important to be technical and to adapt and to build, but also the human aspect and spreading the culture and talking to people and being a people's person professionally. +And also, we need to give space and time to adapt. In some companies where the durability cultures exist, it's easy, obviously. If you have a majority of engineers that follow these practices, let’s say, it becomes very easy for it to spread. But when you're starting from zero, it needs time. People that have been working in other companies that have completely different working mentalities, it's going to be difficult for them to just land in the company and be like, "Ah, okay, now I accept observability as this amazing thing that it is." -You don't have to be an extrovert and have these beautiful conversations. It's a very important aspect of observability. +No, we have to give them space and time to adapt, show them, be patient, be there to help, to implement. Even an engineer that is excellent at his job but has no idea how observability works, we have to be there for them. I think the human aspect of this job is more important than the technical one. It’s important to be technically savvy and to adapt and to build, but also the human aspect and spreading the culture and talking to people and being a people’s person professionally. You don't have to be an extrovert and have these beautiful conversations with everybody; it’s a very important aspect of observability. -[00:24:00] These are all the slides that I had prepared for this presentation, but I just want to say my summary, or let's say the message that I have through this, is that we all need to work as a team to have a great observability culture and a great observability system, and that's going to make our lives much easier. +These are all the slides that I had prepared for this presentation, but I just want to say my summary, or let's say the message that I have through this, is that we all need to work as a team to have a great observability culture and a great observability system, and that's going to make our lives much easier. So that's pretty much it. I don't know if you have any questions, please let me know. -**Derek:** It was a comment from Derek, actually. I think when you're talking about what is important in observability, they mentioned the ability to form rich queries against the data once it's in your analytics backend. It's when you're just asking what's important in the design of a perfect system. You can pop all the perfect data that you want into your backend, but if you don't have a way to quickly and effectively query that to get the answers you want, then you're just going to be frustrated. - -I think that's one of the reasons why we get frustrated with unstructured logs so much is because you end up having to craft these complicated regexes and pattern matching of unstructured strings, and it's really frustrating to try and find what you want. So if we can do our part to make sure the data going in is nicely structured, it makes it a lot easier to get the signals you want out of it. - -**Iris:** Absolutely, absolutely agree. +**Audience Member:** It was a comment from Derek, actually. I think when you're talking about what is important in observability, they mention the ability to form rich queries against the data once it's in your analytics backend. -**Derek:** I had a question too, but I can go to the back if someone else has one. +**Derek:** When you're just asking what's important in the design of like a perfect system, you can pop all the perfect data that you want into your backend, but if you don't have a way to quickly and effectively query that to get the answers you want, then you're just going to be frustrated. I think that's one of the reasons why we get frustrated with unstructured logs so much is because you end up having to craft these complicated regexes and pattern matching of unstructured screams, and it's really frustrating to try and find what you want. -**Iris:** Oh no, go ahead, Derek. +So if we can do our part to make sure the data going in is nicely structured, it makes it a lot easier to get the signals you want out of it. -**Derek:** I have one, but I can go after. +**Iris:** Absolutely, absolutely agree. I had a question too, but I can go to the back if someone else has them. -So, I guess my question would be: what was the observability culture like when you joined your company? And kind of like what was ground zero? Where were you starting from? And then kind of how did you make improvements to that, or how did you kind of watch that culture improve? +**Audience Member:** Oh no, go ahead, Derek. I have one, but I can go after. -**Iris:** So actually, I'm pretty lucky when it comes to that extent where in this company, when I joined, I think the previous team and just a few members that are currently part of my team had done most of the heavy lifting. They had started the spreading of the observability culture and how to do it correctly. +**Derek:** I guess my question would be what was the observability culture like when you joined your company? And kind of like what was Ground Zero? Where were you starting from? And then kind of how did you make improvements to that? Or how did you kind of watch that culture improve? -So when I joined, I think everything was put in place. But of course, we are improving that day-to-day because you can have a culture there, but you have to, let's say, you have to water it every day and you have to keep it and to improve it. +**Iris:** So actually, I'm pretty lucky when it comes to that extent where in this company, when I joined, I think the previous team and just a few members that are currently part of my team had done most of the heavy lifting. They had started the spreading of the observability culture and how to do it correctly. So when I joined, I think everything was put in place. But of course, we are improving that day-to-day because you can have a culture there, but you have to, let’s say, you have to water it every day, and you have to keep it and improve it. -So what do we do? We have presentation sessions with other engineers in the company showing them how to use our tools best of or what our tools can do. We are always open—the team—for just a Slack message to help them on how to create metrics dashboards. Even though the simplest questions, we're always there for them. We also share articles, share presentations for observability, and it's actually interesting because many people read them. +So what do we do? We have presentation sessions with other engineers in the company, showing them how to use our tools best, or what our tools can do. We are always open—the team—for just a Slack message to help them on how to create metrics dashboards. Even the simplest questions, we are always there for them. We are also sharing articles, sharing presentations for observability. And it's actually interesting because many people read them. We are also making sure that we're always on top of the new technologies. In this case, for example, it was OpenTelemetry. It was something that was very well accepted in Farfetch, and some people actually requested it, so we provided it. -And we're also making sure that we're also always on top of the new technologies. In this case, for example, it was open telemetry; it was something that was very well accepted in Firefits, and some people were actually requested, so we provided. And just saying like always providing with the newest technologies, with the newest updates, so nothing is missing. And if, I don't know if it's called "bribing," but for example, we make sure that it's one of our engineers who reads something on the Internet. It's like, "Oh, this new cool feature in Grafana that I saw! Can we have it?" We make sure that we've had it before we always make sure that we're on top of things because that makes people want to use the system more and use it well and implement all the new changes. +Just saying, we are always providing with new technologies, with the newest updates so nothing is missing. And if—let's—I don’t know if it’s called bribing, but for example, we make sure that it’s one of our engineers who reads something on the Internet, it’s like, "Oh, this new cool feature in Grafana that I saw. Can we have it?" We make sure that we have it before we always make sure that we’re on top of things because that makes people want to use the system more and use it well and implement all the new changes. So it's great for us. -So it's great for us. +**Audience Member:** Yeah, that's really good. If I can ask one more quick follow-up question, then I can give someone else a turn. In your slide deck, you talk about how developing ownership by everybody of observability is super important for success because ultimately, like application engineers, they’re the ones who have the context for how their applications run and for the things they need to look for. -**Derek:** That's really good. If I can ask one more quick follow-up question, then I can give someone else a turn. So in your slide deck, you talk about how developing ownership by everybody of observability is super important for success. Because ultimately, like application engineers, they're the ones who have the context for how their applications run and for the things they need to look for. +So how, in your experience, have you gotten application teams to kind of take that ownership without making it feel like it’s just one more thing you’re trying to chuck over the wall at them? Right? Because again, like it’s a unique spot that we’re at as infrastructure platform people because we’re not like the best at writing code. Like, that’s not going to be our strongest skill. So it feels weird to me, at least in the past, saying, "Hey, can y'all go learn this thing and put it into practice, and I’ll coach you from the sidelines, but I can’t do it as well as you." -So how, in your experience, have you gotten application teams to kind of take that ownership without making it feel like it's just one more thing you're trying to chuck over the wall at them? Right? Because again, like it's a unique spot that we're at as infrastructure platform people because we're not like the best at writing code, like we're not—that's not going to be our strongest skill. So it feels weird to me, at least in the past, saying, "Hey, can y'all go learn this thing and put it into practice, and I'll coach you from the sidelines? But I can't do it as well as you." +**Iris:** Yeah, I think what helps, as I said again, we are lucky because we have a very good observability culture already ingrained. So in most of the engineers, they just do it. But yeah, I think what really helped was that we have given full ownership to the teams on building their own dashboards, alerting, and sending their own telemetry signals. So basically, when they come to us like, "Hey, can you help us set up an alert?" We’re like, "Well, say, you are full owners of that part of the observability." -**Iris:** Yeah, I think what helps, as I said again, we are lucky because we have a very good observability culture already ingrained. So most of the engineers just do it. But yeah, I think what really helped was that we have given full ownership to the teams on building their own dashboards, alerting, and sending their own telemetry signals. +That actually helps the teams knowing that they’re owners of that. They want to make it grow and improve it rather than just leaving it there. We’ve also made it very clear that we will not be going there and implementing anything related to monitoring. So if you need monitoring, you have to do it yourself. We are here for you; we provide guidance. Of course, we have tons of documentation on how to do everything. They’re not alone, but we made it very clear that we’re not going to do that for you. It’s simple. And considering that all our incidents are coming from the alerting that is set up on our platform, let’s say they’re kind of forced to implement it. -So basically, when they come to us like, "Hey, can you help us set up an alert?" We're like, "Well, say you are full owners of that part of the observability," and that actually helps the teams knowing that they're owners of that. They want to make it grow and improve it rather than just leaving it there. +But I think just the fact that they’re owners of that and it’s part of their team responsibility coming from the higher structures of the company—"Okay, you’ll have this amazing product that you are building and maintaining, but you also have observability that is your full ownership. So if it’s not good, then it is your responsibility, not somebody else’s." I think that has motivated the teams to really work on that. Sometimes it happens that when a junior enters the team, they’re the ones that have to take care of all the monitoring without knowing well the code. But I think they soon realize how important it is for everyone to put their hands in to contribute; otherwise, it does not work out. -And we've also made it very clear that we will not be going there and implementing anything related to monitoring. So if you need monitoring, you have to do it yourself. We are here for you; we provide guidance, of course. We have tons of documentation on how to do everything. They're not alone, but we've made it very clear that we're not going to do that for you. It's simple. +**Audience Member:** That's great, thank you. I have a question for you. It is, you're saying like you guys hold your ground as far as not being the ones to instrument code for folks. What kind of reactions do you get from teams when you say that? Like, are they like, "Oh, okay, I'll do it," or did they like defensive pushback? What’s it like? How do you deal with it? -And considering that all our incidents are coming from the alerting that is set up on our platform, let's say they're kind of forced to implement it. But I think just the fact that they're owners of that, and it's part of their team responsibility coming from the higher structures of the company, "Okay, you'll have this amazing product that you are building and maintaining, but you also have observability that is your full ownership." So if it's not good, then it is your responsibility, not somebody else's. +**Iris:** Most of the time, they're like, "Oh, okay. Is there a framework? Can you send me to the framework so that I can find it?" We usually also have a team that helps build this framework, so we send it to them. But most of the time, yeah, we just hold our ground. If they say, "Oh, but I really need help," we’re like, "Okay, I'm gonna help you. You need to do it for yourself. I don’t know your code." -So I think that has motivated the teams to really work on that. Sometimes it happens that when a junior enters the team, they're the ones that have to take care of all the monitoring without knowing well the code, but I think they soon realize how important it is for everyone to put their hands in to contribute there; otherwise, it does not work out. +Of course, there are cases that people are not happy, and they could go to a higher structure in the company and complain. But considering that this is not just something that the observability team has decided today, that "Oh, we're not going to build anything for anyone," because it’s their responsibility, it’s not something that is, of course, supported and signed by upper management. They’re like, "Sorry, but you have to do it. It’s documented, and you have to do it." And it’s simple as that. -**Derek:** That's great, thank you! +**Audience Member:** That’s great having support. That’s awesome because I think you've nailed it. That is the key thing because you're right: if you don’t have the management supporting that decision to have an observability team, we're not going to instrument your code—yeah, you're gonna have back-channeling. I mean, I've experienced that. It's really annoying and ends up kind of ruining the thing that you're trying to build as far as that observability culture. -**Iris:** I have a question for you. It is, um, so you're saying like you guys hold your ground as far as like not being the ones to instrument code for folks. What kind of reactions do you get from teams when you say that? Like, are they like, "Oh, okay, I'll do it," or did they—like defensive pushback? Like, what's it like? How do you deal with it? +**Iris:** Yeah, exactly. As I said, in Farfetch, we are very, very happy in this aspect because being forced to instrument somebody’s code—that would be the breaking point for me as an engineer in believing that observability is lived preaching with everyone and talking about it and then being forced to go against these values that we have—that would be the breaking point for me. I have to stay. -**Iris:** Most of the time, they're like, "Oh, okay. Is there a framework? Can you send me to the framework so that I can find it?" And we usually also have a team that helps build this framework, so we send it to them. But most of the time, yeah, we just hold our ground. If they say that, "Oh, but I really need to help me," like, "Okay, I'm gonna help it. I don't know how to do it for yourself; I don't know your code." +**Audience Member:** Yeah, I don't blame you. I have a question. Farfetch is an e-commerce site, correct? -And of course, there are cases that people are not happy, and they could go to a higher structure in the company and complain. But considering that this is not just something that as the durability team have decided today, that, "Oh, we're not going to build anything for anyone because it's their responsibility." It's not something that is, of course, supported and signed by upper management. They're like, "Sorry, but you have to do it; it's documented, and you have to do it." And it's simple as that. +**Iris:** Yes. -**Derek:** It's great having support; that's awesome. Because I think you've nailed it: that is the key thing. Because you're right, like if you don't have the management supporting that decision to like, you know, have an observability team, we're not going to instrument your code. Yeah, you're gonna have back channeling, and I mean, I've experienced that—it's really annoying. +**Audience Member:** I was curious, and Derek, I guess actually this question is inspired by your question in the hotel end users channel this morning: how do you, or do you, like what do you use for client-side instrumentation? -And it ends up kind of ruining the thing that you're trying to build, like as far as that observability culture. +**Iris:** We are using a vendor currently. I’d rather not say, but yeah, we’re using a vendor for that part, which is very, very small. The rest of the platform is all instrumented by our own custom framework. -**Iris:** Yeah, exactly. As I said, yeah, in Firefits, we are very, very happy in this aspect because, yeah, being forced to instrument somebody's code, that would be the breaking point for me as an engineer in believing that observability is lived preaching with everyone and talking about it and then being forced to go against these values that we have. That would be the breaking point for me; I have to stay. +**Audience Member:** Okay. Do you have that integrated at all, or is it kind of like its own view of the client, and then you have trouble correlating signals with your backend? -**Derek:** Yeah, I don't blame you. +**Iris:** Currently, it is a bit segregated, but we're working on changing it because it's such a small part of our platform. But yeah, it's not fully integrated. We're focusing more on our own custom solution that we have locally, and that one is a bit segregated from it and it has only specific information, which is not great. But that’s our goal for 2023 and 2024—to integrate everything together. -**Iris:** I have a question. Firefits is an e-commerce site, correct? And so I was curious—and Derek, I guess actually this question is inspired by your question in the hotel end-users channel this morning. How do you—or do you—like, what do you use for client-side instrumentation? Are you— +**Audience Member:** What does your custom solution involve? What makes it custom, I guess is the question. -**Iris:** Well, we are using a vendor currently. I’d rather not say, but yeah, we're using a vendor for that part, which is very, very small. The rest—the rest of the platform is all instrumented by our own custom framework. +**Iris:** It’s using OpenTelemetry now. We’re using Prometheus, Grafana, Alertmanager—basically all open source—and we built it. Let’s say we have around 100 Kubernetes clusters. We have created our agent based on this open source and just adding them on each of the clusters, collecting information, centralizing it. -**Derek:** Okay, okay. Do you have that integrated at all, or is it kind of like its own view of the client? And then you have trouble correlating signals with your back end? +**Audience Member:** Oh, so custom solution, like you’re talking specifically around tooling and not like—I thought maybe you meant like having some abstraction layer on top of OpenTelemetry, which I’ve seen some organizations do. -**Iris:** Currently, it is a bit segregated, but we're working on changing it because it's such a small part of our platform. But yeah, it's not fully integrated. We're focusing more on, let's say, on our own custom solution that we have locally, and that one is a bit segregated from it, and it has only specific information, which is not great. But that's our goal for 2023 and 2024 to integrate everything together. - -**Derek:** What's your custom solution involve? Like, what makes it custom, I guess, is the question? - -**Iris:** I mean, it's using open telemetry now. We're using thousands, Prometheus, Grafana, Alert Manager—basically all open source—and we built it, let's say, we have around 100 Kubernetes clusters. We have created our agent based on this open source and just adding them on each of the classes, collecting information, centralizing it. +**Iris:** No, no, no. -**Derek:** Oh, so custom solution, like you're talking specifically around tooling and not like—I thought maybe you meant like having some abstraction layer on top of like telemetry, which I've seen some organizations do. +**Audience Member:** Okay, okay, cool. Any other thoughts or questions for Iris? -**Iris:** No, no, no. +**Audience Member:** Yeah, I guess I’m a little curious. So you said, you know, there’s no such thing as like too many traces, and I know some people, you know, prefer to do some form of tail sampling. So they’re just—they’re still getting like a subset of, you know, randomized 200s and then like, you know, whatever traces with errors or latency or whatever attributes that they might be interested in. -**Derek:** Okay, cool. +So I was just curious if you could expound on that a little bit more, especially like, you know, when talking about like increased costs with excessive amounts of data. -**Iris:** Any other thoughts or questions for Iris? +**Iris:** Yeah, so tracing is something that we are working a lot on because it's not one of the telemetry signals that are being used a lot in the company. So we are trying to advocate it and push it a lot. And for your—when I say too few traces, I would say 0.01 sampling, like normal sampling, let’s say. In younger teams, they were considering it extremely low, and they weren’t even caring about it even though some of them you could find very good information there. -**Derek:** Yeah, I guess I'm a little curious. So you said, you know, there's no such thing as like too many traces. And I know some people, you know, prefer to do some form of tail sampling. So they're just—they're still getting like a subset of, you know, randomized 200s and then like, you know, whatever traces with errors or latency or whatever attributes that they might be interested in. +For example, I use it a lot to troubleshoot our Thanos queries—it's amazing! So where we are right now, we increased because we were able to change our backend solution from Cassandra to Grafana Tempo, which is also open source. So of course, because before, we were spending a crazy amount of money for 0.01 tracing. We were spending so much money. I don’t want to say numbers. There was no place for us to grow more; it was completely out of budget. -So I was just curious if you could expound on that a little bit more, especially like, you know, when talking about like increased costs with excessive amounts of data. +So now we implemented Tempo, and we are able to send more traces, and we increased to five percent, increasing that just a few applications. We actually implemented this change because, of course, it’s a custom framework, and they can do it however they want. We already went from 1,000 spans per second to 40,000 from just one application, and this application is not even using most of what they're sending. -**Iris:** Yeah, so tracing is something that we are working a lot because it's not one of the telemetry signals that are being used a lot in the company. So we are trying to advocate it and push it a lot. And for your—when I say too few traces, I would say, uh, 0.01 sampling, like normal sampling, let's say, in younger. So the teams were considering it extremely low, and they weren't even caring about it, even though some of them you could find very good information there. +Imagine we have hundreds of applications, all of this sending—that is going to be millions of spans per second. It’s going to require a lot of computation power, and it’s going to require a lot of storage. It’s going to be a mess, and the cost will be out of this world. -For example, I use it a lot to troubleshoot our Thanos queries; it's amazing. So where we are right now, we increased because we were able to change our back-end solution from Cassandra to Grafana Tempo, which is also open source. So of course, before we were spending a crazy amount of money for 0.01 tracing, we were spending so much money. I don't want to say numbers, so there was no place for us to grow more; it was completely out of budget. +In conclusion, I think that we really need to go there. Like, we’re in the process of doing it and try to see to implement better tracing because just normal sampling is not working. Yeah, we're looking into tail-based—that's why OpenTelemetry was brought up in the first place. -So now we implemented Tempo, and we are able to send more traces, and we increased to five percent and increasing that just a few applications. And actually, uh, implemented this change because, of course, it's a custom framework, and they can do it however they want. And we already went for 1,000 spans per second to 40,000 from just one application, and this application is not even using most of what they're sending. +So the perfect would be to look at the tail-based, and yeah, we were sending too little or basically there was not enough information. Now we're sending too much, so we are trying to find something that is in between. Tail sampling is one of them, and also we're also making changes to the framework so not everything goes through. Some of the information is just not usable, and the engineers are not needed. -And imagine we have hundreds of applications, all of this sending—all that is going to be millions of spans per second. It's going to require a lot of computation power, and it's going to require a lot of storage. It's going to be a mess, and the cost will be out of this world, in conclusion. So I think that we really need to go there, like we're in the process of doing, and try to see—to implement better tracing because just normal sampling is not working. +Yeah, it's a work in progress that is taking a lot of our time. I’d rather spend more money and send more information, obviously, but at the same time, we could not spend millions of dollars a year for information that’s not going to be used. -Yeah, we're looking into a tail base; that's why open telemetry was brought up in the first place. So the perfect—to look at the tail base—and yeah, we were sending too little or basically there was not enough information. Now we're sending too much, so we are trying to find something that is in between. Tail sampling is one of them, and also we're also making changes to the framework so not everything goes through. Some of the information is just not usable, and the engineers are not needed. +**Audience Member:** I see a raised hand. -Yeah, it's a work in progress that is taking a lot of our time. I'd rather spend more money and send more information, obviously, but at the same time, we could not spend millions of dollars a year for information that's not going to be used. +**Audience Member:** I did! I wanted to ask if you’re doing anything around anomaly detection, especially as you increase the volume of information coming in to avoid having to look at everything—look at those anomalies that are out of the time frame? -**Derek:** I see a raised hand. +**Iris:** You mean anomaly detection in the amount? -**John:** I did. I wanted to ask if you're doing anything around anomaly detection, especially as you increase the volume of information coming in, to avoid having to look at everything to look at those anomalies that are out of the time frame. +**Audience Member:** Well, so the scenario of, you know, here’s a high-volume event, but it’s normal, and here’s a high volume at what’s normally a low time of day, right? Just to look for anomalies and everything that you’re receiving without having to have a human look at it. -**Iris:** You mean anomaly detection in the—? +### [00:39:52] Anomaly detection and machine learning in observability -**John:** Yeah, well, so the scenario of, you know, here's a high volume event, but it's normal, and here's a high volume at what's normally a low time of day, right? Just to look for anomalies and everything that you're receiving without having to have a human look at it. +**Iris:** Yeah, we actually have some alerting implemented there based on standard deviation, and it’s like a simple PromQL-based alerting for that. And they’re accurate, but we haven’t gone about that because it’s a custom solution, and we actually need to apply some machine learning there, which we haven’t really been able to get to. -**Iris:** Yeah, we actually have some alerting implemented there based on standard deviation, and it's like a simple PromQL-based alerting for that. And they are accurate, but we haven't gone about that because it's a custom solution, and we actually need to apply some machine learning there, which we haven't really been able to get to. +So it’s doing some alerts with the standard deviation that are okay. They can detect some anomalies and some very big changes, but also are highly inaccurate, so it’s not my favorite, but they are there. We need to work better on it, but yeah, for example, if we have a huge anomaly or a huge flow of sudden that is not predicted, it is okay. Let’s say it’s not perfect. -So it's do some alerts with the standard deviation that are okay; they can detect some anomalies and some very big changes, but also are highly inaccurate. So it's not my favorite, but they are there. We need to work better on it, but yeah, for example, if we have a huge anomaly or a huge flow of sudden that is not predicted, it is okay, let's say, it's not perfect. +I know you can get into some really interesting solutions with that that can come with a high cost, but there are some really interesting things you can do with that too. So I would love to know more if you have any links, any suggestions for me, please let me know because the more solutions we can implement! -I know you can get into some really interesting solutions with that that can come with a high cost, but there's some really interesting things you can do with that too. So I would love to know more if you have any links, any suggestions for me, please let me know, because the more solutions we can implement, hahaha! Like you mentioned, cost is always a worry, obviously, but if it's actually a quality solution that we are introducing, of course, it is a cost that is needed. +**Audience Member:** Like you mentioned, cost is always a worry, obviously, but if it’s actually a quality solution that we are introducing, of course it is a cost that is needed. -And that's it, you know. It's really enjoyable when you start to use tools like that to solve the problems because then you see things you didn't expect to see. We were experimenting with something a few years ago, and we intended to point at the top 10 issuers. And, you know, the engineer mistakenly didn't put the top 10 in, so he was actually looking for anomalies across the entire user set. +**Audience Member:** And that’s it, you know? It’s really enjoyable when you start to use tools like that to solve the problems because then you see things you didn’t expect to see. -So, you know, thousands of customers, and they were in entities. So, you know, you can start to see, "Oh look, they're not sending at this time of day when they usually are! That's a low volume, low center, but they're sending nothing, and they usually send 70 per hour!" And we could see that before it finally blew up from running out of memory. But you know, you really get to say, "Now what can I do with that for my business?" +We were experimenting with something a few years ago, and we intended to point at the top 10 issuers. And you know, the engineer mistakenly didn’t put the top 10 in, so he was actually looking for anomalies across the entire user set. So, you know, thousands of customers, and we could start to see, "Oh look, they’re not sending at this time of day when they usually are—that’s a low volume, low center, but they’re sending nothing when they usually send 70 per hour." -So that was interesting. We presented to them at the team who wanted to own that and didn't want to develop the expertise in that tooling, even though we could show them what it could do. So a 500 gigabyte EC2 instance was running that before it blew up. +And we could see that before it finally blew up from running out of memory. But, you know, you really get to say, "Now what can I do with that for my business?" So that was interesting. We presented to them at the team who wanted to own that but didn’t want to develop the expertise and that tooling, even though we could show them what it could do. -**Iris:** Yeah, a challenge that we have currently is with our metrics as well, and detecting which applications are sending higher technology, more number of times series. And we have dashboards and alerts for that, but nothing is based on machine learning. It's just pure statistical data, and it is being evaluated for standard deviation or simple queries. +So a 500-gigabyte EC2 instance was running that before it blew up. -So some more insight there would be amazing. +**Iris:** Yeah, a challenge that we have currently is with our metrics as well and detecting which applications are sending higher technology, more number of times series. We have dashboards and alerts for that, but nothing is based on machine learning. It’s just pure statistical data, and it is being evaluated for standard deviation or simple query. -**John:** Yeah, that too much data problem is very interesting to have, especially as the open telemetry standards evolve and add more fields. So now you have more fields to consider. +So some more insight there would be amazing! -**Iris:** Yeah, and they're good. It's good that those are coming, absolutely. +**Audience Member:** Yeah, that too much data problem is very interesting to have, especially as the OpenTelemetry standards evolve and add more fields, so now you have more fields to consider. -**John:** One thing that I wanted to ask you is: is your team involved at all in the creation of SLOs? +**Iris:** Yeah, and they’re good. It’s good that those are coming. -**Iris:** Foreign, yes, but also no. Well, let's say we are the ones that are providing the tool for creating SLOs. +**Audience Member:** Absolutely. One thing that I wanted to ask you is, is your team involved at all in the creation of SLOs? -So basically, the teams can implement the structure or basically create alerting dashboards based on it, but we are not the ones that are providing the guidelines and the instructions on how to create them. We just provide the tooling. +**Iris:** Yes, but also no. Well, let's say we are the ones that are providing the tool for creating SLOs. -**John:** Is it like an open-source tool that you guys are using, or is it some like commercial? +So basically, the teams can implement the structure or create alerting dashboards based on it, but we are not the ones that are providing the guidelines and the instructions on how to create them. We just provide the tooling. -**Iris:** We're actually using—we have our own alerting solution, so we are doing the same for SLOs as well, and we are currently using Thanos ruler to send the alerts and to evaluate them, Alert Manager to send the alerts, and we also have a custom tool that reads the SLOs that are written in Terraform and makes them compatible with a Thanos ruler, so Thanos ruler can read them. +**Audience Member:** Is it like an open source tool that you guys are using or is it some commercial? -**John:** Okay, and do you—so the reason why I'm asking specifically about SLOs is because one of the, I'd say, one of the practices a team should aim for is like using observability data to help generate their SLOs or not generate, to, you know, create their SLOs based on observability data. +**Iris:** We’re actually using—we have our own alerting solution, so we are doing the same for SLOs as well. We are currently using Thanos Ruler to send the alerts and to evaluate them, Alertmanager to send the alerts, and we also have a custom tool that reads the SLOs that are written in Terraform and makes them compatible with a SLO’s Prometheus rule. -So I'm just wondering if that's something that your team is planning on providing guidance on? Like, what's—are there any plans around that? +**Audience Member:** And do you—so the reason why I’m asking specifically about SLOs is because one of the—I’d say one of the practices a team should aim for is like using observability data to help generate their SLOs or not generate, to create their SLOs based on observability data. So I’m just wondering if that’s something that your team is planning on providing guidance on? -**Iris:** So basically, we are providing the data and the tooling to do it. Most of the guidance, like how the SLO should be built, most of them are our business, and we're not really involved in that part. +**Iris:** Basically, we are providing the data and the tooling to do it. Most of the guidance on how the SLO should be built, most of them are our business, and we’re not really involved in that part. So we cannot give guidelines. -So we cannot give guidelines, so that's completely different. But yeah, for example, for infrastructure, we provide some basic guidelines on SLOs. But I would say that, yeah, we just want to provide the data; they feed off of our metrics and the tooling that they can do it and how they can actually build them, but not really how we can go and or like how to calculate the SLOs or what is best and what is worse. +So that’s completely different, but yeah, for example, for infrastructure we provide some basic guidelines of SLOs, but I would say that yeah, we just want to provide the data that they feed off of our metrics and the tooling that they can do it and how they can actually build them, but no, not really how they can go and, or like how to calculate the SLOs or what is best and what is worse. We haven’t gotten there, and I don’t think we have plans currently to move into that as we have other teams that give advice or recommendations on that matter. -And we haven't gotten there. I don't think we have the plans currently to move into that as we have other teams that give advice or recommendations on that matter. There is so much to do in observability in our company. I think we've done a pretty good job so far, but there is so much more that we can do. +There is so much to do in observability in our company. I think we’ve done a pretty good job so far, but there is so much more that we can do. -**John:** Is there something that your team is working on implementing, like that you're excited about in the near future? +**Audience Member:** Is there something that like your team is working on implementing that you’re excited about in the near future? -**Iris:** Well, we decided that because of the huge amount of data and we do not have capability to process it as good as we would like, we are currently trying to see if we can—well, let me put some precedent. We are using different tools for different things. We're using Prometheus styles for metrics, we're using open telemetry for traces and open telemetry for metrics to a certain degree as well, Grafana for dashboards, and we're using a different tool for logging. +**Iris:** Well, we decided that because of the huge amount of data and we do not have capability to process it as well as we would like, we are currently trying to see if we can—well, let me put some precedent: we are using different tools for different things. We’re using Prometheus for metrics; we’re using OpenTelemetry for traces and OpenTelemetry for metrics to a certain degree as well, Grafana for dashboards, and we’re using a different tool for logging. -And currently, it's all a mess. So what we're trying to do right now is we want to centralize them in one platform. What we don't know yet if it's going to be an outside platform that we are feeding the data to or like a vendor or if we're going to provide it ourselves. So that's something that we're working on right now: centralizing everything in one place and sending all telemetry signals through one transporter, that is open telemetry, so we could have better correlation, which we are also lacking currently. +Currently, it’s all a mess, so what we’re trying to do right now is we want to centralize them in one platform. What we don’t know yet is if it’s going to be an outside platform that we are feeding the data to or like a vendor, or if we’re going to provide it ourselves. So that’s something that we’re working on right now—centralizing everything in one place and sending all telemetry signals through one transporter, that is OpenTelemetry, so we could have better correlation, which we are also lacking currently. -**John:** Awesome, so exciting times! +**Audience Member:** Awesome, so exciting times! -**Iris:** It is amazing! I love the work that you're doing right now, and it's going to be the next two years at least. I know after that, it's going to be very exciting for us. +**Iris:** It is amazing! I love the work that you’re doing right now, and it’s going to be the next two years at least. I know after that it’s going to be very exciting for us. -**John:** Well, it's very, very awesome. We're coming up on time, but I did want to give folks an opportunity to ask any other final burning questions. +**Audience Member:** Well, it’s very, very awesome. We’re coming up on time, but I did want to give folks an opportunity to ask any other final burning questions. -Nope, everyone's questioned out. Thank you, everyone, so much for joining. And if anyone who's in the audience is interested in giving a presentation for hotel and practice or would like to participate in one of our hotel Q&As, please reach out on Slack. We are more than happy to hear your stories, share your stories so that we can continue to build this amazing open telemetry community. +**Audience Member:** Nope, everyone’s questioned out. -Thank you so much for having me today! +**Iris:** Thank you everyone so much for joining, and if anyone who’s in the audience is interested in giving a presentation for OpenTelemetry or would like to participate in one of our OpenTelemetry Q&As, please reach out on Slack. We are more than happy to hear your stories, share your stories so that we can continue to build this amazing OpenTelemetry community. Thank you so much for having me today! ## Raw YouTube Transcript diff --git a/video-transcripts/transcripts/2023-07-02T17:46:37Z-otel-q-a-feat-jacob-aronoff-of-lightstep-from-servicenow.md b/video-transcripts/transcripts/2023-07-02T17:46:37Z-otel-q-a-feat-jacob-aronoff-of-lightstep-from-servicenow.md index 60c2ed1..296199d 100644 --- a/video-transcripts/transcripts/2023-07-02T17:46:37Z-otel-q-a-feat-jacob-aronoff-of-lightstep-from-servicenow.md +++ b/video-transcripts/transcripts/2023-07-02T17:46:37Z-otel-q-a-feat-jacob-aronoff-of-lightstep-from-servicenow.md @@ -10,405 +10,400 @@ URL: https://www.youtube.com/watch?v=dpXhgZL9tzU ## Summary -In this YouTube video, Jacob Arnoff, a staff engineer at Lightstep, discusses his experience migrating to OpenTelemetry (otel) from OpenTracing and other telemetry solutions within the company. The conversation, facilitated by the host, explores the challenges and strategies involved in the migration process, emphasizing safety and performance improvements. Jacob shares insights into the initial migration from OpenTracing to OpenTelemetry, the issues encountered while migrating metrics, and the advantages of using features like views for managing metrics effectively. The discussion also touches on deployment strategies for collectors in a Kubernetes environment, comparing sidecar, daemon set, and stateful set approaches. Throughout the conversation, Jacob emphasizes the importance of keeping dependencies up to date and how this impacts the overall performance and reliability of observability systems. The session concludes with an invitation for Jacob to share more of his insights in future talks, highlighting the ongoing evolution of telemetry practices. +In this YouTube video, a Q&A session features Jacob Arnoff, a staff engineer at Lightstep, discussing his experiences with migrating to OpenTelemetry (otel) within his organization. The conversation is led by an unnamed host, who highlights the significance of an observability vendor using otel for their own products. Jacob shares insights about the migration process from OpenTracing to OpenTelemetry, detailing the challenges he faced, such as performance issues and the importance of maintaining alerting capabilities during the transition. Key topics include migration strategies, the use of attributes and metrics, the differences between sidecars, deployments, and daemon sets in Kubernetes, and the implementation of the target allocator for efficient data collection. The discussion wraps up with a focus on the evolving nature of telemetry setups and a call for Jacob to potentially give a talk on his experiences. ## Chapters -00:00:00 Welcome and intro -00:00:20 Guest introduction: Jacob -00:02:30 Jacob's background and role -00:03:50 Migration to OpenTelemetry -00:05:41 Migration challenges and strategies -00:09:00 Q&A session begins -00:12:01 Metrics migration details -00:16:02 Performance improvements discussion -00:18:00 Metrics views and aggregation -00:22:02 Logging and span events -00:24:50 Collector setup and architecture +00:00:00 Introductions +00:01:18 Guest introduction: Jacob Arnoff +00:05:54 Migration from OpenTracing to OpenTelemetry +00:10:49 Migration strategies discussion +00:16:23 Performance issues during migration +00:22:37 Discussion on metrics aggregation +00:31:28 Target allocator explanation +00:39:40 Collector setup discussion +00:47:12 Use of stateful sets vs deployments +00:52:07 Wrap-up and parting thoughts -[00:00:20] **Host:** Thank you. Welcome everyone to Hotel Q&A. Thanks for joining. Today we are super lucky to have Jacob Aronoff from Lightstep from ServiceNow join us. It's a cool treat because I tapped Jacob because he was telling me that he had submitted a CFP to one of the, I think it was KubeCon, right Jacob? +## Transcript -**Jacob:** Yeah, it was a CFP on migrating to Hotel from our organization—migrating to Hotel, right? If I understand correctly, which I thought that's a pretty freaking cool story to tell. So I asked him to join and have this Q&A and hear the story because I think it makes for a compelling narrative when an observability vendor talks about using OTEL themselves on their own products. So here we are. So welcome Jacob. Do you want to do like a quick little intro? +### [00:00:00] Introductions -**Jacob:** Sure. Yeah, hi, my name is Jacob Aronoff. I'm a staff engineer at Lightstep on the Telemetry pipeline team. I've been in lead stuff for almost two years, and the first year of that journey was solely focused on Hotel migrations—doing them internally and making them easier for customers, sort of that whole process. So yeah, I could get into this anyway that is interesting. How do you think we want to do this? Very Q&A style or should I just go right into the story? +**Host:** Thank you, welcome everyone to Hotel Q&A. Thanks for joining. Today we are super lucky to have Jacob Arnoff from Lightstep from ServiceNow join us. It's a cool treat because I tapped Jacob because he was telling me that he had submitted a CFP to one of the, I think it was KubeCon, right Jacob? -**Host:** We can let it evolve organically. +**Jacob:** Yeah, it was a CFP on migrating to Hotel from our organization migrating at Hotel, right? If I understand correctly, which I thought that's a pretty freaking cool story to tell. -**Jacob:** Cool. Yeah, I like that. Why don't we start like— I mean I think you're rearing to go, so maybe why don't you describe, like, set the scene for us? +### [00:01:18] Guest introduction: Jacob Arnoff -[00:02:30] **Jacob:** When I joined Lightstep, we were still on OpenTracing for tracing and then a mix of OpenCensus and some hand-rolled StatsD stuff for metrics. So this meant that we had to run a proxy on every single pod that we ran in Kubernetes. A proxy, since it's a sidecar on every pod, means that every single time you run one of your applications, you have to run another little application that's going to read from StatsD and then forward those metrics off. And, you know, for us, as we're building out a metric solution, that destination was Lightstep. +**Host:** So I asked him to join and have this Q&A and hear the story because I think it makes for a compelling narrative when an observability vendor talks about using OTEL themselves on their own products. So here we are. Welcome, Jacob. Do you want to do a quick little intro? -[00:03:50] I came in, and this was sort of at the beginning of metrics Alpha, I think. I was like, hey, it would be great for us to—we know that OpenTelemetry for metrics is going to be a huge effort for us in the next year or two. We wanted to reach stability. The Hotel team had been—we have an Hotel team internal to Lightstep. They've been working on it a lot and really wanted some immediate feedback on how to improve it. So I took on that migration for us. +**Jacob:** Sure. Yeah, hi, my name is Jacob Arnoff. I'm a staff engineer at Lightstep on the Telemetry pipeline team. I've been with Lightstep for almost two years, and the first year of that journey was solely focused on Hotel migrations, doing them internally and making them easier for customers, sort of that whole process. -We also had the theory that doing so would save us a good chunk of money because we would no longer need to run these relatively expensive StatsD sidecars. So I planned it initially to be sort of as safe as possible. I'd done some migrations like this in the past, and there are a few different ways that you can do migrations like this. You can do the all-in-one go, which for us would have been possible— we're in a mono-repo—but it's much more dangerous because you worry about, you know, am I going to push a bug that's going to take everything down, right? +**Host:** That's interesting. Do we want to do this very Q&A style or should I just go right into the story and talk about it? -Obviously, this is application data. It's data about your applications, which we use for alerting. We use to understand how our workloads are functioning in all of our environments, and so it's important that we don't take that down because that would be disastrous for us. But obviously, for an end user, it's going to be the same story. They want the comfort that if they migrate to Hotel, they're not going to lose all of their alerting capabilities immediately. They want a safe and easy migration. +**Jacob:** We can let it evolve organically. -So that was the only one with our initial approach for doing a sort of feature flag-based—just like part of that configuration that you run in Kubernetes. It would disable this sidecar, enable some code that would then swap to OTEL for metrics and then forward it off to where it's supposed to go. So that was sort of the path there. +**Host:** Cool. Yeah, I like that. Why don't we start? I mean, I think you're rearing to go, so maybe why don't you describe, like, set the scene for us. -[00:05:41] Midway through this journey of doing these migrations, I had tested it all out, and staging looked pretty good. I tested the container in our meta environment. So we use to monitor our public environment. I noticed some pretty large performance issues in those 2021 and I had reached out to the Hotel team, and we had worked together to sort of alleviate some of those concerns. One of the ones that we found that was a big blocker was we heavily use attributes on metrics right now, and it was incredibly tedious to go in and figure out which metrics are using all these attributes and getting rid of them. +**Jacob:** When I joined Lightstep, we were still on OpenTracing for tracing and then a mix of OpenCensus and some hand-rolled StatsD stuff for metrics. This meant that we had to run a proxy on every single pod that we ran in Kubernetes. A proxy, since it's a sidecar on every pod, means that every single time you run it, one of your applications, you have to run another little application that's going to read from StatsD and then forward those metrics off. -Well, I had a theory that really this one code path was the problem where we're doing the conversion from our internal tagging implementation through Hotel tags, which had a lot of other logic with it that was pretty expensive to do on pretty much every call. So I was like, you know what? There's no better time than now to begin another migration from OpenTracing to OTEL basically. While we wait on the Hotel folks on the metric side to push out more performant code implementations for us, we can also test out this theory that if we migrate the Hotel entirely, we're actually going to see even more performance benefits. +For us, as we were building out a metric solution, that destination was Lightstep. I came in, and this was sort of at the beginning of metrics Alpha, I think. I was like, hey, it would be great for us to, we know that Hotel for metrics is going to be a huge effort for us in the next year or two. We wanted to reach stability. The Hotel team at Lightstep had been working on it a lot and really wanted some immediate feedback on how to improve it. So I took on that migration for us. -So at that point, I said, okay, I'm going to put a pause in the metrics work while we wait for Hotel, and I'm going to begin on this tracing migration. However, I decided to try a different approach, which is the all-or-nothing approach. Basically, the OpenTracing to OpenTelemetry path was a bit more known. There were a few really small docs and examples, and they are backwards compatible. You’re able to use them in conjunction with each other, so one thing emitting Hotel and one thing emitting OpenTracing is not the end of the world. +We also had the theory that doing so would save us a good chunk of money because we would no longer need to run these relatively expensive StatsD sidecars. I planned it initially to be sort of as safe as possible. I'd done some migrations like this in the past, and there are a few different ways that you can do migrations like this. You can do the all-in-one go, which for us would have been possible since we're in a mono repo, but it's much more dangerous because you worry about, am I going to push a bug that's going to take everything down? -So you can mix those as long as you have the propagators set up correctly. So first step, propagators. Second step, make sure that all of our plugins worked, which at the time they weren't open sourced. Now they are open source, so people can just use them. I just did it all in one swoop. Maybe I had to revert like three times from our staging environment—nothing really major. And then there was one bug that I missed where we were previously doing a lot of in-app sampling because we had a really noisy function call. So I had to implement the custom sampler, which is actually like ten times easier with OTEL than it was with OpenTracing. I was able to get rid of a good like 1000 lines of code or so and some really dangerous hacks, so that was a really good thing. +Obviously, this is application data. It's data about your applications, which we use for alerting. We used to understand how our workloads are functioning in all of our environments, and so it's important that we don't take that down because that would be disastrous for us. But obviously for an end user, it's going to be the same story. They want the comfort that if they migrate to Hotel, they're not going to lose all of their alerting capabilities immediately. They want a safe and easy migration. -And yeah, so then that went out very happy. Also stopping at any time if we want to get back to like more Q&A stuff. +So our initial approach was to do a sort of feature flag-based part of the configuration that you run in Kubernetes. It would disable this sidecar and enable some code that would then swap to OTEL for metrics and then forward it off to where it's supposed to go. That was sort of the path there. -[00:09:00] **Host:** I have so many questions from this already, but do you mind taking like a quick pause? +### [00:05:54] Migration from OpenTracing to OpenTelemetry -**Jacob:** Sure. +Midway through this journey of doing these migrations, I had tested it all out, and staging looked pretty good. I tested the container in our meta environment, so we use it to monitor our public environment. I noticed some pretty large performance issues in those 2021, and I had reached out to the Hotel team, and we had worked together to sort of alleviate some of those concerns. One of the ones that we found that was a big blocker was we heavily use attributes on metrics right now, and it was incredibly tedious to go in and figure out why, you know, figure out which metrics are using all these attributes and getting rid of them. -**Host:** A couple of comments. One thing that came to my mind as you're saying this, I'm like, this is actually really freaking cool story because I think we tend to see like two different types of organizations. We see the ones where there's like zero code instrumented—like this is their first foray into instrumenting their code—and then we see the organizations that have either dabbled in OpenTracing. I think it's a really cool story because this is like a real-life migration story where you can actually provide advice on this is what you can do if you find yourself in this situation, which is really cool. +I had a theory that like really this one code path was the problem where we're doing the conversion from our internal tagging implementation through Hotel tags, which had a lot of other logic with it that was pretty expensive to do on pretty much every call. So I was like, you know what? There's no better time than now to begin another migration from OpenTracing to OTEL, basically. While we wait on the Hotel folks on the metric side to push out more performant code, more performant implementations for us, we can also test out this theory that if we migrate to Hotel entirely, we're actually going to see even more performance benefits. -I want to call that out because I think that's a really important thing, especially if you're starting to get into OpenTelemetry. The other thing that I wanted to ask you about because you said that it’s a monorepo, so in that case, did you find it—and especially since you did the all-or-nothing approach—did you find having a monorepo more challenging than if you'd been dealing with microservices instead? +At that point, I said, okay, I'm going to put a pause in the metrics work while we wait for Hotel, and I'm going to begin on this tracing migration. However, I decided to try a different approach, which is the all-or-nothing approach. The OpenTracing to OpenTelemetry path was a bit more known. There were a few really small docs and examples, and they are backwards compatible. You’re able to use them in conjunction with each other. -**Jacob:** Yeah, well, so we do use microservices. It's just that we have a repo of microservices—sorry, my bad about you guys. In that case, yeah, then how do you know—because I mean yes, you're going for like a big bang approach, but you got to start somewhere. So then like where do you start? +So first step: set up propagators. Second step: make sure that all of our plugins worked, which at the time they weren't open-sourced; now they are open-sourced, so people can just use them. I just did it all in one swoop. Maybe I had to revert like three times from our staging environment, nothing really major. There was one bug that I missed where we were previously doing a lot of in-app sampling because we had a really noisy function call. So I had to implement the custom sampler, which is actually like ten times easier with OTEL than it was with OpenTracing. I was able to get rid of a good like 1000 lines of code or so and some really dangerous hacks. That was a really good thing. -**Jacob:** I started with—and it was the same with how I started the metrics migration—I started with really small services that my team owned that were really low traffic, but enough for it to be constant. The reason that you want to pick a service like that is if it's too low traffic, if you're only getting like one request every minute, one request every like ten minutes, you have to worry about sample rates. You might not have a lot of data to compare against. Really, that's like the big thing that you need to have is some data to compare against. +Yeah, so then that went out. Very happy. Also, stopping at any time if we want to get back to like more Q&A stuff, but I have so many questions from this already. -[00:12:01] I wrote a script early on for the metrics migration that just queried different build tags that are on all of our metrics. So you would say, you know, query all of the metrics for service X, grouped by release tag, and if you see that the standard deviation for the newer build tag is, you know, greater than one—right? So if it's one or more standard deviations away from the previous release, then there's probably something going wrong in your instrumentation library. +**Host:** Do you mind taking a quick pause? -If you assume that your metrics are relatively stable, then if they're not, it's important to know. The other thing I had to check for was that all of the attributes were still present before and after migration, which is another thing that matters. Sometimes they weren't because something might be something that StatsD just adds automatically that we don't really care about, and so those were acceptable. I just like hand waved and said those are fine; we don’t care. +**Jacob:** Sure. -For tracing, it's sort of the same deal where I picked a service that had both internal-only traces—traces that stayed within a single service—and then traces that spanned multiple services with different types of instrumentation, so coming from like Envoy to Hotel to OpenTracing. What you want to see is that the trace before has the same structure as the trace after. So I made another script that checked that those structures were relatively similar and that all of them had the same attributes as well. +**Host:** A couple of comments. One thing that came to my mind as you're saying this: this is actually a really freaking cool story because, like, I think for there we tend to see like two different types of organizations. We see the ones where there's like zero code instrumented; this is like their first foray into instrumenting their code. Then we see the organizations that I think have either that have dabbled in OpenTracing. So I think it's a really cool story because this is like a real-life migration story where you can actually provide advice on this is what you can do if you find yourself in this situation, which is really cool. -**Host:** Right, right. Because tracing attributes, again, I was doing an attribute migration. That was really the point of doing the tracing one, so what matters is that all the attributes stayed the same, right? +I want to call that out because I think that's a really important thing, especially if you're starting to get into OpenTelemetry. The other thing that I wanted to ask you about because you said that it's a monorepo, so in that case, did you find it, and especially since you did the all-or-nothing approach, did you find having a monorepo more challenging than if you'd been dealing with microservices instead? -**Jacob:** Yeah, it's interesting too because you're starting from the point where you were migrating from an existing thing. You have that frame of reference, which I guess is kind of a double-edged sword, right? Because on the one hand, it's like you pretty much know that you've instrumented the things—hopefully. Maybe you’ll discover as you go along that there's like more stuff to instrument, but at least you have a baseline to start from. But then I guess on the other hand, if something's missing, you're like, oh damn, why is that missing? +**Jacob:** Yeah, well, so we do use microservices; it's just that we have like a repo of microservices. Sorry, my bad about you guys. -**Jacob:** Yeah, and those, you know, "why is this missing" stories are the really complicated ones because, of course, sometimes it's easy to just like, you know, oh I forgot to add this thing in this place, and that's usually pretty simple. But sometimes it's like, oh, there's an upstream library that doesn't emit the thing or Hotel. And now I need to—again, this is like early stuff. Most of these have all been upgraded and are fine now. +### [00:10:49] Migration strategies discussion -But there is an example with like our gRPC. Actually, this is like an interesting one. I had done a gRPC migration on a gRPC util package, which I think is now in like token trip—the Hotel gRPC trip. There was an issue with propagation. I was trying to understand, you know, what's going wrong here? And when I looked at the code, it just tells you how early in this story I was doing this migration, where there was supposed to be a propagator. There was just a TODO. +**Host:** In that case, how do you know, because, I mean, yes, you're going for like a big bang approach, but you got to start somewhere. So then where do you start? -**Host:** Oh no. +**Jacob:** I started with, and it was the same with how I started the metrics migration. I started with really small services that my team owned that were really low traffic but enough for it to be constant. The reason that you have to pick a service like that is if it's too low traffic, if you're only getting like one request every minute, one request every like ten minutes, you have to worry about sample rates. You might not have a lot of data to compare against. Really, that's the big thing that you need to have is some data to compare against. -**Jacob:** So, and the TODO was from someone on it. It was from Alex Book, which I'll call him out. So I sent it to Alex, and I was like, hey, this is a funny TODO because I just, you know, took down an entire service's traces in staging. So I spent some time to fix that TODO. It wasn't that difficult; it was just that they were waiting on another thing. I mean, that's how it goes. You're waiting on someone else, which, you know, another person—it's just endless cycles of that type of thing. +I wrote a script early on for the metrics migration that just queried different build tags that are on all of our metrics. You would say, you know, query all of the metrics for service X, grouped by release tag, and if you see that the standard deviation for the newer build tag is like, you know, greater than one, right? So if it's one or more standard deviations away from the previous release, then there’s probably something going wrong in your instrumentation library. If you assume that your metrics are relatively stable, then if they're not, it's important to know. -**Host:** Yeah. +The other thing I had to check for was that all of the attributes were still present before and after migration, which is another thing that matters. Sometimes they weren't because something might be something that StatsD just adds automatically that we don't really care about, and so those were acceptable. I just hand-waved and said those are fine; we don't care. -**Jacob:** But then I got it working, so that was like one of the main blockers for us. +For tracing, it’s sort of the same deal where I picked a service that had both internal-only traces that stayed within a single service and then traces that span multiple services with different types of instrumentation. Coming from Envoy to Hotel to OpenTracing, what you want to see is that the trace before has the same structure as the trace after. So I made another script that checked that those structures were relatively similar and that all of them had the same attributes as well. -**Host:** Nice, nice. +**Host:** Right, right. Because tracing attributes again, I was doing an attribute migration. That was really the point of doing the tracing one, so what matters is that all the attributes stayed the same. -[00:16:02] **Jacob:** And was able to upstream it as well. It wasn't just a fix for ourselves; it was a fix for the community. There were a few things like that. A lot of the metrics work actually resulted in big performance boosts for Hotel metrics, like Hotel Go metrics, and it also has given the specs folks some ideas about how descriptive the API should be or various features. +**Jacob:** Yeah, it's interesting too because you're starting from the point where you were migrating from an existing thing. You have that frame of reference, which I guess is kind of a double-edged sword, right? Because on the one hand, it's like you know you pretty much know that you've instrumented the things. Hopefully, maybe you'll discover as you go along that there's more stuff to instrument, but at least you have a baseline to start from. But then I guess on the other hand, if something's missing, you're like, oh damn, why is that missing? -So things like views and the use of views is something that we did heavily in that early migration because we were worried about—can you just— +**Jacob:** Yeah, and those, you know, "why is this missing" stories are the really complicated ones. Because of course, sometimes it's easy to just like, you know, oh, I forgot to add this thing in this place, and that's usually pretty simple. But sometimes it's like, oh, there's an upstream library that doesn't emit the thing or Hotel, and now I need to, like, again, this is like early stuff. Most of these have all been upgraded and are fine now. -**Host:** Yeah, definitely. Just tell folks what you mean by that. +But there is an example with like our gRPC. Actually, this is like an interesting one. I had done a gRPC migration on a gRPC util package, which I think is now in like OpenTelemetry. There was an issue with propagation. I was trying to understand, you know, what's going wrong here? And when I looked at the code, it just tells you how early in this story I was doing this migration, where there was supposed to be a propagator. There was just a TODO. Oh no. -**Jacob:** Yeah, so a metrics view is something that's run inside of your metrics provider in OTEL, your media provider in Hotel. What it's doing is it's saying you can configure it to do kind of a lot. It could just be drop this attribute whenever you see it. If you're a centralized SRE and you don't want anybody to instrument code with any user ID attributes because that's a super high cardinality thing, it's going to explode your metrics cost, right? So you could just make a view that gets added to your instrumentation that says just don't let this attribute from being recorded—just deny it. +It was from someone on it; it was from Alex Book, which I'll call him out. I sent it to Alex, and I was like, hey, this is a funny TODO because I just, you know, took down an entire service's traces in staging. I spent some time to like fix that TODO. It wasn't that difficult; it was just that they were waiting on another thing. I mean, that's how it goes; you're waiting on someone else, which, you know, another person, it's just endless cycles of that type of thing. -So that's like probably the most common use case. There are other ones though—more advanced use cases for dynamically changing things like the temporality or the aggregation of your metrics. So temporality being cumulative or delta for like a counter. You know, am I recording zero, one, three? I'm trying to—two, three, zero, one, three—or am I recording one and two, right? +But then I got it working, so that was like one of the main blockers for us. Nice, nice. I was able to upstream it as well; it wasn't just a fix for ourselves; it was a fix for the community. -And then your aggregation is going to be about how do you—oh man, being on the spot is so much harder because it's like I want to look it up, but feel free to look it up. That's totally cool. +### [00:16:23] Performance issues during migration -**Host:** Well, aggregation is like how you send off these metrics basically. We had an aggregation that instead of doing—well, for histograms, this is like most useful when you record a histogram. There are a few different types of histograms. Datadog's histograms, StatsD's histogram is not a true histogram because what they're recording is like aggregation samples. +**Host:** And that was, yeah, there were a few things like that. A lot of the metrics work actually resulted in big performance boosts for Hotel metrics, like Hotel Go metrics. It also has given the specs folks some ideas about how descriptive the API should be or various features. -So they give you a min, max, sum, count, average. And so I actually don't even think they give you sum. I looked at this last night. They don't give you sum. They give you like min, max, count, average, and like P95 or something. And so the problem with that is in distributed computing, if you had multiple applications that are reporting a P95, there's no way that you could get a true P95 from that observation with that aggregation. +**Jacob:** Things like views and the use of views is something that we did heavily in that early migration because we were worried about, can you just, yeah, definitely just tell folks what you mean by that. -The reason for that is that in order to get P95, you can't—if you have five P95 observations, there's not an aggregation to say give me the overall P95 from that, right? You need to have something about the original data to actually recalculate it. You could get the average of the P95s, but that's not a great metric. That's not really like—it doesn't really tell you much. It's not really accurate. And if you're going to alert on something, if you're going to page someone at night, you should be paging on accurate measurements. +**Jacob:** So a metrics view is something that's run inside of your metrics provider in OTEL. Your media provider in Hotel. What it's doing is it's saying you can configure it to do kind of a lot. It could just be dropping this attribute whenever you see it. If you're a centralized SRE and you don't want anybody to instrument code with any user ID attributes because that's a super high cardinality thing, it's going to explode your metrics cost, right? -[00:18:00] Initially though, we did have a few people who relied on this min, max, sum, counter instrument, so we used Hotel views in the metrics SDK to configure custom aggregation for our histograms to emit what some would call a distribution or what OTEL calls an exponential histogram, or the min, max, and the min, max count. So we were dual emitting. This works because they're different metric names that we were emitting, so there was no overlap between them. So what we did was we migrated. After we did the metrics migration, we were able to then go back and say any dashboard, any alert, anything that was using a min, max, sum, count metric just changed it to be a distribution instead. +So you could just make a view that gets added to your instrumentation that says just don't let this attribute from being recorded, just deny it. So that's like probably the most common use case. -Because we had enough data in the past, like, you know, a few weeks, months of running Hotel metrics in our public environment, that was possible to do. So that was like one of the key features because we had it, it was ten times easier, and we were able to do it from the application. We didn't have to introduce any other components, which is pretty neat. +There are other ones though, more advanced use cases for dynamically changing things like the temporality or the aggregation of your metrics. Temporality being cumulative or delta for like a counter. Am I recording zero, one, three? I'm trying to, too crazy. Zero, one, three, or am I recording one and two? -[00:22:02] **Host:** Right, right. Cool. Another question that I had for you. So like, when you were doing this migration, traces and metrics were in existence; logs I believe would not have been like the specification would not have been ready—possibly not even in the works? +Aggregation is going to be about how do you, I always struggle to explain all of them, and I'm trying to come back to what I had done in that moment. -**Jacob:** No, it was still really early for that. +**Host:** Talk about temporality. -**Host:** I guess in lieu of logs, there's span events that you could use, so it's not something like that was leveraged as well? +**Jacob:** Oh, aggregation is like, oh man, being on the spot is so much harder because it's like I want to look it up, but feel free to look it up. That's totally cool. -**Jacob:** Definitely. We've heard a long time have you used span events and logs or a lot of things internally. I'm a big fan of them. I am not a huge fan of logging. I find it to be really cumbersome and really expensive. IOPS for tracing and trace logs, whenever possible, I find it easier for myself to reason about. There are other people who are like logging first, and that's great, but that's just not who I am. I like logging for local development and tracing for distributed elements; that makes sense. +Aggregation is like how you send off these metrics, basically. We had an aggregation that instead of doing, well, for histograms, this is like most useful when you record a histogram. There are a few different types of histograms. Datadog's histograms, StatsD's histograms are not a true histogram because what they're recording is like aggregation samples. -But we use this heavily. That was one of the first things that I checked worked. It's actually an interesting bug where we had some custom code or OpenTracing that allowed us to serialize JSON blobs in the span events, and that stopped working because we didn't emit them in the same way. It's a little hazy, but I had to rewrite a processor to make that work and then update some downstream code like in Lightstep as platform to Facebook. +They give you a min, max, sum, count, average. And so I actually don't even think they give you sum. I looked at this last night; they don't give you sum. They give you like min, max, count, average and like P95 or something. -**Host:** Cool. So now how about keeping that in mind—now that logs are more mature, are there any plans to do any conversions? And please correct me if I'm wrong, but my understanding too is that with the log specification maturing more and more, the span events are going to be replaced by logs in some form—like it's going to be the log specification for span events. Have you heard anything around that? +And so the problem with that is in distributed computing, if you had multiple applications that are reporting P95, there's no way that you could get a true P95 from that observation with that aggregation. The reason for that is that in order to get P95, you can't, if you have five P95 observations, there's not an aggregation to say give me the overall P95 from that, right? -[00:24:50] **Jacob:** No, this is a bit outside of where my recent focus has been, so I'm not positive. I think right now the way that we do—I think the thing that we would change is how we collect those logs potentially. Right now we use—how do we do this right now? It changed recently. I don't want to say something incorrect, but we previously did it by just using like Google's logging agent, where they basically are running Fluent Bit on every node in the GKE cluster, and then they send it off to GCP and they just like tail it there. I think this changed though, and I'm not sure what we do now. +You need to have something about the original data to actually recalculate it. You could get the average of the P95s, but that's not a great metric; that's not really telling you much. It's not really accurate. And if you're going to alert on something, if you're going to page someone at night, you should be paging on accurate measurements. -**Host:** Okay, cool. Speaking of GKE, I have many questions on GKE specifically. Do you know if there's like a feature now in newer versions of Kubernetes where there's like some— I think there's some telemetry collection. Do you know if that's been enabled in any of the clusters? +Initially though, we did have a few people who relied on this min, max, sum, count metric. So we used Hotel views in the metrics SDK to configure custom aggregation for our histograms to emit what some would call a distribution, what OTEL calls an exponential histogram, or the min, max and the min, max and count. -**Jacob:** Yeah, so I think that Kubernetes now has the ability to emit like the Hotel traces natively. +So we were dual emitting this. This works because they're different metric names that we were emitting, so there was no overlap between them. So what we did was we migrated after we did the metrics migration. We were able to then go back and say any dashboard, any alert, anything that was using a min, max, sum, count metric, just change it to be a distribution instead. -**Host:** Yeah, yeah. +Because we had enough data in the past, like, you know, a few weeks, months of running Hotel metrics in our public environment, that was possible to do. So that was like one of the key features because we had it, it was ten times easier. And we were able to do it from the application. We didn't have to introduce any other components, which is pretty neat. -**Jacob:** I'm not sure if we're collecting those yet. I don't know what version that's more of a question for the SREs. I don't—Kubernetes came out like I think even last year, starting whatever, like last fall kind of thing. +**Host:** Right, right. Cool. Another question that I had for you. So like when you were doing this migration, traces and metrics were in existence; logs, I believe would not have been, like, the specification would not have been ready, possibly not even in the works. -**Host:** Yeah, that's a really good question that I want to look into because I want to see if really what I would like to do is see if we can collect the traces that we get from those to use the span metrics processor to generate like better Kubernetes metrics from those traces. I'm very focused on infrastructure metrics—like Kubernetes infrastructure metrics—and I find them to be very painful in their current form. +**Jacob:** No, it was still really early for that, so, but I guess in lieu of logs, there's span events that you could use. So it's not something like that was leveraged as well? -It would be really cool; right now, I prefer to use the Prometheus APIs for them currently. It's just a bit more ubiquitous in the observability community to use Prometheus to do that because that's what Kubernetes natively emits, right? +### [00:22:37] Discussion on metrics aggregation -**Jacob:** Right, go ahead. +**Jacob:** Definitely. We've heard a long time have you used span events analogs, or a lot of things internally. I'm a big fan of them. I am not a huge fan of logging; I find it to be really cumbersome and really expensive. IOPS for like tracing and trace logs whenever possible, I find it easier for myself to reason about. -**Host:** Oh, no, go ahead. I'll let you complete the thought. Maybe it answers my questions. +There are other people who are logging first, and that's great, but that's just not who I am. I like logging for local development and tracing for distributed elements; that makes sense. But we use this heavily. That was one of the first things that I checked worked. -**Jacob:** And so that's what we do right now. I use the target allocator, which is, you know, a nutshell component that I work on to distribute those targets, which is, you know, a pretty efficient way of getting all that data. We also use daemon sets as well that we run in our clusters to get that data. In addition to that, so that works pretty effectively. The thing that's frustrating is just Prometheus. Prometheus script failures can be a super common problem, and it gets really annoying when you have to worry about metrics cardinality as well because it can explode. +It's actually an interesting bug where we had some custom code or OpenTracing that allowed us to serialize JSON blobs in the span events, and that stopped working because we didn't emit them in the same way. It's like a little hazy, but I had to like rewrite a processor to make that work and then update some downstream code like in Lightstep as a platform to Facebook. -I actually found a bug in GKE maybe six to eight months ago—six, seven months ago—but they've since fixed where they weren't deleting—they weren't reconciling certificate signing requests in their Kubernetes cluster, which meant that for kube-state metrics, which reports on cluster state, it was omitting because there were so many certificate signing requests left from these abandoned nodes. +**Host:** Cool. Now, how about keeping that in mind? Now that logs are more mature, are there any plans to do any conversions? And please correct me if I'm wrong, but my understanding too is that like with the log specification maturing more and more, that like span events are going to be replaced by logs in some form. Like, it's going to be the log specification for span events. Have you heard anything around that? -Something on the magnitude of like six hundred thousand for like a single metric, which is huge. And so then Prometheus—the Prometheus that I was running—fell over because of that. And that's like a thing that happens constantly in this Prometheus realm, which is just like someone emits a high cardinality metric, Prometheus goes to scrape it, and then it just crashes. +**Jacob:** No, this is a bit outside of where my recent focus has been, so I'm not positive. I think right now the way that we do, I think the thing that we would change is how we collect those logs potentially. Right now we use, how do we do this right now? It changed recently; I don't want to say something incorrect. -**Jacob:** Oh wow. +We previously did it by just using like Google's logging agent, where they basically are running like Fluent Bit on every node in a GKE cluster, and then they send it off to GCP, and they just like tail it there. I think this changed though, and I'm not sure what we do now. -**Host:** That does not sound fun. +**Host:** Okay, cool, cool. Speaking of GKE, I have many questions on GKE specifically. Do you, because I believe there's like a feature now in newer versions of Kubernetes where there's like some, I think there's some telemetry collection; do you know if that's been enabled in any of the clusters? -**Host:** I want to just take a step back because you mentioned the target allocator. I was wondering if you could expand a little bit on that because I know one of our previous Q&A folks also mentioned the target allocator. That was the first time I had heard of it, so I think it'd be like super helpful to just get a little overview. +**Jacob:** Yeah, so I think that Kubernetes now has the ability to emit like the Hotel traces natively. -**Jacob:** Sure, yeah. So the target allocator is a component, part of the Kubernetes operator in Hotel, that does something that Prometheus can't do, which is dynamically shard targets amongst a pool of scrapers. Prometheus has some experimental functionality for sharding, but you still have a problem for querying because Prometheus is a database, not just a scraper. +**Host:** Yeah, yeah. -If you shard your targets, you don't necessarily—you have to do some amount of coordination within those Prometheus instances, which gets expensive. It's like a very experimental feature. Or you could scale Prometheus with something like Thanos or Cortex, which is Grafana's Prometheus scaling solution, I think, right? +**Jacob:** I'm not sure if we're collecting those yet. I don't know what version; that's more of a question for the SREs. I don't, Kubernetes came out like I think even last year, starting whatever, like last fall kind of thing. That's a really good question that I want to look into because I want to see if really what I would like to do is see if we can collect the traces that we get from those are enough to use the span metrics processor to generate better Kubernetes metrics from those traces. -Which works, but you just then have to run like six more components that you then need to monitor, and then if those go down, how do you monitor all these other problems, right? In Hotel, we just basically tack on this Prometheus receiver to get all this data. But because we want to be more efficient than Prometheus, because we don't need to store the data, we tell—we have this component, the target allocator, which goes to do the service discovery from Prometheus. So it says give me all of the targets that I need to scrape, and then the target allocator says with those targets distribute them evenly amongst the set of collectors that are running. +I'm very focused on like infrastructure metrics, like Kubernetes infrastructure metrics, and I find them to be very painful in their current form. It would be really cool. Right now, I prefer to use the Prometheus APIs for them currently. It's just a bit more ubiquitous in the observability community to use Prometheus to do that. -**Host:** Oh, okay. +**Host:** Right, right. Go ahead. -**Jacob:** So that's the main thing. It does some more stuff around job discovery now, or if you're using Prometheus service monitors, which is part of the Prometheus operator, which is a very popular way of running Prometheus in your cluster. It's what a lot of vendors use as well. So if you're on GAE or OpenShift, I think both of those natively use service monitors and pod monitors. +**Jacob:** No, go ahead. I'll let you complete the thought; maybe it answers my questions. -So the target allocator can also pull those service monitors and pod monitors and update the collectors' scrape configs to do that. +**Jacob:** And so that's what we do right now, and I use the target allocator, which is a nutshell component that I work on, to distribute those targets, which is a pretty efficient way of getting all that data. We also use like daemon sets as well that we run in our clusters to get that data in addition to that. So that works pretty effectively. -**Host:** Oh cool, it's awesome. And so related to the Prometheus thread, are you running like Prometheus itself, or are you just scraping the Prometheus metrics and pumping them through to the collector? +The thing that's frustrating is just Prometheus. Prometheus script failures can be a super common problem, and it gets really annoying when you have to worry about metrics cardinality, as well, because it can explode. -**Jacob:** Exactly right. Just no Prometheus instances, just the collector running Prometheus receiver and then sending them off to Lightstep. +I actually found a bug in GKE maybe six to eight months ago, six, seven months ago, but they've since fixed where they weren't deleting or reconciling certificate signing requests in their Kubernetes cluster, which meant that for Kube State Metrics, which reports on cluster state, it was omitting because there were so many certificate signing requests left from these abandoned nodes. -**Host:** Oh, living the dream! That was always my dream. +Something on the magnitude of like six hundred thousand for like a single metric, which is huge. And so then Prometheus, the Prometheus that I was running fell over because of that. And that's like a thing that happens constantly in this Prometheus realm, which is just like someone emits a high cardinality metric, Prometheus goes to scrape it, and then it just crashes. -**Jacob:** That's awesome. +**Host:** Oh wow. -**Host:** Do you use—because I remember Lightstep has like a Prometheus operator that helps facilitate that, so we used to have this thing called the Prometheus sidecar, which you might run. +**Jacob:** Which isn't fun. -You would run it as part of your Prometheus installation, which would then sit on the same pod as your Prometheus instance and read the write-ahead log that Prometheus has for persistence and batching and all these other things. So we would read the write-ahead log and then forward those metrics. +**Host:** Yeah, that does not sound fun. I want to just take a step back because you mentioned the target allocator. I was wondering if you could expand a little bit on that because I know like we actually had one of our previous Q&A folks also mention the target allocator. That was the first time I had heard of it, so I think it'd be super helpful to just get a little overview. -But if your Prometheus is very noisy—as many customers have very noisy Prometheus statistics—it's not really efficient. It can get really noisy, and not that—what's the word? It's not the best thing to run the collector as like the best way to run. +**Jacob:** Sure. Yeah, so the target allocator is a component, part of the Kubernetes operator in Hotel that does something that Prometheus can't do, which is dynamically shard targets amongst a pool of scrapers. Prometheus has some experimental functionality for sharding, but you still have a problem for querying because Prometheus is a database, not just a scraper. -**Jacob:** Okay, so—and it sounds like this thing still requires to have Prometheus installed. +If you shard your targets, you don't necessarily—you have to do some amount of coordination within those Prometheus instances, which gets expensive. It's like a very experimental feature, or you could scale Prometheus with something like Thanos or Cortex, which is Grafana's Prometheus scaling solution, I think, right? -**Host:** Yeah, you would still need to be running a whole computer system. +Which works, but you just then have to run like six more components that you then need to monitor, and then if those go down, how do you monitor? It's all these other problems. -**Jacob:** Oh, okay. I thought it was—I was under the impression it was like a replacement for Prometheus and that it was—maybe I'm thinking of something else. There was a thing that I knew was like a replacement for needing Prometheus, and it was like vendor-neutral, so it wasn't like, oh, you have to use Lightstep to use this thing. +In Hotel, we just basically tack on this Prometheus receiver to get all this data, but because we want to be more efficient than Prometheus, because we don't need to store the data, we tell—we have this component, the target allocator, which goes to do the service discovery from Prometheus. So it says, give me all of the targets that I need to scrape. -**Host:** I think I might just be a Hotel operator collector target allocator trio. +Then the target allocator says, with those targets, distribute them evenly amongst the set of collectors that are running. -**Jacob:** Oh, okay, okay. +**Host:** Oh, okay. -**Host:** But maybe there's another thing out there. +**Jacob:** So that's the main thing that it is doing. It does some more stuff around job discovery now, or if you're using Prometheus service monitors, which is part of the Prometheus operator, which is a very popular way of running Prometheus in your cluster. It's what a lot of vendors use as well. -**Jacob:** Oh, unless maybe that got integrated into the target allocator as part of—anyway, it is a mystery. +### [00:31:28] Target allocator explanation -**Host:** Yeah, okay, cool, cool. +So if you're on GAE or OpenShift, I think both of those natively used service monitors and pod monitors. The target allocator can also pull those service monitors and pod monitors and update the collectors' scrape configs to do that. -**Host:** So then, okay, since we're talking collectors now, for me, like the two million dollar question—the one that I'm always curious about—is collector setup. So what is the collector setup that y'all have chosen? What works for you now? +**Host:** Oh, cool. That's awesome. And so related to the Prometheus thread, are you running like Prometheus itself or are you just scraping the Prometheus metrics and pumping them through to the collector, then? -**Jacob:** Yeah, it's hard to say because we run a lot of different types of collectors. At Lightstep, we run like metrics things, tracing things, internal ones, external ones. There are a lot of different collectors that are running at all times. You have like a separate one that just collects metrics and one that just collects traces. +**Jacob:** Exactly right. Just no Prometheus instances, just the collector running Prometheus receiver and then sending them off to Lightstep. -Right now we don't—it’s all varying flux. Right now we're changing this a lot to run experiments and stuff. Basically, like the best way for us to be able to make features for customers and end-users is by running them ourselves and then using them internally, making sure that they work, and then sending them for the open source realm. +**Host:** Oh, living the dream. That was always like that was always my dream. -So that's what we're trying to do even more of—like we're kind of reaching a point where we dogfood everything, which gets really confusing because you have to like— +**Jacob:** That's awesome. -**Host:** Yeah, I can imagine. +**Host:** That's nice. Do you use, like, because I remember Lightstep has like a Prometheus operator that helps facilitate that, so we used to have this thing. -**Jacob:** Yeah, we're running like in a single path there could be like, I think, two different collectors in two environments that could be running two different images in two different versions. It gets really meta and very confusing to talk about. +**Jacob:** Yeah, we used to have this thing called the Prometheus sidecar, which you might run as part of your Prometheus installation, which would then sit on the same pod as your Prometheus instance and read the write-ahead log that Prometheus has for persistence and batching and all these other things. -**Host:** Yeah, I can imagine. +So we would read the write-ahead log and then forward those metrics, but if your Prometheus is very noisy, as many customers have very noisy Prometheus statistics, it's not really efficient. It can get really noisy, and it's not that—what's the word? It's not the best thing to run. The collector is like the best way to run. -**Jacob:** And, you know, if you're sending from collector A across an environment to collector B, collector B also emits telemetry about itself, which is then collected by collector C. And it just chains—like you basically ensure that you have to like make sure that the collectors are actually working. +**Host:** Okay, so and it sounds like this thing still requires Prometheus to be installed? -**Host:** Yeah, you have to be sure that everything along this path—yeah, you just have to know which thing has the data. +**Jacob:** Yeah, you would still need to be running a whole computer system. -**Jacob:** Right. Well, you shouldn't have to—we do that for you, but like— +**Host:** Oh, okay. I thought it was—I was under the impression it was like a replacement for Prometheus and that it was—maybe I'm thinking of something else. -**Host:** Right. +**Jacob:** There was a thing that I knew of that was like a replacement for needing Prometheus, and it was like vendor-neutral, so it wasn't like, oh, you have to use Lightstep to use this thing. I think I might just be a Hotel operator collector target allocator trio. -**Jacob:** Yeah, and we make like dashboards to help with that. But that's like the problem when it's like we're debugging this stuff is when there's a problem you have to think about like where's the problem actually? Is it in how we collect the data? Is it in how we emit the data? Is it in, you know, the source of how the data was generated? It's one of like a bunch of things. +**Host:** Oh, okay, okay. -**Host:** Yeah, yeah. +**Jacob:** But maybe there's another thing out there. -**Host:** Now, like you need to work on the Hotel operator. So, and I've been reading up on the operator recently, and there's like, I think, four different deployment modes, right? There's sidecar deployment, daemon set, and—what's the other one? +**Host:** Oh, unless maybe that got integrated into like the target allocator as part of—anyway, it is a mystery. -**Jacob:** Yeah, nice, yeah. +**Jacob:** Yeah. -**Host:** So my question is, which mode—or is it like all of the above, depending on the thing that you need to do? +**Host:** Okay, cool, cool. Oh, that's awesome. So then, okay, since we're talking collectors now, I, for me, the two million dollar question, the one that I'm always curious about is collector setup. So what is the collector setup that y'all have chosen? What works for now? -**Jacob:** Yeah, it's all the above depending on what you need to do and your general needs and like how you like to run applications for reliability and stuff. +**Jacob:** It's hard to say because we run a lot of different types of collectors. Yeah, at Lightstep, we run like metrics things, tracing things, internal ones, external ones. There are a lot of different collectors that are running at all times. -**Jacob:** So sidecar is the one that we use the least and is probably used the least if I were to just make a bet. Sidecars are really useful across the industry, you would think. +You have like a separate one that just collects metrics and one that just collects traces. Right now, we don't—it's all varying flux. Right now, we're changing this a lot to run experiments and stuff. Basically, like the best way for us to be able to make features for customers and end users is by running them ourselves and then using them internally, making sure that they work, and then sending them for the open-source realm, and so that's what we're trying to do even more of. -**Host:** Yeah, across the industry, I'd be willing to bet that those are the least popular. +We're kind of reaching a point where we dogfood everything, which gets really confusing because you have to like— -**Jacob:** That's the least popular method. Sidecars are just expensive, and if you're not using them—if you don't really need them, then you shouldn't use them. You’ll really only need them like something that's run as a sidecar, like Istio, which makes a lot of sense to run as a sidecar because it's doing like traffic proxy hooks into your container network to change how that all does its thing. - -And you get a performance hit if you sidecar your collectors for all your services. You just get like a cost. It would just cost you a lot more. And you also wouldn't be able to do as much with like if you're making like Kubernetes API calls for attribute enrichment—that's like the thing that would get exponentially more expensive if you're running it as a sidecar, right? But as like a stateful set of like, you know, five pods, that's not that expensive. +**Host:** Yeah, I can imagine. -But if you have a sidecar on like 10,000 pods, then that's 10,000 API calls made to the Kubernetes API. +**Jacob:** —we're running like in a single path. There could be like, I think, two different collectors in two environments that could be running two different images in two different versions. It gets really meta and very confusing to talk about. -**Host:** Right, right. +**Host:** Yeah, I can imagine. And then, you know, if you're sending from collector A across an environment to collector B, collector B also emits telemetry about itself, which is then collected by collector C. -**Jacob:** What would be the advantage of running your collector as a stateful set versus a deployment? I guess what's the state that you would want to persist? +**Jacob:** Yeah, it's just chains like you basically ensure that you have to make sure that the collectors actually work. -**Jacob:** Yes, this is, I don't know what the right word is, but stateful sets aren't only used for their ability to mount volumes. There are a few other things that are inherent to how stateful sets run that are really valuable in distributed computing. This is an important thing to know for not just like how the collector runs as an application, but how your applications can run, right? +**Host:** You have to be sure that everything along this path—yeah, you just have to know which thing has the data. -Stateful sets have consistent IDs, so if you have a stateful set with 10 replicas, they're all going to be the stateful set name dash counter number, so it goes from like zero to n. So that's a really valuable thing when you want consistent IDs, right? +**Jacob:** Right. Well, you shouldn't have to; we do that for you, but like, right. -As opposed to like with deployments, like when you, like your pods are all like random crap, right? So the pod IDs are done where it’s deployment name dash replica set ID dash pod ID, right? And so with stateful sets, because we have this consistent ID range, we can actually do some extra work with—for the target allocator, which is why we require that. +**Host:** Yeah, and we make like dashboards to help with that, but that's like the problem. When it's like we're debugging this stuff, when there's a problem, you have to think about like where's the problem actually? Is it in how we collect the data? Is it in how we emit the data? Is it in, you know, the source of how the data was generated? It's, you know, one of like a bunch of things. -And so the other thing that stateful sets guarantee is what's called an in-place deployment, which is what daemon sets do as well, where you take the replica, you take the pod down before you create a new one. +**Jacob:** Yeah. -So the reason that this is important is that in a deployment, you normally do a one-up, one-down, right? Or what's called a rolling deployment, a rolling update. And so if we were to do this for—if we were to do this for the— with the target allocator, we would probably get much more unreliable scrapes because you would—and someone actually just asked this question in the operator channel—I mean, I'm going to give them this exact response. +**Host:** Now, like you need to work on the Hotel operator, and I've been reading up on the operator recently. There's like, I think, four different deployment modes, right? There's sidecar deployment, daemon set, and what's the other one? -When a new replica comes up, you have to redistribute all the targets because your hash ring that you place these on has changed. So if you're doing a rolling deployment, if you're doing one up, one down, that's a really expensive operation, because then you have to recalculate all these hashes that you assign. +**Jacob:** Stateful set. -So if you were to do a one down, one up, you would still have to redo this whole thing because you would lose a pod, which means it's taken out of the ring, redistribute. You would gain a new ID, and then you'd have to redistribute again, right? +**Host:** Yes, yes. My question is, which mode, or is it like all of the above depending on the thing that you need to do? -Whereas stateful sets, because it’s a consistent ID range, you don't have to do that at all. And so this means that when we do a one down, one up, it keeps the same targets each time. +**Jacob:** Yeah, it's all the above depending on what you need to do and your general needs and how you like to run applications for like reliability and stuff. -**Host:** Right, right. So it's almost like a placeholder for it. You don't have to recalculate the ring, basically. +So sidecar is the one that we use the least and is probably used the least. If I were to make just like a bet, sidecars are really useful across the industry. You would think? -**Jacob:** Yeah, it's just sort of like a little—what's it called? +**Host:** Yeah, across the industry, I'd be willing to bet that those are the least popular—those are the least popular method. Sidecars are just expensive, and if you're not using them, if you don't really need them, then you shouldn't use them. You'll really only need them like something that's run as a sidecar, it's like Istio, which makes a lot of sense to run as a sidecar because it's doing like traffic proxy hooks into your container network to change how that all does its thing. -**Host:** Yeah, yeah, yeah. +You get a performance hit if you sidecar your collectors, right? For all your services, you just get like a cost; it would just cost you a lot more, and you also wouldn't be able to do as much with like, if you're making like Kubernetes API calls for attribute enrichment, that's like the thing that would get exponentially more expensive if you're running it as a sidecar. -**Jacob:** I can't think of the word. Cool, that's really neat. I didn't know that. +But as like a stateful set of, you know, five pods, that's not that expensive, but if you have a sidecar on like 10,000 pods, then that's 10,000 API calls made to the Kubernetes API. -**Host:** Yeah, it was funny because I was reading about like pods being deployed in stateful sets. I'm like, straight or sorry, not collectors. I'm like, I straight up do not understand what the use case would be, but this makes a lot of sense. +### [00:39:40] Collector setup discussion -**Jacob:** So that's really cool. And so it's not as useful—or this is really only useful for like a tracing use case I would say, or sorry, metrics use case where you're like doing complete test scores. +**Host:** Right, right. What would be the advantage of running your collector as a stateful set versus a deployment? I guess what's the state that you would want to persist? -We would probably run it as a deployment for anything else because a deployment gives you everything that you need pretty much, because the collectors are stateless—they don't need to hold on to anything. Deployments are much more lean as a result. +**Jacob:** This is an important thing to know for not just like how the collector runs as an application, but how like your applications can run. Stateful sets have consistent IDs. If you have a stateful set with 10 replicas, they're all going to be the stateful set name dash counter number, so okay, that's a really valuable thing when you want consistent IDs, right? -**Host:** Yeah, yeah. +As opposed to like with deployments, like when you like your pods are all like random, random crap, right? So the pod IDs are done where it's deployment name dash replica set ID dash pod ID, right? And so with stateful sets, because we have this consistent IDs, we can actually do some extra work for the target allocator, which is why we require that. -**Jacob:** They can just run and roll out, and everybody's happy, and that's how we run most of our collectors—deployment. +The other thing that stateful sets guarantee is what's called an in-place deployment, which is what daemon sets do as well, where you take the replica, you take the pod down before you create a new one. The reason that this is important is that in the deployment, you normally do a one up, one down, right? -**Host:** And then at what point would a daemon set be useful? +Or what's called a rolling deployment, a rolling update. If we were to do this for the target allocator, we would probably get much more unreliable scrapes because you would—and someone actually just asked this question in the operator channel. I'm going to give them this exact response. When a new replica comes up, you have to redistribute all the targets because your hash ring that you place these on is changed. -**Jacob:** Yeah, so daemon sets are really good for things like node scraping, which we do a lot of. So this allows you to scrape like the kubelet that's run on every node. It allows you to scrape the node exporter that's also run on every node, which is another Prometheus daemon set that most people run. +If you're doing a rolling deployment, if you're doing one up, one down, that's a really expensive operation because then you have to recalculate all these hashes that you assign. So if you were to do a one down, one up, you would still have to redo this whole thing because you would lose a pod, which means it's taken out of the ring, redistribute, you would gain a new ID, and then you'd have to redistribute again, right? -**Host:** Right, right. +Whereas stateful sets, because it's a consistent ID range, you don't have to do that at all. This means that when we do a one down, one up, it keeps the same targets each time. -**Jacob:** Yes, the daemon sets guarantee that you’ve got pods running on every node. +**Host:** Right, right. So it's almost like a placeholder for it, like you don't have to recalculate the ring basically because— -**Host:** Right, exactly. Every node that matches its selector, right? +**Jacob:** Yeah, it's just sort of like a little, what's it called? -**Jacob:** Right, right. +**Host:** Yeah, yeah, yeah. I can't think of the word. -**Host:** And so that's really useful for like scaling out. So if you have like a cluster of like 800 plus nodes, it's more reliable to run like a bunch of little collectors that get those tiny metrics rather than a few bigger stateful set pods, right? +**Jacob:** Cool, that's really neat. I didn't know that. -Because your blast radius is much lower, so if one pod goes down, you lose like just a tiny bit of data. But remember like with all this cardinality stuff, that's a lot of memory. +**Host:** It was funny because I was reading about like pods being deployed as stateful sets. I'm like, straight or sorry, not collectors. I'm like, I straight up do not understand what the use case would be, but this makes a lot of sense. So that's really cool, and so it's not as useful, or this is really only useful for like a tracing use case, I would say, or sorry, metrics use case where you're doing complete test scores. -**Jacob:** So if you're doing like a stateful set scraping all these nodes, that's a lot of targets, that's a lot of memory. It can go down much more easily, and you lose more data. Luckily, the collector isn't like Prometheus, where we don't care about that state. So if a collector goes down, it comes back up super fast. So usually, the blip is low, but it does mean that the blip is more flappy, right? +We would probably run it as a deployment for anything else because a deployment gives you everything that you need pretty much. The collectors are stateless; they don't need to hold on to anything. -Where like it could go up and down pretty quickly if you're past the point of saturation. That's why it's good to have like a HPA, horizontal auto-scaler. +Deployments are much more lean as a result. -**Host:** Right, right. +**Host:** Yeah, they can just run and roll out and everybody's happy, and that's how we run most of our collectors, deployment. And then at what point would a daemon set be useful? -**Jacob:** But still, daemon set is a bit more reliable. +**Jacob:** Yeah, so daemon sets are really good for things like node scraping, which we do a lot of. So this allows you to scrape like the kubelet that's run on every node. It allows you to scrape the node exporter that's also run on every node, which is another Prometheus daemon set that most people run. -**Host:** And it sounds like it would be useful again, like from a metric standpoint. +Yes, the daemon sets guarantee that you've got odds running on every node, right? Exactly. Every node that matches its selector, right? -**Jacob:** Yeah, yeah. +**Host:** Right, right. -**Jacob:** Tracing, you could do it for tracing and just send it on like a node port, but tracing workloads, again, because it's all push-based, they are much easier to scale on. +**Jacob:** And so that's really useful for like scaling out. If you have a cluster of like 800 plus nodes, it's more reliable to run like a bunch of little collectors that get those tiny metrics rather than a few bigger stateful set pods. Because your blast radius is much lower, so if like one pod goes down, you lose like just a tiny bit of data. -And you can distribute targets; you can load balance. There are all these other benefits that we get from push-based workloads. Pull-based is like—the reason that Prometheus is so ubiquitous in my opinion is just because it makes local development really easy, where you can just scrape your local endpoint. +But remember, like with all this cardinality stuff, that's a lot of memory. If you're doing like a stateful set scraping all these nodes, that's a lot of targets, that's a lot of memory; it can go down much more easily, and you lose more data. -That's what most back-end development is anyway, so you could like hit endpoint A and then hit your metric set point and then hit endpoint A again—metric standpoint. You can just like check that. It's like a very easy developer loop. +Luckily, the collector isn't like Prometheus, where we don't care about that state. So if a collector goes down, it comes back up super fast, so usually the blip is low. But it does mean that the blip is more flappy, right? Where like it could go up and down pretty quickly if you're past the point of saturation. -No, it also means that you don't have to reach out side of the network. So if you're a really strict like proxy requirements to send data, local dev is much easier for that, which is why like Hotel now has like a really good Prometheus exporter so you could do both, right? +That's why it's good to have like a HPA (Horizontal Pod Autoscaler) on that stuff, but still, daemon set is a bit more reliable. -**Host:** Right, right. +**Host:** Right, and it sounds like it would be useful again from a metrics standpoint. -**Jacob:** If you have that hankering for running Prometheus. +**Jacob:** Yeah, yeah. Tracing, you could do it for tracing and just send it on like a node port. But tracing workloads, again, because it's all push-based, they are much easier to scale on, and you can distribute targets; you can load balance. There are all these other benefits that we get from push-based workloads. -**Jacob:** And then I'm assuming there's a centralized gateway somewhere or— +Pull-based is like the reason that Prometheus is so ubiquitous in my opinion is just because it makes local development really easy, where you just can scrape your local endpoint, and that's what most back-end development is anyway. -**Jacob:** This is part of the collector chain that I was talking about. Again, we're running a lot of experiments. Cool. I can like half talk about it. +So you could hit endpoint A and then hit your metrics endpoint and then hit endpoint A again for the metric standpoint. You can just check that; it's like a very easy developer loop. -Okay, I can be vague. A big effort within Hotel right now is around Arrow, which you might have been hearing about some. There’s been some work done by Lightstep and F5 to improve the processing speed and egress and ingress costs of OTEL data by using Apache Arrow, which is a project for columnar-based data representations. +### [00:47:12] Use of stateful sets vs deployments -And so we're just like doing some proof of concepts or like proof of implementation work to see what the actual performance of this stuff looks like, right? And also, like, you know, check that everything works as expected. +It also means that you don't have to reach out outside of the network, so if you're a really strict proxy requirements to send data, local dev is much easier for that. -**Host:** Yeah, yeah, yeah. +That's why like Hotel now has like a really good Prometheus exporter, so you could do both. -**Jacob:** As well, which it is, but that's—you always have to check. +**Host:** Right, right. And then I'm assuming there's a centralized gateway somewhere or— -**Host:** Yes, absolutely. +**Jacob:** This is part of the collector chain that I was talking about. Again, we're running a lot of experiments. I can like half talk about it. -**Host:** Well, I'd say the main takeaway from this whole story on collectors is like it sounds like it's always going to be an evolving game, which is not a terrible thing to do. +**Host:** Okay. -**Jacob:** No, it's important that you keep your telemetry up to date. I think that like library authors and maintainers are like constantly working on new performance features and new ease of use, like quality of life stuff as well. +**Jacob:** I can be vague. A big effort within Hotel right now is around Arrow, which you might have been hearing about some. There's been some work done by Lightstep and F5 to improve the processing speed and egress and ingress costs of OTEL data by using Apache Arrow, which is a project for columnar-based data representations. -**Host:** Yeah, yeah. +We're just like doing some proof of concepts or like proof of implementation work to see what the actual performance of this stuff looks like, right? -**Jacob:** Especially with OTEL, like we talk about quality of life a lot. +And also, you know, check that everything works as expected, yeah, as well, which it is, but you always have to check. -**Host:** Yeah. +**Host:** Yes, absolutely. Well, I'd say the main takeaway from this whole story on collectors is like it sounds like it's always going to be an evolving game, which is not a terrible thing to do. -**Jacob:** And so that is definitely a focus, and that's why it's important to keep up to date. It makes migrations easier as well. Trying to migrate from like an ancient version of something to the latest version, you're probably missing a lot of breaking changes potentially, and you have to be careful of that. +**Jacob:** No, it's important that you keep your telemetry up to date. I think that like library authors and maintainers are like constantly working on new performance features and new ease of use, like quality of life stuff as well. -**Host:** And on that vein then, how do you ensure that everyone's keeping up to date with the latest versions of OTEL across the org? +**Host:** Yeah, yeah. Especially with OTEL, like we talk about quality of life a lot. -**Jacob:** I think like stuff like Dependabot is pretty good. We use it internally—or not internally, we use it in Hotel. We're keeping up to date with Hotel stuff, so I find it to be really helpful. It's very frustrating sometimes because it's like Hotel packages all update in like lockstep pretty much. +**Jacob:** Yeah, and so that is definitely a focus, and that's why it's important to keep up to date. It makes migrations easier as well. Trying to migrate from like an ancient version of something to the latest version, you're probably missing a lot of breaking changes potentially, and you have to be careful of that. -That means you have to update a fair amount of packages at once, but it does do it for you, which is pretty nice. +**Host:** And on that vein, then how do you ensure that everyone's keeping up to date with the latest versions of Hotel across the org? -**Host:** That's nice. That's nice. +**Jacob:** I think like stuff like Dependabot is pretty good. We use it internally, or not internally, we use it in Hotel. We're keeping up to date with Hotel stuff, so I find it to be really helpful. It's very frustrating sometimes because it's like Hotel packages all update in lockstep pretty much; that means you have to update a fair amount of packages at once, but it does do it for you, which is pretty nice. -**Jacob:** But you should be doing this not just for Hotel but like any dependency, right? Like CVEs happen in the industry constantly, and if you're not staying up to date with vulnerability fixes, then you're opening yourself up to security attacks, which you don't want. +**Host:** That's nice. That's nice. But you should be doing this not just for Hotel but like any dependency. Right? Like CVEs happen in the industry constantly, and if you're not staying up to date with vulnerability fixes, then you're opening yourself up to security attacks, which you don't want. -**Host:** So, yeah, do something about it is my recommendation. +**Jacob:** Yeah, yeah, do something about it is my recommendation. -**Jacob:** That’s fair, that’s fair. +**Host:** That's fair, that's fair. I know we've got like four minutes left. Do you want to give us any like parting thoughts as we wrap up? -**Host:** I know we've got like four minutes left. Do you want to give us any like parting thoughts as we wrap up? I'm interested to hear if there are any questions from the group that we've missed in our discussion here. Any takers with burning questions? Now's the time. If not, I'm going to call on Rhys, but this was great. +I'm interested to hear if there are any questions from the group that we've missed in our discussion here. -I feel like I was like rapidly trying to take notes. I probably will have more as I try to like go back and tidy up some of my notes. But yeah, honestly, I was just trying to like keep up with people with so much information, I feel like because I've been reading up on the operator like the last week and so like more questions than answers, and I feel like I have some answers now. +**Jacob:** Any takers with burning questions? Now's the time. If not, I'm going to call on Rhys. But this was great. -I've definitely learned a lot. I hope folks on this call have learned a lot as well. I think this is like really great information. I think it gives folks like an idea of like what's involved in a migration, things to consider like when you're setting up your collectors, things to avoid doing, keeping up with those latest versions of OTEL—always a good thing. +I feel like I was like rapidly trying to take notes. I probably will have more as I try to go back and tidy up some of my notes. But yeah, honestly, I was just trying to keep up with people; there was so much information. -**Jacob:** Yeah, that's the big one is that like new fixes really often. +### [00:52:07] Wrap-up and parting thoughts -**Host:** Yeah, yeah. +I feel like because I've been reading up on the operator the last week, and so like more questions than answers, and I feel like I have some answers now. -**Jacob:** Like new versions every two weeks for the collector, I'd say it's like a pretty frequent cadence. There are some Prometheus libraries that like don't update like ever. Like Thanos hasn't released since March, right? +I've definitely learned a lot. I hope folks on this call have learned a lot as well. I think this is like really great information. I think it gives folks an idea of what's involved in a migration, things to consider, like when you're setting up your collectors, things to avoid doing, keeping up with those latest versions of Hotel—y'all always a good thing. -**Host:** Which is absurd to me because there's like a bug in compatibility between Prometheus and Thanos right now. +**Jacob:** Yeah, that's the big one is that like new fixes really often. -**Jacob:** Oh really? +**Host:** Yeah, yeah, like new versions every two weeks for the collector, I'd say it's like a pretty frequent cadence. There are some Prometheus libraries that like don't update like ever—like Thanos hasn't released since March, right? Which is absurd to me because there's like a bug in compatibility between Prometheus and Thanos right now. -**Host:** How cheap! +**Jacob:** Oh really? How cheap. -**Jacob:** Yeah, so you can use them together if you're like coding, so not fun. +**Host:** Yeah, so you can use them together if you're like coding, so not fun. -**Host:** Damn! +**Jacob:** Yeah, damn. Alright, last chance for burning questions y'all. -**Host:** All right, last chance for burning questions, y'all. Erica said thanks for the info and your time, Jacob. +**Erica:** Thanks for the info and your time, Jacob. **Jacob:** Thank you, Erica, for hopping on. -**Host:** Yeah, this was really awesome. Thank you so much because like for real, this is a dope, dope topic. So yeah, we'll, you know, reach out to your friendly neighborhood CFP approvers. +**Host:** Yeah, this was really awesome. Thank you so much because like for real, this is a dope topic. + +**Jacob:** We'll, you know, reach out to your friendly neighborhood CFP approvers. -**Jacob:** No, thank you so much. I feel like there was so much more we could have chatted about as well, so we might— +**Host:** No, thank you so much. I feel like there was so much more we could have chatted about as well, so we might— -**Host:** Yeah, yeah. Maybe I can—if I get accepted for the talk, then I can give a sneak peek at least. I can get some feedback on it. +**Jacob:** Yeah, yeah. Maybe I can, if I get accepted for the talk, then I can give a sneak peek at least. -**Jacob:** Actually, you know what, like if you're interested because we have Hotel in practice, which is kind of like giving like a little talk. So if you're interested in doing that even like before you find out whether or not you get accepted, we are happy to have you. +**Host:** I can get some feedback on it. Actually, you know what? Like if you're interested, because we have Hotel in Practice, which is kind of like giving a little talk. So if you're interested in doing that even like before you find out whether or not you get accepted, we are happy to have you. **Jacob:** Yeah, it sounds great. Cool, I'm in. -**Host:** Cool, cool. We'll figure out the behind the scenes on like when to schedule you. +**Host:** Cool, cool. We'll figure out the behind the scenes on like when to schedule you and— -**Jacob:** Sounds good. +**Jacob:** Sounds good. -**Host:** Yay! Cool. All right, and Daniel said thanks, Jacob, as well. And yeah, we're at the top of the hour. Thanks for joining us. +**Host:** Yay, cool. Alright, and Daniel said thanks, Jacob, as well. And yeah, we're at the top of the hour. Thanks for joining us. **Jacob:** Thank you all. -**Host:** Thank you! - -**Jacob:** Yeah, thank you everyone. - -**Host:** Bye! +**Host:** Thank you, everyone. Bye. ## Raw YouTube Transcript diff --git a/video-transcripts/transcripts/2023-09-13T19:26:10Z-opentelemetry-q-a-feat-hazel-weakly.md b/video-transcripts/transcripts/2023-09-13T19:26:10Z-opentelemetry-q-a-feat-hazel-weakly.md index f93e33b..0481a39 100644 --- a/video-transcripts/transcripts/2023-09-13T19:26:10Z-opentelemetry-q-a-feat-hazel-weakly.md +++ b/video-transcripts/transcripts/2023-09-13T19:26:10Z-opentelemetry-q-a-feat-hazel-weakly.md @@ -10,281 +10,336 @@ URL: https://www.youtube.com/watch?v=wMJEgrUnX7M ## Summary -In this YouTube video, Hazel Weekly discusses her experiences with observability and OpenTelemetry, providing insights for novice users. The conversation touches on the initial challenges companies face when adopting observability, including understanding the need for meaningful telemetry and the tendency to over-instrument systems without clear intent. Hazel shares her journey of helping a company reduce unnecessary data collection and implement effective instrumentation, highlighting how cultural and organizational factors can complicate observability efforts. She emphasizes the importance of aligning observability practices with business goals and the need for effective communication between developers and stakeholders. The video also covers specific technical challenges encountered while working with OpenTelemetry, including issues with sampling, baggage, and the ergonomic difficulties of manual instrumentation in various programming languages. Overall, Hazel's insights underscore the complexity of implementing observability in real-world scenarios and the importance of thoughtful design in telemetry practices. +In this YouTube Q&A session, Hazel Weekly discusses her extensive experiences with OpenTelemetry and observability, particularly for novice users in the field. She shares insights on her journey from being a newcomer to becoming a knowledgeable user of OpenTelemetry, emphasizing the importance of meaningful questions in shaping observability practices within organizations. Hazel highlights the challenges of over-instrumentation, where companies collect excessive telemetry data without clear intent, leading to inefficiencies and confusion. She discusses how she guided a previous organization to streamline their telemetry practices, reduce data volume, and implement better instrumentation strategies. The conversation also touches on the technical challenges of OpenTelemetry, including difficulties with sampling, context propagation, and the ergonomic limitations of various programming languages, particularly Haskell and TypeScript. Furthermore, Hazel addresses the complexities of advocating for observability initiatives to executives, emphasizing the need for tangible business value and the importance of aligning developer and business lifecycles. The discussion concludes with thoughts on the regulatory landscape and the potential for collaborative compliance solutions in the observability space. ## Chapters -00:00:00 Welcome and intro -00:00:20 Introduction of Hazel -00:01:21 Observability definition -00:03:00 Instrumentation challenges -00:05:00 Cost-driven changes -00:06:25 Sampling methods -00:09:30 Over-instrumentation issues -00:12:01 Context propagation challenges -00:15:00 Convincing the team -00:18:00 Observability improvements -00:19:01 Anti-pattern discussion -00:23:00 OpenTelemetry support in languages -00:27:00 Baggage challenges -00:30:00 Ergonomics of instrumentation -00:34:30 Libraries for OpenTelemetry -00:38:35 Infrastructure management experience -00:45:00 Making the case to executives -00:50:00 Regulatory compliance challenges -00:56:00 Closing remarks and future events +00:00:00 Introductions +00:01:02 Hazel's experience with observability +00:01:14 Definition of observability +00:03:06 Challenges of over-instrumentation +00:05:10 Reducing telemetry data volume +00:06:12 Importance of asking meaningful questions +00:10:20 Technical challenges with OpenTelemetry +00:19:38 Cultural issues in observability +00:45:08 Advocating for observability to executives +00:54:46 Regulatory landscape and compliance solutions -[00:00:20] **Host:** Thank you. Welcome everyone to the OpenTelemetry Q&A. We have the pleasure of having our good friend Hazel Weekly come talk to us about her experiences with observability. Welcome, Hazel. +## Transcript -**Hazel:** Glad to be here! Very excited for this. Yay! +### [00:00:00] Introductions -**Host:** I guess let's start with first things first because I know we still have a bunch of OpenTelemetry novice users just getting into OpenTelemetry, folks who join our end-user working group. From the perspective of somebody who's new to OpenTelemetry, can you share what that experience was like? What kind of landscape were you coming into? I guess for starters, drawing back on that experience, was the company at that point ready for observability? What kind of kick-started the conversation into OpenTelemetry in the first place? +**Host:** Thank you. Welcome, everyone, to the OpenTelemetry Q&A. We have the pleasure of having our good friend Hazel Weekly come talk to us about her experiences with observability. Welcome, Hazel. -[00:01:21] **Hazel:** I would say, so I'm going to start off with my definition of observability because it's slightly different. The definition from control theory would be like, can you understand this system from the input and the outputs? Fred Herbert might talk about the definition from cognitive safety systems engineering, and that one is much more about the work required and the process required for a group of people to be able to understand everything and actually understand the system. That's how they discovered and think about that. Mine is the process through which you develop the capability of asking meaningful questions and getting useful answers. It's a process because it's evolutionary, and the questions have to be meaningful to you—whatever that means—and the answers don't have to be correct, but they have to be useful. +**Hazel:** Glad to be here. Very excited for this! -[00:03:00] So when I look back at the company when we were starting to think about observability, or rather like distribution, literally, what are the questions that the company is starting to ask from the perspective of the engineers? One of the questions the company was asking from the perspective of the managers and what was it asking from the perspective of the executives? From the engineers at one of the companies that were early adopters of OpenTelemetry, they had it and they had instrumented things with it, but more from the perspective of “we know we need this,” but they didn't necessarily have that motivation yet. They weren't asking the types of sophisticated questions that motivate people to add their own telemetry. They had a bunch of telemetry, and they had a huge amount of instrumentation, and they had just the volume turned up to 11, and no one was actually querying that information. +### [00:01:02] Hazel's experience with observability -What I was able to do was turn down that volume significantly. I essentially said, “You're not using this, and you're not asking any questions from your system.” So when you do, don't turn the volume back up; add that instrumentation in there with thought and intent. That was an interesting hurdle to happen because in a way, people would ask questions to understand the system. They were asking them to solve a problem, and consequently, they just wanted every bit of information ever so that whenever they needed it, they could just go through a giant stack of noise and find a needle, right? That's not really what OpenTelemetry is for. Online distribution is for, can you understand the state of your system to the point that you only collect what you actually need? That is, in and of itself, a second layer of understanding the system, and they didn't have that second layer. +**Host:** Yay! I guess let's start with first things first because I know we still have a bunch of OpenTelemetry novice users just getting into OpenTelemetry, folks who join our end-user working group. From the perspective of somebody who's new to OpenTelemetry, can you share what that experience was like? What kind of landscape were you coming into? And I guess for starters, drawing back on that experience, was the company at that point ready for observability? What kind of kick-started the conversation into OpenTelemetry in the first place? -**Host:** Right, so it sounded like they were instrumenting for the sake of instrumenting and just sort of throwing everything at the wall and hoping something would stick. +**Hazel:** I would say—so I'm going to start off with my definition of observability because it's slightly different. The definition from control theory would be like, can you understand this daily system from the input and the outputs? Fred Herbert might should talk about the definition from cognitive safety systems engineering, and that one is much more about the work required and the process required for a group of people to be able to understand everything and actually understand the system. That's how they like discovered and think about that. Mine is the process through which you develop the capability of asking meaningful questions and getting useful answers. -**Hazel:** Yeah, that also sounds like a very “I came over from Vlogs and I'm used to being able to search everything ever” approach. They also still had logs, which is a surprise. Basically, nobody of course. +So it's a process because it's evolutionary, and the questions have to be meaningful to you, whatever that means. The answers don't have to be correct, but they have to be useful. When I look back at the company when we were starting to think about observability, or rather distribution, literally, what are the questions that the company is starting to ask from the perspective of the engineers? And one of the questions the company is asking from the perspective of the managers, and what is it asking from the perspective of the executives? -**Host:** In terms of like, how did you convince them to kind of tamp it down and direct it, make it more directed so that it could actually work for them? +From the engineers at one of the companies that were the early hands-on OpenTelemetry, they had it, and they had instrumented things with it, but more from the perspective of, we know we need this, but they didn't necessarily have that motivation yet. They weren't asking the types of sophisticated questions that motivate people to add their own telemetry. They had a bunch of telemetry, and they had a huge amount of instrumentation, and they had just the volume turned up to 11. No one was actually querying that information. -[00:05:00] **Hazel:** Convincing them to champion it down and turn the volume down was actually pretty easy. That came down to cost. They were about 300% over budget from the vendor, and so I said, “I need to get you under budget.” I actually have a very funny image from that time period where you see the ingestion at about like two to three hundred million events per day, and you know I'm trying to tweak it down to about like 200, 250, or like, you know, 180 to 250 million. That was with very, very aggressive sampling. Finally, I figured out a bunch of misconfigurations in the sampling information and dropped it to about like three to five million events per day. +### [00:03:06] Challenges of over-instrumentation -Once I had done that, they had concerns about, “Oh, what if I can't find something?” I said, “Well, do you need to ask that question?” Then we were able to actually start the useful dialogue of, “Now that you have a question you need to ask, go and build what you need in order to get that answer.” Naturally, that sort of feedback loop that you kind of need with observability, whether or not you need that feedback loop end-to-end is a different question, but that is currently what OpenTelemetry requires. +So what I was able to do was turn down that volume significantly, and then what I did was essentially say, "You're not using this, and you're not asking any questions from your system." So when you do, don't turn the volume back up; like, add that instrumentation in there with thought and intent. That was an interesting hurdle to happen because in a way, people would ask questions to ask these types of semi-physical questions, asking them to understand the system. They're asking them to solve a problem, and consequently, they just wanted every bit of information ever so that whenever they needed it, they could just go through a giant stack of noise and find a needle, right? And that's not really what OpenTelemetry is for online distribution. It's for, can you understand the state of your system to the point that you only collect what you actually need? -[00:06:25] **Host:** I think that's really great because being able to know what questions to ask makes it a much more meaningful experience. In your words, not searching for that needle in the haystack. I want to go back to sampling for a second. So sampling to a sampling was, we did the sampling through two methods. We had the OpenTelemetry collector, and I'll show you now, but at this company, we only had the Honeycomb Refinery set up until we had that set up. At other companies, we've had the OpenTelemetry collector setup, and I've used that. My preference is actually to use both, regardless of whether or not using Honeycomb. +That is, in and of itself, a second layer of understanding the system, and they didn't have that second layer, right? -The reason for that is because the OpenTelemetry collector doesn't have the best steel sampling configuration, and the OpenTelemetry collector has like the most open-source line economic sort of configuration setup to consent things to multiple sources and multiple sinks so that is actually really useful. I like to use that one, and then aggregate everything and send it to Refinery and then do useful tail sampling. But running two pieces of information is kind of complicated for a lot of people. It's not going to be a huge hurdle of adoption to like my current company. They use DataDog, and one of the challenges there is they're not at the point where they can ask sophisticated questions of their infrastructure. +**Host:** So it sounded like they were instrumenting for the sake of instrumenting and just sort of throwing everything at the wall and hoping something would stick. -I'm really still more at the point of “Is it on?” The question there is, well, even if I set things up for success by switching us to the OpenTelemetry line instrumentation tooling and then still send it to the same place, we now need to run like the collector in multiple places or run like a collector and Refinery just in order to do the same thing that they already do with their built-in tooling. That can be a bit of a lift because rather than saying “install library,” it's “install a library and have like five things also.” +**Hazel:** Yeah, that also sounds like a very—I came over from Vlogs, and I'm used to being able to search everything ever approach. They also still had logs, which is a surprise, basically nobody, of course. -**Host:** Going back to the first organization where they were sending like way too much data, after you convinced them to chill and do more directed instrumentation, what was the next sort of hurdle that you experienced? +**Host:** In terms of how did you convince them to kind of tamp it down and make it more directed so that it could actually work for them? -**Hazel:** There was a notch hurdle there. The way that company structured work was very much the over-platformed thing, this little towards unusual, but they had like developer productivity teams, and they didn't really have a platform team. What they had were engineers that didn't do work. The engineers didn't do infrastructure stuff; they wrote additional code, and they would have a bunch of people working on build tooling and build other stuff like that. +### [00:05:10] Reducing telemetry data volume -[00:09:30] The question was, in the way that they do work in order to adopt OpenTelemetry successfully, you would need to essentially build libraries for things or write stuff into the code in a way that engineers weren't necessarily writing their own instrumentation, and that requires working at a level of abstraction that OpenTelemetry is really, really difficult. You can't propagate certain types of information down to child spans very easily or really at all. Setting up baggage is still very difficult. +**Hazel:** Convincing them to champion it down and turn the volume down was actually pretty easy. That came down to cost. They were about 300% over budget from the vendor, and so I said, "I need to get you under budget." I actually, in the Honeycomb Pond Eaters, have a very funny image from that time period where you see the ingestion at about like two to three hundred million events per day, and you know, I'm trying to tweak and get it down to about like 200, 250, or like, you know, 180 to 250. That was with very, very aggressive sampling. -I'm working with context can be really tricky. If somebody uses an async function, then you might drop that and have to reconnect it somehow, and if the lifetimes aren't like already set up nicely, people can even add their own instrumentation, and then it'll get dropped from something because they don't actually understand the full context of where it is. They're just random code, and they add something, but in OpenTelemetry, you have to understand the call stack, not just the functionality, and those aren't always the same. +Finally, I figured out a bunch of misconfigurations in the sampling information and dropped it to about like three to five million events per day. Once I had done that, they had concerns about, oh, what if I can't find something? I said, "Well, do you need to ask that question?" And then we were able to actually start the useful dialogue of, now that you have a question you need to ask, go and build what you need in order to get that answer. + +That sort of feedback loop that you kind of need with observability—whether or not you need that feedback loop end to end is a different question, but that is currently what OpenTelemetry requires. + +### [00:06:12] Importance of asking meaningful questions + +**Host:** I think that's really great because being able to know what questions to ask makes it for a much more meaningful experience. In your words, not searching for that needle in the haystack. + +I want to go back to a sampler for a second. Do sampling to a sampling was—we did the sampling through two methods. We had the OpenTelemetry collector, and I'll show you now, but at this company, we only had the Honeycomb Refinery set up. Until we had that set up, at other companies, we've had the OpenTelemetry collector setup, and I've used that. My preference is actually to use both, regardless of whether or not using Honeycomb. + +The reason for that is because the OpenTelemetry collector doesn't have the best sampling configuration, and the OpenTelemetry collector has the most open-source, line-economic configuration setup to send things to multiple sources and multiple sinks. That is actually really useful, so I like to use that one and then aggregate everything and send it to Refinery and then do useful tail sampling. + +But running two pieces of information, sorry, is kind of complicated for a lot of people. It's not going to be a huge hurdle of adoption to my current company. They use DataDog, and one of the challenges there is they're not at the point where they can ask sophisticated questions of their infrastructure, and I'm really still more at the point of, "Is it on?" So the question there is, well, even if I set things up for success by switching us to the OpenTelemetry line instrumentation tooling and then still send it to the same place, we now need to run like the collector in multiple places or run a collector and Refinery just in order to do the same thing that they already do with their built-in tooling. So that can be a bit of a lift because rather than saying, "Install Library," it's "Install a library and have like five things also." + +**Host:** Going back to the first organization where they were sending way too much data. After you convinced them to kind of chill and do more directed instrumentation, what was the next sort of hurdle that you experienced? + +**Hazel:** There was a notch hurdle. The way that company structured work was very much the over-platformed thing. This little towards unusual, but they had like a developer productivity team, and they didn't really have a platform team. What they had were engineers that didn't do infrastructure stuff; they did additional code. They would have a bunch of people working on build tooling and build other stuff like that. + +The question was, in the way that they do work in order to adopt OpenTelemetry successfully, you would need to essentially build libraries for things or write stuff into the code in a way that engineers weren't necessarily writing their own instrumentation. That requires working at a level of abstraction that OpenTelemetry is really, really difficult. You can't propagate certain types of information down to child spans very easily or really at all. Setting up baggage is still very difficult. Working with context can be really tricky. If somebody uses an async function, then you might drop that and have to reconnect it somehow. + +If the lifetimes aren't like already set up nicely, people can even add their own instrumentation, and then it'll get dropped from something because they don't actually understand the full context of where it is. They're just random code, and they add something. But in OpenTelemetry, you have to understand the call stack, not just the functionality, and those aren't always the same. **Host:** Interesting. So how do you get over that hurdle? -**Hazel:** At that company, I was never able to fully get over that hurdle. I just made it better. One of the ways that I made it better was essentially following through all of the spans manually and looking for errors. When I found errors, I would trace it down to the source code, and I would do that. I ran into a bunch of very small things of like propagating information wasn't correct. So it was probably the information of like the helper functions rather than the actual call sites to activate that. Then I had to fix like 500 small things. +### [00:10:20] Technical challenges with OpenTelemetry -Even when I did that, I found that there were a bunch of places where these looking as contexts weren't being complicated, so functions were just being launched or, for example, calling tracing, and that was being disconnected from the actual whatever. Sometimes things had to end before they started or start before they ended, and that got confusing because the language that they were using, it's very easy to write code in that logic, and it actually is more correct to do it that way. +**Hazel:** So at that company, I was never able to fully get over that hurdle. I just made it better. One of the ways that I made it better was essentially following through all of the fans manually and looking for errors. When I found errors, I would trace it down to the source code and I would do that. I ran into a bunch of very small things of like propagating information wasn't corrupt. -But OpenTelemetry, in its design, is very much a chicken-and-egg call stack, a tree-shaped, which is really, really ergonomic for a language like Java. It's not necessarily ergonomic for languages that aren't like that. So like React is a really common example because it's a runtime that's asynchronous, and it's really difficult to write code in React to the deadline nicely. +So it was probably the information of like the helper functions rather than the actual call sides to activate that. I had to fix like 500 small things, and even when I did that, I found that there were a bunch of places where use of context weren't being complicated. Functions were just being launched or, for example, calling tracing, and that was being disconnected from the actual whatever. Sometimes things had to end before they started or start before they ended, and that got confusing. -[00:12:01] I suppose a good one closer is a good example of one that doesn't work very well. Watch does work, but for a very interesting reason, and that is because the Rust language cares about lifetimes. So you think about the lifetime of your functions, and so you always know what those are, and those end up being the natural tracing point protocol stack-based tradition library. +Because the language that they're using, it's very easy to write code in that logic, and it actually is more correct to do it that way. But OpenTelemetry in its design is very much a chicken-is-called stack, a tree-shaped, which is really, really ergonomic for a language like Java. It's not necessarily ergonomic for languages that aren't like that. -In terms of what we were able to do and how I was fixing it was really mostly in guacamole and doing some weird clever things to pack into the language runtime in order to make things hit the semantic model differences between OpenTelemetry and other language things about calling code. +So like React is a really common example because it's a runtime that's asynchronous, and it's really difficult to write code in React to the deadline nicely. I suppose a good one closer is a good example of one that doesn't work very well. -**Host:** So were you like a one-woman show doing all of this? +**Host:** Were you like a one-woman show doing all of this? -**Hazel:** So the other person on this was off on parental leave, so for the duration, I was actually one person. +**Hazel:** So the other person on this was off on parental leave. So for the duration then, I was actually one person. **Host:** That is mad impressive. Did you want to like pull your hair out some days? -**Hazel:** It was tricky. A lot of it was that one of the benefits of using OpenTelemetry is that you get like this intuitive sense of what the code is doing at runtime if you play with it enough. They had to sit there and kind of doodle with it and look things up and ask questions and get answers. You have to have that dialogue running, and I knew how to do that, and no one else at the company was doing that except like one or two people. +**Hazel:** It was tricky. A lot of it was that one of the benefits of using OpenTelemetry is that you get like this intuitive sense of what the code is doing at runtime if you play with it enough. They had to sit there and kind of like doodle with it and look things up, and I ask questions and get answers. You have to have that dialogue running, and I knew how to do that. No one else at the company was doing that except like one or two people. + +**Host:** Wow. And so you had like a very small select group of power users, favorite people, and then you had everyone else who may not have even used OpenTelemetry at all or even knew like a vendor in any way whatsoever. + +So I would get an intuitive sense if one was broken and what was wrong or something, and I would say, "Hey, like this is an issue," and then people wouldn't necessarily believe me. + +**Hazel:** Oh, interesting. So how do you end up convincing them that it is an issue, or could you convince them, I guess is the question? + +**Hazel:** So when I was able to, I was able to take advantage of certain people of the things, and one of the ways that I did that was essentially my predicting things, and then my predictions would turn out to be correct. Then I would give a solution to the position, and then the solution would work. + +So like one was the database was very, very spiky, and I mentioned that this was going to cause some sort of issues. One of the reasons that the database was spiky was because we had like basically the disk would slow on a database. We spent up the disk on the database quite a bit, and when that happened, despite this level down, these sets were still very spiky in general but despite going high enough to lock things down. + +There were small things like that added up over time, and people started to believe me a lot more when it came to certain issues. But other ones that were more fundamental in the architecture, I never actually got around to being able to convince people of that, and I actually ended up leaving that company mainly due to that. -**Host:** Wow, and so you had like a very small select power users, favorite people, and then you had everyone else who may not have even like used OpenTelemetry at all or even knew like a vendor in any way whatsoever. +**Host:** Interesting. So did you, by the time you left, did you find there was at least like they were getting more out of their observability at that point? Because of your efforts, did you at least see some positive results because of that? -**Hazel:** Yes, and I would get an intuitive sense if one was broken and what was wrong or something, and I would say, “Hey, like this is an issue,” and then people wouldn't necessarily believe me. +**Hazel:** Yeah, I did. So there was a lot better understanding of where they were with the observability. People had more of an understanding of how to use it. I had like a couple of things so that it was more useful. They had a lot of it, but some things were wrong. -**Host:** Interesting. So how do you end up convincing them that it is an issue, or could you convince them? +Like if the error stack was there, the error stack died at the wrong spot, so it wouldn't actually tell you where the error happened. It would tell you where you defined the OpenTelemetry like the tradition helper, which is pointless. So I fixed that, and then all of a sudden, the errors were correct again, and people were happy about that. Then they would actually use it. -**Hazel:** I was able to take advantage of certain people of the things, and one of the ways that I did that was essentially my predicting things, and then my predictions would turn out to be correct, and then I would give a solution to the position, and then the solution would work. One was the database was very, very spiky, and I mentioned that this was going to cause some sort of issues. One of the reasons that the database was spiky was because we had like basically the disk would slow on a database, and so we sped up the disk on the database quite a bit. +I hadn't fixed a bunch of things, and I had reduced the usage from like 200 to 300 events per day, or even maybe 50 million events per day, to like right. When that happened, then people could continue to add OpenTelemetry and add instrumentation everywhere without running into the physical. They were going to blow their budget even more than their own work, and that allows people to continue to add more instrumentation. -[00:15:00] When that happened, despite this level down, these sets were still very spiky in general, but despite going high enough to lock things down, there were small things like that added up over time, and people started to believe me a lot more when it came to certain issues. But other ones that were more fundamental in the architecture, I never actually got around to being able to convince people of that, and I actually ended up leaving that company mainly due to that. +So then I guess there it seems like there wasn't an issue per se as far as like getting people to add the instrumentation at that point because they were obviously they had already over-instrumented the system previously. -**Host:** Interesting. By the time you left, did you find there was at least like they were getting more out of their observability at that point? Because of your efforts, did you at least see some positive results because of that? +**Hazel:** It's interesting. They had over-instrumented it, but only via auto instrumentation. So there was very little manual stuff done, and the manual stuff that was done was often done in a way that made the questions you can ask relatively useless. -**Hazel:** Yeah, I did. There was a lot better understanding of where they were with the observability. People had more of an understanding of how to use it. I had like a couple of things so that it was more useful. They had a lot of it, but some things were wrong. If the error stack was there, the error stack died at the wrong spot, so it wouldn't actually tell you where the error happened. It would tell you where you defined the OpenTelemetry like the tradition helper, which is pointless. +So like one example was there's like at the top of a watched halfway down is when in the life cycle of the call stack is when they would get the user ID associated with it because of how the database calls worked. But the user ID was never propagated to the top, so it was actually impossible to figure out how many pages a user was visiting, like procession. -So I fixed that, and then all of a sudden the errors were correct again, and people were happy about that. Then they would actually use it. I had fixed a bunch of things, and I had reduced the usage from like two to three hundred events per day or even maybe 50 million events per day to like right. When that happened, then people could continue to add OpenTelemetry and add instrumentation everywhere without running into the physical—they were going to blow their budget even more than their own work. +These types of questions, whether or not I usually like it's on the same thing over and over because the question we wanted to ask was, how effective would database account should be? But would it be effective to put Radish there somewhere? Probably don't have a 60 second something TTL. We had no idea because we couldn't actually ask that question. -That allows people to continue to add more instrumentation. +All the information was there but not actually in a way that made anything possible to look for, right? Because you couldn't look halfway down the span in order to get this span correlated with this one. So that tree—you can only ask for things on the same level and down and filter that way, and so they need to fix that, but no one necessarily knew how to, and no one necessarily knew that they wanted to because they weren't asking that type of question. -**Host:** So then I guess there it seems like there wasn't an issue per se as far as like getting people to add the instrumentation at that point because they were obviously—they had already over-instrumented the system previously. +**Host:** So I bought that up, and then people started to get more of a sense of, oh, here's kind of where we put the information in a way that's visible. To this day, I still don't have like a really good way to explain to people how to think about that other than you do kind of need to display the system. I couldn't understand where the data needs to be in order to be useful for asking questions. -**Hazel:** Yes, it was interesting. They had over-instrumented it, but only via auto-instrumentation, so there was very little manual stuff done, and the manual stuff that was done was often done in a way that made the questions you could ask relatively useless. So like one example was, there's like at the top of the watched halfway down when in the life cycle of the call stack is when they would get the user ID associated with it because of how the database calls worked. +**Hazel:** Yeah, that makes a lot of sense. -But the user ID was never propagated to the top, so it was actually impossible to figure out how many pages a user was visiting, like procession. These types of questions whether or not I usually like it's on the same thing over and over because the question we wanted to ask was, “How effective would database account should be?” But would it be effective to put radish there somewhere? Probably don't have a 60-second something TTL. We had no idea because we couldn't actually ask that question. All the information was there, but not actually in a way that made anything possible to look for. +**Host:** I find this such an interesting sort of— I don't want to well, I guess use case of sorts, scenario, if you will. Because there oftentimes we hear the stories of people just struggling with bringing OpenTelemetry into the organization and then starting to instrument. This feels like it was like one giant anti-pattern of one giant OpenTelemetry anti-pattern, which I think is a very important thing to be able to discuss because, you know, like all tools, OpenTelemetry can be grossly misused, and then you end up with scenarios like this where you're instrumenting but you're not getting anything out of it. -[00:18:00] Because you couldn't look halfway down the span in order to get this span of correlated with this one, so that tree—you can only ask for things on the same level and down and filter that way. They needed to fix that, but no one necessarily knew how to, and no one necessarily knew that they wanted to because they weren't asking that type of question. +### [00:19:38] Cultural issues in observability -So I brought that up, and then people started to get more of a sense of, “Oh, here's kind of where we put the information in a way that's visible.” To this day, I still don't have like a really good way to explain to people how to think about that other than you do kind of need to display the system. I couldn't understand where the data needs to be in order to be useful for asking questions. +**Hazel:** It was really interesting to see how this happened because it's a lot of what happened is due to how the company works and how it thought about working. It was very anarchic in the sense that engineers were very, very self-directed, and they would do things. The company had a huge cultural issue of intricate 80 percent done with Instagram, maybe like 90 percent done, and then they would just drop off or someone like outside traffic and had to do something else, and no one else would pick it up. -[00:19:01] **Host:** Yeah, that makes a lot of sense. So yeah, that's so interesting. I find this such an interesting sort of, I don't want to use the word “use case,” but scenario, if you will. Because oftentimes we hear the stories of people just struggling with bringing OpenTelemetry into the organization and then starting to instrument, and this feels like it was like one giant anti-pattern—one giant OpenTelemetry anti-pattern—which I think is a very important thing to be able to discuss because, you know, like all tools, OpenTelemetry can be grossly misused, and then you end up with scenarios like this where you're instrumenting but you're not getting anything out of it. +No one would like sit there and make sure the things you got like fully finished, and so that essentially happened where we had one employee set up the main amount of instrumentation, and then one or two people added like some things to it, and one or two people added some things to it, but no one did like that last 20 percent of actually tuning things down. -**Hazel:** It was really interesting to see how this happened because a lot of what happened is due to how the company works and how it thought about working. It was very anarchic in the sense that engineers were very, very self-directed, and they would do things. The company had a huge cultural issue of intricate 80 percent done with instrumentation, maybe like 90 percent done, and then they would just drop off or someone outside traffic had to do something else, and no one else would pick it up. +They turned everything on, they turned it all up, they added a bunch of things, and the only instrumentation, and then no one actually necessarily used it. -No one would sit there and make sure the things got fully finished, and so that essentially happened where we had one employee set up the main amount of instrumentation, and then one or two people added like some things to it, and one or two people added some things to it, but no one did like that last 20 percent of actually tuning things down. They turned everything on, they turned it all up, they added a bunch of things, and the only instrumentation, and then no one actually necessarily used it. +**Host:** For example, the television instrumentation library was something they had to write because they used Haskell as a language, and you know what that meant was they had each database query. They had four different spans; it would make a span from the start of the query instead of a transaction, an end of a transaction, end of a query. So you had—and then you have the command of an aquarium itself. So every single database call cost at least four to five spans to appear. -For example, the OpenTelemetry instrumentation library was something they had to write because they used Haskell as a language. What that meant was they had each database query; they had four different spans. It would make a span from the start of the query instead of a transaction, an end of a transaction, and an end of a query. So you had, and then you have the command of a query itself. Every single database call cost at least four to five spans to appear. +**Hazel:** Oh, wow. And there's no way to configure that or turn that down or cool unless things or anything like that. And what happened was that word to create, and then anytime you had like an n plus one pattern just appear instantly in the database, you would have given millions of events. It's like millions of spans all over the place, and then they were like, oh, well, what do we use span events instead? -**Host:** Oh wow. +Most vendors either charge per ingestion or they charge for like until then, actually doesn't shift the cost anywhere. It sometimes meets the user interface slightly better but doesn't shift the cost, which is one of the reasons why people say like, have one wide event when you put things in there. Then, like, you know, have timestamps and don't be afraid to use that. -**Hazel:** And there's no way to configure that or turn that down or cool unless things or anything like that. What happened was that word created, and then anytime you had like an N+1 pattern just appear instantly in the database, you would have given millions of events, millions of spans all over the place. They were like, “Oh, well, what do we use? Span events instead?” Most vendors either charge per ingestion, or they charge for like the telemetry that actually doesn't shift the cost anywhere. +But they were using the SDK essentially as the timestamp functionality to do a Korean event anytime at any time. Mark that something happened, whether you're a crazy fan anytime something happened, a lab formatted all the way down to the auto instrumentation. -It sometimes meets the user interface slightly better, but it doesn't shift the cost, which is one of the reasons why people say, “Have one wide event when you put things in there, then like, you know, have timestamps and don't be afraid to use that.” But they were using the SDK essentially as the timestamp functionality to do a query event anytime, at any time, mark that something happened with a crazy fan anytime something happened, and it formed all the way down to the auto-instrumentation. +**Host:** Huh, so that was tricky. That was probably like 80 percent of like now we have instrumentation in the database, but no one actually said, well, it turns out literally every other database SDK lets you turn 90 percent of the auto instrumentation off, and for a good reason. -That was tricky. That was probably like 80 percent of like, “Now we have instrumentation in the database,” but no one actually said, “Well, it turns out literally every other database SDK lets you turn 90 percent of the auto-instrumentation off,” and for a good reason. Same for like instrumenting the HTTP server, like the web server. You highlight the web server and then you have the web framework, and both of them had almost identical levels of instrumentation. +Same for like instrumenting the HTTP server, like the web server. You highlight the web server, and then you have the web framework, and both of them had almost identical levels of instrumentation. So depending on which endpoint handler was in place, you either got one top level or the other top level or both, and so you had like a massive amount of certifications of the amount of spans that you might not need. -Depending on which endpoint handler was in place, you either got one top-level or the other top-level or both, and so you had like a massive amount of certifications of the amount of spans that you might not need. It turns out it's largely because there wasn't really an ergonomic way to say, “Add this to a span if it exists; otherwise, create your own span.” That pattern is really necessary for libraries, and it's not actually really relevant in the SDK; it's not really mentioned anywhere, and it's not really possible for a lot of languages to do. +It turns out it's largely because there wasn't really an ergonomic way to say, "Add this to a span if it exists, otherwise create your own span." That pattern is really necessary for libraries, and it's not actually really relevant in the SDK. It's not really mentioned anywhere, and it's not really possible for a lot of languages to do. -[00:23:00] **Host:** Right. Yeah, that is an interesting problem to have. So now when you were helping this organization with OpenTelemetry, how well-versed were you in OpenTelemetry at the time? +**Hazel:** Right. -**Hazel:** I was knowledgeable of it, and I thought about it. I had done some things with it, and there had been like a previous company where I had really played with OpenTelemetry and React. I had set up like some very interesting and overly clever TypeScript helpers that made it really easy for people. I had familiarity there, but this company is where I had to really dig into the Refinery, into configuration, into operationalizing it, and to controlling the cost and understanding it from like that to the perspective of the operator rather than the end user. +**Host:** Yeah, that is an interesting problem to have. -So that was an interesting change in perspective, and I ended up reading pretty much the entire RFC for everything, most of the SDK code, and a whole bunch of other things. I dug really, really deeply into it, and to this day, I will still not necessarily understand baggage. Like, I understand it, but it's kind of not well implemented. +So now, when you were helping this organization with OpenTelemetry, how well-versed were you in OpenTelemetry at the time? -**Host:** Yeah, you were mentioning earlier that it's funny because I was reading up on baggage today, and you were mentioning earlier that baggage is still kind of wonky to work with. Specifically, what were some of the challenges in working with baggage that you experienced? Because I think this would definitely be good feedback for the OpenTelemetry folks, the maintainers. +**Hazel:** So I was knowledgeable of it, and I had thought about it. I had done some things with that, and there had been like a previous company where I had really played with OpenTelemetry and React, and I had set up like some very interesting and overly clever TypeScript helpers that made it really easy for people. -**Hazel:** It comes down to baggage is a really generic tool that ends up being used for like five different things, and they mostly come up when you want to stitch together multiple services or when you want to write a library or do something like that or write like a platform for people so that they can instrument things very easily and not have to worry about certain underlying implementation details. +I had familiarity there, but this company is where I had to really dig into the Refinery, into configuration, into operationalizing it, and to controlling the cost and understanding it from like that to the perspective of the operator rather than the end user. That was an interesting change perspective, and I ended up reading pretty much the entire RFC for everything, most of the SDK code, most of like a whole bunch of other things. I dug really, really deeply into it. -You can also use baggage to paper over the API of a lot of things and do what you want to even if you shouldn't necessarily do that. For example, package or just context application in general is more or less the only way you can have a directed acyclical graph. Otherwise, you only really have like a linear constant, and you have like some ability to form links, but even that is pretty under economic. But you need to know where you're linking to and where or what you're linking from. +To this day, I don't necessarily understand baggage. Like I understand it, but it's kind of not well implemented. -Baggage will let you like put things in a header so that you have cost service cost registration. The actual distributive part, the markets, is also the thing that you need if you want to navigate information down to public information upper chain, and there's also what you need if you want to write some sort of demand processor as a way to work around issues. So if you want to have like something added to multiple spans down, then the only way to do that, really, as far as I can tell, is to write a span processor and use this package to pull that information out. +**Host:** Yeah, you were mentioning earlier that it's funny because I was reading up on baggage today. You were mentioning earlier that baggage is still kind of wonky to work with. What were some of the challenges in working with baggage that you experienced? Because I like this would definitely be good feedback for the OpenTelemetry folks, the maintainers. -You would add something to the package and then ask the span processor if it's really thrown the spans, and it'll reconstruct that whole tree and figure everything out and then add that in there. You've more or less rewritten the Refinery in your code base, or rewritten the OpenTelemetry collector in your groupies in order to use packets in order to add certain features that aren't in the SDK yet. +**Hazel:** It comes down to baggage is a really generic tool that ends up being used for like five different things. They mostly come up when you want to stitch together multiple services or when you want to write a library or do something like that or write like a platform for people so that they can instrument things very easily and not have to worry about certain underlying implementation details. + +You can also use baggage to paper over the API of a lot of things and do what you want to even if you shouldn't necessarily do that. For example, baggage or just context application in general is more or less the only way you can have a directed acyclic graph. Otherwise, you only really have like a linear constant. You have like some ability to form links, but even that is pretty under economic. But you need to know where you're linking to and where or what you're linking from. + +Baggage will let you like put things in a header so that you have cost service cost registration. The actual distributive part—the markets is also the thing that you need if you want to navigate information down to public information upper chain. There's also what you need if you want to write some sort of demand processor as a way to work around issues. + +So if you want to have something added to multiple spans down, then the only way to do that, really, as far as I can tell, is to write a span processor and use this package to pull that information out. So you would add something to the package and then ask the span processor to really throw the spans. It'll reconstruct that whole tree and figure everything out and then add that in there. See, more or less rewrite Refinery in your codebase. We'll rewrite the OpenTelemetry collector in your groupies in order to use packets in order to add certain features that aren't in the SDK yet. **Host:** Interesting. Did you find that with other aspects of OpenTelemetry as well that were kind of limiting for you when you were working with it, when you were digging deep? -[00:27:00] **Hazel:** I think the main thing that was limiting outside of that aspect was that it's really hard to make adding instrumentation that's manual ergonomic. It's just difficult, and a lot of that comes down to OpenTelemetry really wants you to like have written the code right there, and you need to kind of know how to do that, and it's very hard to wrap the libraries themselves without adding like that extra layer of interaction that makes things hard. +**Hazel:** I think the main thing that was unlimiting outside of that aspect was that it's really hard to make adding instrumentation that's manual ergonomic. It's just difficult, and a lot of that comes down to OpenTelemetry really wants you to like have written the code right there, and you need to kind of know how to do that. + +It's very hard to wrap the libraries themselves without adding like that extra layer of interaction that makes things hard. So unless your language has a very massive macro system and you can use that ergonomically, it's difficult to do that. -So unless your language has a very massive macro system and you can use that ergonomically, it's difficult to do that. Languages like TypeScript don't. In TypeScript, what you really want is something like a decorator; you can't get that. The other issue that I really ran into ergonomically with OpenTelemetry would have been based around async/await. The async/await one was probably one of the larger beams of my existence, and the reason for that is there's multiple different ways you can write, and especially as you're going to smoke, there is one way that OpenTelemetry works, and that would be if the parent function follows a child function. +Languages like TypeScript don't, and so in TypeScript, what you really want is something like a decorator. You can't get that. The other issue that I really ran into ergonomically with OpenTelemetry would have been based around asynchronous worlds. The asynchronous one was probably one of the larger beams of my existence. -You're going to sleep, pass the context into it, and then waits and outlives the child function and then end-to-end span. So you have like a circuit historic, but actually, this only inside the lifetime of the parent's span. Anything other than that use case gets really weird. +The reason for that is there's multiple different ways you can write, and especially as you're going to smoke. There is one way that OpenTelemetry works, one with—if the parent function follows a child function, you're going to sleep, passes the context into it, and then waits and outlives the child function and then end to tone span. + +That is only inside the lifetime of the parent's span. Anything other than that use case gets really weird. **Host:** What about span links? Aren't they supposed to kind of alleviate that though? -**Hazel:** Span links do work, but you have to know how to use them. The parent still has to pull the child, and then the child has to have the parent ID in order to link to that, or I think vice versa. I don't know if you can do that, but essentially I couldn't write a function generically so I was unaware that it was being called from the parent and then had to link up to the parent. +**Hazel:** Span links do work, but you have to know how to use them. The parent still has to pull the child, and then the child has to have the parent ID in order to link to that—or I think vice versa. I don't know if you can do that. Essentially, I couldn't write a function generically so I was unaware that it was being called from the parent and then had to link up to the parent. + +That was tricky because a lot of languages, you want to write like a library or something, and the library can be very agnostic. So I don't necessarily want to say this function is called in the pharmaceutical context all the time. I want to say this function is called basically in this context that's relevant to the application domain, and sometimes that's interesting context, in which case it's been called asynchronously. But sometimes it's more synchronously, and sometimes it's called in a different context altogether, and I can't express that with OpenTelemetry very easily, if really at all. + +**Host:** What languages were part of that landscape in that organization? + +**Hazel:** This organization had TypeScript and Haskell as its main languages, but I've also done OpenTelemetry with React. I don't know what TypeScript—done with Haskell, done it with a little bit of Go. -That was tricky because a lot of languages, you want to write like a library or something, and the library can be very agnostic. So I don't necessarily want to say this function is called pharmaceutical context all the time; I want to say this function is called basically in this context that's relevant to the application domain, and sometimes that's interesting context, and in which case it's been called asynchronously. But sometimes it's more synchronously, and sometimes it's called a different context altogether, and I can't express that with OpenTelemetry very easily, if really at all. +**Host:** And I thought which one was the least annoying to implement? -**Host:** Got it. And speaking of languages, what languages were part of that landscape in that organization? +**Hazel:** The least annoying to instrument would be probably Python because so many other people use Python, and Python is used in more of a service sound context. OpenTelemetry is often used there. That was actually more ergonomic. Python also has more decorators and meta-programs, how many things like that to make it easier to work with. -**Hazel:** This organization had TypeScript and Haskell as its main languages, but I've also done OpenTelemetry with React. I don't know what TypeScript done with, and I've done it with a little bit of your name. I thought which one was the least annoying to implement? The least annoying to instrument would be probably Python because so many other people use Python, and Python is used in more of a service-sound context, and so OpenTelemetry is often used there. +It's interesting because it's both a front end and a back-end, and so a lot of things that you can do with it may have to run both in a vendor context and a server context. OpenTelemetry works really well on the server side and very not great on the client side. -That was actually more ergonomic. Python also has like more decorators and metaprograms, how many things like that to make it easier to work with. It's interesting because it's both a front end and a back end, and so a lot of things that you can do with it may have to run both in a voucher context and server context. OpenTelemetry works really well on the service side and very not great on the client side. +Dealing with all of those issues can be really tricky, and it may survive in an API that works while I'm bullet. It's really, really hard to, like on the client side, you want to send as little data as possible. On the service side, you want to send as much data as possible to the collectors so the collector can filter it. You have completely different needs there. -Dealing with all of those issues can be really tricky, and it may survive an API that works while I'm bullet is really hard to like on the client side. You want to send as little data as possible; on the service side, you want to send as much data as possible to the collectors so the collector can filter it. And so you have a completely need to there. You also have the issue of like how do I actually send data to the cluster on the client side, which is kind of solved, but not really. You end up like writing an API endpoint that supports things to the collector so that you can transparently authenticate or not authenticate in a way that actually works, and you can start to try and filter out noise signal. +You also have the issue of like, how do I actually send data to the cluster on the client side, which is kind of solved but not really, and you end up like writing an API endpoint that supports things to the collector so that you can transparently authenticate or not authenticate in a way that actually works, and you can start to try and filter out noise signal. -**Host:** I guess someone calling the endpoint or are they just like—because of where did they answer something? +**Host:** Was there another language that you've worked with that was even more annoying? -**Hazel:** I was reacting. The most annoying one to instrument or was there another language that you've worked with that was even more annoying? Haskell was the most annoying one to instrument. +**Hazel:** Haskell was the most annoying one to instrument. **Host:** How big is like the OpenTelemetry support around Haskell? -**Hazel:** The OpenTelemetry support for Haskell is actually pretty decent, which is really funny because Haskell has a bigger usage community-wise than Master does for OpenTelemetry. Yeah, and so Haskell is an ergonomic language; it's very expressive, it's very interesting, but the one time of Haskell is absolutely absurd and makes OpenTelemetry extremely difficult. +**Hazel:** The OpenTelemetry support for Haskell is actually pretty decent, which is really funny because Haskell has a bigger usage community-wise than Master does for OpenTelemetry. Haskell is a funny lazy language, which means that if you write the code, you don't have any control over the lifetime of how the code is executed or actually evaluated. -So Haskell is a funny lazy language, which means that if you write the code, you don't have any control over the lifetime of when the code is executed or actually evaluated or how deeply it's evaluated, and OpenTelemetry really, really wants you to explicitly call things, have it start, then have that happen and turn like point. +OpenTelemetry really, really wants you to explicitly call things, have it start then, and have that happen and turn like point, and so in Haskell, laziness makes it interesting because you end up peppering the laziness with the wants of strict functions, which are the tracing stuff. -In Haskell, laziness makes it interesting because you end up peppering the laziness with the ones of strict functions, which are the tracing stuff, and that can mess with your memory usage, can mention your performance, your interest, and it can match a couple of other things. It's very, very difficult in that language to have something include transparent; you want the tracing to be transparent. +To that can mess with your memory usage, your performance, you're interested. It can mess with a couple of other things, and it's very, very difficult in the language to have something include transparent, where you want the tracing to be transparent. Otherwise, the tracing is really tricky, and it brings a bunch of other things, and so making it transparent actually required the use of unstable primitives exposed by the internal implementation details of the compiler and the runtime. -Otherwise, the tracing is really tricky and brings a bunch of other things, and so making it transparent actually requires the use of unstable primitives exposed by the internal implementation details of the compiler and the runtime. +**Host:** Interesting. I feel like I learned something new today. I want to go back to another point that you had made earlier on, and hopefully I understood it correctly. I think you made a point saying that generally organizations do tend to like create libraries around OpenTelemetry. Did I understand you correctly? -[00:34:30] **Host:** Interesting. I feel like I learned something new today. I want to go back to another point that you had made earlier on, and hopefully, I understood it correctly. I think you made a point saying that generally organizations do tend to like create libraries around OpenTelemetry. Did I understand you correctly? +**Hazel:** Yeah, and it's not necessarily the organizations tend to do that; it's that that is one of the best ways to multiply efforts when you have like a platform team when you're thinking of things in a platform engineering manner. -**Hazel:** Yeah, and it's not necessarily that organizations tend to do that; it's that that is one of the best ways to multiply efforts when you have like a platform team when you're thinking of things in a platform engineering manner. So in general, if you have like something people need to care about life testing or instrumentation or runtime performance or some aspect of the code that isn't necessarily functionality, getting everyone to care about that equally is really hard. +So in general, if you have like something people need to care about—like testing or instrumentation or runtime performance or some aspect of the code that isn't necessarily functionality, getting everyone to care about that equally is really hard. -It'll live on to have the same level of knowledge; it's really hard. In fact, it's kind of not even so much as impossible, but I think it's, I think, oh, people try too hard to hit, and consequently, it ends up being very, very useful from an organization perspective to accelerate your developers by having like libraries built around things or my platforms putting on things. +It'll live on to have the same level of knowledge. It's really hard. In fact, it's kind of not even so much as impossible, but I think people try too hard to hit, and consequently, it ends up being very, very useful from an organization perspective to accelerate your developers by having like libraries built around things or platforms putting on things. -So you have like maybe a standard template and then your instrumentations just don't work and your CI/CD is just dealt with, and a whole bunch of other things you're done with it, and you can use it, but these structures are already set up to give people capabilities for handling things, taking care of like distribution, so that cross-service instrumentation to support the box. All those things would be things that I would want to deal with from a platform team perspective of making telemetry easier to do. +So you have like maybe a standard template, and then your instrumentations just don't work, and your CI/CD is just dealt with, and a whole bunch of other things are done with it, and you can use it, but these structures are already set up to give people capabilities for handling things, taking care of like distribution, so that cross-service instrumentation can support the box. -**Host:** Yeah, that makes a lot easier at home really hard. So was there, at this previous organization, or even other orgs, even subsequent organizations, is that something that—it's one thing to, I think, have the aspiration to abstract that stuff from folks to, you know, make sure that they're at the same level when it comes to instrumenting. It's another to like be able to achieve that goal. Do you feel that in any of the organizations where you worked that that has been actually achieved with these kinds of libraries? +All those things would be things that I would want to deal with from a platform team perspective of making telemetry easier to do. -**Hazel:** I've been able to get things to a point where it was achievable; whether or not I actually achieved it is a different question. But so what I mean by that is I was able to figure out ways to like wrap and abstract away the vast majority of the setup for OpenTelemetry when I did TypeScript or Haskell, and I was able to write wrappers around functions in a way that the wrappers are transparent. +**Host:** Yeah, that makes a lot easier at home. Really hard. -So we could have like a bunch of environment variables properly in and a bunch of like sort of stand up for things dealt with in a way that people just needed to set up their application in a certain way, and then you got like the very base skeleton of the tracer and all those things were just ready to go. They can like get the, you know, without the span sort of function and have like a more limited understandable API where you didn't need to pass in certain types of contexts manually. +So was there, at this previous organization or even subsequent organizations, is that something that—it's one thing to, I think, have the aspiration to abstract that stuff from folks to, you know, make sure that they're at the same level when it comes to instrumenting. -I've been able to do that for both, and I've even been able to do that in a way that works isometrically in JavaScript or TypeScript. +It's another to, like, be able to achieve that goal. Do you feel that in any of the organizations where you worked that that has been actually achieved with these kinds of libraries? -**Host:** What does that mean? +**Hazel:** I've been able to get things to a point where it was achievable. Whether or not I actually achieved it is a different question. But what I mean by that is I was able to figure out ways to wrap and abstract away the vast majority of the setup for OpenTelemetry when I was in TypeScript or Haskell. -**Hazel:** It means you can have the same code running in server and the browser, and it works. That involved a mild amount of crimes; it involved a lot of crimes. It involves a lot of crimes. I abused how Node modules cached a bunch of things, and then I imported certain globals and then the runtime certain stable code here and there. If you import everything in the right order in the right place and then you rely on trade shaking, then your bundle on the server can be different than your bundle on the client, and you can end up splitting that out in a way that doesn't break the global singleton pattern of observability of the OpenTelemetry stuff and also not explained anywhere. +I was able to write wrappers around functions in a way that the wrappers are transparent so we could have like a bunch of environment variables properly in and a bunch of like sort of stand up for things dealt with in a way that people just needed to set up their application in a certain way, and then you got like the very base skeleton of the tracer and all those things. -[00:38:35] **Host:** Switching gears a bit to more managing the infrastructure side of OpenTelemetry, how was that experience for you in contrast to having to, I guess, previous roles of being more, I guess, holistically focused or having to like clean up the OpenTelemetry mess? How was that in contrast? +She's just ready to go, right? And they can like get the, you know, without the span sort of function and have like a more limited understandable API where you didn't need to pass in certain types of contents manually. I've been able to do that for both, and I've even been able to do that in a way that works isometrically in JavaScript or TypeScript. -**Hazel:** Running OpenTelemetry is interesting because like you have to open the OpenTelemetry collector to run, and the collector, like, then you have to set it up. If you go outside of essentially any of the very small documentation examples, you end up having to read the RFCs for things or you end up having to read the technical design document, and they are not obtuse, but they're understandable by like a select amount of people, and you have to really dig in to understand that. +**Host:** And what that means is you can have the same code running in server and the browser, and it works? -To like even for the tail sampling configuration of the OpenTelemetry collector, like an entire document of how to do it is the most difficult thing I've ever seen. It has like different layers of things; each one has its own entire language and specification setup, and it's wild. The Honeycomb Refinery is much better in that regard, but running Refinery in production is a janky mouse in a lot of ways, and that largely comes down to Refinery is really a tool that they built for themselves, and then they made it available to other people. +**Hazel:** Yes, and that involved a mild amount of crimes. It involved a lot of crimes. It involves a lot of crimes. I abused how node modules cached a bunch of things and then in chapter return globals and then in Iran certain stable code here and there. -So here's Refinery; if you know how it works, great, you just run it. But otherwise, you can run Refinery successfully if you're an organization with high-performance CI/CD, the ability to implement things and look at them, and you have that ability to like dial things down very rapidly. Otherwise, Refinery is not going to work super well for you, and to the organization that I ran Refinery in, split the difference; they had some amount of rapid CI/CD and some men of that, and I was able to look into logs and stuff like that and get some things down. +If you import everything in the right order in the right place and then you rely on trade shaking, then your bundle on the server can be different than your bundle on the client, and you can end up splitting that out in a way that doesn't break the global singleton pattern of observability of the OpenTelemetry stuff and also not explained anywhere. -But they used its pattern and Refinery due to how broken the OpenTelemetry stuff was. We ran into essentially every error case and edge case of it. For example, one of the edge cases that we had was many spans were not like, you know, a minute or two long. Finished spans were hours long or even days, and we had some spans that were multiple weeks in length, and we also had some spans that had over three to five million events in them. +**Host:** Switching gears a bit to more managing the infrastructure side of OpenTelemetry, how was that experience for you in contrast to having to, I guess, to previous rules of being more, I guess, holistically focused or having to lean up the cleanup the OpenTelemetry mess? How was that in contrast? -What ended up being was you would have something that would go, you know, I started to ask, and then that starts to span, and so this would be like an asynchronous job. Into the asynchronous job would, for every single user in the database, do a whole bunch of validation stuff, and so that would be about like 1,000 demands per user times 500,000 users or 20,000 spans per user times 40,000 users. +**Hazel:** Running OpenTelemetry is interesting because you have to open OpenTelemetry collector to run, and The Collector, like, then you have to set it up. If you go outside of essentially any of the very small documentation examples, you end up having to read the RFCs for things or you end up having to read the technical design document, and they are not obtuse, but they're understandable by like a select amount of people. -It's ridiculous! Or like for every single item in the database, check its consistency in one span, so that would take like 20 hours, and that would break every possible configuration of Refinery. You just can't have this span that's not broken and also have Refinery store 50 million things in memory. +You have to really dig in to understand that to like even for the tail sampling configuration of the OpenTelemetry collector, like an entire document of how to do it is the most difficult thing I've ever seen. It has like different layers of things; each one has its own entire language and specification setup, and it's wild. -I got things improved, and ultimately I ended up saying this entire way we do OpenTelemetry in like this with an asynchronous job is actually now close enough to streaming services that it doesn't work that way. The wheel dealer streaming services is to do it the way Honeycomb does, which is you just send a snapshot of this span every minute, so any rule of things in the code, and that requires you to have written your code in a way that you can roll things up and then send it every minute. +The Honeycomb Refinery is much better in that regard, but running Refinery in production is a janky mouse in a lot of ways, and that largely comes down to Refinery is really a tool that they built for themselves and then they made it available to other people. -You need to completely rewrite how you do most of your task handling to have some sort of task handling manager that sits there and can collect a bunch of things in every minute, do that, and start a new span. That is not really possible the way most people write asynchronous jobs, and fixing that is really tricky because there's not really a wheel if it's now; you're not actually just rewriting the code. +Here's Refinery. If you know how it works, great; you just run it. But otherwise, you can run Refinery successfully if you're an organization with high-performance CI/CD, the ability to implement things and look at them, and you have that ability to like dial things down very rapidly. -It requires you to really just start a million different tasks, and each task is linked together casually by like span information. Then the collector sort of like does some things in there, or maybe your span processor can roll that up, but otherwise, you can't actually do that. +Otherwise, Refinery is not going to work super well for you, and to the organization that I ran Refinery in, it split the difference. They had some amount of rapid CI/CD and some men of that, and I was able to look into logs and stuff like that and get some things down. But they used its pattern, and Refinery, due to how broken the OpenTelemetry stuff was, we ran into essentially every error case and edge case of it. -Now, you should do your tasks in that manner anyway because then it allows you to have a right-hand lock, and then you can ensure your task consistency is a longer think. It's not interested in the middle; you should do that anyway because that's how you do it if you understand distributed systems. But you run into this thing in OpenTelemetry; it was written by systems-minded people, and so a lot of design decisions that they make make sense if you know how to write a standard stability system. +For example, one of the edge cases that we had was many spans were not like, you know, a minute or two long—finished spans were hours long or even days. We had some spans that were multiple weeks in length, and we also had some spans that had over three to five million events in them. -Of course, you use a write-ahead log; of course, you want to just spawn an event from like respond one task somewhere for like 20 hours. Who does that? Everyone does that until they know better. +What ended up happening was you would have something that would go, you know, I started to ask, and then that starts to span. So this would be like an asynchronous job, and the asynchronous job would, for every single user in the database, do a whole bunch of validation stuff. -[00:45:00] **Host:** That's a really good point. We've got about six minutes left, but before we wrap up, I did want to talk briefly about some of your experience in trying to make the case for OpenTelemetry to executives because that can be challenging. +That would be about like 1000 demands per user times 500,000 users, or 20,000 spans per user times 40,000 users. It's ridiculous. For every single item in the database, check its consistency in one span. -**Hazel:** Yeah, so there's two sides of it: there's making the case for any sort of monitoring or observability in general, and I was making the case for more OpenTelemetry specifically. The second case usually happens when you have some pre-existing stuff, and you're arguing, “We should not use the existing stuff; we should use this other thing instead.” +That would take like, you know, 20 hours, and that would break every possible configuration of Refinery. You just can't have this fan that's not broken and also have Refinery store 50 million things in memory. -What you're arguing for there is a migration, two migrations: a technical migration and a social migration, and those are very tricky. You want those both migrations to happen, and so you need to be able to essentially show that the time and effort saved by via this migration will pay for itself in about four months. If you can't show that it will pay for itself in about four months, then you probably shouldn't actually do it, and you should figure out how to make it pay for itself in four months and then do it. +I got things improved, but ultimately I ended up saying this entire way we do OpenTelemetry in like this with an asynchronous job is actually not close enough to streaming services that it doesn't work that way. -Otherwise, you'll end up with a migration that takes like a year or something, and it has no perceivable difference in benefit, and then people are not going to be sold on it. People need to actually tangibly feel the benefit, or the migration is going to feel on the social aspect technically. +The wheel dealer streaming services is to do it the way Honeycomb does, which is you have like—you just send a snapshot of this fan every like minute. Show any rule of things in the code, and that requires you to have written your code in a way that you can roll things up and then send it every minute. -For conventional organizations that you want to get like any sort of monitoring in general, that relies on being able to convince people that the life cycle leads to encompass more at the life cycle, and the easiest way to do that is, in my opinion, you tie the code lifecycle to the business value delivery lifecycle. +So you need to completely rewrite how you do most of your task handling to have some sort of task handling manager that sits there and can collect a bunch of things in every minute, do that, and start a new span. That is not really possible the way most people write as you're going to stats, and fixing that is really tricky because there's not really a wheel if it's now. -So what can happen in organizations is you have like the developer lifecycle, which is over here, and it's, “I write code, I commit code, I merge code,” and you have the business lifecycle, which is we could do some market research, or we get like some seamless research, and then we design a product, and we get the final features, and then that final feature is implemented, and then we see what the feedback is. +We're not actually just rewriting the code; it requires you to really just start a million different tasks, and each task is linked together casually by like span information, and then the collector is sort of like does some things in there, or maybe your span processor can roll that up. -At no point do those two overlap or cross. So that leads to people to my product owners handing features over to developers and developers writing tune and just handing lots of production. Instead of playing frisbee, you need to merge those. When those get merged, then essentially the developers have to care about things all the way to the point where it delivers value to the customer, which is absolutely what every executive already wants to have happen. +But otherwise, you can't actually do that. Now, you should do your tasks in that manner anyway because then it allowed to have a right-hand lock, and then you can ensure your task consistency is a longer thing. It's not interested in the middle. You should do that anyway because that's how you do it if you understand distributing systems. -The state they have within separated is what they think is like has to be that way, and the second is it's possible and even desirable to get developers to care about like the outcomes from the customers, get them talking to salespeople, and get them talking to product people. +But you run into this thing in OpenTelemetry; it was written by systems-minded people, and so a lot of design decisions that they make make sense if you know how to write a standard stability system. -That will not only improve things for the product, improve the internet, but it'll actually improve the productivity of the developers and their experience and their ability to do this. That usually sounds executives, and you need this. +**Host:** Of course, you use a writer had a lack of question journal things. A question like, "Who does that?" Everyone does that until they know better. -The next checking after that is once they have, “You need this,” if they just look for like observability, they're going to get one of like three or four different vendors and probably going to end up an expensive vendor. You need to pick the vendor based on what the business needs and not necessarily what you like the best. Like for example, my current company, we have advanced monument compliance requirements. +**Hazel:** That's a really good point. -We have it for only one environment that provides more than one codebase, and so because of that, no one in the company can use anything that isn't FedRAMP compliant, which significantly limits the amount of advantage we can have to basically one, and I don't necessarily want to use that vendor necessarily, but I have no plans to migrate off of it. If I were to migrate off of it, it would be in a very interesting manner. +We've got about six minutes left, but before we wrap up, I would like to talk briefly about some of your experience in trying to make the case for OpenTelemetry to executives because that can be challenging. -What I would do is I would split the environment into two environments and have them be identical, one FedRAMP certified, one not FedRAMP certified, and then essentially would have a FedRAMP program, but that would be like a separate sales funnel, and anyone who actually needs it can have that because the FedRAMP is valuable in two aspects for the business. One is that it gets them certain customers, and one is that it gets some certain conversations, and for the conversations, those people may not be on federal, but they may want to eventually when it means that I can run the same codebase in both locations and use a different vendor and the one that isn't FedRAMP and that one but that whole migration for the current company would be allowed two years of work. +### [00:45:08] Advocating for observability to executives -Are we going to do that? Who knows? Maybe. But there's a lot of other questions to ask first and a lot more that we need to do before we can even get to that point. +**Hazel:** Yeah, to OpenTelemetry. So there's two sides of it: there's making the case for any sort of monitoring or observability in general, and I was making the case for more OpenTelemetry specifically. -[00:50:00] **Host:** Yeah, speaking as a vendor, we've had a lot of conversations about FedRAMP certification, and being that not 100% of our employees are in the U.S., we'd have to do exactly what you described. We tried to do that before for HIPAA, and it created kind of a weak experience for some customers, so yeah, it's a tricky problem. +The second case usually happens when you have some pre-existing stuff, and your article and we should not use the existing stuff when we should use this other thing instead. -**Hazel:** Interesting. I think one of the most interesting things about HIPAA is that any sort of regulatory environment—there's like one regulatory environment, and then there's like FedRAMP, and the overlap between people that need any regulatory environment and people that need or really want FedRAMP is basically a circle. +What you're arguing for there is a migration—two migrations: a technical migration and the social migration, and those are very tricky. You want those both migrations to happen, and you need to be able to essentially show that the time and effort saved by this migration will pay for itself in about four months. -So if you're going to go like, “Oh, we're going to have like a HIPAA environment, we're going to have a FIPS environment, we could have like, you know, SOC 2, like it used to be SOC 2,” whatever. But if you have one of those, you probably would actually get as much benefit, if not more, if you went straight to FedRAMP, even though it's like three times as expected to do that at least. +If you can't show that it will pay for itself in about four months, then you probably shouldn't actually do it, and you should figure out how to make it pay for itself in four months and then do it. Otherwise, you'll end up with a migration that takes like a year or something and it has no perceivable difference in benefit, and then people are not going to be sold on it. -And the ongoing maintenance burden of that is ridiculous, but it gets you on the other customers you need, like HIPAA plus FedRAMP or FIPS plus FedRAMP, and that is a really interesting business thing that's non-obvious. Any type of regulatory environment completely changes the game for OpenTelemetry, what you can do with it, how it works, your vendors in general, and even like your hiring practices, and it's really weird that you can't really have an asset, but you can't just get HIPAA; you really should probably go all the way to a FedRAMP just because of who the market is that needs one or both. +People need to actually tangibly feel the benefit, or the migration is going to feel on the social aspect technically. For a conventional organization, you want to get like any sort of monitoring in general. That relies on being able to convince people that the lifecycle leads to encompass more at the lifecycle. -**Host:** I totally agree with you based on like our experience with that HIPAA server because it turned out there were companies camped on it that just had a desire for like more privacy, stricter controls because of how their company culture and their data worked without actually having the need for the federal regulation. +The easiest way to do that is, in my opinion, you tie the code lifecycle to the business value delivery lifecycle. What can happen in organizations is you have like the developer lifecycle which is over here, and it's I write code, I commit code, I merge code. And you have the business lifecycle, which is we do some market research, or we get like some seamless research, and then we design a product, and we get the final features, and then that final feature is implemented, and then we see what the feedback is. -I think you're right that it makes it easier to sell. I think probably we wouldn't end up going in that direction again unless we managed to partner with somebody who was actively like bringing in FedRAMP customers, like some agency that worked with the government or whatever, like we would have to be. But I wish we had known that at the start, and we didn't. +At no point do those two overlap or cross, so that leads to people to my product owners handing features over to developers, and developers writing tune and just handing lots of production. Instead of playing frisbee, you need to merge those. When those get merged, then essentially the developers have to care about things all the way to the point where it delivers value to the customer, which is absolutely what every executive already wants to have happen. -One thing that would be interesting there would be to have OpenTelemetry have some sort of agency that—and that agency gets FedRAMP compliance. Then what you can do with that is you can have that agency essentially facilitate the onboarding of OpenTelemetry vendors or just other observability vendors into the space, and you can use that to sort of start bridging the gap and giving people more capabilities and sharing that my FedRAMP specific architecture concerns in the way that spreads their compliance load among multiple companies. +The state they have within separated is what they think is like has to be that way, and the second is it's possible and even desirable to get developers to care about the outcomes from the customers. -**Host:** That would be a really interesting idea. +Get them talking to salespeople and get them talking to product people, and that will not only improve things for the product, improve the internet, but it'll actually improve the productivity of the developers and their experience and their ability to do this. -**Hazel:** Yes, it could maybe make it work through the project, and I would say for us as a small vendor, that really is a problem that if we are not allowed to have our engineers who are in other countries touch the system, we may only have one engineer working in a particular area, and so we have to hire a second person in the U.S., which is undoable. +That usually sounds executives, and you need this. The next checking after that is once they have you need this, if they just look for like observability, they're going to get one of like three or four different vendors and probably going to end up an expensive vendor. -There are certain ways, and you would need somebody to talk to someone who's experienced in FedRAMP to negotiate that, which would have to be—100% U.S. citizenship is a non-seer own requirement. It gets nuanced, and it gets tricky. +You need to pick the vendor based on what the business needs and not necessarily what you like the best. Like for example, my current company, we have advanced monument compliance requirements. We have it for only one environment. I'm not that provides more than one codebase, and so because of that, no one in the company can use anything that isn't FedRamp compliant, which significantly limits the amount of advantage we can have to basically one. -[00:56:00] **Host:** Oh, interesting. Well, I think we have to leave it at that before we got into a funny side. Thank you again, Hazel, for sharing your insights. This has been really cool. I don't think we get to talk to too many people who have had like such advanced use cases for OpenTelemetry, and being able to share that with the community is really good because I think a lot of us get, you know, we do like those intro tutorials. We’re like, “Hey, that's pretty easy!” and then you get into like the guts of it, and then that's when you start getting into some of those gnarly use cases that can be really tricky in life. +I don't necessarily want to use that vendor necessarily, but I have no plans to migrate off of it. If I were to migrate off of it, it would be in a very interesting manner. What I would do is I would split the environment into two environments and have them be identical, one FedRamp certified, one not FedRamp certified. + +Essentially, I would have a FedRamp program, but that would be like a separate sales funnel, and anyone who actually needs it can have that because the FedRamp is valuable in two aspects for the business: one is that it gets them certain customers, and one is that it gets some certain conversations. For the conversations, those people may not be on federal, but they make one eventually when it means that I can run the same codebase in both locations and use a different vendor, and the one that isn't FedRamp and a FedRamp verified vendor. + +That migration for the current company would be allowed two years of work. + +**Host:** So are we gonna do that? Who knows? Maybe. But there's a lot of other questions to ask first and a lot more that we need to do before we can even get to that point. + +**Hazel:** Yeah, speaking as a vendor, we've had a lot of conversations about FedRamp certification, and being that not 100% of our employees are in the U.S., we'd have to do exactly what you described. We tried to do that before for HIPAA, and it created kind of a weak experience for some customers. + +So, yeah, it's a tricky problem. + +**Hazel:** Interesting. I think we have to leave it at that before we got into a funny side. Thank you again, Hazel, for sharing your insights. This has been really cool. I don’t think we get to talk to too many people who have had such advanced use cases for OpenTelemetry, and being able to share that with the community is really good because I think a lot of us get—we do like those intro tutorials. + +We're like, "Hey, that's pretty easy," and then you get into like the guts of it, and then that's when you start getting into some of those gnarly use cases. It can be really tricky in life. **Hazel:** Yeah, definitely. -**Host:** So definitely appreciate that feedback and the viewpoint, and you will be back for OpenTelemetry in practice, I believe, on September 14th. Looking forward to that! +**Host:** I definitely appreciate that feedback and the viewpoint, and you will be back for OpenTelemetry in practice, I believe, on September 14th. Looking forward to that! + +Do you know what topic you'll be discussing? -**Hazel:** No idea. I'm going to figure it out, but it's going to be a lot of fun! +**Hazel:** No idea. I'm gonna figure it out, but it's gonna be a lot of fun! **Host:** Awesome! Really looking forward to that. Thank you again, as always, super, super awesome insights. Yeah, and we will hopefully see everyone for OpenTelemetry in practice with Hazel on September 14th. Thank you! diff --git a/video-transcripts/transcripts/2023-09-16T04:37:50Z-otel-in-practice-presents-context-abstraction-threading-the-needle-with-hazel-weakly.md b/video-transcripts/transcripts/2023-09-16T04:37:50Z-otel-in-practice-presents-context-abstraction-threading-the-needle-with-hazel-weakly.md index 13186ed..75bad8e 100644 --- a/video-transcripts/transcripts/2023-09-16T04:37:50Z-otel-in-practice-presents-context-abstraction-threading-the-needle-with-hazel-weakly.md +++ b/video-transcripts/transcripts/2023-09-16T04:37:50Z-otel-in-practice-presents-context-abstraction-threading-the-needle-with-hazel-weakly.md @@ -10,272 +10,278 @@ URL: https://www.youtube.com/watch?v=1a7GyarlGAQ ## Summary -In this YouTube video, Hazel discusses the complexities of observability and distributed systems, focusing on concepts like context, abstraction, and the challenges of building observability frameworks. The presentation begins with a primer on observability, exploring its definitions and the importance of asking meaningful questions to derive useful answers about system performance and behavior. Hazel highlights the role of context in linking disparate components of a system and the difficulties faced when attempting to instrument code for observability without intrusive methods. The talk also touches on issues related to graph structures in tracing, life cycle problems, and the limitations of current frameworks like OpenTelemetry. The session concludes with a Q&A, where Hazel addresses audience inquiries related to post-processing of trace data and the evolution of observability practices. The video emphasizes collaboration among engineers to enhance system understanding and improve observability. +In this YouTube video, Hazel discusses the intricacies of observability, focusing on concepts such as context abstraction and distributed tracing within software systems. Hazel, who is involved in innovation and developer experience, provides definitions and insights into observability, emphasizing the importance of asking meaningful questions to gain useful insights about systems. The presentation delves into the historical context of observability, its components, and the challenges faced when implementing observability practices, particularly around context management in OpenTelemetry. Key points include the differences between definitions of observability, the significance of context in tracing, and the problems that arise when trying to maintain observability in complex, distributed systems. The session also features a Q&A segment where attendees engage with Hazel, reinforcing the clarity of her explanations and the practical challenges developers face in this domain. ## Chapters -00:00:00 Welcome and intro -00:01:40 Definitions of observability -00:04:20 Personal definition of observability -00:06:50 Context in observability -00:09:28 Types of context in OpenTelemetry -00:12:30 In-process context explanation -00:15:00 Cross-cutting concerns in instrumentation -00:19:00 Life cycle problem in tracing -00:24:01 Graph rewriting problem -00:35:56 Conclusion and Q&A +00:00:00 Introductions +00:01:47 What is observability? +00:03:22 Definitions of observability +00:05:30 Discussion on context abstraction +00:09:50 Importance of context in tracing +00:15:50 Challenges in context management +00:20:00 Discussion on distributed systems +00:25:30 Issues with observability practices +00:40:00 Q&A session begins +00:45:00 Closing remarks and future sessions -**Speaker:** Foreign +## Transcript -I am having a lot of fun hoping to run Innovation and developer experience in my current company. I also serve on the board of directors of the hospital foundation and I have a lot of opinions online. You can find me pretty much everywhere, and I am really excited to be here today talking about contact abstraction and typing the needle. +### [00:00:00] Introductions -I'm gonna go over kind of like a primer of observability and talk to you a little bit about how I think about it and sort of what that context is, and the historical why do we need distribution and how does that tie into observability. Then we're going to talk about some components of the social filtration, mainly context, and then we're going to talk about abstraction and one of the difficulties that you can run into when you're trying to actually in real life build down libraries or build on platform team or build out something other than just manually instrumenting all of your code and writing everything from scratch. +**Speaker:** foreign -Don't you started with observability? You have essentially your system, you have everything, and you want to figure out what's going on. Over time, people have run into this problem, and different groups of people have come up with different definitions for observability. +I am having a lot of fun hoping to run Innovation and developer experience in my current company. I also serve on the board of directors of the hospital foundation, and I have a lot of opinions online. You can find me pretty much everywhere, and I am really excited to be here today talking about contact abstraction and typing the needle. -[00:01:40] One definition that you've heard journey majors and a lot of other people open telemetry and tracing backgrounds and distribution system background to talk about is the one from consumer theory, and that is the ability to measure the internal state of a system based on its external objects. This is a definition that you see repeated in a lot of places. It's a great definition, I like it a lot, but it isn't necessarily the entire story. +So, I'm gonna go over kind of like a primer of observability and talk to you a little bit about how I think about it and sort of what that contest is and the historical why do we need distribution and how does that tie into observability. Then we're going to talk about some components of the social filtration, mainly context, and then we're going to talk about abstraction and one of the difficulties that you can run into when you're trying to actually in real life build down libraries or build on a platform team or build out something other than just manually instrumenting all of your code and writing everything from scratch. -One other definition comes from a different theory of practice which is cognitive systems engineering, and so for them their definition is feedback that provides insight into process and the work required to start to mean from available data. We like this definition for a lot of reasons, mainly that it does a really good job at showing that observability is really this back and forth dialogue. You need to not just have, you know, oh, you can observe everything. Well, that's cool once everything is built out, but like how do you get there? What does it mean? And the whole process of people are involved too is something that this definition kind of highlights, and I love any technical definition that includes people. It's my favorite. +Don't you started with observability? You have essentially your system; you have everything, and you want to figure out what's going on. Over time, people have run into this problem, and different groups of people have come up with different definitions for observability. -Speaking of which, my personal definition is I'm not biased, but I like this one the most. I took the other two definitions from cognitive systems and from contour theory. They're kind of blending them together. For me, my definition of observability is the process through which one develops the ability to ask meaningful questions and get useful answers. +### [00:01:47] What is observability? -The way I think about that is it's a process, which means it has to be evolved and you start from somewhere, but you're not going to stay there. You're going to continually figure out what observability means to you, and it may mean something different today than it does tomorrow, than it will not show you than it did last week. Additionally, meaningful questions is really the thing that observability to me hinges around. +One definition that you've heard Journey majors and a lot of other people, open Telemetry and tracing backgrounds, and distribution system backgrounds talk about is the one from consumer theory, and that is the ability to measure the internal state of a system based on its external objects. This is a definition that you see repeated in a lot of places; it's a great definition. I like it a lot, but it isn't necessarily the entire story. -Can you ask a meaningful question of your system, which may or may not involve computers, may or may not around people? It may involve like a whole bunch of different things, and are those questions meaningful to business, meaningful to you, meaningful to what you need out of that learning? And in asking those questions, can you get a useful answer? +One other definition comes from a different theory of practice, which is cognitive systems engineering, and so for them, their definition is feedback that provides insight into process and the work required to start to make meaning from available data. We like this definition for a lot of reasons, mainly that it does a really good job at showing that observability is really this back and forth dialogue. You need to not just have, you know, "Oh, you can observe everything." Well, that's cool once everything's built out, but how do you get there? What does it mean? The whole process of people being involved too is something that this definition kind of highlights, and I love any technical definition that includes people; it's my favorite. -[00:04:20] I like this definition for me because it includes every aspect of observability in my opinion. If a CEO wants to say, "Hey, I have this meaningful question, which is I want to know where we are in the market compared to where we want to be in the market and how feasible am I going to get there?" Can they answer that to build like what we need in order to approach our problems for the next five years? I only said in Los Angeles, so I'm keeping them happy and healthy. That's a meaningful question. An observable system will let you answer that. +### [00:03:22] Definitions of observability -For engineers, a lot of that observability questions, it's going to come down to things about the system, things about code, and we're going to dig a lot more into that today than the other questions. We should all be working together, and we should all be able to collaborate in order to actually solve all these problems into these answers. +Speaking of which, my personal definition is I'm not biased, but I like this one the most. This one is I took the other two definitions from cognitive systems and from contour theory, blending them together. For me, my definition of observability is the process through which one develops the ability to ask meaningful questions and get useful answers. -For me, a meaningful question is one that maximizes how much you learn about the system that you're participating in. If you have that system and you want to know more about it, what does that mean? How do you get there? What are you doing right? +The way I think about that is it's a process, which means it has to evolve, and you start from somewhere, but you're not going to stay there. You're going to continually figure out what observability means to you, and it may mean something different today than it does tomorrow, and then it will not show you than it did last week. Additionally, meaningful questions are really the thing that observability, to me, hinges around. Can you ask a meaningful question of your system? This may or may not involve computers; it may or may not revolve around people. It may involve a whole bunch of different things, and are those questions meaningful to business, meaningful to you, meaningful to what you need out of that learning? + +In asking those questions, can you get a useful answer? I like this definition for me because it includes every aspect of observability, in my opinion. If a CEO wants to say, "Hey, I have this meaningful question, which is I want to know where we are in the market compared to where we want to be in the market and how feasible am I going to get there," can they answer that to build what we need in order to approach our problems for the next five years? I only said in Los Angeles, so I'm keeping them happy and healthy. That's a meaningful question; an observable system will let you answer that. + +For engineers, a lot of those observability questions are going to come down to things about the system, things about code, and we're going to dig a lot more into that today than the other questions. We should all be working together, and we should all be able to collaborate in order to actually solve all these problems and answer these questions. -For programming, we have all my camera disappeared. Brilliant. I love silent problems. Alright, so I'm going to figure out what happened there super briefly and then fix that, and if I can't fix it, then I'll just tell everyone what's on the slide. Oh well. Alright, let me share my screen again and I'll just tell you what was on the slide. Sorry about that. I debugged everything and we're learning information. Awesome. +### [00:05:30] Discussion on context abstraction + +For me, a meaningful question is one that maximizes how much you learn about the system that you're participating in. If you have that system and you want to know more about it, what does that mean? How do you get there? What are you doing right? -[00:06:50] So imagine on the slide that you have a typical system in which you have someone and they're running a web server, and they are asking themselves, "What is this web server doing?" and should they look at the ones and the lots? And once you have no data whatsoever, absolutely nothing, nothing's there, and they're like, "Okay, now what? What do I do?" +For programming, we have... oh, my camera disappeared. Brilliant! I love silent problems. Alright, so I'm going to figure out what happened there super briefly and then fix that, and then get back. If I can't fix it, then I'll just tell everyone what's on the slide. -Back in the good old days, well, what you probably would have done was you would have just killed production, taken the whole thing down, launched the debugger, and just stepped through the entire system step by step, solving everything. You can ask a lot of meaningful questions that way, and it can get a lot of useful answers. Everything's solved, and all you had to do is completely change production in order to do it, which is, you know, maybe not the best approach, but it is, you know, it's in vogue; that's what we did for a long time, and in many ways it still works. +Oh well, alright, let me share my screen again, and I'll just tell you what was on the slide. Sorry about that; I debugged everything, and we're learning information. Awesome! -But one of you have to constraint of, "I want to observe without interruption." I want to, you know, keep everything running and ask me to focus into the system. Now I'm considering the system running as part of that question. +So imagine on the slide that you have a typical system in which you have someone and they're running a web server, and they are asking themselves, "What is this web server doing?" and should they look at the logs? Once you have no data whatsoever, absolutely nothing's there, and they're like, "Okay, now what? What do I do?" -Let me try, you know, adding some French statements, which again, we're seeing a little bit of lack of observability here in the slides. That's fine. So if we have a French statement, and we've added them everywhere, now we have a real login. This is awesome. So you can imagine us taking the web server and getting those alarms and looking through, and you can see, "Oh hey, this one usually locked in this one." +Back in the good old days, well, what you probably would have done was you would have just killed production, taken the whole thing down, launched the debugger, and just stepped through the entire system step by step, solving everything. You can ask a lot of meaningful questions that way, and it can get a lot of useful answers. Everything's solved, and all you had to do is completely change production in order to do it, which is, you know, maybe not the best approach, but it is, you know, it's in vogue. That's what we did for a long time, and in many ways, it still works. -I use a middle transaction; they're able to do something, and then they locked out. That's awesome. We have everything there. We have it done, and we're ready to go, right? Is everything there? Well, you have the web server and you have the back end, and unless you have very carefully, you know, done everything with your logging, you'll notice that the front end and the back end, nothing's tied together. +But one of you have to constraint of, "I want to observe without interruption. I want to, you know, keep everything running and ask meaningful questions about the system," and now I'm considering the system running as part of that question. -The pen might say, "Oh, you should check down," and the back end might say, "This database middle transaction, this little visual transaction, this individual transaction," and you have nothing to tie those two disparate pieces of information together. You're like, "Okay, I still can't really understand anything about the system. I know what each individual piece is doing, but I have nothing to tie it all together." +So, let me try adding some French statements, which again we're seeing a little bit of lack of observability here in the slides. That's fine. So if we have a French statement and we've added them everywhere, and we now have a real login, this is awesome! You can imagine us taking the web server and getting those alarms and looking through, and you can see, "Oh hey, this one usually logged in this one." -[00:09:28] Context really ties the system together. Context, in OpenTelemetry, is for cross-cutting concerns. The documentation for OpenTelemetry says that context is a propagation mechanism which carries execution script values across API boundaries and between logically associated execution units. That is awesome. +I use a middle transaction; they were able to do something, and then they logged out. That's awesome; we have everything there, we have it done, and we're ready to go, right? Is everything there? Well, you have the web server and you have the back end, and unless you have very carefully, you know, done everything with your logging, you'll notice that the front end and the back end are tied together. -Let's break that down just a tiny bit. So context, let's skip on the middle bit, carries values. Context is the blue, the terms of metadata into unified trees. What does that mean for you? What does that mean when you're trying to understand it and instrument it? +The front end might say, "Oh, you should check down," and the back end might say, "This database middle transaction, this little visual transaction, this individual transaction." You have nothing to tie those two disparate pieces of information together to like, "Okay, I still can't really understand anything about the system. I know what each individual piece is doing, but I have nothing to tie it all together." -So let's break down and decent context. They're going to have for each span, they're going to have a whole bunch of context in there. You'll have like this span ID, which identifies that, the trace ID, some trace files, and a trace state. They're going to have essentially everything that identifies to spam, of which there are many spans in one trace and many traces in one system. +### [00:09:50] Importance of context in tracing -Scan links or another type of context, and they say, "Hey, this advantage is related to market." So you can think of traces as a graph, spans are nodes, and then links to the edges. If you have a graph that's a tree and you're like sort of building it down, and all of a sudden you need to take this one span over here and this one span over here and say, "But they're kind of linked together," you build that link, and that's an edge in that graph. +Context really ties the system together. Context in open Telemetry is for cross-cutting concerns. The documentation for open Telemetry says that context is a propagation mechanism which carries execution script values across API boundaries and between logically associated execution units. That is awesome! Let's break that down just a tiny bit. -Links can be really confusing. They can be kind of weird. They're not always integrated very well in the whole setup, but they are needed for especially in some types and patterns, and they are what turn OpenTelemetry traces from a tree into a graph. They are the most expensive thing you have in there and also the hardest thing to work with at the same time. +So context... let's skip on the middle bit. It carries values; context is the blue terms of metadata into unified trees. What does that mean for you? What does that mean when you're trying to understand it and instrument it? -Currently, you can only add them at span creation time, but people want to be able to add them anytime, and that may happen in the future. You'll be able to have a non-directed graph and a little drag, which will be fascinating. +So let's break down the decent context. They're going to have for each span, they're going to have a whole bunch of context in there. You'll have this span ID, which identifies that, the trace ID, some trace flags, and a trace state. They're going to have essentially everything that identifies to span, of which there are many spans in one trace and many traces in one system. -Another type of context is baggage. Packets want to keep valued parents downstream. It is widely regarded as a questionable thing to use. I know many people who might ban it in their code base. It's a bit of a on, but also very, very powerful and flexible. It is one of the ways to send information downstream, but you still can't use it to send information upstream, and I'll get more into what I mean by that later. +Span links are another type of context, and they say, "Hey, this advantage is related to market." You can think of traces as a graph: spans are nodes, and links are the edges. If you have a graph that's a tree and you're like sort of building it down, and all of a sudden you need to take this one span over here and this one span over here and say, "But they're kind of linked together," you build that link, and that's an edge in that graph. -[00:12:30] The package is there; you may eventually need it. I kind of hope you don't, but it's there. And when you have all these different types of context and you have these different services and you have the different processes, how do you take this context and actually use it to glue together everything and make it usable? +Links can be really confusing; they can be kind of weird. They're not always integrated very well in the whole setup, but they are needed for especially in some types and patterns, and they are what turn open Telemetry traces from a tree into a graph. They are the most expensive thing you have in there and also the hardest thing to work with at the same time. -You have the in-process context, and then you have the cross-process, and how to complicate things from one thing to another. Then you have fun process, which is in everything, and actually taking it in the next service and going down that way. So I'm going to go over all of them in order. +Currently, you can only add them at span creation time, but people want to be able to add them anytime, and that may happen in the future. You'll be able to have a non-directed graph, and a little drag, which will be fascinating. -So there's some code here. Sorry about that. I've never even probably known as context in process. It's the normal context; it's very typical, very standard, and this is what you've seen in the documentation. This is the in-process context. You create an octaves fan, and then in that disadvantage, and work, you create another active span, and then that context and links the two actually happens pretty much automatically, and you never really need to think about it. +Another type of context is baggage packets. It's widely regarded as a questionable thing to use. I know many people who might ban it in their codebase; it's a bit of an on but also very, very powerful and flexible. It is one of the ways to send information downstream, but you still can't use it to send information upstream, and I'll get more into what I mean by that later. -The library and the SDK will handle all of that for you. Sometimes, however, you do need to add it explicitly. This is an invisible example showing you that you need to do it, especially when it comes to making links and doing things that way. +The baggage is there; you may eventually need it. I kind of hope you don't, but it's there. When you have all these different types of context and you have these different services and you have the different processes, how do you take this context and actually use it to glue together everything and make it usable? -I'm going to one second. Oh, amazing. I'm going to watch my screen shoot, stop sharing, and then share something different. It turns out I accidentally made my syntax highlight and only work in Firefox. That's what that was. Cool, yay. +You have the in-process context, and then you have the cross-process, and that complicates things from one thing to another. Then you have the fun process, which is everything in and actually taking it in the next service and going down that way. -There you go. You can see here that we have the current span, we have these span contents, and we get that context and pull it out manually. So then we can create a span with a link. We need to do that in order to propagate things manually for this type of context information. But typically, you don't really need to do that, and things through it just naturally work with the links you need to do normally. +So I'm going to go over all of them in order. There's some code here; sorry about that. I've never even probably known as context in process. It's the normal context; it's very typical, very standard, and this is what you've seen in the documentation. This is the in-process context. You create an active span, and then in that disadvantage, and work, you create another active span, and that context links the two. -[00:15:00] That can be a little tricky. Cross-process, there's two steps involved. You serialize the context. Foreign, it can actually secretly, don't tell anyone I told you this, but there's no reason context needs to be an adder. You can do anything with any transform mechanism that you want to. It's just really serializing the context needs to happen somewhere, and then that needs to be attached to the payload of your call. +Actually, it happens pretty much automatically, and you never really need to think about it. The library and the SDK will handle all of that for you. Sometimes, however, you do need to add it explicitly, and this is an invisible example showing you that you need to do it, especially when it comes to making links and doing things that way. -I know some people who have actually serialized the context and embedded it into an RPC account as a way of actually instrumenting and tracing embedded systems. Some people have done that with their own bootloaders, which is actually really cool. It doesn't need to be a matter; a little wild if you want to use the library should not write everything yourself down. All the OpenTelemetry stuff will stick it in each TV header for you. +I'm going to... one second. Oh, amazing! I'm going to watch my screen shoot. Stop sharing and then share something different. It turns out I accidentally made my syntax highlight and only work in Firefox; that's what that was. Cool! Yay! -So what that looks like is you have from the sending service, the sending service is going to inject what's the publication object, the current context into some output, and so that will do the serialization process. Then you'll have a transparent and a trace state, and then output, and you can use that over the internet. The transparent is actually the only thing that you need. The twisted has a bunch of other stuff, but you don't actually kneel in and you may not want it. +There you go; you can see here that we have the current span, we have these span contents, and we get that context and pull it out manually. So then we can create a span with a link. We need to do that in order to propagate things manually for this type of context information, but typically you don't really need to do that, and things just naturally work with the links you need to do normally. -So the transparent is the trace ID and everything required to actually link the span from the net system over the trace state has on the other fun goodies that you may want but may not want. Some people I know have been using twisted and can't touch specification because of issues it has. +That can be a little tricky. Cross-process: there are two steps involved. You serialize the context. Foreign it can actually... secretly don't tell anyone I told you this, but there's no reason context needs to be an adder. You can do anything with any transformation mechanism that you want to; it's just really serializing the context that needs to happen somewhere. -Then I'll get to later the context one process. This is the second process. We've yielded everything up on the internet, and now I'm ready to receive that and actually integrate it. You'll have, you get the fabricator, you ingest everything, and you deserialize the context and you extract it, and then you did that. It's not a context, and you make it your current context. You don't need to make it the current context, but typically that's what people do. +Then that needs to be attached to the payload of your call. I know some people who have actually serialized the context and embedded it into an RPC account as a way of actually instrumenting and tracing embedded systems. Some people have done that with their own bootloaders, which is actually really cool. -Sometimes, people use the trace state in order to determine whether or not they should make the contents the current context, and you can maybe want to do that sometimes if you don't always own your systems end to end. Someone may be putting something in the middle somewhere, and that can be really tricky. +### [00:15:50] Challenges in context management -I know that some OpenTelemetry vendors, for example, have run into this problem of you need to really make sure that you're contacted your contacts before actually making it your current context. So that's a lot of fun. Once you have that context, then you've started it, you set your span and just done working from there. +It doesn't need to be a matter, but a little wild if you want to use the library. You should not write everything yourself; all the open telemetry stuff will stick it in each TV header for you. -So that is moving the context, wrapping everything together, and you have all of that, and that is great, but that is all really about manual instrumentation. That assumes that at every single point in every service you are writing all of your code, you're writing all of your instrumentation, you're writing everything from scratch, and you're writing it right there in line with all the code in an intrusive manner. +So what that looks like is you have from the sending service, the sending service is going to inject what's the publication object, the current context, into some output. That will do the serialization process, and then you'll have a transparent trace state and then output. You can use that over the internet. The transparent is actually the only thing that you need. The twisted has a bunch of other stuff, but you don't actually need it, and you may not want it. -If you don't want that intrusive style and you want to be able to build something for other people to use, this is where you run into a lot of problems, and we'll get into this now. This is, in my opinion, one of the weakest aspects of fishing in general, and it's a sign of the image journey in the ecosystem. I'm gonna have a lot of hope that can be improved, and I'm looking forward to seeing what happens there. +The transparent is the trace ID and everything required to actually link the span from the net system over. The trace state has a bunch of other fun goodies that you may want but may not want. Some people I know have been using twisted and can't touch specification because of issues it has; then I'll get to later. -[00:19:00] Do you remember earlier when I said that context is about to cause any concerns when they have a cross-Canadian concern and you tie everything together? What you also have is cross-cutting breakage because one of my favorite, um, laws out there is Hiram's law, which tells them with a sufficient number of users of an API, it does not matter what you promise in the contact. All observable behaviors of your system will be dependent on by someone, which means that the more instrumentation that you add into your system, the more things that you're adding that can be observed by yourself but also the rest of the system. +The context one process: this is the second process. We've yielded everything up on the internet, and now I'm ready to receive that and actually integrate it. You'll have to get the fabricator; you ingest everything, and you deserialize the context and extract it. Then you did that; it's not a context, and you make it your current context. -So paradoxically, when you come on service boundaries, in theory, you can only depend on the public API and service. It's a very standard service-oriented architecture and things like that. The tracing actually doesn't do that; it's out of hand intentionally invisible, not apparently API. +You don't need to make it the current context, but typically that's what people do. Sometimes people use the trace state in order to determine whether or not they should make the context the current context, and you can maybe want to do that sometimes if you don't always own your systems end to end, and someone may be putting something in the middle somewhere. -If you've been with anything anywhere, it'll propagate to the rest of the system, and you'll end up having a lot of essentially convention and hand-holding and little manual pieces put together that will break every time you change something. Those out-of-band information become in-band semantics, and this problem is something that you see in a lot of different ways. +That can be really tricky. I know that some open Telemetry vendors, for example, have run into this problem of needing to really make sure that your context is your context before actually making it your current context. So that's a lot of fun. Once you have that context, then you've started it, you set your span, and you're done working from there. -More OpenTelemetry, it's actually specifically about this context modification, but in terms of verification, you see this in terms of the internal implementation details of a function changing the proof required to show the scratch verified for property. And in distributed systems, you can see this in the end-to-end property of networks and in lots of other places. +That is moving the context, wrapping everything together, and you have all of that, and that is great, but that is all really about manual instrumentation. That assumes that at every single point in every service, you are writing all of your code, you're writing all of your instrumentation, you're writing everything from scratch, and you're writing it right there in line with all the code in an intrusive manner. -You can see it probably one of the easiest ones is different agility of snapshot testing for fun and design systems. If you've ever worked on those, you know that you can meet the tiniest little change. Everything's the worst, and somehow you just broke your entire test suite because some lines moved here and there in the code itself or something changed here and there in the code itself or in like the HTML, but the actual appearance and everything's fine, and you end up dealing with this frustration and description of the out-of-band information becoming in-band semantics, and that mismatch causing wreckage. +If you don't want that intrusive style and you want to be able to build something for other people to use, this is where you'd like to run into a lot of problems, and we'll get into this now. This is, in my opinion, one of the weakest aspects of fishing in general, and it's a sign of the image journey in the ecosystem. I'm going to have a lot of hope that can be improved, and I'm looking forward to seeing what happens there. -In OpenTelemetry, the package is not simple enough to cause a lot of honey. It's abused. One context is key, and when you have this cross-process propagation, you lose the ability to have your out-of-band and inbound semantics match up and be enforced across boundaries. +Do you remember earlier when I said that context is about to cause any concerns when they have a cross-cutting concern, and you tie everything together? What you also have is cross-cutting breakage because one of my favorite launch out there is Hiram's law, which tells them with a sufficient number of users of an API, it does not matter what you promise in the context. All observable behaviors of your system will be dependent on by someone, which means that the more instrumentation that you add into your system, the more things that you're adding that can be observed by yourself but also the rest of the system. -Additionally, in Winter designing options, or you're wrapping them, that same Alabama information will be very painful because you can't necessarily build anything that actually lets you do this in a reusable way unless you build a shared library of semantic conventions. +### [00:20:00] Discussion on distributed systems -Speaking of which, good options are both transparent and opaque. What I mean by that is they're transparent in that you don't have to know that they're there, any condition about them at that level of abstraction. A good way to think about that is, for example, HTTP requests; they just happen and they're just there, and you don't really have to know that they're implemented on top of a bunch of wire or glue, terrible crimes, and weird packet header loss information, blah blah blah. +Paradoxically, when you come on service boundaries, in theory, you can only depend on the public API and service; it's a very standard service-oriented architecture and things like that. The tracing actually doesn't do that; it's out of hand intentionally invisible, not apparently API, and if you've been with anything anywhere, it'll propagate to the rest of the system. You'll end up having a lot of essentially convention and hand-holding and little manual pieces put together that will break every time you change something. -There's a lot of trauma behind that, but you don't have to know that. With that said, you can't know that, and the transparency there is that you can reason about everything at that level as if the abstraction was in there, and you can write things that are aware of the fact that they actually work through it in order to know and understand the underlying system. +Those out-of-band information become in-band semantics, and this problem is something that you see in a lot of different ways. More open Telemetry is actually specifically about this context modification, but in form of verification, you see this in terms of the internal implementation details of a function changing the proof required to show the scratch verified for property. -Having that absorption means that you don't need to know the system underneath it, but that the system underneath it doesn't make it less understandable of inflammatory, and out-of-band information in general meets both of those goals. +In distributed systems, you can see this in the end-to-end property of networks, and in lots of other places, you can see it. Probably one of the easiest ones is different agility of snapshot testing for fun and design systems. If you've ever worked on those, you know that you can meet the tiniest little change; everything's the worst, and somehow you just broke your entire test suite because some lines moved here and there in the code itself or something changed here and there in the code itself or in the HTML, but the actual appearance and everything's fine, and you end up dealing with this frustration and description of the out-of-band information becoming in-band semantics and that mismatch causing wreckage. -Hard to achieve, the Louisiana ban information is not actually semantically embedded in the system and becomes relevant to the system, and so you can't build an abstraction that lets you operate without knowing the internal implementation details, and you don't actually have a way to observe the internal implementation details and know what they are without disapproved forms memorizing those conventions. +In open telemetry, a package is not simple enough to cost a lot of honey; it's abused. One context is key, and when you have this cross-process propagation, you lose the ability to have your out-of-band and inbound semantics match up and be enforced across boundaries. Additionally, in Winter designing options or you're wrapping them, that same Alabama information will be very painful because you can't necessarily build anything that actually lets you do this in a reusable way unless you build a shared library of semantic conventions. -That gives you this quandary: what do you do with that? This can manifest in a couple interesting ways. But a different problem of abstraction is I'll show you this one, and this is a problem that I run into the most, which is what I call the life cycle problem. +Speaking of which, a good option is both transparent and opaque. What I mean by that is they're transparent in that you don't have to know that they're there; any condition about them at that level of abstraction. A good way to think about that is, for example, HTTP requests; they just happen, and they're just there. You don't really have to know that they're implemented on top of a bunch of wire, or glue, terrible crimes, and weird packet header loss information, blah, blah, blah. -[00:24:01] That is you have tracing which is a graph, but you don't actually have a way to really manipulate the graph very well, and that shows up in two ways. The livestock of how much the first flame, which is that sending information up a graph or back in time, so to speak, isn't super possible or convenient or really ergonomic in any way. +There's a lot of trauma behind that, but you don't have to know that. With that said, you can't know that, and the transparency there is that you can reason about everything at that level as if the abstraction was in there. You can write things that are aware of the fact that they actually work through it in order to know and understand the underlying system. -Then adding, and then adding things after the fact is not possible either. So you can't like pack to things as a graph and like incrementally build up or out of beyond build up a view of the graph that's more complete. +Having that absorption means that you don't need to know the system underneath it, but the system underneath it doesn't make it less understandable of inflammatory, and out-of-band information in general meets both of those goals. It's hard to achieve; the Louisiana ban information is not actually semantically embedded in the system, and it becomes relevant to the system. -So here's an example. We have a web server, a service, and a client. The web server has, you know, one event, a one span, and then a second span. Two fish has one span, and then a plan has one span, and they have overlapping timelines. There are problems that I ran into trying to deal with building this. +You can't build an abstraction that lets you operate without knowing the internal implementation details, and you don't actually have a way to observe the internal implementation details and know what they are without disapproved forms memorizing those conventions. That gives you this quandary: what do you do with that? -So the first span in the web server can pass information to the second span when creating it. That's totally fine; that works. That's amazing. That's awesome. However, it has no real way to pass information to the second span after creating it. +This can manifest in a couple of interesting ways, but a different problem of abstraction is I'll show you this one, and this is a problem that I run into the most, which is what I call the life cycle problem. That is, you have tracing, which is a graph, but you don't actually have a way to really manipulate the graph very well, and that shows up in two ways: the livestock of how much the first flame, which is that sending information up a graph or back in time, so to speak, isn't super possible or convenient or really ergonomic in any way. -So after you've created that challenge span, the root doesn't really have any way to continue to add information. It doesn't have any way to modify that, and it doesn't have any way to like dig down into the tree and continue to add things. If you have intuitively instruments and everything, you do have the reference to this span, and in theory, you can work with it. +Then adding things after the fact is not possible either, so you can't, like, pack two things as a graph and incrementally build up or out of beyond build up a view of the graph that's more complete. Here's an example: we have a web server, a service, and a client. The web server has, you know, one event, a once span, and then a second span. The client has one span, and then the plan has one span, and they have overlapping timelines. -One of your building apps into new building libraries, you don't have that; so you don't have a way to send information down. They have baggage, but it almost feels like a workaround. It doesn't feel like it's embedded in this idea of how do I add things into the system and build this graph. +There's a problem that I ran into trying to deal with building this. So the first span in the web server can pass information to the second span when creating it; that's totally fine. That works; that's amazing; that's awesome! However, it has no real way to pass information to the second span after creating it. After you've created that challenge span, the root doesn't really have any way to continue to add information. -Conversely, the challenge span doesn't have any good way to send information to the parent span conveniently at all. It's possible if you know how to do it; it's possible if you know how to build a workaround for that. You can even build conventions to make it easier, but it's not absolutely convenient in any way. +### [00:25:30] Issues with observability practices -The lump server is able to pass information to the service and then the client via contact configuration, which I talked about earlier, and that's possible not to create networks. However, the service and the client essentially have no way to pass any information back up to the web server. Forget about it. +It doesn't have any way to modify that, and it doesn't have any way to, like, dig down into the tree and continue to add things. If you have intuitively instruments and everything, you do have the reference to this span, and in theory, you can work with it. -Another thing that's interesting here is if you notice the life cycle of the web server watched is actually shorter than the life cycle of the whole trees, which means that Taylor sampling is not going to work as you might expect. Taylor sampling really wants you to have a full stitched treat with a scooped corn snack life cycle to the web server or the root span has to be the longest span and everything else has to be inside of it and organized nicely so that you can make a decision once you've collected everything. +One of you building apps into new building libraries, you don't have that, so you don't have a way to send information down. They have baggage, but it almost feels like a workaround. It doesn't feel like it's embedded in this idea of how do I add things into the system and build this graph. -When you have these Instagram timelines and you don't actually know how long everything is going to be, if you don't know how long to wait to make your final decision, you end up discussing until it becomes heuristic faced rather than, "Oh, and let's actually do things." +Conversely, the challenge span doesn't have any good way to send information to the parent span conveniently at all. It's possible if you know how to do it; it's possible if you know how to build and work around for that. You can even build conventions to make it easier, but it's not absolutely convenient in any way. -But you can really bring you, if those heuristics are used as part of filtering your traces and actually trying to sample things. +The lump server is able to pass information to the service and then the client via context configuration, which I talked about earlier, and that's possible not to create networks. However, the service and the client essentially have no way to pass any information back up to the web server. There's test; that's not happening; forget about it. -So Taylor sampling distributing choices is actually really, really hard. Another problem is that once each fan has ended, you have no way to add information to it. So even though it is technically possible to try and add information to expand from the parent, you can add a permission to a challenge fan in some ways; that only works if the child span hasn't ended yet. +Another thing that's interesting here is if you notice the life cycle of the web server watched is actually shorter than the life cycle of the whole trees, which means that your sampling is not going to work as you might expect. Tail sampling really wants you to have a full stitched tree with a scooped corn snack life cycle; the web server or the root span has to be the longest span, and everything else has to be inside of it and organized nicely so that you can make a decision once you've collected everything. -You can have a really weird and rich conditions essentially. Speaking of that, also happens. Yeah, you can add information to the service, and also the child and parents have the same problem too. Even in context and even in process versus under severe process, you still run into the same issue. +When you have these Instagram timelines, and you don't actually know how long everything is going to be, if you don't know how long to wait to make your final decision, you end up discussing until it becomes heuristic-based rather than, "Oh, let's actually do things," but you can really, really bring you if those heuristics are used as part of filtering your traces and actually trying to sample things. -This means that you have a ton of internal implementation details of the instrumentation, the shape of the code, how the code works, the life cycle of their code, the column staff of everything to keep track of, which if you are the engineer writing everything, interviewing with the whole system and working end to end with it, that may not be a huge deal for you, and that may actually be there. +Tail sampling and distributing choices are actually really, really hard. -That's one of the reasons why OpenTelemetry is so deeply in hand with people owning their systems end to end and having that feedback. +Another problem is that once each span has ended, you have no way to add information to it. So even though it is technically possible to try and add information to span from the parent, you can add a permission to a challenge span in some ways that only works if the child span hasn't ended yet, and you have no way to know one or not adding that information worked. -There's actually, in some ways, available compensation because you can't build an abstraction unless a lot of people work on top of that. So you have to own your system end to end intrusively rather than being able to build these abstractions that let you instrument things more effectively. +You can end up with code that adds information to a span; it works great, more express financial production; it's awesome. Then someone optimizes the John function; I mean, it should take half the time, and used to, and then everything starts breaking silently because you didn't actually know. Then you're trying to add things to span after it ended, and you can have a really weird and rich condition, essentially. -The second problem that you run into a lot is what I call the graph rewriting problem. So we dig back into this livestock problem. You have a bunch of information based on life cycle. +Speaking of that also happens. Yeah, you can add information to the service, and also the child and parents have the same problem too. Even in context and even in process versus under severe process, you still run into the same issue, which means that you have a ton of internal implementation details of the instrumentation, the shape of the code, how the code works, the life cycle of their code, the column staff of everything to keep track of, which if you are the engineer writing everything, interviewing with the whole system and working end to end with it, that may not be a huge deal for you, and that may actually be there. -So when you make lesbians, lens fans, and how that interfaces with the constant of the stimulate system in general, the graph rewriting problem is this one. The rule of thumb when doing instrumentation is that you have like this final instrumentation that can build this scaffolding. You would take all of your manual instrumentation and hang it on that. +That's one of the reasons why open to my materials so deeply in hand with people owning their systems end to end and having that feedback. There's actually in some ways available compensation because you can't build an abstraction unless a lot of people work on top of that. So you have to own your system end to end intrusively rather than being able to build these abstractions to let you instrument things more effectively. -So in theory, you're not really creating one span. You usually understand that auto instrumentation already set up, and you're just adding attributes in my spots, which sometimes that doesn't work, especially when you're trying to write abstract code. You end up wanting to create this fan only abstract; couldn't do that. +The second problem there that you run into a lot is what I call the graph rewriting problem. So we dig back into this livestock problem. You have a bunch of information based on life cycle, so when you make lesbians lens fans and how that interfaces with the constant of the stimulate system in general. -In general, the more you can just add attributes to an existing span created by Honor instrumentation but not sad, where do libraries go in this, and how do they work, and how do you actually do this? +The graph rewriting problem is this one: the rule of thumb when doing instrumentation is that you have like this final instrumentation that can build this scaffolding. You would take all of your manual instrumentation and hang it on that. In theory, you're not really creating one span; you're using spans that auto instrumentation already set up, and you're just adding attributes in my spots. -So let's look at this as an example. We have a tree. It's fast client, and then it smash client has some action. Someone called make request on the client side, and then the auto instrumentation has that git endpoint. The next step on the client side to half the function call, and then you have the honor instrumentation of the actual that response. Awesome. +Sometimes that doesn't work, especially when you're trying to write abstract code. You end up wanting to create this span, only abstract couldn't do that. But in general, the more you can just add attributes to an existing span created by honor instrumentation, but not sad where do libraries go in this, and how do they work, and how do you actually do this? -That's great. That's cool. Then we go over into the server, and the server has the server endpoint that has three manaway functions that it has a post endpoint for the sending for an internal call. Then that has the same chain; you post on one thing, and then you go to the next one, then you keep going and keep going, and you can run into the same middleware, and you're going to do all that. +Let's look at this as an example. We have a tree, a fast client, and then this mash client has some action. Someone called make request on the client side, and then the auto instrumentation has that git endpoint. The next step on the client side is to have the function call, and then you have the honor instrumentation of the actual response. Awesome! That's great! That's cool! -If you're going to run into a couple problems here, the first problem is how do you collapse spans? You don't actually have a good one to do that. What if I don't actually want all of the middleware to be their own spans, and I want to have just one span for everything? Just add information to the development to it with attributes as I go down to me. If you want some spam events, maybe I just want the attributes, maybe I want things like that, and I don't actually need this fan for like the middleware around, or maybe I don't need a fan for some other things. +Then we go over into the server, and the server has the server endpoint that has three man away functions. It has a post endpoint for sending for an internal call, then that has the same chain. You post on one thing, and then you go to the next one, then you keep going and keep going, and you can run into the same middleware and you're going to do all that. -I don't have a way to like collapse that tree down, and that can be activated by the Fall. That's that if you want to search through your system, typically the root span is where you search, or the same level of a span is when you search when you're trying to like index through your data. +If you're going to run into a couple problems here, the first problem is how do you collapse spans? You don't actually have a good one to do that. What if I don't actually want all of the middleware to be their own spans, and I want to have just one span for everything? Just add information to it with attributes as I go down to me. If you want some spam events, maybe I just want the attributes, maybe I want things like that, and I don't actually need this span for like the middleware around, or maybe I don't need a fan for some other things. -So if you have dealer at different span levels, you essentially have no way to correlate those two pieces of datum and do a query in which you aggregate things across those spans. That's one reason why a lot of good practice is to have very, very wide events and a very wide spans and less of them so that way you have all the data at the same level and you can do all the correlation and grouping and aggregation and everything else there that you need to in order to effectively look through your data and find something. +I don't have a way to collapse that tree down, and that can be activated by the Fall. If you want to search through your system, typically the root span is where you search, or the same level of a span is when you search when you're trying to like index through your data. -But if you have all of your instrumentation and your honor instrumentation building a nested tree of like 10 or 20-inch spans before you get to run any of your code, where do you stick in so that it's at the level that it needs to be? +If you have dealer at different span levels, you essentially have no way to correlate those two pieces of datum and do a query in which you aggregate things across those spans. That's one reason why a lot of good practices is to have very wide events and very wide spans and less of them so that way you have all the data at the same level and you can do all the correlation and grouping and aggregation and everything else there that you need to in order to effectively look through your data and find something. -On top of that, every library, auto instrumentation, and things like that reinvent their weird set of configuration. I want you to try and make it to that they can help solve some of these problems. So you'll have, if you look through all the SDKs or all the contributors contributing libraries for OpenTelemetry, you'll notice this problem over and over. +But if you have all of your instrumentation and your honor instrumentation building a nested tree of like 10 or 20 inch spans before you get to run any of your code, where do you stick in so that it's at the level that it needs to be? -You're like, "Oh, how do I use, for example, the express server auto instrumentation?" It'll say, "You can just use it, and also here's like 15 different options. Do you want to silence some endpoints? Do you want to filter some endpoints? Do you want to silence submit awareness? Do you want to whatever?" Here's a whole bunch of functions you can use. Here's some callbacks, and we're going to call this compact before and after this; we're going to call these to these endpoints. +On top of that, every library, auto instrumentation, and things like that reinvent their weird set of configuration. I want you to try and make it to that they can help solve some of these problems. -Essentially, it gives you every possible hook point to you can think of to enter into the system and stick your own code there as if you haven't written it yourself. But when you don't have this way to write an abstraction and you end up having to sit there and write the code yourself in all these intuition endpoints and you need to have people having foresight in their libraries to put these insertion points for you, you have this very, very weird platform of really people wanting to like more or less patch the source to go to the libraries and the other instrumentation, and they want to have written all of it themselves graph effectively. +If you look through all the SDKs or all the contributors contributing libraries for open Telemetry, you'll notice this problem over and over. You're like, "Oh, how do I use, for example, the express server auto instrumentation? It'll save you can just use it, and also here's like 15 different options. Do you want to silence some endpoints? Do you want to filter some endpoints? Do you want to silence submit awareness? Do you want to whatever? Here's a whole bunch of functions you can use; here's some callbacks, and we're going to call this compact before and after this. We're going to call these to these endpoints," and essentially it gives you every possible hook point you can think of to enter into the system and stick your own code there as if you haven't written it yourself. -Why can't we deal with that, right? What's going on? I think I cut out for a brief second there. Did everyone catch the last things that I said? Awesome. +But when you don't have this way to write an abstraction, and you end up having to sit there and write the code yourself in all these intuition endpoints, and you need to have people having foresight in their libraries to put these insertion points for you, you have this very, very weird platform of really people wanting to, like, more or less patch the source to go to the libraries and the other instrumentation, and they want to have written all of it themselves graph effectively. -[00:35:56] So in conclusion, there are some problems here. There are a lot of things going on. There's some advice on what to do, and I'm gonna go through all of that now. +Why can't we deal with that, right? What's going on? I think I cut out for a brief second there. Did everyone catch the last things that I said? -Observability is the process through which one develops the ability to ask meaningful questions and get useful answers. Meaningful questions for you are going to be about learning about the system: how do you collectively get together and advance your understanding of the system, its behavior, how it interacts with you, and everything, and how do you get that knowledge as quickly as possible? +Awesome! -Those are meaningful questions. Continue can use for answers from them. If you can, your system is becoming observable. However, observability is interesting because the more you can observe, the more correlations you're going to infer with your data, and the harder it's going to be to discover contributing factors. +So, in conclusion, there are some problems here. There are a lot of things going on; there's some advice on what to do, and I'm gonna go through all of that now. -So just because you can find a needle, it doesn't mean you need to build a haystack. When you have more and more data and you keep shoving more things in there, then you have more and more out-of-band data, and you just add more and more layers of things that can give you observing money into the system. +Observability is the process through which one develops the ability to ask meaningful questions and get useful answers. Meaningful questions for you are going to be about learning about the system. How do you collectively get together and advance your understanding of the system, its behavior, how it interacts with you, and everything, and how do you get that knowledge as quickly as possible? Those are meaningful questions; continue to use for answers from them. -You should be really careful to look at everything and go, "Is this actually an intervention or understanding of the system, or is this adding noise?" Context plus configuration is traces; it is the glue that turns and back of data points into a unified trace. +If you can, your system is becoming observable. However, observability is interesting because the more you can observe, the more correlations you're going to infer with your data, and the harder it's going to be to discover contributing factors. -When working with context, it interfaces with everything that has to do with OpenTelemetry and in many ways remains invisible the entire time using until you're wondering where the entire or how to use it, or maybe you're confused. +So just because you can find a needle doesn't mean you need to build a haystack. When you have more and more data and you keep shoving more things in there, then you have more and more out-of-band data, and you just add more and more layers of things that can give you observing money into the system, you should be really careful to look at everything and go, "Is this actually an intervention or understanding of the system, or is this adding noise?" -But if you know how to use context and know what to think about when using context, it can become your friend. One way to map contacts and map different aspects of OpenTelemetry together is thinking about the shape of those contacts of those concepts. +Context plus configuration is traces; it is the glue that turns and backs of data points into a unified trace. When working with context, it interfaces with everything that has to do with open Telemetry and, in many ways, remains invisible the entire time using until you're wondering where the entire or how to use it, or maybe you're confused. -So traces are a graph; spans are your nodes and links are your edges. Despite choices being a graph, they're often actually mostly thought of and best most ergonomically designed as a card stock, which is a training that is like perhaps multi-branched. +But if you know how to use context and know what to think about when using context, it can become your friend. One way to map context and map different aspects of open Telemetry together is thinking about the shape of those concepts. -That tree is very scooped in. The top node is always the whitest and the longest, and everything is only encapsulated all the way down, and that's the most natural and ergonomic way to do OpenTelemetry and tracing, and everything's going to push you towards that. +So traces are a graph; spans are your nodes, and links are your edges. Despite traces being a graph, they're often actually mostly thought of and best ergonomically designed as a cardstock, which is a training that is like perhaps multi-branched. That tree is very scooped in; the top node is always the widest and the longest, and everything is only encapsulated all the way down, and that's the most natural and ergonomic way to do open Telemetry and tracing. Everything's going to push you towards that, but traces are a graph, and you can actually access all of that graph power when you need it, and sometimes you wish you didn't. -But traces are a graph, and you can actually access all of that graph power when you need it, and sometimes you wish you didn't. Observability and abstraction are natural enemies. They are the chief people in high school that love to take in on each other the entire time they're doing everything. +Observability and abstraction are natural enemies. They are the chief people in high school that love to take in on each other the entire time they're doing everything. Even though we love open Telemetry and we love observability and we love my other sister meditation, the problem of needing a large amount of out-of-band context but also wanting the ability to add and annotate a very complicated problem of your entire system, its business logic, the flow of information throughout the system; those are all problems that we don't actually have good answers for in general. -So even though we love OpenTelemetry and we love observability and we love my other sister meditation, the problem of needing a large amount of out-of-band context but also wanting the ability to add and annotate is a very complicated problem of your entire system, its business logic, the flow of information throughout the system. +We don't have good leverage for that; we don't have good approaches for that, and it stands beyond open Telemetry and beyond observability. This is just one of those ways that it becomes a very, very obvious problem. Two ways of having an average problem are the life cycle problem, which we talked about, which is where you have this graph, you have the whole power of a graph where you don't have a graph that is like convergent; you have a graph that you kind of have to satanically know the shape of as you're building out, and you can't really patch it up and picture them later. -Those are all problems that we don't actually have good answers for. In general, we don't have good leverage for that, and it stands beyond OpenTelemetry and beyond observability. This is just one of those ways that it becomes a very, very obvious problem. +### [00:40:00] Q&A session begins -Two ways of having an average problem are the life cycle problem, which we talked about, which is where you have this graph, you have the whole power of a graph where you don't have a graph that is like convergent. You have a graph that you kind of have to satanically know the shape of an Azure building out, and you can't really patch it up and picture them later. +Speaking of that, graph reminding it is really difficult, if not impossible. It's very much convention and ad hoc based. -Speaking of that graph, reminding it is really difficult, if not impossible. It's very much convention and ad hoc based. +Are there any questions, thoughts, or anything else? -Does anyone have any questions, thoughts, or anything else? +**Speaker:** foreign -**Speaker:** Hazel +I just want to say, Hazel, that the way that you describe these concepts in distributed tracing and open Telemetry are like I think the clearest I've heard. So thank you for that. I definitely really appreciate that. -I just want to say, Hazel, that the way that you describe these concepts in distributed tracing and OpenTelemetry are like I think the clearest I've heard. So thank you for that. I definitely really appreciate that. I think it's like everything now like just fully makes sense. +I think it's like everything now, like just fully makes sense. So yeah, I just want to echo that. That's definitely the vibe; it's like, "Oh yeah, cool, now I know what I'm writing about for the next six months." It's like the stuff that Hazel has put so clearly; it's very good. -So yeah, I just want to echo that, but that's definitely the vibe; it's like, "Oh yeah, cool, now I know what I'm writing about for the next six months." The stuff that Hazel has put so clearly is very good. +**Speaker:** foreign -I’m really happy to hear that. The question that I had is I'm curious how you feel like post-processing fits into that picture. Like with the collector, a lot of your times revolved around like developers, DevOps people who are working with the code, like how they're able to propagate data. But what about that? +I'm curious how you feel like post-processing fits into that picture, like with the collector. A lot of your times revolved around like developers, DevOps people who are working with the code. Like, how they're able to propagate data, but what about that? Like, not so much like tail-based sampling where you're deciding right at the moment, but like maybe a little further down the line being able to connect data points, you know, once you're holding five minutes of trace data. -Like not so much like tail-based sampling, where you're deciding right at the moment, but like maybe a little further down the line being able to connect data points, you know, once you're holding five minutes of trace data? +**Speaker:** foreign -**Speaker:** Foreign +One thing that's interesting there is that open Telemetry feels in many ways like an event-based emitting style of instruments in your code, but it's actually not that. The collector and other ways of post-processing data is where you start to feel that sort of impedance mismatch in particular. -Yeah, so one thing that's interesting there is that OpenTelemetry feels in many ways like an event-based emitting style of instruments in your code, but it's actually not that. The collector and other ways of post-processing data is where you start to feel that sort of impedance mismatch in particular. +Because of that, you can run into certain problems. A great example of that is that links can't be added after expanded there, and so even as far as I know, like once you create this span, you can't add a link to it even in push processing. -And so because of that, you can run into certain problems. A great way, a great example of that is that links can't be added after expanded there. And so even, as far as I know, like once you create this fan, you can't add a link to it, even in push processing. +So if you're trying to do that, things are going to be really tricky. Maybe post-processing can fit that, but I don't know that. -If you're trying to do that, things are going to be really tricky. Maybe post-processing can fit that, but I don't know that that's now I'm gonna go like research that. +**Speaker:** foreign -**Speaker:** Hazel +Wow, okay, cool. Yeah, I mean, I realize I have not done it. I've been like digging through it, haven't done it, so wow, okay. -That's wow, okay, cool. Yeah, I mean, I realize I have not done it. I've been like digging through it; I haven't done it. So wow, okay. +So there is interesting historical context there in that there used to be a method in the open Telemetry specification that you can add a link to this span as long as this span is active. Then they took that method out several years ago, and it's been an open discussion in the GitHub tracker for about three or four years now of this is absolutely useful. We do end up needing this a lot, and can we do that? -So there is interesting historical contents there in that there used to be a method in the OpenTelemetry specification of you can add a link to this fan as long as this man is active. Then they took that method out several years ago, and it's been an open discussion in the GitHub tracker for about three or four years now of this is absolutely useful. We do end up needing this a lot, and can we do that? +Apparently, as far as I know, the only way to do that is that span creation in the options of the... for each band, you can add links in there, and that's the only time, which means certain types of Instagram patterns or certain types of that type of thing are really, really difficult to do. -Apparently, as far as I know, the only way to do that is that's fan creation in the options of the, for each band, you can add links in there, and that's the only time. +Also means, you know, when you can approach process and then after that, pretty much... -Which means certain types of Instagram patterns or certain types of like that type of thing are really, really difficult to do, and also means, you know, when you can approach process and then after that pretty much. +**Speaker:** foreign -**Speaker:** Foreign +Another way to think about the open Telemetry commentary that's interesting is I actually view the counter and how to set up libraries and baggage and other things like that all as different types of convention-based out-of-band information and handling. -And then another way to think about the OpenTelemetry commentary that's interesting is I actually view the counter and how to set up libraries and baggage and other things like that all as different types of convention-based out-of-band information and handling. +So when you build up conventions in your company or in your system or in anything like that, you can start to say, "Oh yeah, if you set this information here, then it'll be there," or you can actually just assume that we'll have this information because the clutter will add it here. -So when you build up conventions in your company or in your system or in anything like that, you can start to say, "Oh yeah, if you set this information here, then it'll be there," or "You can actually just assume that we'll have this information because the clutter will add it here." +As an example, when it's metal and a blog post in which they talked about, they set up their collectors to actually add to every span what data center that span came from, what machine that span came from, what cluster it was in, and all this actual information, so everyone can always clear all that information. -So as an example, when it's metal and a blog post in which they talked about, they said of their collectors to actually add to every span what data center that span came from, what machine that span came from, what, and then where everything was, what cluster it was in, and all this actual information. So everyone can always clear all that information. +You don't know that it's there because you read the documentation in the internal tooling, and you can build internal tooling that works on top of that natural convention rather than something you can actually express in your telemetry itself. -You don't know that, like, it's there because you read the documentation in the internal tooling, and you can build internal tooling that works on top of that natural convention rather than something you can actually express in your telemetry itself. +**Speaker:** foreign Anything else? -**Speaker:** Hazel +### [00:45:00] Closing remarks and future sessions -Well, I guess if there are no further questions, thank you so much for coming on, for coming on twice, and for putting this together. I know presentations are definitely a lot more work to put together compared to just, you know, sitting there answering questions. +Well, I guess if there are no further questions, thank you so much for coming on twice and for putting this together. I know presentations are definitely a lot more work to put together compared to just, you know, sitting there answering questions. -So definitely appreciate the time that you took to put this together and to share your knowledge with folks. I think there's, you know, so much out there on like the novice hotel stuff, but when it starts to get into the gnarly bits, we're definitely lacking in that information. +I definitely appreciate the time that you took to put this together and to share your knowledge with folks. I think there's, you know, so much out there on the novice hotel stuff, but when it starts to get into the gnarly bits, we're definitely lacking in that information. -So we definitely appreciate you sharing that, and call out to anybody if you or you know anybody who is interested in sharing some hotel goodies in hotel in practice, we would absolutely love to have you. +We definitely appreciate you sharing that and call out to anybody. If you or you know anybody who is interested in sharing some hotel goodies in hotel in practice, we would absolutely love to have you. -Also, if you want to share your hotel story, we would love to have you for Open A. And I think we've got a session coming up later this month, right, Rhys? +Also, if you want to share your hotel story, we would love to have you for open A. I think we've got a session coming up later this month, right, Rhys? -**Speaker:** Rhys +**Speaker:** foreign -We do. We have a special discussion actually coming up on the 28th. It's going to be about the evolution of observability practices, and actually Nika is going to be part of that panel. We'll do a full announcement. +We do. We have a special discussion actually coming up on the 28th. It's going to be about the evolution of observability practices, and actually, Nika is going to be part of that panel. We'll do a full announcement. -So thank you, Hazel, for everything, and I guess we'll see y'all on the internet. Thank you. +So, thank you, Hazel, for everything, and I guess we'll see you all on the internet. Thank you! ## Raw YouTube Transcript diff --git a/video-transcripts/transcripts/2023-09-20T19:30:01Z-otel-in-practice-context-propapagtion-in-distributed-tracing-with-doug-ramirez.md b/video-transcripts/transcripts/2023-09-20T19:30:01Z-otel-in-practice-context-propapagtion-in-distributed-tracing-with-doug-ramirez.md index 9172544..e35619d 100644 --- a/video-transcripts/transcripts/2023-09-20T19:30:01Z-otel-in-practice-context-propapagtion-in-distributed-tracing-with-doug-ramirez.md +++ b/video-transcripts/transcripts/2023-09-20T19:30:01Z-otel-in-practice-context-propapagtion-in-distributed-tracing-with-doug-ramirez.md @@ -10,227 +10,222 @@ URL: https://www.youtube.com/watch?v=Of_ygNU_Cbw ## Summary -In this YouTube video, Doug Ramirez from Uplight presents a session on distributed tracing and context propagation in microservices architecture, focusing on the OpenTelemetry framework. Doug explains the importance of distributed tracing for observability, especially as microservices become more prevalent, and discusses key concepts like spans, traces, and trace context. He provides a minimal coding example to illustrate how to implement these concepts using OpenTelemetry, emphasizing the use of standardized HTTP headers for trace context propagation. Doug also addresses privacy considerations when implementing tracing and encourages developers to familiarize themselves with relevant documentation. The session concludes with a Q&A, where Doug shares further insights and resources for those looking to enhance their observability practices. +In this YouTube video, Doug Ramirez, a principal architect at Uplight, discusses the implementation of distributed tracing and context propagation using OpenTelemetry. This session is part of the "Hotel in Practice" series, following a previous Q&A session Doug participated in. He explains distributed tracing as a crucial observability concept that helps visualize how various microservices interact, especially as organizations shift from monolithic to microservices architectures. Doug outlines key concepts such as spans, traces, and the importance of context propagation through HTTP headers, specifically the W3C's trace parent and trace state headers. He provides a minimal coding demonstration using a fictitious API for an "Internet Band Database" to showcase how to implement tracing effectively. Doug emphasizes leveraging existing OpenTelemetry tools to facilitate this observability process and encourages practitioners to integrate logging, metrics, and tracing together for comprehensive observability. The session concludes with a discussion on documentation and future improvements for the sample code. ## Chapters -00:00:00 Welcome and intro -00:00:50 Doug Ramirez introduction -00:01:50 Distributed tracing overview -00:03:00 Context propagation explanation -00:05:00 Trace and span definitions -00:06:30 W3C trace context specification -00:08:20 HTTP headers for tracing -00:12:35 Live coding demonstration -00:15:00 Initial performance issues -00:23:00 Observability with Jaeger -00:32:55 Q&A and closing remarks +00:00:00 Introductions +00:01:54 What is distributed tracing? +00:03:28 Explanation of spans and traces +00:05:12 Trace context and its importance +00:13:00 Live coding demonstration begins +00:14:44 Example API for band reviews +00:20:48 Injecting trace context into service calls +00:31:42 Observing traces in Jaeger +00:39:42 Discussion on logging and metrics integration +00:46:48 Upcoming events and announcements -**Speaker 1:** Thank you. Hi everyone, thanks for joining us for Hotel in Practice and for being patient and sticking around. We've got Doug Ramirez from Uplight joining us for a second time. This time in the Hotel in Practice capacity. He has joined us previously for the Hotel Q&A session, and in case you missed that, we do have a blog post summarizing that session in the Hotel blogs in OpenTelemetry.io under the blog section. +## Transcript -I guess without further ado, here's Doug. +### [00:00:00] Introductions -[00:00:50] **Doug:** Thank you. Hi everyone, I am Doug Ramirez. I'm a principal architect at Uplight. We do cool things to try to save the planet. Just a shameless plug, uplight.com/careers, we're hiring. We're a B Corp, we do cool things. But let's move on to OpenTelemetry and distributed tracing. +**Speaker 1:** Thank you. Hi everyone, thanks for joining us for Hotel in Practice and for being patient and sticking around. We've got Doug Ramirez from Uplight joining us for a second time. This time in the Hotel in Practice capacity. He has joined us previously for the Hotel Q&A session, and in case you missed that, we do have a blog post summarizing that session in the Hotel Blogs in OpenTelemetry.io under the blog section. So, I guess without further ado, here's Doug. -[00:01:50] I have a few things that I want to try to do today, and those are to introduce this concept of distributed tracing context propagation. I want to share a very, very minimal example of distributed tracing in action, trying to peel away a lot of the extraneous stuff and noise around this so that you can just see in as few lines of code how this is implemented. Then also share some resources around just more resources for people to go and explore distributed tracing. +**Doug:** Thank you. Hi everyone, I am Doug Ramirez. I'm a principal architect at Uplight. We do cool things to try to save the planet. Just a shameless plug: Uplight.com/careers, we're hiring. We're a B Corp, we do cool things. But let's move on to OpenTelemetry and distributed tracing. -So what is distributed tracing? It's an observability concept that gives us a big picture of how our myriad of services interact with each other. If you think about the three main pillars of observability—logs, metrics, tracing—those have all been around for a while. Distributed tracing has been around for a while, but as more and more of microservices architecture becomes a thing and becomes popular, it really elevates the need for the ability to trace things that span process and service boundaries. +I have a few things that I want to try to do today, and those are to introduce this concept of distributed tracing context propagation. I want to share a very, very minimal example of distributed tracing in action, trying to peel away a lot of the extraneous stuff and noise around this so that you can just see in as few lines of code how this is implemented. Then, also share some resources around just more resources for people to go and explore distributed tracing. -There's a lot of things that monoliths give us, and with microservices, it's really—I really think that if you're going to move towards microservices, you really need to make sure that you're treating distributed tracing as a first-class citizen as part of that implementation. +### [00:01:54] What is distributed tracing? -[00:03:00] Context propagation is the mechanism that gives us the ability to observe things that happen in a service as a transactional request spans another service and another service, and kind of your call chain of microservices. Context propagation is the thing that gives us the ability to watch and see what's happening. +So, what is distributed tracing? It's an observability concept that gives us a big picture of how our myriad of services interact with each other. If you think about the three main pillars of observability: logs, metrics, tracing, those have all been around for a while. Distributed tracing has been around for a while, but as more and more of microservices architecture becomes a thing and becomes popular, it really elevates the need for the ability to trace things that span process and service boundaries. -A couple of concepts that are important to know about when you're thinking about distributed tracing and context propagation are spans and traces. In my experience, oftentimes I think people flip the two. But to be clear, a span is basically a unit of operation, a unit of work. You can think a database query is a great example of a span. That's a unit of work that a service does. +There's a lot of things that monoliths give us, and you know with microservices, it's really... I really think that if you're going to move towards microservices, you really need to make sure that you're treating distributed tracing as a first-class citizen as part of that implementation. Context propagation is the mechanism that gives us the ability to observe things that happen in a service as a transactional request spans another service and another service, and kind of your call chain of microservices. Context propagation is the thing that gives us the ability to watch and see what's happening. -There's a collection of these units of work that make up the transaction that may happen in the service. A span is nothing more than that unit of work, and a span ID is the thing that uniquely identifies that unit of work. A trace is just a collection of those spans, and it tells the story of how a request traverses all of these different services that essentially enable a use case or a feature or a capability. Each trace has a unique identifier, which is globally unique and random. +### [00:03:28] Explanation of spans and traces -[00:05:00] I put some links in the readme here so that you can read more about the reasoning behind that, but I think it's important for people to know. I know this has come up in conversations that I've had with people around what a trace ID is, and it is important that it is globally unique and random for reasons that are explained better in the documentation from the W3C, which I'll get into in a moment. +So, a couple of concepts that are important to know about when you're thinking about distributed tracing and context propagation are spans and traces. In my experience, oftentimes I think people flip the two. But to be clear, a span is basically a unit of operation, a unit of work. You can think a database query is a great example of a span; that's a unit of work that a service does. There's a collection of these units of work that make up the transaction that may happen in the service. A span is nothing more than that unit of work, and a span ID is the thing that uniquely identifies that unit of work. -A trace context is essentially kind of like the envelope or baggage that it's passed between services so that they can understand what uniquely identifies the request that's coming into them, have a uniquely identifiable transaction request that's going out of that service to another service. Essentially, in order for all of this to work, there has to be an agreement on what that format and what that specification is for a trace context. +A trace is just a collection of those spans, and it tells the story of how a request traverses all of these different services that essentially enable a use case or a feature or a capability. Each trace has a unique identifier, which is globally unique and random. I put some links in the README here so that you can read more about the reasoning behind that, but I think it's important for people to know. I know this has come up in conversations that I've had with people around what a trace ID is, and it is important that it is globally unique and random for reasons that are explained better in the documentation from the W3C, which I'll get into in a moment. -One of the things that we get as developers, which is awesome, is a recommendation from the W3C around what a trace context should look like. We also get the benefit of a lot of the work that the OpenTelemetry community has done to codify those recommendations and specifications into SDKs and into other things like the OpenTelemetry collector that give us as developers a really easy way to implement distributed tracing context propagation. +### [00:05:12] Trace context and its importance -[00:06:30] There's an agreed-upon format that is recommended by the W3C on how to do this. There are software libraries and specifications that the OpenTelemetry community gives us to make all of this really easy, and I'll show that in a moment. +A trace context is essentially kind of like the envelope or baggage that gets passed between services so that they can understand what uniquely identifies the request that's coming into them and have a uniquely identified transaction request that's going out of that service to another service. Essentially, in order for all of this to work, there has to be an agreement on what that format and what that specification is for a trace context. One of the things that we get as developers, which is awesome, is a recommendation from the W3C around what a trace context should look like. -With that specification that the W3C gives us, part of that recommendation is to use HTTP headers to allow a service to send information to another service so that that trace context can be propagated. There are two headers that the W3C recommends for doing this: a trace parent and trace state. +We also get the benefit of a lot of the work that the OpenTelemetry community has done to codify those recommendations and specifications into SDKs and into other things like the OpenTelemetry collector that give us as developers a really easy way to implement distributed tracing context propagation. There's an agreed-upon format that is recommended by the W3C on how to do this. There's software in libraries and specifications that the OpenTelemetry community gives us to make all of this really, really easy, and I'll show that in a moment. -Trace parent is where we package up the unique identifiers that describe a unit of work that's happening in one service, the trace that's associated with it, to send to another service so that they can unpack that tag, that unit of work in the other service with information so that as that information is emitted to an observability platform, we can tie all these things together and we can see how this call chain of things that happen that span services all work together. +With that specification that the W3C gives us, part of that recommendation is to use HTTP headers to allow a service to send information to another service so that the trace context can be propagated. There are two headers that the W3C recommends for doing this: `traceparent` and `tracestate`. `traceparent` is where we package up the unique identifiers that describe a unit of work that's happening in one service, the trace that's associated with it, to send to another service so that they essentially can unpack that tag, that unit of work in the other service with information so that as that information is emitted to an observability platform, we can tie all these things together and we can see how this call chain of things that happen that span services all work together. -[00:08:20] Back in the day with monoliths, it was pretty easy to observe and debug things that spanned different subcomponents of your monolithic architecture, and we kind of lose that ability with microservices. But we get it back by leveraging this trace context concept and the headers that are specified from that W3C recommendation to pass that information between services. +Back in the day with monoliths, it was pretty easy to observe and debug things that span different subcomponents of your monolithic architecture, and we kind of lose that ability with microservices. But we get it back by leveraging this trace context concept and the headers that are specified from that W3C recommendation to pass that information between services. -Okay, service A calls another service. It simply uses a header to package up some information. It's an agreed-upon format for other services to unpack and use to tag the spans and units of work that those other services are using. +Okay, service A calls another service; it simply uses a header to package up some information. It's an agreed-upon format for other services to unpack and use to tag the spans and units of work that those other services are using. There are some details in here about what the format of that is. -There's some details in here about what the format of that is. Let me reduce my font here so you can see what an example of a trace parent would look like. There's a version—I'm not going to get into the version today, nor will I get into the trace flags—but the really important part of this is to understand that this ID here, this unique identifier for the trace ID, is the thing that will allow us to tie all of this work together so that we can observe how microservices are all working together to enable some feature, satisfy some use case. +Let me reduce my font here so you can see what an example of a trace parent would look like. So, there's a version. I'm not going to get into the version today, nor will I get into the trace flags. But the really important part of this is to understand that this ID here, this unique identifier for the trace ID, is the thing that will allow us to tie all of this work together so that we can observe how microservices are all working together to enable some feature, satisfy some use case. -In addition to the trace parent header, the W3C recommendation also offers another additional header called trace state, which is really nothing more than a simple list of key-value pairs that you can add along with the context that's being propagated between services so that you can attribute that to things that may be unique to whatever it is you're doing. +In addition to the trace parent header, the W3C recommendation also offers another additional header called `tracestate`, which is really nothing more than a simple list of key-value pairs that you can add along with your context that's being propagated between services so that you can attribute that to things that may be unique to whatever it is you're doing. Vendors may use `tracestate` to provide observability into what they're doing, but you can just think of it almost just like a little bit of baggage that rides along with the trace context that moves between the services. -Vendors may use trace state to provide observability into what they're doing, but you can just think of it almost as a little bit of baggage that rides along with the trace context that moves between the services. +I have some links in here to the W3C trace context specification and recommendation. One of the things that I think is worth calling out and I would encourage anybody who's going to go down this path is to make sure that you're familiar with the privacy considerations. You know, in this day and age with PII and GDPR and a lot of the other things that we really need to pay attention to as it relates to the potential leakage of personal information, there are some considerations that are worth noting. -I have some links in here to the W3C trace context specification and recommendation. One of the things that I think is worth calling out, and I would encourage anybody who's going to go down this path, is to make sure that you're familiar with the privacy considerations. In this day and age with PII and GDPR and a lot of the other things that we really need to pay attention to as it relates to the potential leakage of personal information, there are some considerations that are worth noting. +I will say that in general, following the W3C recommendation, using the specifications in the libraries that we get from OpenTelemetry, this is a very safe thing to do. But just to kind of like be clear, spend some time and just make sure you get yourself familiar with those privacy considerations. -I will say that in general, following the W3C recommendation using the specifications in the libraries that we get from OpenTelemetry, this is a very safe thing to do. But just to be clear, spend some time and just make sure you familiarize yourself with those privacy considerations. - -I know that I ran through that quickly because I know we're kind of pressed for time, but essentially, this thing called distributed tracing, again, it's a way to observe how a transaction or a request traverses multiple microservices. Again, the good news here is that we have a specification that's widely adopted and it's widely known. We have tools that we can use from OpenTelemetry to employ this. - -For anyone who's thinking about, like, I'm building a bunch of microservices, I need to observe how they're operating, how do I do it? The good news is that that problem's been solved for you. There's a de facto standard, there's tools that you can use, and as you'll see in a moment, it's actually quite easy to do. +I know that I ran through that quickly because I know we're kind of pressed for time, but essentially, you know, this thing called distributed tracing again, it's a way to observe how a transaction or a request traverses multiple microservices. Again, the good news here is that we have a specification that's widely adopted and it's widely known. We have tools that we can use from OpenTelemetry to employ this. So for anyone who's thinking about, like, I'm building a bunch of microservices, I need to observe how they're operating, how do I do it? The good news is that that problem's been solved for you. There's a de facto standard, there's tools that you can use, and as you'll see in a moment, it's actually quite easy to do. Let me pause there because I know that was probably a fire hose of stuff, but I want to see if anybody who's on the call right now has any questions about these concepts before I jump into some code. -[00:12:35] Okay, we're going to run with scissors now and we're going to do some live coding. What I've done here is, again, what I tried to do was to just create the most minimal example of a couple of microservices all working together to demonstrate how you can use the W3C recommendation and use the OpenTelemetry libraries to do this stuff. - -In this scenario, we're building the—it's kind of like IMDb. Instead of the Internet Movie Database, it's the Internet Band Database. I want to create this source of truth when it comes to band names, and also in this scenario, we have—we're leveraging a platform that just handles and reviews just a generic platform for reviews. +### [00:13:00] Live coding demonstration begins -We've created a band's API that our front-end developers are going to use to build web and mobile applications on top of, and rather than building a reviews platform ourselves, we're just going to leverage this fictitious reviews platform. +Okay, we're going to run with scissors now, and we're going to do some live coding. What I've done here is, again, what I tried to do was to just create the most minimal example of a couple of microservices all working together to demonstrate how you can use the W3C recommendation and use the OpenTelemetry libraries to do this stuff. -In this case, we've exposed an endpoint that allows something to call and say, "Hey, give me this band that has a unique ID and give me the reviews associated with it." So I'm going to call the endpoint to get a band and its reviews, and wow, it took six seconds. That's a long time. Let me run it again just to see what's happening here. Twelve seconds is a long time. +In this scenario, we're building the... it's kind of like IMDb. Instead of the Internet Movie Database, it's the Internet Band Database. I want to create this source of truth when it comes to band names. Also, in this scenario, we have... we're leveraging a platform that just handles and reviews just a generic platform for reviews. We're creating a band's API that our front-end developers are going to use to build web and mobile applications on top of. Rather than building a reviews platform ourselves, we're just going to leverage this fictitious reviews platform. -You can imagine we're all familiar with this, right? Like you're building the service, you've got people that are writing front ends to it, and they're saying, "Hey Doug, I'm calling the endpoint to get a band and it's taking a long time," and that's not a lot of information to go on and debug. +In this case, we've exposed an endpoint that allows something to call and say, "Hey, give me this band that has a unique ID and give me the reviews associated with it." So, I'm going to call the endpoint to get a band and its reviews, and wow, it took six seconds. That's a long time. Let me run it again just to see what's happening here. Twelve seconds is a long time. -[00:15:00] I think we're all familiar with those scenarios where somebody contacts you and says, "Hey, you know, we've been noticing that this doesn't seem to be performing very well." We have to do that guessing game of, "Okay, well when did you call this service? Do I start grabbing a bunch of logs?" It becomes very tedious. +### [00:14:44] Example API for band reviews -So one of the things that helps is to employ some tracing and some distributed tracing. What I'm going to do now is say, "All right, hey, from a developer, I'm going to add some stuff to my service and all I need you to do is to make sure you pass me some information when you call me so I know that you're calling me." +You can imagine we're all familiar with this, right? Like you're building the service, you've got people that are writing front ends to it, and they're saying, "Hey Doug, I'm calling the endpoint to get a band and it's taking a long time." And that's not a lot of information to go on to debug. You know, I think we're all familiar with those scenarios where somebody contacts you and says, "Hey, you know, we've been noticing that this doesn't seem to be performing very well." We have to do that guessing game of, like, "Okay, well when did you call this service? Do I start grabbing a bunch of logs?" It becomes very tedious. -Then we can kind of debug this and see what's happening. The first thing I want to do is I want to go into my bands API, and this is just a very, very simple FastAPI application, just a very simple model for a band. I have an endpoint here that lets a caller get a band by its ID. I have a little helper function here that does some work to go actually access a data repository of reviews, which happens to be this other review service that I mentioned before. +One of the things that helps is to employ some tracing and some distributed tracing. So, what I'm going to do now is say, "Alright, hey, from a developer, I'm going to add some stuff to my service, and all I need you to do is to make sure you pass me some information when you call me so I know that you're calling me, and then we can kind of debug this and see what's happening." -So let me bring up my cheat sheet here. Let's start by bringing in some code that will provide some visibility with tracing. What I'm doing here is a couple of things. One is I'm going to bring in the library that's going to help me essentially tease out information from the header that I talked about a moment ago, this trace parent header, and inject it into essentially a hook that will call out to another service and emit that tracing signal. +The first thing I want to do is I want to go into my Bands API, and this is just a very, very simple FastAPI application, just a very simple model for a band. I have an endpoint here that lets a caller get a band by its ID. I have a little helper function here that does some work to go actually access a data repository of reviews, which happens to be this other review service that I mentioned before. -In this case, I'm going to use Jaeger, all populated about the OpenTelemetry collector. But in this case, I'm just going to bring in some code with the SDK, give the band service, the band's API, the ability to unpack that header and inject that information to emit a trace so that we can see what's happening. +Let me bring up my cheat sheet here. Let's start by bringing in some code that will provide some visibility with tracing. What I'm doing here is a couple of things. One is I'm going to bring in some... with the help of the OpenTelemetry SDK, I'm going to bring in the library that's going to help me essentially tease out information from the header that I talked about a moment ago, this `traceparent` header, and inject it into, essentially, a hook that will call out to another service and emit that tracing signal. -Oh yeah, so the get reviews. So let's go to our get reviews. Excuse me, this is the function that does the call to the reviews service. What I want to do here is say, "Okay, for my when I call the get review service, I want to start some kind of trace so I can see what's happening." +In this case, I'm going to use Jaeger, all populated by the OpenTelemetry collector. But in this case, I'm just going to bring in some code with the SDK to give the Bands API the ability to unpack that header and inject that information to emit a trace so that we can see what's happening. -In this case, I'm starting a span in my service, and what I want to do is I want to actually—I don't want to do this just yet. Sorry, let's just do this, and then back over here I'll explain what I'm doing here. +Oh yeah, so the get reviews. So, let's go to our get reviews. Excuse me, this is the function that does the call to the reviews service. What I want to do here is say, "Okay, for my... when I call the get reviews service, I want to start some kind of trace so I can see what's happening." In this case, I'm starting a span in my service. What I want to do is... I don't want to do this just yet. Sorry, let's just do this, and then back over here, I'll explain what I'm doing here. -What I'm doing now is I'm adding a couple things. I am creating the ability, or what I'm doing is I'm bringing in a very, very small library. This is just a helper library that is essentially provisioning the tracer that we get as part of the OpenTelemetry SDK. +What I'm doing now is I'm adding a couple of things. I am creating the ability... or what I'm doing is I'm bringing in a very, very small library. This is just a helper library that is essentially provisioning the tracer that we get as part of the OpenTelemetry SDK. What this tracer will do is this is the thing that will build up the traces and send it to an observability platform, in this case, it's going to be Jaeger. -What this tracer will do is this is the thing that will build up the traces and send it to an observability platform. In this case, it's going to be Jaeger. In case you're wondering, there's some default stuff that happens, like the OpenTelemetry SDK knows to send to a certain port that a service is listening to—in this case, it's 4317—and Jaeger's running locally, so it will pick it up. +In case you're wondering, there's some default stuff that happens, like the OpenTelemetry SDK knows to send to a certain port that a service is listening to. In this case, it's 4317, and Jaeger's running locally, so it will pick it up. That's a lot of hand waving, but in the interest of time, just know that the SDK is handling a lot of this stuff for you. -That's a lot of hand waving, but in the interest of time, just know that the SDK is handling a lot of this stuff for you. Back in my band service, what I want to do is say, "Okay, anytime somebody calls the service, what I want to do is I want to allow the caller to send that trace context in the header." +Back in my Bands service, what I want to do is say, "Okay, anytime somebody calls the service, what I want to do is I want to allow the caller to send that trace context in the header." In my service, what I want to do is I want to tease out that `traceparent` header because I know that there's a standard and a specification for that. I'm going to build up a carrier, and I'm going to inject that or I'm going to extract that carrier from the payload, and then I'm going to inject that trace context from my caller into my context so that when I build up a span, the OpenTelemetry SDK is going to do a bunch of magic and work for me and essentially emit that trace to the observability platform. -In my service, what I want to do is I want to tease out that trace parent header because I know that there's a standard and a specification for that. I'm going to build up a carrier, and I'm going to inject that trace context from my caller into my context so that when I build up a span, the OpenTelemetry SDK is going to do a bunch of magic and work for me and essentially emit that trace to the observability platform. +### [00:20:48] Injecting trace context into service calls -So I have this now built up in my code. Now the band's API can receive a request, build up a trace loop in that trace context from the caller, and when it calls its own internal function, we're going to spin up another span, which is a unit of work to make a query to the reviews platform. +So, I have this now built up in my code. The Bands API now can receive a request, build up a trace loop in that trace context from the caller, and when it calls its own internal function, we're going to spin up another span, which is a unit of work to make a query to the reviews platform. So, this will start to give us some visibility into what the caller is doing and what the Bands service is doing. -This will start to give us some visibility into what the caller is doing and what the band service is doing. So now let's go. What I want to say to the caller, then, the person who's building this application, is to say, "Hey, I'm adding some stuff to our service because I know you said it was slow and things don't seem right. Send this bit of information so that we can start to really observe what's happening here." +So now, let's go... So what I want to say to the caller then, the person who's building this application, is to say, "Hey, I'm adding some stuff to our service because I know you said it was slow and things don't seem right. Send this bit of information so that we can start to really observe what's happening here and kind of dig into what the problem may or may not be." -If I go back to my client, what I want to do is I want to essentially say, "Hey, help me help you by sending this header." When you send this header, manufacture a trace and a span on your end, add that header to the call that you make to me, and then we can kind of start to dig into what the problem may or may not be. +If I go back to my client, what I want to do is I want to essentially say, "Hey, help me help you by sending this header," and when you send this header, manufacture a trace and a span on your end, add that header to the call that you make to me, and then we can kind of start to dig in to see what's happening here. -So now at this point, we're kind of wired up now to get a bit more observability into what these different services are doing. If I run my—if the client now calls me, I can see it took five seconds. Oh, I forgot one thing. Sorry. +So, now at this point, we're kind of wired up now to get a bit more observability into what these different services are doing. So, if I run my... if the client now calls me, I can see it took five seconds. Oh, I forgot one thing. Sorry. I also just kind of want to print out to the console a nice little helper thing just to link me right to Jaeger where these traces are being sent to, so I can start to see what's happening. -I also just kind of want to print out to the console a little helper thing just to link me right to Jaeger, where these traces are being sent to, so I can start to see what's happening. Now the client calls me again; we should be able to start digging into what's going on here. Wow, 15 seconds is a long time. +So now the client calls me again, we should be able to start kind of digging into what's going on here. Wow, 15 seconds is a long time. -[00:23:00] Now I can kind of dig into this and I can see—and I don't—I'm not going to spend too much time on Jaeger, but Jaeger is an open-source platform that helps visualize traces and spans. Now that I have the caller following that W3C recommendation, passing that trace parent with some unique identifiers of the units of work it's doing, sending it to the band service, the band service is taking that information, its context has been propagated to it, it's leveraging that to emit information about what it's doing, and now we can observe what's happening. +So now I can kind of dig into this, and I can see... and I'm not going to spend too much time on Jaeger, but Jaeger is an open-source platform that essentially helps visualize traces and spans. So now that I have the caller following that W3C recommendation, passing that `traceparent` with some unique identifiers of the unit of work it's doing, sending it to the Bands service, the Bands service is taking that information. Its context has been propagated to it. It's leveraging that to emit information about what it's doing, and now we can observe what's happening. -I can see that the get bands endpoint was called, and I can see that the get reviews function unit of work was run. If I look at this, the spans table here, I can see that the majority of time that was spent here was getting the reviews. So that's good helpful information to know. I can see, like, okay, get reviews is taking a long time; let's dig into that. +I can see that the get bands endpoint was called, and I can see that the get reviews function unit of work was run. If I look at this, the spans table here, I can see that the majority of time that was spent here was getting the reviews. So that's good, that's helpful information to know, right? I can see, like, "Okay, get reviews is taking a long time. Let's dig into that." -So let's go back to our code and I can say, "Hey guys, building folks building the client, I see something going on in my service. Let me dig into it. The get reviews is taking a long time." +So, let's go back to our code and I can say, "Hey guys, building folks building the client, I see something going on in my service. Let me dig into it. The get reviews is taking a long time." -"Hey, what's this debugging thing? Somebody did something while they were debugging. I just have a random timer here. Let me just get rid of this. Sorry about that, folks." +Hey, what's this debugging thing? Somebody did something while they were debugging. I just have a random timer here. Let me just get rid of this. Sorry about that, folks. Right, okay. Hey, I just deployed a new version of my service. Call me again. Still taking a long time; something doesn't seem right. -Right. Okay, hey, I just deployed a new version of my service. Call me again. Still taking a long time, something doesn't seem right. So now in this fictitious world, I'm going to turn around to the folks that work on the reviews platform and say, "Hey, you know, we're looking into an issue. There was some random code that was sleeping in our end. Can you take a look and see what's going on in your platform? Because when we're calling you, like, it's taking a really long time to get reviews; like, our service seems to be working pretty fast." +So now, in this fictitious world, I'm going to turn over and turn around to the folks that work on the reviews platform and say, "Hey, you know, we're looking into an issue. There was some random code that was sleeping on our end. Can you take a look and see what's going on in your platform? Because when we're calling you, like, it's taking a really long time to get reviews. Our service seems to be working pretty fast." -So now let's go and do the same thing on the reviews end. Very similar bit of refactoring. Let's go to the reviews, especially. Let me bring—all right, let's bring all this in. +So now let's go and do the same thing on the reviews end. Very similar bit of refactoring. Let's go to the reviews. Especially, let me bring... yeah, let's bring all this in. So now the same service here is going to bring in this helper function to start up the tracer again, leveraging the OpenTelemetry SDK to do that. We'll bring in this trace context propagator so that we can have this service do interesting things with the trace context. -Let's see. Here we want to do something that's a little bit different than the band service does, right? Yeah, I think that's right. So now the review service can also receive that trace parent, that trace context, unpack it, and associate it with its span of work. In addition to that, we want to start a span at the internal—imagine this would be a piece of code that would more than likely read from a data repository, a data store associated with the review service. +Let's see. Here we want to do something that's a little bit different than the Bands service does, right? Yeah, I think that's right. -So now I can say, "Okay folks that are building the front end of the mobile app, we've released a new band's API, we've released a new reviews API that we think is going to help give us some visibility into what's going on." +So now the review service can also receive that `traceparent`, that trace context, unpack it, and associate it with its span of work. In addition to that, we want to start a span at the internal... imagine this would be a piece of code that would more than likely read from a data repository, a data store that's associated with the review service. -So they're going to keep doing their development. They're going to start making calls to us. Four seconds isn't too bad. Let's try again. It's still taking too long, so now we got to figure out, okay, well what's going on? +So now I can say, "Okay, folks that are building the front end of the mobile app, we've released a new Bands API, we've released a new Reviews API that we think we're going to help give us some visibility into what's going on." -So let's go back to our observability platform, Jaeger, and what is happening? Not seeing the trace from the reviews out of this. This is what you get when you try and run with scissors and do live coding. What am I missing here? +So they're going to keep going doing their development. They're going to start making calls to us. Four seconds isn't too bad. Let's try again. It's still taking too long. So now we've got to figure out, okay, well, what's going on? -Trace parents. So this is getting—oh, I know what we did. All right folks, there it is. +So let's go back to our observability platform, Jaeger, and what is happening? Not seeing the trace from the reviews out of this. This is what you get when you try and run with scissors and do live coding. -The reason that the review service wasn't getting the context propagated is because I didn't add that in the—so in the band service that's calling the review service, we need to propagate that context. Be quiet, Siri. +What am I missing here? Trace parents. So this is getting... oh, I know what we did. Alright folks, there it is. -Here we go. All right. I know I'm jumping around quite a bit. In the band service, it's getting that trace context from the caller, it's attaching it to a span, it's making this call to get reviews. We want to see what's happening in that unit of work, so we're starting up a new span. +Okay, the reason that the review service wasn't getting the context propagated is because I didn't add that in the... so in the Bands service, it's getting that trace context from the caller. It's attaching it to a span. It's making this call to get reviews; get reviews. We want to see what's happening in that unit of work, so we're starting up a new span. But we also want to take the context from the caller and inject it into a header so that when the Bands service calls the Review service, that context gets propagated to it as well. -But we also want to take the context from the caller and inject it into a header so that when the band service calls the review service, that context gets propagated to it as well. So now when the caller calls the band service, it's spinning up this trace ID, this unique identifier, sending it to the band service. The band service is like, "Thank you very much. I can attach this to my span, my units of work. I'm going to send it downstream to the thing that I rely on." +So now when the caller calls the Bands service, it's spinning up this trace ID, this unique identifier, sending it to the Bands service. The Bands service is like, "Thank you very much. I can attach this to my span, to my units of work. I'm going to send it downstream to the thing that I rely on." It's going to attach this information to its units of work, and we should see all of that come together in Jaeger. Boom! Alright, service is still running slow, but at least now we have some visibility into what's happening. -It's going to attach this information to its units of work, and we should see all of that come together in Jaeger. Boom! All right, service is still running slow, but at least now we have some visibility into what's happening. +Oops, and if I go to my trace graph, sorry, my spans table... actually, where is it? I can see here that the majority of the time seems to be sent, being sent, being spent, sorry, in getting the reviews from the reviews database. -Oops, and if I go to my trace graph—sorry, my spans table—actually, where is it? I can see here that the majority of the time seems to be spent getting the reviews from the reviews database. So when it's getting a review, by getting the reviews for a band by the band ID, it's taking a long time. +So when it's getting a review by getting the reviews for a band by the band ID, it's taking a long time. So let's go back and start digging into that code there to see what might be happening. -So let's go back and start digging into that code there to see what might be happening. So in the review service, if I look at this function that is grabbing data from some kind of data repository, again, somebody put in some kind of sleep. Maybe they were debugging, testing some timeouts; who knows what, but let's pull that out. +So in the review service, if I look at this function that is grabbing data from some kind of data repository, again, somebody put in some kind of sleep. Maybe they were debugging, testing some timeouts, who knows what, but let's pull that out. -Now we can say, "Hey folks that are building the mobile and web app, we think we fixed it for real this time. Can you try again?" Wow, that's better. +Now we can say, "Hey folks that are building the mobile and web app, we think we fixed it for real this time. Can you try again?" Wow, that's better! -Let's go back and see what happened in Jaeger. Okay, things are taking milliseconds. That feels better. I like this. Now I can observe what's happening across all of these different things. Yay team! +### [00:31:42] Observing traces in Jaeger -Just to kind of walk through the code one last time: the client is packaging up this header based on the W3C recommendation. It's passing that header along to our bands API. Our bands API is leveraging the OpenTelemetry SDK and saying, "Hey, let me pull this stuff out and inject it into a carrier so that I can send it downstream." +Let's go back and see what happened in Jaeger. Okay, things are taking milliseconds. That feels better. I like this! Now I can observe what's happening across all of these different things. Yay team! -[00:32:55] By the way, when I'm creating my spans, I'm going to leverage that information. When the review service gets a call into it, it can tease out that trace parent information and inject it into its spans by attaching that context. And now we have distributed tracing with context propagation because we've got this awesome stuff from the OpenTelemetry community and we've got this awesome recommendation from the W3C. +Just to kind of walk through the code one last time: the client is packaging up this header based on the W3C recommendation. It's passing that header along to our Bands API. Our Bands API is leveraging the OpenTelemetry SDK and saying, "Hey, let me pull this stuff out and inject it into a carrier so that I can send it downstream." -Let me stop there and ask if anybody has any questions about the basic minimal implementation of these concepts in these handful of FastAPI services, and maybe this is a good time just to kind of open up to general questions as well because I know we're coming up on time. +By the way, when I'm creating my spans, I'm going to leverage that information when the Review service gets a call into it. It can tease out that trace parent information and inject it into its spans by attaching that context. Now we have distributed tracing with context propagation because we've got this awesome stuff from the OpenTelemetry community and we've got this awesome recommendation from the W3C. -**Audience Member:** Doug, I had a question for you. How did you find in terms of finding the information, the documentation around context propagation? How was that experience for you in terms of finding that information in the Hotel docs? +Let me stop there and ask if anybody has any questions about the basic minimal implementation of these concepts in these handful of FastAPI services. -**Doug:** It was pretty good actually. So in the readme for this repo, which I hope that other people can help me test the documentation, I would love for some folks to go and clone this repo and try to spin all these services up and make some calls and see if they can get it running locally. It was good. +**Audience Member:** I had a question for you. How did you find in terms of finding the information, the documentation around context propagation? How was that experience for you in terms of finding that information in the hotel docs? -The OpenTelemetry documentation has some very good details around things that I glossed over today that I would really like to go into further detail around. I'm sure that anybody who watches this is probably going to start asking questions like, "Well, how does that actually work?" There's a lot of magic under the hood, but the OpenTelemetry documentation goes into a lot of detail around the things that I just waved my hands around, like the tracers, the trace exporters, the context propagation, the shape and format of spans, how these things all link together. +**Doug:** It was pretty good, actually. So in the README for this repo, which I hope that other people can help me test the documentation, I would love for some folks to go and clone this repo and try and spin all these services up and make some calls and see if they can get it running locally. It was good. The OpenTelemetry documentation has some very good details around things that I glossed over today that I would really like to go into further detail around. -I found it was pretty easy to find that, and I also think that the W3C documentation is also very good as well. So I was able to kind of bring everything together pretty quickly. It didn't take me very long to build this reference implementation of these few applications. +I'm sure that anybody who watches this is probably going to start asking questions like, "Well, how does that actually work?" There's a lot of magic under the hood, but the OpenTelemetry documentation goes into a lot of detail around the things that I just waved my hands around, like the tracers, the trace exporters, the context propagation, the shape and format of spans, how these things all link together. -**Audience Member:** Yeah, OpenTelemetry also publishes a bunch of integration packages which lets you hook up to like Node middleware or HTTP clients automatically, so you can just have everything instrumented and doing the correct header insertion and things like that without really having to worry about doing it manually like in this demonstration. +I found it was pretty easy to find that, and I also think that the W3C documentation is also very good as well. So, I was able to kind of bring everything together pretty quickly. It didn't take me very long to build this reference implementation of these few applications. -**Doug:** Yes, that's a really good point. One of the things that I did as part of this reference implementation was to do a very explicit implementation of these concepts. We don't have time, and I wanted to do that thing that you always see in a demo where somebody says, "Do all this stuff," and at the end of the demo, they say, "Oh, by the way, you don't have to do anything. You just do this other thing and it all gets handled for you." +**Audience Member:** OpenTelemetry also publishes a bunch of integration packages which lets you hook up to node middleware or HTTP clients automatically, so you can just have everything instrumented and doing the correct header insertion and things like that without really having to worry about doing it manually. -There is a FastAPI instrumentor which will remove the need to do a lot of this explicit unpack, this explicit extraction and injection of the context. I didn't have time today to show that, and I kind of wanted to make it very clear to a developer who's learning these concepts to walk through the code and see something very procedural that shows this. +**Doug:** Yes, that's a really good point. One of the things that I did as part of this reference implementation was to do a very explicit implementation of these concepts. We don't have time, and I wanted to do that thing that you always see in a demo where somebody says, "Do all this stuff," and at the end of the demo, they say, "Oh, by the way, you don't have to do anything. You just do this other thing, and it all gets handled for you." -For example, there's a header, and that header is very important, and it is called a trace parent, and it has a specification. Here's an example of teasing out that header in code. The OpenTelemetry library has this thing where it can extract this context for you. Here's the line of code that does this. When I make a call to another service, I want to pass that along very explicitly. +There is a FastAPI instrumenter which will remove the need to do a lot of this explicit unpack, this explicit extraction, and injection of the context. I didn't have time today to show that, and I kind of wanted to make it very clear to a developer who's learning these concepts to walk through the code and see something very procedural that shows this. For example, there's a header, and that header is very important, and it is called a `traceparent`, and it has a specification. Here's an example of teasing out that header in code. -There was a lot of intention behind trying to make this application a bit more verbose, even though it's very minimal, to try to show somebody who's maybe new to this what does this actually look like. If you use the FastAPI instrumentor, the good news is a lot of this gets handled for you. +The OpenTelemetry library has this thing where it can extract this context for you. Here's the line of code that does this. When I make a call to another service, I want to pass that along very explicitly. + +So, there was a lot of intention behind trying to make this application a bit more verbose, even though it's very minimal, to try to show somebody who's maybe new to this, "What does this actually look like?" If you use the FastAPI instrumenter, the good news is a lot of this gets handled for you. My recommendation would be for anyone who's getting started with microservices is to make sure that you include distributed tracing from the very, very beginning. I would almost suggest that you spend a bit of time and implement this explicitly so that you understand what's happening, so that other developers understand what's happening. -Once you solve the problem of observability, then level up and start to leverage some of these other things that OpenTelemetry gives us, like the FastAPI instrumentor and these other things that we get from the community. +Once you solve the problem of observability, then level up and start to leverage some of these other things that OpenTelemetry gives us, like the FastAPI instrumenter and these other things that we get from the community. I feel like that just kind of helps bake the concepts in for people. I know that for me, I'm a very tactile learner, so it helps me to see what's happening before I go and start to let some libraries handle that magic for me, if that makes sense. Your mileage may vary, but my recommendation would be to go ahead and do it explicitly. It's a handful; it's a few lines of code. You can share it with people when you're doing code reviews. You can explain to them what's happening. It's not mysterious; it's very explicit. -I feel like that just kind of helps bake the concepts in for people. I know that for me, I'm a very tactile learner, so it helps me to see what's happening before I start to let some libraries handle that magic for me, if that makes sense. Your mileage may vary, but my recommendation would be to go ahead and do it explicitly. +**Audience Member:** Do you have any questions for Doug? -It's a handful of lines of code, you can share it with people when you're doing code reviews, you can explain to them what's happening. It's not mysterious, it's very explicit. +### [00:39:42] Discussion on logging and metrics integration -**Audience Member:** Do we have any questions for Doug? +**Audience Member:** Thank you, Doug, for sharing that. One question from me is that how does it look when you... like so the observability problem in production, I suppose apart from this OpenTelemetry tracing part, I suppose we also need to serve logging and metrics. Did you find any difficulty integrating the different parts of observability? -**Audience Member:** Thank you, Doug, for sharing that. One question from me is that how does it look when you, like, so the observability problem in production, I suppose, apart from this OpenTelemetry tracing part, I suppose we also need to serve logging and metrics, and do you find any difficulty integrating the different parts of observability? +**Doug:** The last time I spoke here, I mentioned that one of the paths that we took and had been taking at Uplight has been to actually begin with the log signal. The reason is that in terms of introducing these concepts, the problems that OpenTelemetry solves, the problems that the W3C helps us with solving, is to begin with logging. -**Doug:** The last time I spoke here, I mentioned that one of the paths that we took and have been taking at Uplight has been to actually begin with the log signal. The reason is that in terms of introducing these concepts that these problems that OpenTelemetry solves, these problems that the W3C helps us with solving is to begin with logging so that you—because logging, I think, is very, very familiar with most developers. +Because logging, I think, is very, very familiar with most developers. I think, you know, junior and intermediate developers are familiar with this concept of logging, like a print statement. So what I've recommended and what we did at Uplight when we started down this path was to start using the logging SDK, even though it's newer than the work for the signals of metrics and traces, but to start with logging so that you can at least begin to practice including the packages and libraries into your code using OpenTelemetry to emit that logging signal. -I think junior and intermediate developers are familiar with this concept of logging, like a print statement. What I've recommended and what we did at Uplight when we started down this path was to start using the logging SDK, even though it's newer than the work for the signals of metrics and traces, but to start with logging so that you can at least begin to practice including the packages and libraries into your code using OpenTelemetry to emit that logging signal. +Get that logging signal landing in your APM or whatever your observability platform is, and once you make that connection of like, "I'm writing code, I'm writing a statement that says logger.debug, my message is landing in an APM, it's structured, it's being... it's honoring the specification that the OpenTelemetry community gives us," then it's really easy to then level up to add tracing, and you get trace-log correlation for free. -Get that logging signal landing in your APM or whatever your observability platform is, and once you make that connection of like, "I'm writing code, I'm writing a statement that says logger.debug, my message is landing in an APM," it's structured, it's honoring the specification that the OpenTelemetry community gives us. +And that's one of the things I would like to actually do with this little reference implementation is to add in some trace-log correlation so that people can see, like, "How do I do the thing that I'm so familiar with doing?" Like, "I like to send out a log. I like to do a print statement." Like, that's so... we always do that all day, every day. -Then it's really easy to then, you know, level up to add tracing, and you get trace-log correlation for free. That's one of the things I would like to actually do with this little reference implementation is to add in some trace-log correlation so that people can see, like, how do I do the thing that I'm so familiar with doing? +If you start with logging and you get the SDKs into your repo and you have that information flow into your APM, then adding the layer of maturity around traces and the trace-log correlation is almost free because you've already got everything packaged up into your application already. -I like to send out a log; I like to do a print statement. That's so we always do that all day, every day. If you start with logging and you get the SDKs into your repo and you have that information flow into your APM, then adding the layer of maturity around traces and the trace-log correlation is almost free because you've already got everything packaged up into your application already. +That might not work for everybody. To me, that feels like a very natural way to evolve into using the specification, using the SDKs, and really kind of getting into observability nirvana where you can walk up to your APM and say, "I have questions about the health and behavior of my services, and I have a lot of them. What's happening?" And you can get those answers if you do this. -That might not work for everybody. To me, that feels like a very natural way to evolve into using the specification, using the SDKs, and really kind of getting into observability nirvana, where you can walk up to your APM and say, "I have questions about the health and behavior of my services, and I have a lot of them. What's happening?" +**Audience Member:** Doug, it sounds like you need to do a follow-up presentation on logging. -You can get those answers if you do this. +**Doug:** I would love to, and I think that I would love some help from other people to take this repo and clone it or fork it or just play with it. And for, you know, to help me test the documentation. Some other things I would like to do is to go into the code itself and provide a bit more kind of verbose commentary around what's happening. Again, I think it's helpful to have a very explicit implementation of these concepts, but for example, I would really like to have some comments here that tell somebody, "Hey, when you call the extract method, this is what it's doing for you." -**Audience Member:** Doug, it sounds like you need to do a follow-up presentation on logging. +Because right now, it's magic; it's just happening, and that's awesome. But I think for people to really understand what OpenTelemetry is doing and to understand how all this works, I think it would be helpful to have in this reference implementation some pretty verbose comments. Not the kind of thing you might want to put in your production code, but for a reference implementation, I think it would be helpful, and I would love to do another one of these. -**Doug:** I would love to, and I think that I would love some help from other people to take this repo and clone it or fork it or just play with it. I definitely encourage others to poke around as well. I think it would be cool to contribute this back to OpenTelemetry. +**Audience Member:** And you know, I've been looking through your repo as you've been presenting, and I really like this example. It's simple, but it's got enough complexity that it illustrates what you're trying to convey. -Although, to be honest, I don't know if other OpenTelemetry folks have a better idea on the process around that because I think I'm newer to the process. But anyway, I think that, yeah. +So, yeah, I think I would love to, at some point when I have some spare time, to poke around. I definitely encourage others to poke around as well. Maybe... I don't know as far as... I think it'd be cool to contribute this back to OpenTelemetry, although to be honest, I don't know if other OpenTelemetry folks have a better idea on the process around that because I think I'm newer to the process. -I know that there are, like, there's a demo app and there's other reference implementations, but one of the problems I was trying to solve here was how do I strip away all the noise and give a developer with a modern operating system and Docker the ability to get clone, fire up these services, and watch what's happening so they can really dig into these concepts and understand how powerful they are in their simplicity. +But anyway, I think that... yeah, and I know that there are like, there's a demo app and there's other reference implementations, but one of the problems I was trying to solve here was how do I strip away all the noise and give a developer with a modern operating system and Docker the ability to clone, fire up these services, and watch what's happening so they can really dig into these concepts and understand how powerful they are in their simplicity. -It's pretty amazing; all we're doing is just sending a header, and these SDKs are doing these wonderful things for us. It's pretty simple, but it's unbelievably powerful. +It's pretty amazing. All we're doing is just sending a header, and these SDKs are doing these wonderful things for us. It's pretty simple, but it's unbelievably powerful. So, the idea for this reference implementation was to try to just highlight that as best we could with as few lines of code as possible so that people can understand this and so that people can go and be successful with building this level of maturity in their observability in their code. -The idea for this reference implementation was to try to highlight that as best we could with as few lines of code as possible so that people can understand this and so that people can go and be successful with building this level of maturity in their observability in their code. +**Audience Member:** I totally agree, and I think there's a lot of value in that. Like you mentioned, the OpenTelemetry demo app is awesome because it shows all of the things, but there is definitely value in simpler implementations like this one. -**Audience Member:** I totally agree, and I think there's a lot of value in that. Like you mentioned, the OpenTelemetry demo app is awesome because it shows all of the things, but there is definitely value in simpler implementations like this one. +So, yeah, well, on that note, I know we're a little bit over time. Thank you everyone who was able to join us today, especially given that we started late. Thanks, Doug, for working on the fly with our reduced time, and for lunch. That's never an easy task, so definitely appreciate the time that you put into making this happen today. -So yeah, well, on that note, I know we're a little bit over time. Thank you everyone who was able to join us today, especially given that we started late. Thanks Doug for working on the fly with our reduced time and for lunch—that's never an easy task. +Thanks everyone for joining us once again, and keep your eyes open for our next Hotel in Practice. I don't think we have one scheduled yet, but we'll post on the various CNCF Slack channels for that. -So definitely appreciate the time that you put into making this happen today. Thanks everyone for joining us once again, and keep your eyes open for our next Hotel in Practice. I don't think we have one scheduled yet, but we'll post on the various CNCF Slack channels for that. +**Rin:** Do you have any additional upcoming announcements that folks should be aware of? -Rin, do you have any additional upcoming announcements that folks should be aware of? +### [00:46:48] Upcoming events and announcements -**Rin:** Sure, yeah. So first, there's an OpenTelemetry in Practice meetup. If you came here through CNCF, a lightweight way to get notified whenever we have something and not have to look for the Slack. If you're not in the CNCF Slack and you want to talk further to any of the OpenTelemetry in Practice folks, Doug, the audience, we're happy to add you. That information is in the chat. +**Rin:** Sure, yeah. So, first, there's an OpenTelemetry in Practice meetup. If you came here through CNCF, it's a lightweight way to get notified whenever we have something and not have to look for the Slack. If you're not in the CNCF Slack and you want to talk further to any of the Hotel in Practice folks, Doug, the audience, we're happy to add you. That information is in the chat. -If you'll be at KubeCon EU, several of us will be, and we'd love to connect with you there. There will be a blog post on the OpenTelemetry blog on Monday with all of the activities. Yes, I am happy to do a follow-up with Doug because of KubeCon EU; that will probably be around the end of May. We can figure out what format we want to do based on hopefully talking with you all in the Slack. +If you'll be at KubeCon EU, several of us will be, and we'd love to connect with you there. There will be a blog post on the OpenTelemetry blog on Monday with all of the activities. Yes, I am happy to do a follow-up with Doug because of KubeCon EU. That will probably be around the end of May, and we can figure out what format we want to do based on hopefully talking with you all in the Slack. Thank you, everybody. diff --git a/video-transcripts/transcripts/2023-09-25T04:00:25Z-otel-in-practice-fireside-chat-december-2022.md b/video-transcripts/transcripts/2023-09-25T04:00:25Z-otel-in-practice-fireside-chat-december-2022.md index b4af504..3a55da7 100644 --- a/video-transcripts/transcripts/2023-09-25T04:00:25Z-otel-in-practice-fireside-chat-december-2022.md +++ b/video-transcripts/transcripts/2023-09-25T04:00:25Z-otel-in-practice-fireside-chat-december-2022.md @@ -10,332 +10,374 @@ URL: https://www.youtube.com/watch?v=TYrIP_LkwgU ## Summary -The video features a discussion on OpenTelemetry, specifically focusing on its implementation and usage within various tech environments. Key speakers include Adriana, who shares her experience with HashiCorp products and demonstrates how to run a hotel demo app using Nomad and OpenTelemetry; Jess, who presents a fun project that visualizes data in heat maps using OpenTelemetry; and Rhys, who talks about how end users can engage with the OpenTelemetry community. The main points include the importance of observability in production systems, practical implementations of OpenTelemetry, and ways for users to contribute to the community and provide feedback. The session encourages interaction and showcases tools and techniques that enhance understanding and application of OpenTelemetry in real-world scenarios. +In this video from the OpenTelemetry in Practice series, the format diverges from the usual deep dive discussions as presenters share shorter talks. Adriana, who has a background in Kubernetes and previously worked at HashiCorp, discusses observability in HashiCorp products and demonstrates running the OpenTelemetry demo application using Nomad, showcasing its integration with various services like Vault and Console. Jess (Jessatron) presents a fun and creative project that uses OpenTelemetry to generate heat maps from images, specifically designed for educational purposes. Rhys talks about ways for end users to engage with the OpenTelemetry community, highlighting the End User Working Group's initiatives to foster involvement and communication between users and maintainers. Overall, the session emphasizes community engagement, practical applications of OpenTelemetry, and innovative uses of observability tools. ## Chapters -00:00:00 Welcome and intro -00:01:00 Adriana's background -00:03:30 HashiCube demo -00:05:00 Nomad console overview -00:08:20 Job specs in Nomad -00:12:01 Environment variables in Nomad -00:15:30 Health checks explanation -00:21:10 Transition to Jess's demo -00:21:10 Jess's heat map demo -00:29:00 Discussion on observability tools -00:36:00 End user working group overview -00:41:01 Q&A session +00:00:00 Introductions +00:01:46 Guest introduction: Adriana +00:05:30 Observability in HashiCorp products +00:10:24 Demo of OpenTelemetry demo app +00:20:00 Transition to Jess's presentation +00:21:36 Guest introduction: Jess (Jessatron) +00:22:48 Demo of heat maps with OpenTelemetry +00:31:12 Discussion on educational uses of heat maps +00:32:00 Guest introduction: Rhys +00:33:06 End User Working Group initiatives -**Speaker 1:** Thank you. The OpenTelemetry and Practice series is traditionally a pretty serious talk series where we, every month, go deep into implementing OpenTelemetry in a particular language. We talk to an end user who is actually implementing it in production, and we talk to usually a maintainer of the project who can talk about the best ways to implement it. This is a little bit different of a format because we decided to do three or four short talks that didn't fit well together. +## Transcript -[00:01:00] **Speaker 1:** Adriana is going to lead off today with talking about observability for HashiCorp products. She currently works at Lightstep and used to work fully in a Hashi shop. Yes. And then a little bit later we'll have Rhys talking about how end users can get involved with the project, which I expect to be highly interactive. Jess "Jessatron" is going to give us a talk about using OpenTelemetry to make ASCII art in heat maps, which is cool. You are trying to get your team to learn about OpenTelemetry and pay attention. +### [00:00:00] Introductions -**Speaker 2:** Adriana, you ready to...? +**Speaker 1:** Thank you. So the OpenTelemetry and practice series is traditionally a pretty serious talk series where every month we go deep into implementing OpenTelemetry in a particular language. We talk to an end user who is actually implementing it in production, and we talk to usually a maintainer of the project who can talk about the best ways to implement it. This is a little bit different of a format because we decided to do three or four short talks that didn't fit well together. -**Speaker 2:** Yeah, I'll start. So I'm ad hoc-ing this. As Rin mentioned, I used to work at a Hashi shop, so I have a Kubernetes background and was thrust into this HashiCorp world where my previous employer ran their containerized workloads on Nomad. I found myself in a situation where I had to quickly understand this Nomad thing because I was managing a platform team that supported Nomad, Console, and Vault for the entire enterprise. +### [00:05:30] Observability in HashiCorp products -**Speaker 2:** That's how I began my dabblings in the Hashi world. From that, I ended up becoming a HashiCorp Ambassador because I dug in, blogged about it, and continued to do so. At the same time, at this former company, I was also managing an observability team. I came to learn about observability around the same time that I came to understand Nomad. These were two things I was doing at the same time, which led me into OpenTelemetry. +**Speaker 1:** Adriana is going to lead off today with talking about observability for HashiCorp products. She currently works at Lightstep and used to work fully in a Hashi shop. Yes. Then a little bit later we'll have Rhys talking about how end users can get involved with the project, which I expect to be highly interactive. Jess "Jessatron" is going to give us a talk about using OpenTelemetry to make ASCII art in heat maps, which is cool. You are trying to get your team to learn about OpenTelemetry and pay attention. -[00:03:30] **Speaker 2:** Now that I work at Lightstep, I've had a chance to contribute more in the OpenTelemetry community. As I mentioned before, I've worked in the comms area, so I've contributed some content to the doc site. I've made some contributions to the demo app, and I was actually pleasantly surprised to learn about the demo app, which I think launched relatively recently, right? The demo app came out, and I like it because it showcases what you can do with OpenTelemetry. +### [00:01:46] Guest introduction: Adriana -**Speaker 2:** I think the original version of the demo app that came out, you could run everything in Docker Compose locally. More recently, the team created Helm charts for this one, and it got me thinking, "Hey, if this thing runs in Kubernetes, wouldn't it be super cool to get this thing to run on Nomad?" It's been part of my wishlist ever since I learned about the OpenTelemetry demo app Helm charts. I finally carved out some time to do this, and I got it up and running in a local Hashi environment on my machine. +**Speaker 1:** Adriana, you ready to? -**Speaker 2:** There's this really cool tool called HashiCube, which is similar to running Minikube or K3s or whatever, like one of these local Kubernetes dev environments—a similar sort of thing for the Hashi stack. This was created by a third-party company called Serbain. I love using this tool because it gives me a full-fledged Hashi environment, similar to what you would get, I mean obviously smaller scale, to what you would get in a data center. +**Adriana:** Yeah, I'll start. So I'm ad-hocing this. As Rin mentioned, I used to work at a Hashi shop. I actually have a Kubernetes background and was thrust into this HashiCorp world where my previous employer ran their containerized workloads on Nomad. I found myself in a situation where I had to quickly understand this Nomad thing because I was managing a platform team, which supported Nomad, Console, and Vault for the entire enterprise. -[00:05:00] **Speaker 2:** But it replicates the type of setup because in real life, Nomad by itself is kind of useless. You need the power of Console for service discovery, Vault for secrets management, along with Nomad to get all this stuff to really have a robust system to manage your containerized workloads. I have this HashiCube running on my machine locally, which I can show you. I'll do a little screen share. +**Adriana:** That's how I began my dabblings in the Hashi world. From that, I ended up becoming a HashiCorp Ambassador because I dug in, blogged about it, and continued to do so. At the same time, at this former company, I was also managing an observability team. I came to learn about observability at around the same time that I came to understand Nomad. These were two things that I was doing at the same time, learning about at the same time, which led me into OpenTelemetry. -**Speaker 2:** Well, first I'll show you my HashiCube. Oh wait, am I sharing the right one? I'm hearing your HashiCube. I have so many windows. The correct HashiCube... I don't know. Over time... Yeah, this—well, okay, so it says this is my HashiCube startup sequence. +**Adriana:** Now that I work at Lightstep, I've had a chance to contribute more in the OpenTelemetry community. As I mentioned before, I've worked in the comms area, contributed some content to the doc site, made some contributions to the demo app, and I was actually pleasantly surprised to learn about the demo app, which I think launched relatively recently. -**Speaker 2:** What it is, so it's basically using Vagrant to provision either a VM, a VirtualBox VM, or I'm on an M1 Mac, and VirtualBox and M1 Macs don't play nice. I think there's possibly a version of VirtualBox that plays nice with M1 Macs now, but at the time, it didn't. The maintainers of HashiCube basically stand up all these Hashi products in a single Docker image. +### [00:10:24] Demo of OpenTelemetry demo app -**Speaker 2:** This version of HashiCube that I'm running, where you can see the tail end of the startup sequence here, it's running Vault, Console, and Nomad all in one Docker image. If I share, I'll share my Nomad console here so you can see. The problem is when I lose sight of all my windows. +**Adriana:** I like it because it showcases OpenTelemetry, like what you can do with OpenTelemetry. I think the original version of the demo app that came out, you could run everything in Docker Compose locally. Then more recently, the team created Helm charts for this one. That got me thinking, "Hey, if this thing runs in Kubernetes, wouldn't it be super cool to get this thing to run on Nomad?" -**Speaker 2:** Oh, here we go. Okay, so this is Nomad, and these are all of the services that make up the hotel demo app running on Nomad locally on my machine, which is pretty freaking exciting. I basically had to go through the Helm chart. For the hotel demo Helm chart, there's a folder in the Helm chart repo that shows the rendered YAML. +**Adriana:** It's been part of my wish list ever since I learned about the OpenTelemetry demo app Helm charts. I finally carved out some time to do this, and I got it up and running in a local Hashi environment on my machine. There's this really cool tool called HashiCube, which is similar to running Minikube or K3s or whatever, like one of these local Kubernetes dev environments. It's a similar sort of thing for the Hashi stack, which was created by a third-party company called Serbian. -**Speaker 2:** I basically started with the rendered YAMLs and went through the process of translating these Kubernetes manifests for each service into the equivalent Nomad job spec, which is what you see here. For example, we're looking at the feedback service. This is running on Nomad. If we go over here, we can see there's a lot of stuff here. +**Adriana:** I love using this tool because it gives me a full-fledged Hashi environment, similar to what you would get in a data center, but obviously smaller scale. In real life, Nomad by itself is kind of useless, so you need the power of Console for service discovery, Vault for secrets management, along with Nomad to get all this stuff to really have a robust system to manage your containerized workloads. -[00:08:20] **Speaker 2:** We can see the logs of the true flag service. Nothing terrible is happening. Here we go. It's running. We can see from our dashboard here that all of the services are up and running. It's made up of all these different services written in different languages. Plus, we also have— I had to convert, I had to Nomadify Redis, Postgres, we had the hotel collector. What else? There was one more. There was Prometheus and Grafana, and I used Trafic for load balancing so I can expose my services to the outside world. +**Adriana:** I have this HashiCube running on my machine locally, which I can show you. I'll do a little screen share. Well, first I'll show you my HashiCube. Oh wait, am I sharing the right one? -**Speaker 2:** What that looks like then, to be able to access the hotel demo app's endpoints—and hopefully my computer won't crash here—I have an endpoint called hotel-dash-demo.localhost, which is accessible. This is just something accessible off my localhost. +**Adriana:** I have like so many windows. The correct HashiCube? I don't know. -**Speaker 2:** The Jaeger UI is exposed as part of my demo app configuration, so we see Jaeger is up and running here. We've got the demo app's UI running here, so we can buy stuff, we can check stuff out. Let's buy this telephone. Super. +**Adriana:** Okay, so this is my HashiCube startup sequence. What it is, is basically using Vagrant to provision either a VirtualBox VM, or I'm on an M1 Mac and VirtualBox and M1 Macs don't play nice. I think there's possibly a version of VirtualBox that plays nice with M1 Macs now, but at the time it didn't. -**Speaker 2:** If we check Jaeger, we can look to see—sorry, the screen sharing thing is getting in my face. We can see our traces. I think the hamsters are running really, really hard here. We can see it's produced some traces here that we can see. +**Adriana:** The maintainers of HashiCube basically stand up all these Hashi products in a single Docker image. This version of HashiCube that I'm running, where you can see the tail end of the startup sequence here, is running Vault, Console, all in one Docker image. -**Speaker 2:** In addition, when I Nomadified this, there were also some Grafana dashboards—some cool dashboards that come pre-loaded, pre-configured. You can see there's some stuff going on here with the hotel collector. Then there's also this demo dashboard which gets some stats from the recommendation service. +**Adriana:** If I share my Nomad console here, you can see the problem when I lose sight of all my windows. Oh, here we go. Okay, so this is Nomad, and these are all of the services that make up the hotel demo app running on Nomad locally on my machine, which is pretty freaking exciting. -**Speaker 2:** That's basically it. I can show you also what a job spec looks like in Nomad. I want to see more traces. +**Adriana:** I basically had to go through the Helm chart for the hotel demo Helm chart. There's a folder in the Helm chart repo that shows the rendered YAML, so I basically started with the rendered YAMLs and went through the process of translating these Kubernetes manifests for each service into the equivalent Nomad job spec, which is what you see here. -**Speaker 1:** Oh, you want to see more traces? +**Adriana:** For example, looking at the feedback service, this is running on Nomad. If we go over here, we can see there's a lot of stuff here. We can see the logs of the true flag service. Nothing terrible is happening. Here we go, it's running. -**Speaker 2:** Yeah, right. I picked the saddest trace. That's what happens when you're ad hoc-ing it. Okay, let me—let's pull up a more interesting trace. I think the recommendation service has interesting traces that we can see. Checkout's usually good. +**Adriana:** We can see from our dashboard here that all of the services are up and running. It's made up of all these different services written in different languages. Plus, I also had to convert, I had to "nomadify" Redis, Postgres, we had the hotel collector, and there was one more. There was Prometheus and Grafana, and I used Traefik for load balancing so I can expose my services to the outside world. -**Speaker 2:** Cool. Let's find some traces. Oh yeah, look at all those! Look at all those services! Yeah, there's some colors! Look at it go to all the different services! +**Adriana:** What that looks like then, to be able to access the hotel demo app's endpoints, and hopefully my computer won't crash here, I have an endpoint called hotel-dash-demo.localhost, which is accessible. This is just something accessible off my localhost. -**Speaker 2:** This is actually pretty cool. You're right, I picked the dinkiest to showcase all this, right? And all this is happening on my machine in this Docker image that's running all the Hashi things, which is super cool. +**Adriana:** The Jaeger UI is exposed as part of my demo app configuration, so we see Jaeger is up and running here. We've got the demo app's UI running here, so we can buy stuff, we can check stuff out. Let's buy this telephone. -**Speaker 2:** I did want to show also what a Nomad job spec looks like. Let me just share my screen for that. So Nomad is like—it's an alternative to Kubernetes, right? +**Adriana:** If we check Jaeger, we can look to see... Sorry, the screen sharing thing is getting in my face. We can see our traces. I think the hamsters are running really, really hard here. We can see it's produced some traces here that we can see. -**Speaker 1:** Exactly, exactly. +**Adriana:** In addition, when I modified this, there are also some Grafana dashboards, so some cool dashboards that come pre-loaded, pre-configured. You can see there's some stuff going on here with the hotel collector, and there's also this demo dashboard which gets some stats from the recommendation service. -**Speaker 2:** Yeah, and it's interesting too because it doesn't just run containerized workloads. It can run VM workloads; it can even run a JVM or IIS. So it's kind of cool that way. +**Adriana:** That's basically it. I can show you also what a job spec looks like in Nomad. -**Speaker 2:** Here we go. Sorry, I was looking for the one. Okay, so let me pick—let's hear it. A job spec is like a Kubernetes deployment? +**Adriana:** I want to see more traces. -**Speaker 1:** Yeah, exactly. +**Adriana:** Oh, you want to see more traces? Yeah, right. I picked the saddest trace. Yeah, that's what happens when you're ad-hocing it. Okay, let me pull up a more interesting trace. I think the recommendation service has interesting traces that we can see. Checkout's usually good. -**Speaker 2:** So here, I've got a Kubernetes deployment. I'll put it side by side here. Alright, cool. Okay, so this is the feature flag service on the right. We've got the feature flag service definition. This is our Nomad job spec. +**Adriana:** Cool, let's find some traces. -**Speaker 2:** Unlike Kubernetes, where you have all these different YAML files that make up the various objects that make up your manifest, everything is self-contained in one job spec file, which is written in HCL—HashiCorp Configuration Language. +**Adriana:** Oh yeah, look at all those services. Yeah, there's some colors. Look at it go to all the different services. Yeah, this is actually pretty cool. You're right, I picked the dinkiest to showcase all this. -**Speaker 2:** I liken it to, I mean if you've used Terraform, this looks familiar. I find HCL is like if JSON was slightly improved—that's what it would look like. I do find it's easier to read than JSON. But this is basically what it looks like here. +**Adriana:** All this is happening on my machine in this Docker image that's running all the Hashi things, which is super cool. I did want to show also what a Nomad job spec looks like. Let me just share my screen for that. -**Speaker 2:** For example, over here in our network definition, you see that we define two ports: 8081 and 50053, which lo and behold, we defined those over here in our service YAML for the feature flag service. +**Adriana:** Nomad is like an alternative to Kubernetes, right? -**Speaker 2:** If we scroll over to the task definition, we're saying that we're using the Docker driver. This is saying, "Okay, we're using a containerized workload." Let's find the equivalent in the manifest. +**Adriana:** Exactly. It's interesting too because it doesn't just run containerized workloads; it can run VM workloads. It can even run like a JVM or IIS. So it's kind of cool that way. -**Speaker 2:** Alright, here we go to our deployment, and we can see where we define our image. I've defined an image pull timeout because sometimes my network doesn't like to behave, so I don't want Nomad to crap out after five minutes and say, "Sorry, I can't pull your image, sucker," and then fail my deployment. +**Adriana:** Here we go. Sorry, I was looking for the one. Okay, so let me pick... -**Speaker 2:** Over here, we've got the ports configuration, basically saying—well, just like here, we're saying that this container requires these two ports. Well, on the left side here, in our HCL, we see that our HTTP and gRPC ports that we defined up here, that's what they're pointing to. +**Adriana:** A job spec is like a Kubernetes deployment? -[00:12:01] **Speaker 2:** The other thing I wanted to point out was over here in this end definition—these are our environment variables, which for the most part correspond to the ones that we define in our deployment. I did exclude some, like these Kubernetes ones, because obviously we're not running this in Kubernetes; we're running them in Nomad. +**Adriana:** Yeah, exactly. So here I'll... I've got a Kubernetes deployment. I'll put it side by side here. -[00:15:30] **Speaker 2:** But the non-Kubernetes ones, I translated over to Nomad land. I wanted to point out also that over here in what's called the template stanza, these groupings and the curly braces are called the stanzas. I'm defining two additional environment variables, but they're defined in a slightly different way because they're referencing services from other Nomad job specs. +**Adriana:** All right, cool. Okay, so this is the feature flag service. On the right, we've got the feature flag service definition. This is our Nomad job specs. Unlike Kubernetes where you have all these different YAML files that make up the various objects for that make up your manifest, everything is self-contained in one job spec file which is written in HCL, which is HashiCorp Configuration Language. -**Speaker 2:** To be able to reference, for example, our database service, to get the information, the IP address and the port of the database service, we can't hard code that information, right? Because that stuff can potentially change, especially the IP. +**Adriana:** I liken it to... I mean, if you've used Terraform, this looks familiar. I find HCL is like if JSON was slightly improved. That's what it would look like. I do find it's easier to read than JSON. But this is basically what it looks like here. -**Speaker 2:** In order to get that information to define this environment variable, we basically have to look up this service in Console, which, as I mentioned, is for service discovery. To find out what the information of that service is, you refer to it by service name. +**Adriana:** For example, in our network definition, you see that we define two ports, 8081 and 50053, which lo and behold we defined those over here in our service YAML for the feature flag service. -**Speaker 2:** If we open over here, I'm going to open this FF Postgres. This is the job spec for Postgres. I have a service named FF Postgres service which, if we look over here—there we go—basically says, "Hey Console, I want to pull up a service called FF Postgres service, and pretty please tell me the address and port number of the service so I can plug it into this environment variable." +**Adriana:** If we scroll over to this task definition, we're saying that we're using the Docker driver. This is saying, "Okay, we're using a containerized workload. Let's find the equivalent in the manifest." -**Speaker 2:** We use the template stanza to do this kind of dynamic definition, and we have to tell it that the destination is going to be an environment variable. One of the things you can do with the template stanza is use it for configuration files. +**Adriana:** Here we go to our deployment, and we can see where we define our image. I've defined an image pull timeout because sometimes my network doesn't like to behave, so I don't want Nomad to crap out after five minutes and say, "Sorry, I can't pull your image, sucker," and then fail my deployment. -**Speaker 2:** So similarly, very similar functionality to a config map in Kubernetes. Then here is where we define our resources. This is in megahertz for CPU, and our memory is in megabytes. +**Adriana:** Over here we've got the ports configuration basically saying, well, just like here we're saying that this container requires these two ports. Well, on the left side here in our HCL, we see that our HTTP and gRPC ports that we defined up here, that's what they're pointing to. -**Speaker 2:** The other thing I wanted to point out is here I've got some rules on restarting the service. There are no dependencies that you can define in your services in Nomad, like you know how you would do in Docker Compose, saying, "Hey, this service is dependent on that service." You don't have that kind of dependency definition in Nomad. +**Adriana:** The other thing that I wanted to point out was over here in this template stanza. Basically, these groupings and the curly braces, they call them the stanzas. I'm defining two additional environment variables, but they're defined in a slightly different way because they're referencing services from other Nomad job specs. -**Speaker 2:** Basically, the jobs start when they start, which means that, for example, this feature flag service is actually relying on the database and the hotel collector to be up and running. Well, what if the feature flag service starts before the hotel collector and the database startup? +**Adriana:** Here, to be able to reference, for example, our database service to get the information, so the IP address and the port of the database service, we can't hard code that information because that stuff can potentially change, especially the IP. -**Speaker 2:** If we don't put some restart rules in place, then when it starts up and these two services aren't up, or one of these two services isn't up, it'll crap out and then that's it—it's dead. So what we want to do is put some restart rules in place, basically saying within the span of two minutes, we're going to try to restart this thing 10 times with a 15-second interval between restarts. +**Adriana:** In order to get that information to define this environment variable, we have to use... We basically have to look up this service in Console, which, as I mentioned, is for service discovery. -**Speaker 2:** Either after 10 attempts or two minutes, whichever comes first, if this fails to restart, we basically say, "Okay, we're just gonna wait a little bit and reattempt that again." The default mode for this normally is fail. If it doesn't successfully restart within 10 attempts in two minutes, it would completely fail. That means your whole application deployment fails, right, because of all these dependent services. +**Adriana:** To find out what the information of that service is, you refer to it by service name. So if we open over here, I'm going to open this FF Postgres. This is the job spec for Postgres. I have a service named FF Postgres service, which if we look over here... Here we go. -**Speaker 2:** This gives you some protection saying, "Okay, we'll keep trying. We'll keep trying until things are up and running," which ends up being very convenient. +**Adriana:** It basically says, "Hey Console, I want to pull up a service called FF Postgres service, and pretty please tell me the address and port number of the service so I can plug it into this environment variable." -**Speaker 2:** The final thing I wanted to point out was that we also have these checks here. These are basically health checks. They're similar to the types of health checks that you would see in Kubernetes, like liveness probes and readiness probes. +**Adriana:** We use the template stanza to do this kind of dynamic definition, and we have to tell it that the destination is going to be an environment variable because one of the things that you can do with the template stanza as well is you can use it for configuration files. So similarly, it's very similar functionality to a config map in Kubernetes. -**Speaker 2:** I wanted to show you what that looks like in Console really briefly. Let me just pull that up. +**Adriana:** Here is where we define our resources. This is in megahertz for CPU, and our memory is in megabytes. The other thing that I wanted to point out is here I've got some rules on restarting the service. -**Speaker 1:** Yeah, and we're going to move on to Jess Citron after the Console showing, so if you have more, Rihanna, think them up, please. +**Adriana:** Because there's no dependencies that you can define in your services in Nomad, like you know how you would do in Docker Compose saying, "Hey, this service is dependent on that service?" You don't have that kind of dependency definition in Nomad. -**Speaker 2:** Yes! I'll show this very briefly, but this basically shows all of our services. If we look at the feature flag service, we have two services, one for our gRPC and one for our REST API. +**Adriana:** Basically, the jobs start when they start, which means that, for example, this feature flag service is actually relying on the database and the hotel collector to be up and running. Well, what if the feature flag service starts before the hotel collector and the database startup? -**Speaker 2:** This shows us that, hey, we're able to hit our endpoints, so it sends a signal back to Nomad like, "All systems go! Hurray!" That's basically it in a nutshell. +**Adriana:** If we don't put some restart rules in place, then when it starts up and these two services aren't up or one of these two services isn't up, it'll crap out, and then that's it. It's dead. -**Speaker 1:** Okay, follow-up questions? Last parting thoughts about— +**Adriana:** What we want to do is put some restart rules in here, basically saying within the span of two minutes, we're going to try to restart this thing ten times with a 15-second interval between restarts. -**Speaker 2:** Nice job! +**Adriana:** Either after ten attempts or two minutes, whichever comes first, if this fails to restart, we're basically going to say, "Okay, we're just going to wait a little bit and reattempt that again." The default mode for this is fail. -**Speaker 1:** Yeah, I agree. Nice job! That was a super good demo and a good example of how to use the demo app. +**Adriana:** If it doesn't successfully restart in ten attempts within two minutes, it would completely fail, and that means your whole application deployment fails because of all these dependent services. -[00:21:10] **Speaker 2:** So Jess "Jessatron" is actually going to show us something totally useless but super fun for training people on how to make ASCII art into a heat map using OpenTelemetry, right? +**Adriana:** This gives you some protection saying, "Okay, we'll keep trying, we'll keep trying until things are up and running," which ends up being very convenient. -**Speaker 2:** Right! So I made this thing for Christmas, I guess for Christmas gimmicks. It's in a repo, honeycombio/happy-ollie-days, which is cute. You can clone this repo or run it and get pod, and if you give it your Honeycomb API key—this is Honeycomb. The UI is Honeycomb-specific because it had to be specific to the display, but it does use OpenTelemetry. +**Adriana:** The final thing that I wanted to point out was that we also have these checks here. These are basically health checks. They're similar to the types of health checks that you would see in Kubernetes, like liveness probes and readiness probes. -**Speaker 2:** I'm going to give it a Honeycomb API key. Oh, it's still in my paste buffer. Great! And then I'm going to run this little app, and this app is in Node, TypeScript. So it has created a bunch of spans in a trace using—where's my—here we go—start active. +### [00:20:00] Transition to Jess's presentation -**Speaker 2:** So it does a start active span in Node, and for each, it's figured out what spans it wants to send, and then it does a start span for each of them and supplies a bunch of attributes, and then ends each one. They all go into one trace. +**Adriana:** I wanted to show you actually what that looks like in Console really briefly. Let me just pull that up. -**Speaker 2:** It's given me a link to the datasets. Oh, it's a good thing I'm following that link because apparently, my dataset is called Fufu Free. But then in the readme, it describes how to do this. If you do a heat map on the height field and you get the last 10 minutes, you're starting to see something. +**Adriana:** Yeah, and we're going to move on to Jess Citron after the console showing. If you have more, Adriana, think them up, please. -**Speaker 2:** But then I need to pick graph—no, no, no, up here—granularity five seconds. Oh, that's so cute! Isn't it cute? And the cool part is that you can use this program. I mean, I don't know what it's going to look like in any other system because I've customized the heights to the bucket sizes that Honeycomb uses in heat maps, so you'd have to tweak the code to make it do something in somebody else's heat maps. +**Adriana:** Yes, I'll show this very briefly, but this basically shows all of our services. If we look at the feature flag service, we have two services, one for our gRPC and one for our HTTP. -**Speaker 2:** Oh, but it actually pulls this out of—where is it? Images? Now there should be an images directory in here. Oh, input—don't peak.png. So if you give it a PNG that's between 25 and 50 pixels tall, and you probably want it between 1 and 200 pixels wide, then it can convert that into a heat map. +**Adriana:** This shows us that, hey, we're able to hit our endpoints, so it sends a signal back to Nomad like "all systems go, hurray hurray." -**Speaker 2:** Oh, it's converting the heat map based on the blue channel in the PNG. It uses the other one to get some attributes, which does some fun things in Honeycomb because you can pick bubble up and be like, "What is different about...?" +**Adriana:** That's basically it in a nutshell. -**Speaker 2:** Oh, check this out! Okay, my new favorite feature of Honeycomb—wow! That stupid little do-me-anything is gone! Okay, which is very useful sometimes, I'm sure, but not in here. +**Speaker 1:** Follow-up questions? Last parting thoughts? -**Speaker 2:** Now that I've gotten rid of that, what is different about the spans in the second reindeer? Honeycomb does its little "What is different?" analysis, and it says the reindeer name that one has a reindeer name of Dancer. +**Audience Member:** Nice job! -**Speaker 2:** Then if you group by the reindeer name, you can go back and like, "Oh, oh, wait, hold on. I've got to do the trick of—wait, wait, give me the right 10 minutes." +**Speaker 1:** Yeah, I agree. Nice job! That was a super good demo and a good example of how to use the demo app. -**Speaker 2:** It wants to like—the Santa and his reindeer are like moving on. I need to switch to absolute time. Okay, now let's group by reindeer name. Okay, now it's lost the granularity, so let's fix that granularity—five seconds. +**Jess:** So just, Jessatron is actually going to show us something totally useless but super fun for training people or teaching how to make ASCII art into a heat map using OpenTelemetry. -**Speaker 2:** Okay, but now—results tab, and we have the different reindeer names. So there's no reindeer name. This one's Dancer, and this one is Rudolph, and this one is Prancer. +### [00:21:36] Guest introduction: Jess (Jessatron) -**Speaker 2:** Yeah, so it's cute that way. It's got different fields, and those are based—in case you want to use this yourself and supply your own image—those are based on the red channel in the PNG, and then there's a little key for how much red between 0 and 255 is in that color and what the fields are. +**Jess:** Right, right. I made this thing for Christmas, for Christmas gimmicks, I guess. I don't know. It's in a repo, honeycombio/happy-ollie-days, which is cute. You can clone this repo or run it, and if you give it your Honeycomb API key... -**Speaker 2:** So that's also encoded in a PNG, and you can do it yourself. This gets officially released on Monday, and it'll be advertised and stuff, but I'm not—none of that stuff am I telling you how to fix it yourself, so y'all are getting that. +**Jess:** This is Honeycomb. The UI is Honeycomb specific because it had to be specific to the display, but it does use OpenTelemetry. I'm going to give it a Honeycomb API key. Oh, it's still in my paste buffer. Great. -**Speaker 2:** There's one other trick in this data, which is if you do a max of stack heights and group by stack group, then you get some nonsense, but if you change the graph settings to use a stack graph, then you start to get something. +**Jess:** Then I'm going to run this little app. This app is in Node, TypeScript, so it has created a bunch of spans in a trace using... where's my... here we go. Start active. -**Speaker 2:** If you get the order by right, stack group descending—come on, do it! A Christmas tree! Oh my God, that's awesome! And you can also generate yourself because that is based on house.png, obviously. +**Jess:** It does a start active span in Node, and then for each... it's figured out what spans it wants to send, and then it does a start span for each of them and supplies a bunch of attributes, and then ends each one. They all go into one trace. -**Speaker 2:** It used to be a house; now it's a Christmas tree! And of course, there are limitations. You have to have all the colors on top of one another, and you can't have a color that's both under and over, and you don't control what the colors actually show up as. +### [00:22:48] Demo of heat maps with OpenTelemetry -**Speaker 2:** But if you're clever, you can make your own PNGs. Oh yeah, I've got some groups on that you can group by. It is great by stack group. That wasn't necessary, but here's the names of them—star and background and tree and stuff like that. +**Jess:** It's given me a link to the datasets. Oh, it's a good thing I'm following that link because apparently my dataset is called Fufu Free, but then in the readme, it describes how to do this. -**Speaker 1:** Yes, where do people go to find this app? +**Jess:** If you do a heat map on the height field and you get the last 10 minutes, you're starting to see something. But then I need to pick graph... no, no, up here. Granularity five seconds. -**Speaker 2:** It is at honeycomb.io/happy-ollie-days. You can download this, and if you have a free Honeycomb account, or if you get it to work in another tool, that would be awesome. I would love to hear about it. +**Jess:** Oh, that's so cute! Isn't it cute? The cool part is that you can use this program. I mean, I don't know what it's going to look like in any other system because I've customized the heights to the bucket sizes that Honeycomb uses in heat maps, so you'd have to tweak the code to make it do something in some other heat maps. -**Speaker 2:** Oh, thanks, Johnny! Yeah, that's that. We'll find stop share. Maybe not. Daniel's like, "I'm running, not walking, to try this out!" It's fun! It's true! +**Jess:** Oh, but it actually pulls this out of... where is it? Images now? There should be an images directory in here. Oh, input. Don't peak.png. So if you give it a PNG that's between 25 and 50 pixels tall and you probably want it between 1 and 200 pixels wide, then it can convert that into a heat map. -**Speaker 1:** I have a question, which is how do you think this can be useful to people as a tool to teach about heat maps and observability? +**Jess:** Oh, it's converting the heat map based on the blue channel in the PNG. It uses the other one to get some attributes, which does some fun things in Honeycomb because you can pick bubble up and be like, "What is different about... oh, oh, check this out!" -[00:29:00] **Speaker 2:** Yeah, because you can—I mean, you can ask, "How do I draw it?" And the answer is that I send a different number of spans, which, of course, is it this one? Which, of course, I also stick all the intermediate data on the spans because that's what I debug it. +**Jess:** Okay, my new favorite feature of Honeycomb... wow, that stupid little... do me anything is gone! Okay, which is very useful sometimes, I'm sure, but not in here. -**Speaker 2:** No, that's clearly the wrong field. Show me the trace. I haven't made the trace cute yet. Someday, I'll make this have a drawing in it too. +**Jess:** Now that I've gotten rid of that, what is different about the spans in the second reindeer? Honeycomb does its little "what is different" analysis, and it says the reindeer name... that one has a reindeer name of Dancer. -**Speaker 2:** Span, you know, count amount—how many? Oh well, I send multiple spans per pixel if I want a dark color because Honeycomb says darker is more and lighter is fewer, so there's just one span in these. +**Jess:** Then if you group by the reindeer name, you can go back and like, oh, oh, wait, hold on. I've got to do the trick of... wait, wait, give me the right 10 minutes. -**Speaker 2:** Oh, you know what? I can find that field if I do "Bubble Up—what's different about these that only have one span at a time?" There it is! Okay, so this spans at once; these only have one span at a time. +**Jess:** It wants to... like the Santa and his reindeer are moving on. I need to switch to absolute time. Okay, now let's group by reindeer name. -**Speaker 2:** Some of the darkest ones have 10 spans at a time. If you think, if you can use it to be like—how does a heat map? Mostly, it's just entertaining, and it does really illustrate bubble up. +**Jess:** Okay, now it's lost the granularity, so let's fix that granularity. Five seconds. Okay, but now... results tab, and we have the different reindeer names. -**Speaker 2:** Letter—all of these, most of these are letters, and some of them have different letters. +**Jess:** There's no reindeer name. This, there's Dancer. This one is Rudolph, and this one is Prancer. -**Speaker 2:** Yeah, so Bubble Up is pretty well illustrated by this, I think. +**Jess:** Yeah, so it's cute that way. It's got different fields, and those are based... in case you want to use this yourself and supply your own image, those are based on the red channel in the PNG. -**Speaker 1:** I want to say what Bubble Up is for folks who aren't Honeycomb users. Bubble Up is Honeycomb's "What is different?" +**Jess:** There's a little key for how much red between 0 and 255 is in that color and what the fields are. So that's also encoded in a PNG, and you can do it yourself and make your own... this gets officially released on Monday, and it'll be advertised and stuff, but I'm not... none of that stuff am I telling you how to fix it yourself, so y'all are getting that. -**Speaker 2:** Exactly! +**Jess:** There's one other trick in this data, which is if you do a max of stack heights and group by stack group, then you get some nonsense. But if you change the graph settings to use a stack graph, then you start to get something. -**Speaker 1:** So you can draw the box, and it does the statistical analysis on what fields are different. It's just cute! +**Jess:** If you get the order right, stack group descending... come on, do it! A Christmas tree! -**Speaker 2:** That was super good! There's not question three. Do you want to go ahead and talk about end-user working group work? +**Jess:** Oh my God, that's awesome! You can also generate yourself because that is based on house.png, obviously. It used to be a house; now it's a Christmas tree. -**Speaker 1:** I mean, yes, but also, how do I follow up? We're doing systematic and important work and slowly marching through things. +**Jess:** Of course, there's limitations. You have to have all the colors on top of one another, and you can't have a color that's both under and over, and you don't control what the colors actually show up as. But if you're clever, you can make your own PNGs. -**Speaker 2:** Jessica, to do the fun creative stuff! +**Jess:** Oh yeah, I've got some groups on that. You can group by... oh, it is great by stack group. That wasn't necessary, but here's the names of them: star and background and tree and stuff like that. -**Speaker 1:** Yes! This is—I mean, I have some like one or two fun graphics, but it's not going to be anything as cool as Jess "Jessatron." +**Speaker 2:** Yes, where do people go to find this app? -**Speaker 2:** So you'll have to make two! +**Jess:** It is at honeycomb.io/happy-ollie-days. You can download this, and if you have a free Honeycomb account, or if you get it to work in another tool, that would be awesome. I would love to hear about it. -**Speaker 1:** Yeah, I just wanted to talk real quick on something that we've surfaced just from talking to a wide variety of end users. A lot of people are not really aware of how to navigate the community on how they can get involved, or that they even can get involved. +**Speaker 3:** Oh, thanks, Johnny! -**Speaker 1:** So I'm not sure—I know there are a couple contributors on the call. I'm not sure everyone else is an end user, but hopefully this will be of some utility for you. +**Jess:** Yeah, so that's that. -**Speaker 1:** So a couple of things we'll go over: this time of the community at large, some of the different parts of it, and then we'll talk about how you can get involved if you're interested in some of the different ways that you can get involved. +**Speaker 3:** We'll find stop share. Maybe not. -**Speaker 1:** We've mentioned a few times the end user working group that a lot of us are a part of. What is the end-user working group? We have two primary goals: one is to foster a sense of vendor-agnostic community for end users, and two is to create a feedback loop between the end users and project maintainers with the overarching goal of improving the project software. +**Speaker 4:** Daniel's like, "I'm running, not walking, to try this out." -**Speaker 1:** What are some of the working group activities that we have implemented? So first of all, this is one of them. This is Rin's child, but we are very excited—OpenTelemetry in Practice is what it is. So every month, keep an eye out for more fun talks, presentations, and casual conversations. +**Jess:** It's fun. It's true. And I have a question, which is how do you think this can be useful to people as a tool to teach about heat maps and observability? -**Speaker 1:** We have the monthly discussion groups now with a maintainer per session, and in all regions—so America, EMEA, which I just found out stands for Europe, Middle East, and Africa, and APAC, which I'm actually not sure where those things were, but it's Asia and the Pacific region. Imagine where AC stands for! +**Jess:** Because you can... I mean, you can ask, like, "How do I draw it?" And the answer is that I send a different number of spans, which, of course, is this one, which of course I also stick all the intermediate data on the spans because that's what I debug it. -**Speaker 1:** Those are all on the OpenTelemetry public calendars. We also have end-user interview and feedback sessions where we will talk to an end user about their adoption, implementation, and challenges that they face, and get the feedback shared back to the OpenTelemetry maintainers. +**Jess:** No, that's clearly the wrong field. Show me the trace. I haven't made the trace cute yet. Someday I'll make this have a drawing in it too. -**Speaker 1:** If you're interested, if you're an end user and you're interested in sharing feedback in one of these sessions, please reach out to myself, Rin, or Adriana. We would be happy to get you on the schedule. +**Jess:** Span count... how many... oh well. I send multiple spans per pixel if I want a dark color because Honeycomb says darker is more and lighter is fewer, so there's just one span in these. -**Speaker 1:** We are also new—for end-user interviews moving forward, we're going to turn them into profiles for the OpenTelemetry blog to make the implementation and adoption that other end users have done in their organizations more discoverable. +**Jess:** You know what? I can find that field if I do Bubble Up. What's different about these that only have one span at a time? There it is! -**Speaker 1:** We also have a community survey, so that's another way that you can contribute. If you don't really want to get too involved but you still want to have a way to share your thoughts and opinions on how using OpenTelemetry is for you. +**Jess:** Okay, so these only have one span at a time, and some of the darkest ones have ten spans at a time. -**Speaker 1:** Some of the stuff that's upcoming for the working group: we want to extend our user study function, and also the governance committee is going to work to implement a project management function for the specifications SIG to streamline the feedback loop from all the feedback that we're gathering at these sessions and activities and drive prioritization for work based on user feedback. +**Jess:** If you think... if you can use it to be like, "How does a heat map..." Mostly it's just entertaining, and it does really illustrate like bubble up. -[00:36:00] **Speaker 1:** In-person meetups might be a thing coming to a town near you. Maybe we'll see. I think this would be a really fun thing to do, especially as a lot of us were just at KubeCon, and it seems like people are pretty into meeting in person again, so we’ll see. +**Jess:** All of these, most of these are letters, and some of them have... oh, they have different letters. -**Speaker 1:** Six—probably most of you are maybe fully familiar with these, but I'll just go over them. Special interest groups: the goal is to improve the workflow and manage the project more efficiently. Each SIG meets regularly. You can access the meeting notes and recordings through the public calendar, and that's pretty much a thing for pretty much all components, languages of OpenTelemetry. +**Jess:** Yeah, so Bubble Up is pretty well illustrated by this, I think. -**Speaker 1:** There's also a governance committee and a technical committee, which I won't get too deep into. You can kind of see the roles of each committee here. +**Speaker 5:** I want to say what Bubble Up is for folks who aren't Honeycomb users. Bubble Up is Honeycomb's "what is different," so you can draw the box and it does the statistical analysis on what fields are different. -**Speaker 1:** Show me for a second, and I can also share the slide deck too because there are some links to the resources that I mentioned in here. +**Jess:** It's just cute! -**Speaker 1:** Documentation—yes, we have a lot of questions about documentation. If you do have questions, you can always talk to the COMSIG at hotel-dash.coms channel. +**Speaker 5:** That was super good. -**Speaker 1:** Some languages we are aware are a little bit more comprehensive than others. There is standardization and improvement work in progress at the moment. If you would like to help contribute, feel free to jump into the channel, attend one of their meetings, or open an issue directly in the repo. +**Speaker 6:** There are no questions? -**Speaker 1:** I personally find it kind of interesting to see what OTEPs have been proposed. They are open selling to enhancement proposals. It's a process for proposing changes to the spec, and they have to be cross-cutting changes that introduce new behavior or otherwise modify requirements. +### [00:31:12] Discussion on educational uses of heat maps -**Speaker 1:** It's kind of fun, personally, I think, to go and look at what people are proposing or wanting to do. There’s some interesting stuff in there. +**Speaker 7:** Do you want to go ahead and talk about end user working group work? -**Speaker 1:** Also, getting involved, the first one I want to cover is how do I get help using OpenTelemetry. There are multiple ways: CNCF Slack—you have to sign up for an account with the CNCF Slack instance. +**Rhys:** I mean, yes, but also how do I follow up? We're doing systematic and important and slowly marching through things. -**Speaker 1:** But once you get in there, there’s pretty much an ozone channel for whatever it is you’re looking for—languages, components, collector, etc. There are also vendor-specific channels, or you can go to the general OpenTelemetry vendor channel if you're having problems with a specific vendor that you're using. +**Speaker 6:** Jessica, do the fun, creative stuff. -**Speaker 1:** And of course, GitHub—you can open issues, jump in with comments on anything that's open. And we also have the end-user discussion groups, which I mentioned earlier. If you want to join and ask questions or share how you're using OpenTelemetry or help other people who are using OpenTelemetry in their organizations, that's another great way to get involved. +**Rhys:** Yes, this is... I mean, I have some like one or two fun graphics, but it's not going to be anything as cool as Jess's. -**Speaker 1:** As far as contributions, we welcome any and all code and incoming contributions. If there's a specific language you're interested in or a specific component, go check out the SIG notes, see what they're talking about, or you can hop into any of the meetings and just kind of check it out. +### [00:32:00] Guest introduction: Rhys -**Speaker 1:** Blog posts could be something that you know—it could be something like fun and so-called useless, but really, you know, if it brings so much joy, is it really useless? +**Rhys:** I just wanted to talk real quick on something that we've surfaced just from talking to a wide variety of end users. A lot of people are not really aware of how to navigate the community on how they can get involved or that they even can get involved. -**Speaker 1:** So something fun that Jess did could be anything OpenTelemetry-related. Basically, we would love to see it. +**Rhys:** I'm not sure. I know there's a couple contributors on the call. Not sure everyone else's end user, but hopefully this will be of some utility for you. -**Speaker 1:** As I mentioned, documentation—you are welcome to join the end user working group and give suggestions on things you’d like us to help with or do for you as an end user. +**Rhys:** A couple things we'll go over: this time of the community at large, some of the different parts of it, and then we'll talk about how you can get involved if you're interested in some of the different ways that you can get involved. -**Speaker 1:** Of course, you can also just share feedback, and there are different ways to do that. If you don't really want to get engaged with an interview, you are welcome to take part in our survey, which I thought I linked somewhere. +### [00:33:06] End User Working Group initiatives -**Speaker 1:** I have linked the survey in here. If you would like to share feedback that way, that is great. If you would like to participate in the end-user working group by way of the discussion group or end-user interview, we would be very happy to have you. +**Rhys:** We've mentioned a few times the end user working group that a lot of us are a part of. What is the end user working group? We have two primary goals. One is to foster a sense of vendor-agnostic community for end users, and two is to create a feedback loop between the end users and project maintainers, with the overarching goal of improving the project software. -**Speaker 1:** And that's it! That's all I have. Thank you so much. I should have added more Jingle Bells and Christmas stuff. I feel not very festive, but I promise I am! +**Rhys:** What are some of the working group activities that we have implemented? So first of all, this is one of them. This is Rin's child. But we are very excited. "Hotel in Practice" is what it is, so every month keep an eye out for more fun talks, presentations, and casual conversations. -**Speaker 2:** Fair! +**Rhys:** We have the monthly discussion groups now with a maintainer per session and in all regions: America, EMEA (which I just found out stands for Europe, Middle East, and Africa), and APAC (which I'm actually not sure where those things were, but it's Asia and the Pacific region). -[00:41:01] **Speaker 1:** Justice! Thank you, Rhys! And saying that, for those not following the chat, Johnny's saying they participate in some conferences related to DevOps, and they like to share about the power of observability and how OpenTelemetry can help us make our lives easier. +**Rhys:** Imagine where AC stands for. Those are all on the OpenTelemetry public calendars. We also have end user interview and feedback sessions where we will talk to an end user about their adoption, implementation, and challenges that they face and get the feedback shared back to the OpenTelemetry maintainers. -**Speaker 1:** Do folks have questions? Since we've got 10 minutes, we're happy to entertain if you have random questions about OpenTelemetry. We may or may not be able to answer them, but there’s lots of knowledge in this room. +**Rhys:** If you're interested, if you're an end user and you're interested in sharing feedback in one of these sessions, please reach out to myself, Rin, or Adriana. We would be happy to get you on the schedule. -**Speaker 2:** Yeah! And if anyone is having specifically a question about Rhys's presentation— +**Rhys:** We are also new for end user interviews. Moving forward, we're going to turn them into profiles for the OpenTelemetry blog to kind of make the implementation and adoption that other end users have done in their organizations more discoverable. -**Speaker 1:** Absolutely. +**Rhys:** We also have a community survey, so that's another way that you can contribute if you don't really want to get too involved but you still want to have a way to share your thoughts and opinions on how using OpenTelemetry is for you. -**Speaker 2:** Well, actually, I was going to say about OpenTelemetry.net. We have a maintainer on the line. Is that Alan? +**Rhys:** Some of the stuff that's upcoming for the working group: we want to extend our user study function, and also the governance committee is going to work to implement a project management function for the specifications SIG to streamline the feedback loop from all the feedback that we're gathering at these sessions and activities and drive prioritization for work based on user feedback. -**Speaker 1:** Yeah! +**Rhys:** In-person meetups might be a thing coming to a town near you. Maybe we'll see. I think this would be a really fun thing to do, especially as a lot of us were just at KubeCon, and it seems like people are pretty into meeting in person again, so we'll see. -**Speaker 2:** Yes! There are two—Mike Blanchard! +**Rhys:** Six, probably most of you are maybe fully familiar with these, but I'll just go over them. Special interest groups: the goal is to improve the workflow and manage the project more efficiently. Each SIG meets regularly. You can access the meeting notes and recordings through the public calendar. -**Speaker 1:** Hello, Mike! Very important spot! +**Rhys:** That's pretty much a thing or working group for pretty much all components. Languages of OpenTelemetry. There's also a governance committee and a technical committee, which I won't get too deep into. -**Speaker 2:** That's why I'm like, "It's okay if I call him out!" I will say that I tried to pull down the Honeycomb. +**Rhys:** You can kind of see the roles of each committee here. Show me for a second, and I can also share the slide deck too because there are some links to the resources that I mentioned in here. -**Speaker 1:** The Happy Holidays happy hour! +**Rhys:** Documentation? Yes, we have a lot of questions about documentation. If you do have questions, you can always talk to the ComSIG at hotel-dash.comms channel. Some languages we are aware are a little bit more comprehensive than others. -**Speaker 2:** Oh, it is! It's going to take a little bit of work, but I was going to try to see how easy it would be to get that reporting into a different tool than New Relic and see what it would look like. +**Rhys:** There is standardization and improvement work in progress at the moment. If you would like to help contribute, feel free to jump into the channel, attend one of their meetings, or open an issue directly in the repo. -**Speaker 1:** I'm sure it would look probably amazing! +**Rhys:** I personally find it interesting to see what OTEPs have been proposed. They are open selling to enhancement proposals. It's a process for proposing changes to the spec, and they have to be cross-cutting changes that introduce new behavior changes or otherwise modify requirements. -**Speaker 2:** It will look mangled initially! +**Rhys:** It's kind of fun personally, I think, to go and look at what people are proposing or wanting to do. There's some interesting stuff in there. -**Speaker 1:** If you want to work at that, I'm happy to work with you on—it’s I know where in the code it's choosing the height that works, and I also know how I use the browser tools in Honeycomb to figure out what height it was needing. +**Rhys:** Getting involved: the first one I want to cover is how do I get help using OpenTelemetry? There are multiple ways. CNCF Slack, you have to sign up for an account with the CNCF Slack instance. But once you get in there, there’s pretty much an ozone channel for whatever it is you’re looking for: languages, components, collector, etc. -**Speaker 2:** Some of those tricks will also work in New Relic, so happy to work with you on that if you want because I would love to see what it looks like somewhere else. +**Rhys:** There are also vendor-specific channels, or you can go to the general OpenTelemetry vendor channel if you're having problems with a specific vendor that you're using. -**Speaker 1:** I don't know if they're still doing them, but last year Grace and I were doing a lot of Legos for New Relic. I've got some of them here, and I wonder if they’re like amazing art! +**Rhys:** Of course, GitHub, you can open issues, jump in with comments on anything that's open. We also have the end user discussion groups, which I mentioned earlier. If you want to join and ask questions or share how you're using OpenTelemetry or help other people who are using OpenTelemetry in their organizations, that's another great way to get involved. -**Speaker 2:** Oh, that's true! Grace is the New Relic social media person and a huge Lego fan. +**Rhys:** As far as contributions, we welcome any and all code and incoming contributions. If there's a specific language you're interested in or a specific component, go check out the SIG notes, see what they're talking about, or you can hop into any of the meetings and just kind of check it out. -**Speaker 1:** But yeah, I think Legos tie in very well with ASCII art, and you know, lots of tech people are Lego fans! +**Rhys:** Blog posts could be something that you've... you know, it could be something fun and so-called useless, but really, you know, if it brings so much joy, is it really useless? -**Speaker 2:** True, true! That works! +**Rhys:** So something fun that Jess did could be anything OpenTelemetry-related. We would love to see it. -**Speaker 1:** Other questions that folks have? We can also go ahead and end the call early. +**Rhys:** As I mentioned, documentation: you are welcome to join the end user working group and give suggestions on things you'd like us to help with or do as for you as an end user. -**Speaker 2:** Yeah! +**Rhys:** Of course, you can also just share feedback, and there are different ways to do that. If you don't really want to get engaged with an interview, you are welcome to take part in our survey. I thought I linked it somewhere. -**Speaker 1:** Ellen, I'm Jessatron at Honeycomb if you want to get in touch. +**Rhys:** I have linked the survey in here. If you would like to share feedback that way, that is great. If you would like to participate in the end user working group or by way of the discussion group or end user interview, we would be very happy to have you. -**Speaker 2:** Yes! And I'll post the link to Jessatron's calendar. +**Rhys:** And that's it. That's all I have. Thank you so much. I should have added more Jingle Bells and Christmas stuff. I feel not very festive, but I promise I am. -**Speaker 1:** Sounds good! +**Speaker 1:** Fair, Jess. Thank you, Rhys. -**Speaker 2:** Yeah, nice to meet y’all! +**Speaker 1:** And saying that, for those not following the chat, Johnny's saying they participate in some conferences related to DevOps, and they like to share about the power of observability and how OpenTelemetry can help us make our lives easier. -**Speaker 1:** You too! Well, thanks everyone! It was great to see you all! +**Speaker 1:** Do folks have questions? Since we've got 10 home minutes, we're happy to entertain if you have random questions about OpenTelemetry. We may or may not be able to answer them, but there's lots of knowledge in this room. -**Speaker 2:** Yeah, great to see you! +**Speaker 2:** Yeah, and if anyone is having a specific question about Rhys's presentation, absolutely. -**Speaker 1:** Great to have a chatty group! Thank you all for joining, and a couple of you for putting yourself on video—that's always great for us! +**Speaker 3:** Well, actually, I was going to say about OpenTelemetry.net. We have a maintainer on the line. Is that Alan? -**Speaker 2:** Yeah! +**Alan:** Yes. -**Speaker 1:** I'll Zoom to see faces. Cool! Thank you! +**Speaker 3:** Yes, there are two. Mike Blanchard. Hello, Mike. Very important spot. That's why I'm like, it's okay if I call him out. -**Speaker 2:** Yeah, feel free to reach out to any of us with questions or if there's anything you want to follow up on. Happy holidays! +**Speaker 2:** I will say that I tried to pull down the Honeycomb, the Happy Holidays happy hour. + +**Speaker 3:** Oh, it is... it's going to take a little bit of work, but I was going to try to see how easy it would be to get that reporting into a different tool than New Relic and see what it would look like. + +**Speaker 4:** I'm sure it would look probably amazing. + +**Speaker 2:** It will look mangled initially. + +**Jess:** I'm like, if you want to work at that, I'm happy to work with you on... I know where in the code it's choosing the height that works, and I also know how I use the browser tools in Honeycomb to figure out what height it was needing. + +**Jess:** Some of those tricks will also work in New Relic, so happy to work with you on that if you want because I would love to see what it looks like somewhere else. + +**Speaker 3:** I don't know if they're still doing them, but last year Grace and I were doing a lot of Legos for New Relic. I've got some of them here, and I wonder if they're like amazing art. + +**Speaker 4:** Oh, that's true. Grace is New Relic's social media person and a huge Lego fan. + +**Speaker 3:** But yeah, I think Legos tie in very well with ASCII art, and you know lots of tech people are Lego fans, so... + +**Speaker 5:** True, true. + +**Speaker 6:** Other questions that folks have? + +**Speaker 7:** We can also go ahead and end the call early. + +**Speaker 8:** Yeah, Ellen, I'm Jessatron at Honeycomb if you want to get in touch. + +**Speaker 7:** Yes, and I'll post the link to Jess's calendar. + +**Speaker 6:** Sounds good! + +**Speaker 7:** Yeah, nice to meet y'all! + +**Speaker 8:** You too! + +**Speaker 6:** Well, thanks everyone. It was great to see you all! + +**Speaker 7:** Yeah, great to see you. + +**Speaker 6:** It was fun to have a chatty group. Thank you all for joining, and a couple of you for putting yourself on video. That's always great for us. + +**Speaker 8:** Yeah, I'll Zoom to like see faces. + +**Speaker 6:** Cool. Thank you! + +**Speaker 7:** Yeah, feel free to reach out to any of us with questions or if there's anything you want to follow up on. Happy holidays! ## Raw YouTube Transcript diff --git a/video-transcripts/transcripts/2023-09-28T04:00:36Z-otel-end-user-discussions-amer-january-2023.md b/video-transcripts/transcripts/2023-09-28T04:00:36Z-otel-end-user-discussions-amer-january-2023.md index b90692d..4368c5a 100644 --- a/video-transcripts/transcripts/2023-09-28T04:00:36Z-otel-end-user-discussions-amer-january-2023.md +++ b/video-transcripts/transcripts/2023-09-28T04:00:36Z-otel-end-user-discussions-amer-january-2023.md @@ -10,160 +10,284 @@ URL: https://www.youtube.com/watch?v=qn4x0DgG5SI ## Summary -In this YouTube video, Reese, who works with New Relic and the OpenTelemetry user working group, hosts a discussion focused on challenges and strategies related to enabling observability across different programming languages within organizations. Participants, including Derek, Dan, and others, explore how to become effective enablers of OpenTelemetry despite varying levels of expertise in different programming languages. Key topics include defining the role of an enabler, strategies for championing OpenTelemetry adoption, addressing clock drift in telemetry data, and optimizing data pipelines. The group discusses practical solutions such as creating communities of practice, leveraging auto-instrumentation, and implementing routing processors to efficiently handle telemetry data. The session emphasizes the importance of clear communication, sharing success stories, and understanding the unique needs of different teams to foster a culture of observability. +In this YouTube video, Reese, who works for New Relic and is involved with OpenTelemetry, leads a discussion about the role of enablers in technology adoption, particularly in the context of different programming languages. Participants, including Derek, Dan, and Gerasi, share their experiences in promoting OpenTelemetry within their organizations, addressing challenges like language proficiency and the importance of creating champions within teams for better adoption. They highlight strategies for overcoming resistance to change, the significance of auto-instrumentation, and the necessity of effective communication about the benefits of observability tools. The conversation also delves into technical aspects, such as dealing with clock drift in telemetry data and optimizing data pipelines, with suggestions for future improvements like the introduction of connector components in OpenTelemetry. The video concludes with a light-hearted moment as Reese shares her kitten, Taco, and encourages viewers to reach out for further questions or support. ## Chapters -00:00:00 Welcome and introduction -00:01:00 Role of enabler in OpenTelemetry -00:03:40 Strategies for enabling observability -00:06:03 Challenges with team adoption -00:08:00 Importance of auto instrumentation -00:10:20 Community involvement and support -00:12:50 Observability group and knowledge sharing -00:15:00 Documentation and real-world cases -00:18:00 Clock and time drift issues -00:20:05 Best practices for data bifurcation +00:00:00 Introductions +00:01:17 What is an enabler? +00:03:22 Discussion on language challenges +00:05:30 Guest introduction: Derek +00:09:54 Strategies for creating champions +00:12:36 Importance of auto-instrumentation +00:18:00 Metrics for outage recovery +00:27:14 Discussion on clock drift +00:39:06 Scaling collector deployments +00:48:02 Light-hearted moment with Taco -**Reese:** Of course, I was looking for near to the sessions. My name is Reese. My day job is with New Relic, and I also do a lot of work with the OpenTelemetry and User Working Group, including hosting these sessions. Alrighty, let's see. Okay, let's start with this one. Does your role in Compass being an enabler, how do you deal with helping in languages you're not an expert in? +## Transcript -**Derek:** Um, usually I run these as whoever put the topic or question in. If you can unmute and maybe give us a little bit more detail. It looks like Derek has a question on how do you define enabler. +### [00:00:00] Introductions -[00:01:00] **Derek:** Yeah, this one's mine. Happy to expand on this. Um, so like basically the role I have in my company is kind of twofold. The first is like, you know, maintain OpenTelemetry collector deployments and kind of be responsible for the data pipeline receiving data and sending that to the various backends. But it's also like what I call the enabler, which is like being a champion for OpenTelemetry, driving adoption within the company, you know, troubleshooting, helping teams troubleshoot issues that they have in their respective services, you know, things like that. Um, so like in my company, for instance, we're probably like 70% .NET, 10% Java, 10% Go, 5% Python. You know, like we're primarily focused in one area. So like we build examples in .NET. I personally am more of like a .NET developer. My team is as well. Uh, you know, a great thing about OpenTelemetry, in my opinion, is that like the concepts are extremely similar across languages. But like, you know, I don't really know how to write Java, you know, it's not too far away from .NET. But for instance, um, so sometimes it's like troubling for me to provide specific feedback or like look at a PR and like really understand the scope of everything that's happening. And I'm curious, like if teams have this problem at the companies that they're working in. Do you try to like, you know, I don't know, create champions in those languages you're not familiar in or, you know, communities of practice around that? Um, just, you know, if anyone's experiencing this and maybe how you're solving for it. I hope that makes sense. +**Reese:** Of course, I was looking forward to the sessions. My name is Reese, my day job is with New Relic, and I also do a lot of work with the OpenTelemetry User Working Group, including hosting these sessions. Alrighty, let's see. Okay, let's start with this one: does your role in Compass being an enabler, how do you deal with helping in languages you're not an expert in? -[00:03:40] **Another Speaker:** There totally, yeah. As a former enabler, um, I can tell you that my strategy was, um, really to kind of first lead the way by showing how to enable observability within my domain of expertise, which at the time was Go. And, you know, as I was demonstrating the benefits of using OpenTelemetry, or I guess at the time it was OpenTracing, if that gives you an idea of how long ago this was. Um, but you know, one of the benefits, as I was demoing the benefits, people would come in, ask me questions, and a lot of the times these people work in different teams and different languages. Um, and you know, they would be interested in starting out, but they wouldn't be familiar with the concepts of OpenTracing, OpenTelemetry. And so, you know, my role, as I thought, was to kind of pair with those people who were experts in whatever languages that they were trying to become enablers in and, you know, really work alongside them until they felt comfortable with the concepts of the observability platform I was using. So I think that's kind of my strategy. It worked out pretty well. I think we managed to deploy things to, you know, like three or four different teams within the organization, and it worked pretty well. +**Derek:** Um, usually I run these is whoever put the topic or question in, if you can unmute and maybe give us a little bit more detail. It looks like Derek has a question on how do you define enabler. -**Derek:** Yeah, okay, cool. That's kind of I guess what I mean by like champions. So like basically leveling up, you know, maybe individuals or groups within those smaller subsets and then letting them kind of run with it. Um, do you find when you did that that those people, you know, as like I wasn't familiar with OpenTracing, but like, you know, as the spec evolves and as new features get added, do they come back to you? Do they seek information on their own? Do they jump in the OpenTelemetry Slack channels and ask questions or, uh, you know, were they routing things through you and your experience? +### [00:01:17] What is an enabler? -**Another Speaker:** Um, it's kind of a mixed bag, I think. I think some people feel more comfortable with just talking to the person they know. And so, you know, to a certain extent, I became a bit of a router for questions. Um, but you know, I did see a couple of people who were really into, um, you know, talking to the community directly. And so they just went into the Slack channels or reached out to people directly in GitHub or whatever. So it’s a bit of a mixed bag. Um, you know, I think at some point as you direct people to specific issues in GitHub or specific spec changes, people tend to get more comfortable with the idea that they can go there themselves. But um, yeah. +**Derek:** Yeah, this one's mine. Happy to expand on this. Um, so like basically the role I have in my company is kind of twofold. The first is like, you know, maintain OpenTelemetry collector deployments and kind of be responsible for the data pipeline, receiving data and sending that to the various backends. But it's also like what I call the enabler, which is like being a champion for OpenTelemetry, driving adoption within the company, you know, troubleshooting, helping teams troubleshoot issues that they have in their respective services, you know, things like that. -**Derek:** Cool. And then maybe one more follow-up if you don't mind. Like how long would you say like it took for those individuals you worked with to like, I don't know, get up to speed, whatever that means in your opinion? You know, is that like a matter of weeks? Was that like a longer period? You know, how would you engage like how easy it was for them to sort of like understand the domain when it is maybe not their primary focus? +**Derek:** Um, so like in my company for instance, we're probably like 70% .NET, 10% Java, 10% Go, 5% Python. You know, like we're primarily focused in one area so like we build examples in .NET. I personally am more of like a .NET developer; my team is as well. You know, a great thing about OpenTelemetry in my opinion is that like the concepts are extremely similar across languages, but like, you know, I don't really know how to write Java. You know, it's not too far away from .NET, but for instance, sometimes it's like troubling for me to provide specific feedback or like look at a PR and like really understand the scope of everything that's happening. -[00:06:03] **Another Speaker:** Yeah, um, again, sorry to give you maybe a mixed answer. But again, it depended on individuals. I think some people who are really keen on trying to learn new ways of doing things were really excited about the prospect of trying out, um, you know, something new. And then, you know, those people kind of got up to speed a lot faster. The people that were, you know, um, you know, maybe more used to doing things a certain way took a little bit longer. Um, and then, you know, if I'm honest, I think some people, you know, I was trying to get distributed tracing adopted. Some people just never really understood what the benefits were. And I think that, you know, it really is a mixed bag. So it really depends on where people are in your organizations and how resistant to change they are, I think. +**Derek:** I'm curious, like if teams have this problem at the companies that they're working in. Do you try to like, you know, I don't know, create champions in those languages you're not familiar in or, you know, communities of practice around that? Just, you know, if anyone's experiencing this and maybe how you're solving for it. I hope that makes sense. -**Derek:** Yeah, cool. No, thank you. I appreciate that because I feel like we struggle with sometimes exactly that, like they don't really understand the purpose or they don't understand how it maybe has value outside of their individual team. You know, like when we start connecting services or something. Um, so just trying to, you know, come up with any insight I can into like improving, like the culture and that sort of thing. So appreciate the response. +**Dan:** There totally, yeah. As a former enabler, I can tell you that my strategy was really to kind of first lead the way by showing how to enable observability or, you know, within my domain of expertise, which at the time was Go. And you know, as I was demonstrating the benefits of using OpenTelemetry, or I guess at the time it was OpenTracing, if that gives you an idea of how long ago this was. Um, but you know, one of the benefits as I was demoing the benefits, people would come in, ask me questions, and a lot of the times these people work in different teams and different languages. -[00:08:00] **Another Speaker:** Yeah, I think, you know, I think it's really hard to take someone from their day-to-day job, which, you know, most people are already kind of strained on doing whatever it is that they're doing for the business, um, and asking them to do more by, you know, heading off into this completely, what kind of appears unnecessary aspect of their jobs, right, to implement OpenTelemetry. Um, and it's not really until people have those aha moments of, you know, understanding how much better their lives will be in the future, um, to that they can really wrap their head around the benefits. I think one of the things that I've really helped try to help people understand is, you know, how do you get started as quickly as possible? And I think that's one of the main areas of OpenTelemetry that I'm really, really excited about is all of the auto instrumentation work. Um, because I think that allows people to get some amount of benefit without a tremendous amount of investment up front. And a lot of the time I find even, you know, even if it only gets you 60 or 70% of the way there, um, it tends to give people enough of visibility into what their systems are doing through auto instrumentation that they can then like get excited about the prospect of investing more time in doing this work. +### [00:03:22] Discussion on language challenges -**Derek:** Yeah, makes sense. Awesome. Dan, I don't really have an answer. All I can say is I sympathize with you. Uh, as someone who like codes even probably a lot less than, you know, what you described in your role, it feels weird to kind of want to lead the way as an enabler for these things in your work without necessarily being the person who's able to kind of just, "Alright, let me do a couple examples for you. Let me kind of show you from start to finish how this looks." So I don't know, I'm trying to figure it out too. Probably further behind you actually. +**Dan:** Um, and you know, they would be interested in starting out but they wouldn't be familiar with the concepts of OpenTracing, OpenTelemetry. And so, you know, my role as I thought was to kind of pair with those people who were experts in whatever languages that they were trying to become enablers in and, you know, really work alongside with them until they felt comfortable with the concepts of the observability platform I was using. So I think that's kind of my strategy. It worked out pretty well. I think we managed to deploy things to, you know, like three or four different teams within the organization and it worked pretty well. -**Another Speaker:** So in my previous organization, um, like when we were trying to bring OpenTelemetry into the development teams, we faced some interesting challenges where the teams recognized the importance of OpenTelemetry, but they were constantly fighting fires. So they didn't have time to instrument their code, which was kind of ironic because instrumenting their code would probably help them fight those fires more effectively. And so like my team was, uh, their job was to be that enabler. So we, you know, defined the best practices and taught teams how to instrument their code. They kept trying to use my team as like, "Oh, well you guys know this stuff, so why don't you instrument our code for us?" Which we had to push back really hard on because like we don't know your code. So you have to instrument your code because you know it best, right? +**Derek:** Yeah, okay, cool. That's kind of, I guess, what I mean by like champions. So like basically leveling up, you know, maybe individuals or groups within those smaller subsets and then letting them kind of run with it. Um, do you find when you did that that those people, you know, as like I wasn't familiar with OpenTracing, but like, you know, as the spec evolves and as new features get added, do they come back to you? Do they seek information under their own? Do they jump in the hotel Slack channels and ask questions or, uh, you know, were they routing things through you and your experience? -[00:10:20] **Derek:** I feel like everything, like we're all talking about, like we all have, like somehow are in the same boat. And I've just, it's like super interesting because I feel like I've experienced all of what you guys have said. +**Dan:** Um, it's kind of a mixed bag. I think some people feel more comfortable with just talking to the person they know and so, you know, to a certain extent I became a bit of a router for questions. Um, but you know, I did see a couple of people who were really into, um, you know, talking to the community directly and so they just went into the Slack channels or reached out to people directly in GitHub or whatever. So it’s a bit of a mixed bag. Um, you know, I think at some point as you direct people to specific issues in GitHub or specific spec changes, people tend to get more comfortable with the idea that they can go there themselves. -**Another Speaker:** Yeah, what a general mentioned is something that I'm experiencing as well. Um, and even if it was wider than just one company, it goes even throughout the community. So sometimes I, um, I try to help a few open source projects to instrument, to add these orientations to their code base because I wanted to use to have good references of what other people can do, you know, so I could then point other people to that code base and say, "You know, this is a very well-instrumented application you can use it as reference for instrumenting your own application." Um, the problem with that was some projects were, they wanted to have good instrumentation, but they didn't actually, um, they weren't ready for it, right? And this is something that I'm experiencing within the company, uh, within, you know, other companies as well is that we can try to get a network and instrument things for them, but if they are not ready to have their code instrumented, it's just not gonna work. I don't know. +### [00:05:30] Guest introduction: Derek -**Another Speaker:** Um, I could, by the way, you sound a little distant. I was able to hear you, um, but just that way. Um, I think that's true. Like I feel like first somehow we need to like sell the, like if teams maybe recognize the benefit earlier, like it helps in all these avenues. Like it helps with them understanding the purpose and therefore like maybe wanting to contribute more how to solve the problem. So maybe, I don't know, I'm just thinking out loud here and maybe like pushing to show more like here are some examples where doing this solves some problems. Like if you're experiencing similar problems and perhaps like this would be a really good idea for you. Um, maybe like, you know, I don't know, honing in on that kind of an attack might be beneficial. +**Derek:** Um, yeah. Cool. And then maybe one more follow-up if you don't mind. Like how long would you say like it took for those individuals you worked with to like, I don't know, get up to speed, whatever that means in your opinion. You know, is that like a matter of weeks? Was that like a longer period? You know, how would you engage like how easy it was for them to sort of like understand the domain when it is maybe not their primary focus? -**Another Speaker:** Yeah, so is my microphone better now? Am I closer? +**Dan:** Yeah, um, again, sorry to give you maybe a mixed answer, but again, it depended on individuals. I think some people who are really keen on trying to learn new ways of doing things were really excited about the prospect of trying out, um, you know, something new, and then, you know, those people kind of got up to speed a lot faster. The people that were, you know, um, you know, maybe more used to doing things a certain way took a little bit longer. Um, and then, you know, if I'm honest, I think some people, you know, I was trying to get distributed tracing adopted—some people just never really understood what the benefits were, and I think that, you know, it really is a mixed bag. So it really depends on where people are in your organizations and how resistant to change they are, I think. -**Another Speaker:** Yeah, that's much better. +**Derek:** Yeah, cool. No thank you. I appreciate that because I feel like we struggle with sometimes exactly that, like they don't really understand the purpose or they don't understand how it may be has value outside of their individual team, you know, like when we start connecting services or something. Um, so just trying to, you know, come up with any insight I can into like improving like the culture and that sort of thing, so appreciate the response. -[00:12:50] **Another Speaker:** Okay, that's good. Uh, yeah, so, um, I just came with a performing conversation with another person that, um, that person was mentioning something exactly like that, you know? Um, and one thing that they're doing and I found very interesting is, um, they have like an observability group, and that group is, uh, the reference group in terms of observability. And they can watch what other people, what the whole company is doing in terms of observability. And what he mentioned was, um, there was one team using this specific programming language, and they found that one specific metric helped them recover very fast from an outage. So what the observability group did then was to spread that information to the whole company and saying, "So teams that are using that programming language, you should now have to include this metric here." Um, otherwise, you know, because that metric helped that team to recover very fast from an outage. So, um, I see the role of the enabler here, like the observability enabler, as also to spread information, to get information from one title and break this item and bring to the whole company what should the whole community for that matter. +**Dan:** Yeah, I think, you know, it's really hard to get to take someone from their day-to-day job, which you know most people are already kind of strained on doing whatever it is that they're doing for the business, um, and asking them to do more by, you know, heading off into this completely what kind of appears unnecessary aspect of their jobs, right, to implement OpenTelemetry. Um, and it's not really until people have those aha moments of, you know, understanding how much better their lives will be in the future, um, to that they can really wrap their head around the benefits. -**Another Speaker:** Um, would it be, you know, as far as something like the community could do with this, um, would maybe some documentation around like real-world cases like the one Gerasi mentioned, um, do you think that might be helpful if you could share like similar stories with your team? +**Dan:** I think one of the things that I've really helped try to help people understand is, you know, how do you get started as quickly as possible? And I think that's one of the main areas of OpenTelemetry that I'm really, really excited about is all of the auto instrumentation work, um, because I think that allows people to get some amount of benefit without a tremendous amount of investment up front. And a lot of the time I find even, you know, even if it only gets you 60 or 70% of the way there, um, it tends to give people enough of visibility into what their systems are doing through auto instrumentation that they can then like get excited about the prospect of investing more time in doing this work. -[00:15:00] **Another Speaker:** It might. I mean, we try to do that already. I think it's like a combination of everything that's mentioned here. Maybe like, I don't want to say like what we're doing isn't working. I think it is working. It's just like progress is slower than I maybe would prefer, you know? I mean, yeah, I think like, you know, having a collection of like good news stories that, you know, or whatever you want to call it, like, uh, you know, like, "Hey, I work at company X and we were able to solve this problem and, you know, reduce mean time resolution blah blah blah blah." Like, yeah, I do think like that, you know, you know, if people are looking to get into the space, I think those are great. And I think like having those examples of like specifics, um, you know, like as my role here as like in the observability group for my company, like I go and I read blogs and stuff, but I don't think like everybody at my company would do that. Like maybe they are reading blogs specific to like the .NET runtime or something, you know, I don't know. Um, so yeah, I think it would help. I just, I don't know what the best, I guess format would be. I'm just wondering, would it be beneficial to have like periodic, like some sort of forum, um, where like every month or whatever, or every quarter where folks from organizations that are looking to adopt OpenTelemetry practices, OpenTelemetry in general, um, have a chance to like interact with folks in the community for like a pointed Q&A, um, to like just to sort of, you know, ease their concerns? Like, or would you all feel like this is too much of an overlap of what this group already does? I'm just wondering. +**Reese:** Makes sense. Awesome. Dan, I don't really have an answer; all I can say is I sympathize with you. Uh, as someone who like codes even probably a lot less than you know what you described in your role, it feels weird to kind of want to lead the way as an enabler for these things in your work without necessarily being the person who's able to kind of just, "Alright, let me do a couple examples for you. Let me kind of show you from start to finish how this looks." So I don't know; I'm trying to figure it out too. Probably further behind you actually. -**Another Speaker:** Um, my first thought is like it does have some overlap with this group. Um, I mean, I think, I don't know, maybe I would need to think about that a bit more. Well, one thing like Reese mentioned, like she's recording this meeting now, which I think is good. Like people can go back and like get insight into the past, which before I think was like a, there's some good topics here that are brought up that like aren't necessarily easy to get answers because there aren't really answers, like they're more gray, if you will. Um, I don't know, I would have to think about that. +### [00:09:54] Strategies for creating champions -**Another Speaker:** Yeah, um, no, I think it's a very valid, um, observability concern. Um, and yeah, I would like to noodle on this more as well. Um, and that said, I just realized, well, I realized a few minutes ago that I haven't been time boxing this, so I apologize. Um, but it sounds like, um, we might be good on this topic. Dan, was there anything else you wanted to add before we move to the next one? +**Dan:** So in my previous organization, like when we were trying to bring OpenTelemetry into the development teams, we faced some interesting challenges where the teams recognized the importance of OpenTelemetry, but they were constantly fighting fires. So they didn't have time to instrument their code, which was kind of ironic because instrumenting their code would probably help them fight those fires more effectively. And so, like while my team was, uh, their job was to be that enabler, so we, you know, defined the best practices and taught teams how to instrument their code, they kept trying to use my team as like, "Oh, well you guys know this stuff, so why don't you instrument our code for us?" Which we had to push back really hard on because like we don't know your code, so you have to instrument your code because you know it best, right? + +**Derek:** I feel like everything, like we're all talking about, like we all have like somehow are in the same boat. And I've just, it's like super interesting because I feel like I've experienced all what you guys have said. + +**Dan:** Yeah, what a general mentioned is something that I'm experiencing as well. Um, and even if it was wider than just one company, it goes even throughout the community. So sometimes I, um, I try to help a few open source projects to instrument, uh, to add these orientation to their code base because I wanted to use to have good references of what other people can do, you know, so I could then point other people to that code base and say, you know, this is a very well-instrumented application; you can use it as a reference for instrumenting your own application. + +**Dan:** Um, the problem with that was some projects were, they wanted to have a good instrumentation but they didn't, they didn't actually, um, they weren't ready for it, right? And this is something that I'm experiencing within the company, uh, within, you know, other companies as well, is that we can try to get a network and instrument things for them, but if they are not ready to have their code instrumented, it's just not gonna work. + +**Derek:** Yep. Um, I could, by the way, you sound a little distant. I was able to hear you, um, but just that way. Um, I think that's true. Like I feel like first somehow we need to like sell the, like if teams maybe recognize the benefit earlier, like it helps in all these avenues. Like it helps with them understanding the purpose and therefore like maybe wanting to contribute more how to solve the problem. So maybe I don't know, I'm just thinking out loud here and maybe like pushing to show more like here are some examples where doing this solves some problems. Like if you're experiencing similar problems, perhaps like this would be a really good idea for you. + +**Dan:** Um, maybe like, you know, I don't know, honing in on that kind of an attack might be beneficial. + +**Derek:** Yeah, so is my microphone better now? Am I closer? + +**Dan:** Yeah, that's much better. + +### [00:12:36] Importance of auto-instrumentation + +**Derek:** Okay, that's good. Uh, yeah, so, um, I just came with a performing conversation with another person that, um, that person was mentioning something exactly like that, you know? Um, and one thing that they're doing and I found very interesting is, um, they have like an observability group and that group is, uh, the reference group in terms of observability. And they can watch what other people, what the whole company is doing in terms of observability. + +**Derek:** And what he mentioned was, um, there was one team using this specific programming language and they found that one specific metric helped them recover very fast from an outage. So what the observability group did then was to spread that information to the whole company and saying, "So teams who are using that programming language, you should now have to include this metric here, um, otherwise, you know, because that metric helped that team recover very fast from an outage." + +**Dan:** So, um, I see the role of the enabler here, like the observation enabler, is also to spread information, to get information from one title and break this item and bring to the whole company what should the whole community for that matter. + +**Derek:** Would it be, you know, as far as something like the community could do with this, um, would maybe some documentation around like real-world cases, like the one Gerasi mentioned, um, do you think that might be helpful? If you could share like similar stories with your team? + +**Dan:** It might. I mean, we try to do that already. I think it's like a combination of everything that's mentioned here. Maybe like, I don't want to say like what we're doing isn't working; I think it is working. It's just like progress is slower than I maybe would prefer, you know? + +**Dan:** I mean, yeah, I think like, you know, having a collection of like good news stories that you know, or whatever you want to call it, like, uh, you know, like, "Hey, I work at company X and we were able to solve this problem and, you know, reduce mean time resolution blah blah blah blah." Like, yeah, I do think like that, you know, you know, if people are looking to get into the space, I think those are great. And I think like having those examples of like specifics, um, you know, like as my role here in the observability group for my company, like I go and I read blogs and stuff, but I don't think like everybody at my company would do that. Like maybe they are reading blogs specific to like the .NET runtime or something, you know? I don't know. + +**Derek:** Um, so yeah, I think it would help. Um, I just, I don't know what the best, I guess format would be. + +**Dan:** I'm just wondering, would it be beneficial to have like periodic, like some sort of forum, um, where like every month or whatever, or every quarter, where folks from organizations that are looking to adopt OpenTelemetry practices, OpenTelemetry in general, um, have a chance to like interact with folks in the community for like a pointed Q&A? Um, to like just to sort of, you know, ease their concerns. Like, or would you all feel like this is too much of an overlap of what this group already does? I'm just wondering. + +**Derek:** Um, my first thought is like it does have some overlap with this group. Um, I mean, I think, I don't know, maybe I would need to think about that a bit more. + +**Dan:** Well, one thing, like Reese mentioned, like she's recording this meeting now, which I think is good. Like people can go back and like get insight into the past, which before I think was like, there are some good topics here that are brought up that aren't necessarily easy to get answers because there aren't really answers. Like they're more gray, if you will. + +**Derek:** I don't know, I would have to think about that. + +**Dan:** Um, no, I think it's a very valid, um, observability concern. Um, and yeah, I would like to noodle on this more as well. Um, and that said, I just realized, well, I realized a few minutes ago that I haven't been time boxing this, so I apologize. Um, but it sounds like, um, we might be good on this topic. Dan, was there anything else you wanted to add before we move to the next one? **Dan:** No, I'm good. -**Another Speaker:** Okay, can I hop in real quick, just share one last thought? +**Derek:** Can I hop in real quick, just share one last thought? + +**Dan:** Of course. + +### [00:18:00] Metrics for outage recovery + +**Derek:** Um, this is kind of just a more general approach for trying out our organization, but we're kind of trying to drive change across various laterals through the idea of like a service catalog and scorecard for services where various like subject matter experts can define like a set of standards in a particular domain that should apply for services and kind of provide a rubric for those things. + +**Derek:** So what does like, you know, sufficient or like give a service a grading in a particular set of criteria, right? C a A plus, right? And you don't have to be responsible for implementing what is like an A look like in your service, but you're allowed to, you know, you can define those standards and then the actual service owners that are responsible for that service can consult that rubric, that scorecard, kind of grade their own services according, and then kind of refer to whatever standardized documentation you provide as a subject matter expert for figuring out, okay, how do I take my service that is maybe a great C in observability to like a grade A, right? + +**Derek:** And so in that kind of a situation, you're setting forth the standards, you're providing a clear rubric that allows service owners to kind of grade themselves, and you're providing more information if they're looking to kind of level up their service. So this is kind of something that is very new at our organization; we're going to try to drive change in a bunch of different places—things like SRE practices, how you tune your services, Kubernetes, code quality, test coverage, observability stuff. So we're just gonna try that out and see if that makes it kind of easier to gamify making strides, um, and just improving services and things like that. + +**Derek:** So I don't know if that's helpful at all, but it kind of separates implementation from like leading the way and improving something. + +**Dan:** Alright, Derek, I don't know if you can see this, but Dan has put a thumbs up. + +**Derek:** Oh cool, yep. Alrighty, so since the next two have equal votes, we'll just go with the, uh, in order: how are you dealing with clock/time drift? + +**Dan:** Yeah, these are actually all mine, but today, um, yeah. Um, yeah, so like I, uh, I don't know. I noticed, well, I noticed because our backend doesn't accept data that's like in the future. I think it's 10 minutes in the future. Um, obviously people are creating data points from the future because their times on their servers are not correct. Um, one, I guess, are people experiencing this problem? Like the naive side of me wants to just say, well, like they should just fix their servers, which I do agree with. + +**Dan:** Um, if you are experiencing this, do you notify services? Do you implement something in the collector to understand that it's happening? + +**Gerasi:** Uh, yeah, that—so, um, I can probably talk a little bit about how Jaeger deals with that. Um, and the way that Jaeger does or used to do is, um, it tries to detect—well, so I guess the first realization is that, uh, clock skew is going to happen, um, and you just have to account for it. There is no way for clocks to be synchronized, especially on a microservices architecture. -**Another Speaker:** Of course. +**Gerasi:** Um, and one way of dealing with that is Jaeger, by default, assumes that, um, you don't have asynchronous processing. So one trace is, uh, very synchronous, so the parent span or the very first span, um, and at most when the last span finishes, you know, so—and if that's not the case, then Jaeger, by default, will try to adjust the parent spans of that span to be as big or as long as the longest span that it has. -[00:18:00] **Another Speaker:** Um, this is kind of just a more general approach for trying out our organization, but we're kind of trying to drive change across various laterals through the idea of like a service catalog and scorecard for services where various like subject matter experts can define like a set of standards in a particular domain that should apply for services and kind of provide a rubric for those things. So what does like, you know, sufficient or like give a service a grading in a particular set of criteria, right? C a A plus, right? And you don't have to be responsible for implementing what is like an A look like in your service, but you're allowed to, you know, you can define those standards. And then the actual service owners that are responsible for that service can consult that rubric, that scorecard, kind of grade their own services according and then kind of refer to whatever standardized documentation you provide as a subject matter expert for figuring out, okay, how do I take my service that is maybe a great C in observability to like a grade A, right? And so in that kind of a situation, you're setting forth the standards, you're providing a clear rubric that allows service owners to kind of grade themselves, and you're providing more information if they're looking to kind of level up their service. So this is kind of something that is very new at our organization. We're going to try to drive change in a bunch of different places: things like SRE practices, how you tune your services in Kubernetes, code quality, test coverage, observability stuff. So we're just gonna try that out and see if that makes it kind of easier to gamify making strides and just improving services and things like that. So I don't know if that's helpful at all, but it kind of separates implementation from like leading the way and improving something. +**Gerasi:** Um, it did generate a lot of confusion among users, especially for users who do have asynchronous processing, so much that we ended up doing a flag to disable this behavior on the Jaeger side, on the Jaeger collector side. And I think we at some point thought about not having that behavior by default, so only users who would know what they're doing would then enable the clock skew. I can't remember what is the feature name, but they wouldn't have this clock fixing feature enabled. Um, on the collector side, we're not doing anything that I know of. Perhaps Alex Cannon can share if he knows whether we are doing something like that there. -**Derek:** Alright, Derek, I don't know if you can see this, but Dan has put a thumbs up. +**Gerasi:** And I guess the short answer is, um, the owner of the system that generates the telemetry data is in a way better position to understand the clock drift than, uh, any general-purpose tool like the collector. So the collector cannot, in a it's not in a very good position to detect and fix this issue for you. -**Dan:** Oh, cool. Yep. +**Dan:** Um, so yeah, great, thank you. Um, so like, I don't know, for to me at least—and maybe this is not true—but, um, like there's sort of like two use cases here. There's like a lower mild kind of clock skew where it's like, I don't know, a couple seconds or something, and yeah, sure, maybe it makes the tracing vision look a little awkward or something. And then there's like the really bad, you know, ones that are like in this case, like I know they were like 10 minutes off. -**Derek:** Alrighty, so since the next two have equal how more votes, we'll just go with the, uh, in order. How are you dealing with clock/time drift? +**Dan:** Um, so I was thinking of like creating a processor that detects some, you know, incoming data point that is like some, some like distance—I don't know what the right word is—some, you know, if it's more than five minutes or some threshold, uh, like just, I don't know, creating a data point that like captures, you know, the service name or something and, and like, you know, just at least identifies it like, "Hey, this is a, this is something that's like way out of sync with what we consider real-time." I don't know if like something like that would be useful. -[00:20:05] **Another Speaker:** Yeah, these are actually all mine, but today, um, yeah. Um, yeah, so like I, I don't know. I noticed, well, I noticed because our backend doesn't accept data that's like in the future. I think it's 10 minutes in the future. Um, obviously people are creating data points from the future because there are times on their servers are not correct. Um, one, I guess are people experiencing this problem? Like the naive side of me wants to just say, well, like they should just fix their servers, which I do agree with. Um, if you are experiencing this, do you notify services? Do you implement something in the collector to understand that it's happening? +**Gerasi:** Um, right now our backend, there's no good way to like correlate like the error that our backend throws with like the data that's coming through, so I don't even know like what data is missing unless a service owner were to reach out to me. Um, it could be specific to my backend, maybe my use case. -**Another Speaker:** Uh, yeah, that so, um, I can probably talk a little bit about how Jaeger deals with that. Um, and the way that Jaeger does, or used to do, is, um, it tries to detect, well, so I guess the first realization is that, uh, clock skew is going to happen. Um, and you just have to account for it. There is no way for clocks to be synchronized, especially on a microservices architecture. Um, and one way of dealing with that is Jaeger tries to, um, it first assumes that, um, you don't have asynchronous processing. So one trace is very synchronous. So the parent span or the very first span, um, and, uh, at most when the last span finishes, you know, so, and if that's not the case, then Jaeger by default will try to adjust the parent spans of that span to be as big or as long as the longest span that it has. Um, it did generate a lot of confusion among users, especially for users who do have asynchronous processing, so much that we ended up doing a flag to disable this behavior on the Jaeger side, on the Jaeger collector side. And I think we at some point thought about not having that behavior by default so only users who would know what they were doing would then enable the clock skew. I can't remember what is the feature name, but they wouldn't have this clock fixing feature enabled. Um, on the collector side, we're not doing anything that I know of. Perhaps Alex can share if he knows whether we are doing something like that there. But, uh, and I guess the short answer is, um, the owner of the system that generates the telemetry data is in a way better position to understand the clock drift than any general-purpose tool like the collector. So the collector cannot, in a, it's not in a very good position to detect and fix this issue for you. +**Dan:** Um, I know, so I was just like maybe just like having more visibility into the issue would like be a first step. Does that sound like a weird use case? Like I don't know, creating a processor that would kind of detect that? -**Derek:** Um, so yeah, great. Thank you. Um, so like I don't know for, to me at least, and maybe this is not true, but, um, like there's sort of like two use cases here. There's like, like a lower mild kind of clock skew where it's like, I don't know, a couple seconds or something. And yeah, sure, maybe it makes the tracing vision like look a little awkward or something. And then there's like, like the really bad, you know, ones that are like in this case, like I, I know they were like 10 minutes off. Um, so I was thinking of like creating a processor that detects some, you know, incoming data point that is like some, some like distance, I don't know what the right word is, some, you know, if it's more than five minutes or some threshold, uh, like just, I don't know, creating a data point that like captures, you know, the service name or something and, and like, you know, just at least like identifies it like, "Hey, this is a, this is something that's like way out of sync with what we consider real-time." I don't know if like something like that would be useful. +**Gerasi:** I think it's definitely an interesting use case. If you know, you could have a processor that just detects things that are out in the future. Um, I would guess that the hard part is, you know, you still wouldn't catch all of your clock drifts. I guess you would catch a very obvious ones that you kind of know about, um, that you've seen in the past. -**Another Speaker:** Um, right now our backend, there's no good way to like correlate like the error that our backend throws with like the data that's coming through. So I don't even know like what data is missing unless a service owner were to reach out to me. Um, it could be specific to my backend maybe my use case. Um, I know, so I was just like maybe just like having more visibility into the issue would like be a first step. Does that sound like a weird use case? Like I don't know, creating a processor that would kind of detect that? +**Dan:** But yeah, so I guess then, yeah, that's like, I guess because in my case I'm like dropping data because the backend's rejecting it, it would be good to know that. But if there was like say it was a 10-minute threshold and it was, I don't know, three minutes delayed and I didn't want to detect that or I couldn't detect that, now it still cause issues with the interpretation of the data. -**Another Speaker:** I think it's definitely an interesting use case. If you know you could have a processor that just detects things that are out in the future. Um, I would guess that the hard part is, you know, you still wouldn't catch all of your clock drifts. I guess you would catch a very obvious ones that you kind of know about, um, that you've seen in the past. But yeah, so I guess then, yeah, that's like, I guess because in my case I'm like dropping data because the backend's rejecting it. It would be good to know that, but if there was like say it was a 10-minute threshold and it was, I don't know, three minutes delayed and I didn't want to detect that or I couldn't detect that, now it still cause issues with the interpretation of the data. +**Dan:** Uh, but it would be like a secondary concern. Like maybe I should just shore up like the data loss, if you will, and then I don't know, just be better at, you know, having that, like Gerasi, is that how you pronounce your name? Having like the service owners sort of interpret the drift or the skew because they're the experts, like you said. -**Another Speaker:** Uh, but it would be like a secondary concern. Like maybe I should just shore up like the data loss if you will and then, I don't know, just be better at, you know, having that like, like Gerasi, is that how you pronounce your name? Having like the service owners sort of interpret the drift or the skew because they're the experts, like like you said. +**Gerasi:** Um, are you talking specifically about metrics or are you talking about any telemetry data type? -**Another Speaker:** Um, are you talking specifically about metrics or are you talking about any telemetry data type? +**Dan:** I would have to double check. I want to say the data that I was looking at in this case was spans, um, but in my opinion, it would be for any type. -**Derek:** Um, I would have to double-check. I want to say the data that I was looking at in this case was spans. Um, but in my opinion, it would be for any type. +**Gerasi:** Yeah, so for metrics, there's some metrics, um, um, stores like, you know, every single metrics-based storage will block new metrics that are from the past, so not from the future I suppose, but from the past. Um, so you would have a warning on some logs already because of that before using primitives. -**Another Speaker:** Yeah, so for metrics, there's some metrics, um, stores like, you know, every single Prometheus-based storage will block new metrics that are from the past, so not from the future I suppose, but from the past. Um, so you would have a warning on some logs already because of that before using Prometheus. Um, for traces, uh, it might be relatively easy to, to find it out if the clock drift can be detected by looking at a trace, right? So by looking at the spans of a trace. And I guess it would not be too hard to make a processor that detects that. +### [00:27:14] Discussion on clock drift -**Another Speaker:** Now for logs, I don't know. I wouldn't even know if we would want to have something like that for logs, um, because you generate such a huge amount of logs on one specific server and supposedly all of the logs on that server are having the same timestamps for things are a lot at the same time. Um, and, uh, so perhaps just, yeah, I don't know. Perhaps this data, it's hard to think about logs in this case because it would be having a bunch of logs at once, uh, and then you have to compare those logs with a batch of logs from another server, right? Coming from another server on the collector. So your comparison base would be huge. +**Gerasi:** Um, for traces, uh, it might be relatively easy to find it out if the clock drift can be detected by looking at a trace, right? So by looking at the spans of a trace, and I guess it would not be too hard to make a processor that detects that. -**Derek:** Yeah, I guess my take there would be just do the simplest thing that would work, which in this case, if the log timestamps are still in the future, you could flag it. You know, you could create a metric that would at least capture that information. But again, this would only capture clock drift that's in the future. Nothing clear would happen if it was like drifting in the past. +**Gerasi:** Now for logs, I don't know. I wouldn't even know if we would want to have something like that for logs because you generate such a huge amount of logs on one specific server and supposedly all of the logs on that server are having the same timestamps for things that are a lot at the same time. -**Another Speaker:** Yeah, makes sense. Okay, cool. Thanks for the input, guys. I'm good with this one. Reese and other people don't have comments. +**Gerasi:** And, uh, so perhaps just, yeah, I don't know. Perhaps this data, it's hard to think about logs in this case because it would be having a bunch of logs at once, uh, and then you have to compare those logs with a batch of logs from another server, right? Coming from another server on the collector. -**Reese:** Excellence. Alrighty, best practices for bifurcating data in a pipeline. For example, have a filter processor with positive and negative condition clarity around fan-in. +**Dan:** So your comparison base would be huge. -**Another Speaker:** Yeah, so this is also mine, and I don't really know how to phrase this in a simple way. Um, maybe I'll tell you a really weird case use case that I have. And yeah, anyway, so like I have data flowing. Um, let's say I have an OTLP, I have a collector, I have an OTLP receiver, I have some processors configured, and then I have a backend, let's call it backend A, and then I have another pipeline, also OTLP receiver. Well, let's just say we're using metrics here. Um, some processors in backend B. So I want to like have, um, so the same set of data is flowing into both of these pipelines, and then for like pipeline one, I want say just to have histograms go to backend A, and for pipeline two, I wanted to have non-histograms go to backend B. Is configuring two pipelines, um, like that the right thing to do? +**Gerasi:** Yeah, I guess my take there would be just do the simplest thing that would work, which in this case, if the log timestamps are still in the future, you could flag it, you know? You could create a metric that would at least capture that information. But again, this would only capture clock drift that's in the future; nothing clear would happen if it was like drifting in the past. -**Another Speaker:** Um, I was trying to, like, it works. Like I have this configured, it works. I was thinking about like, is this the most CPU or memory efficient in terms of like, uh, I'm not, I'm not, I guess like the fan and fan out of how like the receivers and stuff work in the exporters work. I was trying to like, you know, understand if I'm doing it in like the optimal way. Does that question make sense at all? Is that a use case anyone else has? +**Dan:** Yeah, makes sense. Okay, cool, thanks for the input, guys. I'm good with this one. Reese and other people don't have comments. -**Another Speaker:** Um, there's a new type of component for the collector that is coming up, and that's called the connector. So it's, it's, uh, so the collector right now has four types of components, right? Extensions and then pipeline-specific components like receivers, processors, and exporters. And there's going to be a fourth one, or a fifth, uh, which is connectors. So connectors they can act as receivers and exporters. So you have one pipeline that receives OTLP for instance. Then there is some processing and, um, you end up with an exporter. An exporter for that pipeline is going to be a connector. Now the connector then connects with another pipeline, um, and as a receiver, right? So it's an exporter in one pipeline and a receiver on the next pipeline. Um, so what you can do is you can translate signals. So you can translate spans to metrics for instance. But in your case here, it would work in a way that you have like one receiver and then, uh, two exporters or two connectors as part of the same pipeline. One that is going to connect this data or this pipeline with one that is histogram specific and one that is going to connect with a non-histogram specific with the other one. And then each one of those connectors would then be able to filter what is interesting to be passed through this disconnect and block everything else. So it ended up with, uh, three pipelines in there. And so one that receives the raw data, one that deals with the histogram data, and one that deals with other data. So this is what we have planned for the future. So the basic building blocks for that are merged already. So, um, from what I remember, so Alex can correct me if I'm wrong, um, but, uh, the basic building blocks are there, and it's, we're all looking forward to seeing connectors being ready to be used because we have so many use cases in mind to implement. +**Reese:** Alrighty, best practices for bifurcating data in a pipeline. For example, have a filter processor with positive and negative condition clarity around fan in. -**Another Speaker:** Now what we have today for that specific case is the routing processor. So we have a routing processor, and it doesn't really work the way that you want here, but it works in a very similar way. So what we do is we route the data points based on their characteristics, and we rush them to specific exporters. But we do that by making another network connection, so it's very inefficient, but it works. So those are the two possible solutions for that. With the router, I'll obviously look into this. Thank you. I didn't know what it is. Maybe this is a specific use case, but what would be the advantage of like the router solution versus like having a filter processor that like, you know, having the two pipelines and having the filter processor just like get rid of a, you know, part of like the part A of the set or part B of the set? Would there be an advantage to you either one of those? I guess that would be a third solution. +**Dan:** Yeah, so this is also mine and I don't really know how to phrase this in a simple way. Um, maybe I'll tell you a really weird case use case that I have. And yeah, anyway, so like I have data flowing. Um, let's say I have an OTLP, I have a collector, I have an OTLP receiver, I have some processors configured, and then I have a backend—let's call it backend A—and then I have another pipeline, also OTLP receiver. -**Another Speaker:** Yeah, I guess, um, that's what I'm doing right now. Um, and it's working. Uh, there were some bugs in the filter processor that have been resolved, so now it seems to be working perfectly as far as I know. It just, I just felt like I was, because I had like two receivers receiving the same data, like I was doubling up on the memory that, you know, my collector was using. Uh, I haven't done any profiling, but maybe that's something I should do. Um, but I'll definitely look at the routing processor. Um, actually take a look at the connectors because I've made a very similar question to the author of the connectors on the PR that they introduced, and I think, I think indeed the answer is that with the connector, you wouldn't only have one receiver and only one copy of the data point in memory. It only keeps one copy of the data point in memory. Let me try to find a PR, and you can get more information from there. +**Dan:** Well, let's just say we're using metrics here, um, some processors in backend B. So I want to like have, um, so the same set of data is flowing into both of these pipelines and then for like pipeline one, I want say just to have histograms go to backend A and for pipeline two, I wanted to have non-histograms go to backend B. Is configuring two pipelines like that the right thing to do? Um, I was trying to like, it works; like I have this configured, it works. I was thinking about like, is this the most CPU or memory efficient in terms of like, uh, I'm not, I'm not, I guess like the fan and fan out of how like the receivers and stuff work in the exporters work. I was trying to like, you know, understand if I'm doing it in like the optimal way. Does that question make sense at all? Is that a use case anyone else has? -**Another Speaker:** Okay, awesome. That would be great. Thank you. That sounds like a great feature. I'll definitely follow that. Thank you. +**Gerasi:** Um, there is a new type of component for the collector that is coming up and that's called the connector. So it's, uh, so the collector right now has, uh, four types of components, right? Extensions and then pipeline-specific components like receivers, processors, and exporters, and there's going to be a fourth one or a fifth, uh, which is connectors. -**Another Speaker:** Yeah, that's neat. I haven't heard about the connectors, so I'm interested to learn more as well. Well, Gerasi is getting us that PR. Does anyone else have anything they'd like to add or ask while we still have 10 minutes on the clock? +**Gerasi:** So connectors, they can act as receivers and exporters. So you have one pipeline that receives OTLP, for instance, then there is some processing and, um, you end up with an exporter. An exporter for that pipeline is going to be a connector. Now the connector then connects with another pipeline, um, and as a receiver, right? So it's an exporter in one pipeline and a receiver on the next pipeline. -**Another Speaker:** I do if no one else does that, if someone else obviously go first since I've asked enough. +**Gerasi:** Um, so what you can do is you can translate signals, so you can translate spans to metrics, for instance. But in your case here, it would work in a way that you have like one receiver and then, uh, two exporters or two connectors as part of the same pipeline. One that is going to connect this data or this pipeline with one that is histogram specific, and one that is going to connect with a non-histogram specific with the other one. -**Another Speaker:** I think you're in the clarity and go ahead. +**Gerasi:** And then each one of those connectors would then be able to filter what is interesting to be passed through this disconnector and block everything else. So it ended up with, uh, three pipelines in there and so one that receives the raw data, one that deals with a histogram data, and one that deals with other data. So this is what we have planned for the future. -**Another Speaker:** Okay, um, I asked this a couple of these sessions ago. Um, I was trying to, I'm trying to find, I had opened a GitHub issue. Um, it was very large in scope, so I don't think I have any responses on it. Um, it was, it was, it was about like understanding more about, um, um, I'll just pop this in the Zoom chat, uh, if you're curious. Um, it was more about just like figuring out like how to scale my collector deployment correctly. And Gerasi, I saw that you had like a post yesterday that you posted on one of the channels that talks a little bit about that. Um, there were some other things I had put in here, like, um, understanding like when the num consumers setting, uh, for instance, should be modified. Um, I'm curious if you guys have any like insight into like when I should be, I guess, horizontally scaling my pod versus when I should be modifying the configuration of, uh, an individual pod if that, or an individual collector if that makes sense. Like how do I, what wonder what criteria do I use to determine one versus the other? Does that make any sense? It's probably a loaded question as well. +**Gerasi:** So the basic building blocks for that are merged already, so from what I remember. So Alex can correct me if I'm wrong, but, uh, the basic building blocks are there and we're all looking forward to seeing connectors being ready to be used because we have so many use cases in mind to implement now. -**Another Speaker:** So are you asking like at what point should the configuration be relegated to the pod versus to the collector? +**Dan:** What we have today for that specific case is the routing processor. So we have a routing processor and it doesn't really work the way that you want here, but it works in a very similar way. So what we do is we route the data points based on their characteristics and we rush them to specific exporters, but we do that by making another network connection, so it's very inefficient, but it works. -**Another Speaker:** Uh, so like I have it like an HPA configured, so like as my traffic increases, you know, presumably I spin up new pods. Um, but why are there settings like the number of consumers? Like should I be, is that purely for non, you know, deployments that can't automatically scale? Um, more so like when do I, when do I create more collectors or when do I change collector-specific configuration? +**Dan:** So those are the two possible solutions for that. With the router, I'll obviously look into this. Thank you. I didn't know what it is. Maybe this is a specific use case, but what would be the advantage of like the router solution versus like having a filter processor that like, you know, having the two pipelines and having the filter processor just like get rid of a, you know, part of like the part A of the set or part B of the set? Would there be an advantage to either one of those? I guess that would be a third solution. -**Another Speaker:** Um, I'm sorry, I missed the very beginning of the question because I was looking for the issue. I found the issue and I linked here, and I linked directly to the comment that I've made that is close to your question. Um, you can read then a dance and search that. Um, but coming back to this question here, I'm, and forgive me if I'm, if I'm missing some context from the very beginning of the question, but, um, the way that I would, I would say this, the way that I would tackle is, um, if you only have stateless connect or components in your collector, you can then just dot use a specific metric and scale up or scale adding more replicas would be the solution. Um, the other answer to that is, you know, your workload, um, and you would probably have to think about that not only as in the scaling to attend to the demands, uh, like the load that I have, but also scaling to improve my high availability. So if a node goes down, what am I going to, what are the effects across my observability pipeline? So how would I want to isolate failures on my observability pipeline? So is it better to have one per namespace? Is that, and is that good enough? Or, um, or should I have a one per tenant? Or perhaps a, even within tenants in my cluster, perhaps I should have different layers of collectors, um, because, you know, failing one specific branch of your collection of collectors isn't not going to be so critical as something that is going to fail only, you know, if you have everything going through one collector, then it's really going to be an epic failure at one point. Um, I guess there's no easy answer to that. +**Gerasi:** Yeah, I guess, um, that's what I'm doing right now, um, and it's working. Uh, there were some bugs in the filter processor that have been resolved, so now it seems to be working perfectly as far as I know. It just, I felt like I was, because I had like two receivers receiving the same data, like I was doubling up on the memory that, you know, my collector was using. -**Another Speaker:** Yeah, I know there's no like one size fits all. I guess I was just trying to like, okay, like so obviously like I have some like bait, like some base throughput, like, you know, that I want to kind of, I want to serve like, you know, I get a certain amount of data points per second or whatever, uh, base, and then like I get spikes, right? So I have to like have a minimum deployment that can handle those spikes. Yeah, but then there are like some connections that are, I think like, like have like sticky sessions. So I need to be careful about like if one of those things like burst real high then like a given, you know, a given pod or given collector if you will will will like, I don't know, have to be able to handle that all right. +**Dan:** Uh, I haven't done any profiling, but maybe that's something I should do. Um, but I'll definitely look at the routing processor. Um, actually take a look at the connectors because I've made a very similar question to the author of the connectors on the PR that they introduced. And I think, I think indeed the answer is that the connector is using connectors. You wouldn't only have one receiver and only one copy of the data point in memory; it only keeps one copy of the data point in memory. -**Another Speaker:** Um, so in most of the cases, you're gonna use gRPC for the connection between collectors or between your workload and a collector. Um, and the good thing about gRPC is when a connection fails, uh, it will attempt to connect to another, um, to another backend automatically, right? So if you have your deployment done correctly, and correctly here means on Kubernetes, you wouldn't be having a headless service, and your clients would be then connecting to the headless service so that the client has a list of known backends up front. So whenever one of them fails, it just fails over to the next one. And, uh, so that’s one way of dealing with the sticky problems, the sticky session problems. So most other connections, they are long-lived when we talk about the observability pipeline. So there are gRPC connections, and gRPC connections, they are long-lived by design. Which also means that if you have like only three collectors at the very beginning of your life cycle of your cluster, for instance, and then you keep adding more collectors to that, but you don't increase the number of clients, then you're not going to see any effects until the clients reconnect, uh, due to some failure, right? So you're not going to see any advantage. The moments that you scale up, you're only going to see advantages the moment things start to fail and then you start seeing the other collectors receive some traffic or when you have new clients. So new clients are going to be load balanced through the new nodes, so then you start seeing that. +**Gerasi:** Let me try to find a PR and you can get more information from there. -**Another Speaker:** Yeah, so I guess that's, there's that. Uh, one other thing you may consider is, um, charting your pipeline based on the type of processing that it's doing. Alright, so one pattern that we've seen before, and, uh, that I've recorded at some point is having one pipeline per data point. So you have one metrics pipeline, you have one logs pipeline, and you have one traces pipeline because the types of workloads or, you know, the workload for metrics is way different than the workload for traces. Um, the way that they work is really different, so you might want to split by that. And you also might want to split by the type of processing that you do on the telemetry data that has been generated by your workload. So if you have more PI information being generated by nodes or by pods on one specific namespace, then it makes sense to have one collector on that namespace with a specific processor to remove this PI and then goes to the next layer of collectors that deals with more generic data. +**Dan:** Okay, awesome. That would be great. Thank you. That sounds like a great feature. I'll definitely follow that. Thank you. + +**Gerasi:** Yeah, that's neat. I haven't heard about the connectors, so I'm interested to learn more as well. + +**Dan:** Well, Gerasi is getting us that PR, does anyone else have anything they'd like to add or ask while we still have 10 minutes on the clock? + +**Derek:** I do if no one else does that would—if someone else obviously go first since I've asked enough. + +**Dan:** I think you're in the clarity and go ahead. + +**Derek:** Okay, um, I asked this a couple of these sessions ago. Um, it was trying to—I’m trying to find—I had opened a GitHub issue. Um, it was very large in scope, so I don't think I have any responses on it. Um, it was, it was about like understanding more about, um, I'll just pop this in the Zoom chat if you're curious. Um, it was more about just like figuring out like how to scale my collector deployment correctly. + +**Derek:** Um, Gerasi, I saw that you had like a post yesterday that you posted on one of the channels that talks a little bit about that. Um, there were some other things I had put in here like, um, understanding like when the num consumers setting, uh, for instance, should be modified. Um, I'm curious if you guys have any like insight into like when I should be, I guess, horizontally scaling my pod versus when I should be modifying the configuration of an individual pod if that or an individual collector, if that makes sense. + +**Dan:** Are you asking like at what point should the configuration be relegated to the pod versus to the collector? + +**Derek:** Uh, so like I have it like an HPA configured so like as my traffic increases, you know, presumably I spin up new pods. Um, but why are there settings like the number of consumers? Like should I be—is that purely for non-deployments that can't automatically scale? + +**Derek:** Um, more so like when do I create more collectors or when do I change collector-specific configuration? + +**Gerasi:** Um, I'm sorry, I missed the very beginning of the question because I was looking for the issue. I found the issue, and I linked here and I linked directly to the comment that I've made that is close to your question. Um, you can read then and search that. + +### [00:39:06] Scaling collector deployments + +**Gerasi:** Um, but coming back to this question here, and forgive me if I'm missing some context from the very beginning of the question, but, um, the way that I would say this, the way that I would tackle it is, um, if you only have stateless components in your collector, you can then just use a specific metric and scale up or scale adding more replicas would be the solution. + +**Gerasi:** The other answer to that is, you know your workload, um, and you would probably have to think about that not only as in the scaling to attend the demands, uh, like the load that I have, but also scaling to improve my high availability. + +**Gerasi:** So if a node goes down, what am I going to—what are the effects across my observability pipeline? So how would I want to isolate failures on my observability pipeline? So is it better to have one per namespace? Is that good enough? Or, um, or should I have a one per tenant? Or perhaps a, even within tenants in my cluster, perhaps I should have different layers of collectors. + +**Gerasi:** Um, because, you know, failing one specific branch of your collection of collectors isn't going to be so critical as something that is going to fail only, you know, if you have everything going through one collector, then it's really going to be an epic failure at one point. Um, I guess there's no easy answer to that. + +**Derek:** Yeah, I know there's no like one size fits all. I guess I was just trying to like, okay, like so obviously like I have some like base throughput, like you know, that I want to kind of, I want to serve, like, you know, I get a certain amount of data points per second or whatever, uh, base, and then like I get spikes, right? + +**Derek:** So I have to like have a minimum deployment that can handle those spikes, yeah, but that—but then there are like some connections that are, I think like, like have sticky sessions. So I need to be careful about like if one of those things like burst real high, then like a given, you know, a given pod or given collector, if you will, will, will like, I don't know, have to be able to handle that. + +**Gerasi:** Um, so in most of the cases, you're going to use gRPC for the connection between collectors or between your workload and a collector. Um, and the good thing about gRPC is when a connection fails, uh, it will attempt to connect to another, um, to another backend automatically, right? + +**Gerasi:** So if you have your deployment done correctly and correctly here means on Kubernetes, you wouldn't be having a headless service and your clients would be then connecting to the headless service so that the client has a list of known backends up front. So whenever one of them fails, it just fails over to the next one. And, uh, so that's one way of dealing with the sticky session problems. + +**Gerasi:** So most other connections they are long-lived when we talk about the observability pipeline. So there are gRPC connections and gRPC connections they are long-lived by design, which also means that if you have like only three collectors at the very beginning of your life cycle of your cluster, for instance, and then you keep adding more collectors to that, but you don't increase the number of clients, then you're not going to see any effects until the clients reconnect due to some failure, right? + +**Gerasi:** So you're not going to see any advantage the moments that you scale up; you're only going to see advantages the moment things start to fail and then you start seeing the other collectors receive some traffic or when you have new clients. So new clients are going to be load balanced through the new nodes, so then you start seeing that. + +**Derek:** Yeah, so I guess there's that. + +**Gerasi:** One other thing you may consider is charting your pipeline based on the type of processing that it's doing. + +**Gerasi:** Alright, so one pattern that we've seen before and that I've recorded at some point is having one pipeline per data point. So you have one metrics pipeline, you have one logs pipeline, and you have one traces pipeline because the types of workloads or, you know, the workload for metrics is way different than the workload for traces. + +**Gerasi:** Um, the way that they work is really different. So you might want to split by that and you also might want to split by the type of processing that you do on the telemetry data that has been generated by your workload. So if you have more PII information being generated by nodes or by pods on one specific namespace, then it makes sense to have one collector on that namespace with a specific processor to remove this PII and then goes to the next layer of collectors that deals with more generic data. **Derek:** Yeah, so you're only affecting like, you're only slowing down for like that subset, right? -**Another Speaker:** Exactly, yeah. +**Gerasi:** Exactly, yeah. + +**Derek:** Okay, yeah. Okay, cool. Awesome. And then I know we're like almost at time, so just like harping on one specific setting, that num consumers thing, like is there when, when would I ever want to change that? Do you know? + +**Gerasi:** I'm sorry, um, consumers as in? + +**Derek:** Yeah, there's on the OTLP exporter there's a, maybe this is too technical; maybe I should just open an issue just for this. Uh, there's a property called num consumers. I think it's on the OTLP exporter. Um, it seemed like I got better at the root book when I increased it, um, and I was just wondering what like why wouldn't I increase that? Like what trade-off am I sacrificing? + +**Gerasi:** Okay, um, sorry, I see now. So this is about the signing queue, uh, this is a blocking queue, and, um, the way that it works is, um, whenever things are being sent from one place to another, from an exporter and into a backend, um, it's placed on a queue and then things are picked from the queue like from like workers. + +**Gerasi:** And, uh, this is, you know, number of consumers is basically the number of workers that are picking things up from the queue. Now, um, I'm not the author of this component here, of this Helper, but, um, we had a similar component on Jaeger, and I can tell you by experience that we don't actually know what is the optimal value for that; you have to find that out by yourself. + +**Gerasi:** Um, and it's complicated because, um, in Java for instance, uh, the number of workers would be typically, uh, closely related to the number of processors in a specific, like CPU processors in a machine, a CPU using a machine. Now, with Go, it's not like that. So one worker thread is not a thread; it's a goroutine, which is not related to an OS thread, a Linux thread. So there's no relation between or no very explicit relation between a number of consumers with the number of processors on a specific laptop or a machine server bare metal. -**Derek:** Okay, yeah, so yeah, okay, cool. Awesome. And then I know we're like almost at time, so just like harping on one specific setting, that num consumers thing, like is there when when would I ever want to change that, do you know? +**Gerasi:** Um, so you have to play with that number. I think I kind of played with this information on an article recently, um, and the idea is that the more, um, if you have backends that are taking very long to answer to you, having more consumers here means that you have more HTTP connections with the backend that you're sending data to, and it might be a problem with the backend that you're— I mean, if you're the backend is having trouble with a specific amount of connections and you're adding more connections, you're adding more load to a server that is already overloaded. -**Another Speaker:** I'm sorry, um, consumers as in? +### [00:48:02] Light-hearted moment with Taco -**Derek:** Yeah, there's on the OTLP exporter, there's a, maybe this is too technical, maybe I should just open an issue just for this. Uh, there's a property called num consumers. I think it's on the OTLP exporter. Um, it seemed like I got better at the root book when I increased it. Um, and I was just wondering what like why wouldn't I increase that? Like what trade-off am I sacrificing? +**Gerasi:** In that case, you want to decrease the number of consumers, but what it means is it is processing less data, um, simultaneously, concurrently. So what it effectively means is it increases the concurrency of data being sent to the remote server, so in theory it is related to the CPU number of CPUs that you have, uh, but at the same time, I think it is more important to the backend that we are talking to. -**Another Speaker:** Okay, um, sorry. I see now. So this is about it's about the signing queue. Uh, this is a blocking queue, and, um, the way that it works is, um, whenever things are being sent from one place to another from an exporter and into a backend, um, it's placed on a queue and then things are picked from the queue like from like workers. And, uh, this is, you know, the number of consumers is basically the number of workers that are picking things up from the queue. Now, um, I'm not the author of this component here of this helper, but, um, I, we had a similar component on Jaeger, and I can tell you by experience that we don't actually know what is the optimal value for that. You have to find that out by yourself. Um, a, and it's complicated because, um, in Java for instance, uh, the number of workers would be typically closely related to the number of processors in a specific, like CPU processors in a machine, a CPU using a machine. Now with Go, it's not like that. So one worker thread is not a thread, it's a Go routine, which is not related to an OS thread, a Linux thread. So there's no relation between or no very explicit relation between a number of consumers with the number of processors on a specific laptop or a machine server bare metal. Um, so you have to play with that number. I, I think I kind of played with this information on an article recently. Um, and the idea is that the more, um, if you have backends that are are taking very long to answer to you, having more consumers here means that you have more HTTP connections with the backend that you're sending data to. And it might be a problem with the backend that you're, you're, I mean if you're the backend is having trouble with a specific amount of connections and you're adding more connections, you're adding more load to a server that is already overloaded. In that case, you want to decrease the number of consumers, but what it means is it is processing less data, um, simultaneously, concurrently. So what it effectively means is it increases the concurrency of data being sent to the remote server. So in theory it is related to the CPU number of CPUs that you have, uh, but at the same time, I think it is more important to the backend that we are talking to. +**Derek:** Okay, yeah. So awesome. That's like super helpful. I think the main takeaway for me is like I should probably be doing some more specific testing around like my expected, you know, like how much data I'm receiving and expecting to send to my back ends and just, you know, trying to optimize for my specific use case rather than like having a generic, uh, this type of situation, you know, you should set it to X, Y, or Z. -**Derek:** Okay, yeah, so awesome. That's like super helpful. I think the main takeaway for me is like I should probably be doing some more specific testing around like my expected, you know, like how much data I'm receiving and expecting to send to my back ends and just, you know, trying to optimize for my specific use case rather than like having a generic, uh, this type of situation, you know, you should set it to X, Y, or Z. +**Gerasi:** Absolutely. -**Another Speaker:** Absolutely, okay, awesome, really, really appreciate the insight. Thank you. +**Derek:** Okay, awesome. Really, really appreciate the insight, thank you. -**Derek:** Thank you. +**Reese:** Thank you all. I want to be respectful of everyone’s time. Um, Dan, thank you so much for all your questions. Um, Alex, Gerasi, it was great to have you on. And, um, if anyone has any questions about these sessions or anything else they are curious about, myself, Adriana, as well as Rin are all on the End User Working Group, so feel free to reach out to any of us on CNCF Slack. -**Reese:** Thank you all. All or a few minutes, a few minutes or so. I want to be respectful of everyone's time. Um, Dan, thank you so much for all your questions. Um, Alex, Gerasi, it was great to have you on. And, um, if anyone has any questions about these sessions or anything else they are curious about, myself, Adriana, as well as Rin are all on the End User Working Group, so feel free to reach out to any of us on CNCF Slack. Um, and for, I just have to drop real quick, but, um, Alex and C wanted to see the kitten. I'm just gonna do a quick kitten show. This is Taco. +**Reese:** Um, and for—I just have to drop real quick, but um, Alex and C wanted to see the kitten. I'm just gonna do a quick kitten show. This is Taco. -**Another Speaker:** Thank you. Hi, Taco! +**Dan:** Thank you. -**Reese:** And alright, thank you all so much. +**Reese:** Hi Taco! And alright, thank you all so much! ## Raw YouTube Transcript diff --git a/video-transcripts/transcripts/2023-10-02T15:08:33Z-otel-in-practice-presents-a-hands-on-otel-migration-story-with-jacob-aronoff.md b/video-transcripts/transcripts/2023-10-02T15:08:33Z-otel-in-practice-presents-a-hands-on-otel-migration-story-with-jacob-aronoff.md index 280d7a7..22b8df8 100644 --- a/video-transcripts/transcripts/2023-10-02T15:08:33Z-otel-in-practice-presents-a-hands-on-otel-migration-story-with-jacob-aronoff.md +++ b/video-transcripts/transcripts/2023-10-02T15:08:33Z-otel-in-practice-presents-a-hands-on-otel-migration-story-with-jacob-aronoff.md @@ -10,266 +10,294 @@ URL: https://www.youtube.com/watch?v=pHHINe9D94w ## Summary -In this YouTube video from the "otel in practice" series, Jacob Arof, a staff software engineer at Lightstep, discusses the migration process from StatsD and OpenTracing to OpenTelemetry (otel). Jacob shares insights from his experience leading this migration, highlighting various strategies such as the "All or Nothing," "Slow Tail," and "Bridge" approaches to transitioning to OpenTelemetry while addressing challenges like organizational buy-in and compatibility issues. He emphasizes the importance of getting leadership support, the need for thorough testing, and managing the migration's complexity across multiple services. The session encourages audience participation, allowing viewers to ask questions and share their experiences. Jacob concludes by discussing the future steps of the migration journey, including potential log migrations and adapting to new infrastructure metrics. The video serves as a valuable resource for those navigating similar transitions to OpenTelemetry. +In this video from the "otel in practice" series, Jacob Arof, a staff software engineer from Lightstep, discusses his experience leading a migration from StatsD and OpenTracing to OpenTelemetry. The session covers the various migration paths, including an "All or Nothing" approach and a "Slow Tail" method, weighing their pros and cons. Jacob emphasizes the importance of obtaining organizational buy-in for migration efforts and outlines specific challenges faced during the process, such as performance issues and maintaining compatibility. He encourages audience participation and shares strategies for overcoming resistance from developer teams when transitioning to new observability tools. The discussion concludes with insights on future plans, including potential log migrations and enhancements to metrics collection. The video aims to provide practical advice for teams considering similar migrations in their observability practices. ## Chapters -00:00:00 Welcome and intro -00:01:30 Migration overview +00:00:00 Introductions +00:01:50 Migration overview 00:03:40 Challenges in migration -00:04:50 Importance of organizational buy-in -00:08:00 Migration paths discussion -00:10:15 All or Nothing approach -00:12:15 Slow Tail method -00:15:16 Bridge approach -00:17:34 Q&A session -00:32:50 Future migration plans +00:05:30 Discussion on migration paths +00:07:55 All or Nothing approach +00:12:10 Slow Tail migration method +00:22:45 Discussion on performance issues +00:30:15 OpenTelemetry protocol discussion +00:40:20 Managing collectors in production +00:48:35 Future plans for log migration -**Speaker 1:** Welcome everyone to Otel in Practice. Really stoked for such a good turnout. I think this is one of our best turnouts. Yay! And for anyone who could make it today, there will be a recording. So if you have friends who are like, "Damn it, I missed this," you can let them know that we will be posting the recording and prettying it up once it's available. +## Transcript -**Speaker 1:** Jacob presented, I want to say last month for Otel Q&A. We worked together at Lightstep from ServiceNow. And I'll let Jacob take it away. +### [00:00:00] Introductions -**Jacob:** Thank you! My name is Jacob Arof. I am a staff software engineer. I work on our Telemetry pipeline team. Our team is sort of in charge of our internal and external open-source Telemetry efforts, whether it's like OTL SDKs and Go, or The Collector, or the operator. Sort of anything that you would be using to collect and send data to your vendor is what we do. +**Speaker 1:** Welcome everyone to Otel in Practice. Really stoked for such a good turnout. I think this is one of our best turnouts! Yay! And for anyone who could make it today, there will be a recording. So if you have friends who are like, "Damn it! I missed this," you can let them know that we will be posting the recording and prettying it up once it's available. Jacob presented, I want to say, last month for Otel Q&A. We worked together at Lightstep from ServiceNow. I'll let Jacob take it away. -[00:01:30] **Jacob:** So today I'm going to be talking about a migration that we led last year and a little bit a year and a half ago, for when we migrated from StatsD to OpenTelemetry and also from OpenTracing to OpenTelemetry. I'm going to talk about the various migration paths that are out there, some of the things that went well and went wrong in our migration. But I also would like this to be, you know, feel free to ask as many questions as you have. My slides are pretty light, so I'm really interested to hear, you know, where people struggle currently, what type of challenges you're running into, what features you're interested in. +**Jacob:** Thank you! So yeah, my name is Jacob Arof. I am a staff software engineer. I work on our Telemetry pipeline team. Our team is sort of in charge of our internal and external open source Telemetry efforts, whether it's like OTL SDKs in Go, The Collector, or the Operator. Sort of anything that you would be using to collect and send data to your vendor is what we do. -**Jacob:** So please, you know, raise a hand or comment in the chat, and I'd be happy to answer any questions and go down any interesting avenues that you might have. So with that, I will share my screen. Can we all see my slides? Yes? Great, thank you very much. +### [00:01:50] Migration overview -**Jacob:** So yeah, today I'll be talking about that Otel migration story. So first, let's talk about a problem. I've been in the industry for many years, and I've had to lead a few of these migrations where your hand-rolled metrics or tracing or logging library just isn't cutting it anymore. Something where it's not performant enough, maybe the maintenance is really expensive, maybe there's a feature that everybody wants that you just can't implement. You know, whatever it might be, you want to do a migration. +Today, I'm going to be talking about a migration that we led last year and a little bit over a year and a half ago, for when we migrated from StatsD to OpenTelemetry and also from OpenTracing to OpenTelemetry. I'm going to talk about the various migration paths that are out there, some of the things that went well and went wrong during our migration. But I also would like this to be, you know, feel free to ask as many questions as you have. My slides are pretty light, so I'm really interested to hear, you know, where people struggle currently, what type of challenges you're running into, what features you're interested in. So please, you know, raise a hand or comment in the chat, and I'd be happy to answer any questions and go down any interesting avenues that you might have. -[00:03:40] **Jacob:** It can be very challenging and daunting depending on the amount of services that your company runs, depending on what the organization's willingness to undergo this migration might be. So all of these are problems that I've faced in various jobs, not just where I am now, but you know, a few past roles as well. It can be really hard to make this happen, especially when you, as maybe an SRE or a DevOps person, or maybe just an engineer, just want to make this work. When you feel very passionate about making it happen and you just can't seem to get the buy-in that you need, so that is sort of the theme for today. +With that, I will share my screen. Can we all see my slides? Yes? Great, thank you very much! -**Jacob:** And there is a solution, right? It's like migrating to this new thing. Otel's metric API—I'm going to be talking about metrics, traces—but you know, this is really true about a lot of stuff. For Otel, the metric API, the traces API, and even the logging API is accepted by most major vendors out of the box now. The API supports all these instrument types in all of your favorite languages, probably very few that don't. The performance and compatibility is always top of mind for us throughout the stack of Otel components. Everybody is always thinking about performance. +So yeah, today I'll be talking about that OTEL migration story. -[00:04:50] **Jacob:** So, you know, with all this in mind, you're like, "Yeah, this sounds great. I want to start using this." But there's a problem, right? Using these new tools, migrating these new tools is hard. How do you break up the work? How do you know all of your Telemetry is working the same as before? What are the risks of these different migration models? What even are the different migration models? +First, let's talk about a problem. I've been in the industry for, you know, many years. I’ve had to lead a few of these migrations where your hand-rolled metrics or tracing or logging library just isn't cutting it anymore. Something where it's not performant enough; maybe the maintenance is really expensive. Maybe there's a feature that everybody wants that you just can't implement. You know, whatever it might be, you want to do a migration. It can be very challenging and daunting depending on the amount of services that your company runs, depending on what the organization's willingness to undergo this migration might be. -**Jacob:** And probably the most important one here is how do you convince leadership that this is worth doing? You can try and brute force your way into doing a migration, but that to me is a recipe for burnout. Doing a migration in general, where you know there's a new architecture that you think is really going to improve quality of life for your team, for your company, without getting organizational buy-in is how you spend months in a project that may never see the light of day. And that is really demoralizing and very difficult to work around. +### [00:03:40] Challenges in migration -**Jacob:** So the first thing that you have to do when you want to migrate to a new tool is make the case. It's not even about showing a proof of concept, it's just can you convince people that this is worth doing? Sometimes a proof of concept helps, but that's not the key to success. The key to success is really getting a group of people who agree with you, that you're able to sell one-on-one, who can help you really make it clear to the stakeholders, you know, in your leadership that this is worth doing. +All of these are problems that I've faced in various jobs, not just where I am now, but you know, a few past roles as well. It can be really hard to make this happen, especially when you, as maybe an SRE or a DevOps person, or maybe just an engineer, just want to make this work. When you feel very passionate about making it happen and you just can't seem to get the buy-in that you need. So that is sort of the theme for today. -**Jacob:** For me, it was pretty straightforward. We're a main OpenTelemetry contributor. The Otel metrics API and SDK wanted to go into their 1.0 stable, and they really wanted some quick feedback from someone who has a lot of traffic to test it out and see where the edges are. So in that case, it was very easy to sell the migration. In past jobs, the selling point is usually, you know, you look at maintenance cost and performance overall. I worked with a hand-rolled solution in a few jobs ago, and when it came time to do a migration, the thing that really sold people was just counting the amount of tickets and hours that we had to spend on, you know, new features and maintenance on our internal library, as well as the amount of times where, you know, we've had an incident because something related to our instrumentation is just incorrect, which you know does happen a lot with your own hand-rolled stuff. +And there is a solution, right? It's like migrating to this new thing. OTEL's Metric API, I'm going to be talking about metrics, traces, but you know, this is really true about a lot of stuff. For OTEL, the Metric API, the Traces API, and even the Logging API is accepted by most major vendors out of the box now. The API has support for, you know, all these instrument types in all of your favorite languages, probably. There are very few that don't. And the performance and compatibility are always top of mind for us throughout the stack of OTEL components. Everybody is always thinking about performance. -**Jacob:** So that is maybe a good basis to go off of. Does anyone have any questions sort of before I go into more specifics about this section? I'm going to do like a five count, something I learned from a teacher of mine. I'm just going to—we're going to do five seconds of silence until someone raises their hand, or I'll just keep going. +So, you know, with all this in mind, you're like, "Yeah, this sounds great! I want to start using this." But there's a problem, right? Like, using these new tools, migrating these new tools is hard. How do you break up the work? How do you know all of your telemetry is working the same as before? What are the risks of these different migration models? What even are the different migration models? And probably the most important one here is how do you convince leadership that this is worth doing? -**Speaker 1:** No cool. Oh, is there something in the chat? +### [00:05:30] Discussion on migration paths -**Jacob:** Oh, that's just me moving it a little. Cool, no worries. Thank you. So let's talk about the migration paths. +You can try and, you know, brute force your way into doing a migration, but that to me is a recipe for burnout. Doing a migration in general, where you know there’s a new architecture that you think is really going to improve the quality of life for your team, for your company, without getting organizational buy-in is how you spend months on a project that may never see the light of day. That is really demoralizing and very difficult to work around. -[00:08:00] **Jacob:** The first one is what I call the All or Nothing. This method has you entirely rip out your existing instrumentation in favor of Otel. This would be, you know, a lot of people talk about, you know, replacing the engine of a running plane or running car. A lot of people will say that this would be like, you know, just getting a new plane and have everyone hop to the new one while it's flying. +So the first thing that you have to do when you want to migrate to a new tool is make the case. It’s not even, you know, show a proof of concept; it’s just can you convince people that this is worth doing. Sometimes a proof of concept helps, but that’s not the key to success. The key to success is really getting a group of people who agree with you that you're able to sell one-on-one who can help you really make it clear to the stakeholders, you know, in your leadership that this is worth doing. -**Jacob:** So it's difficult, but you know, maybe it has its benefits. One of the pros is that, you know, once you've pushed your code and you've confirmed things are working and you have a good enough CI/CD system, your work is really done, right? It just rolls out and everybody's happy. This reduces the time for split brain. Split brain is what happens when you're on two systems that may not be compatible together. You could imagine, you know, if you're on StatsD or if you're on Prometheus even. +For me, it was pretty straightforward. We’re, you know, a main OpenTelemetry contributor. The OTEL Metrics API and SDK wanted to go into their, you know, 1.0 stable, and they really wanted some quick feedback from someone who has a lot of traffic to test it out and see where the edges are. So in that case, it was very easy to sell the migration. -**Jacob:** If most of your metrics come from Prometheus, they don't have periods in them. They have, you know, pretty strict requirements about their shape overall. You can't do things like up-down counters; that's not a type that they have. They used to not have a proper histogram; they now do. They now have an exponential histogram support, which is experimental, but you know, there are things that Otel has that Prometheus or StatsD just does not have and doesn't have the ability to do. +In past jobs, the selling point is usually, you know, you look at maintenance cost and performance overall. I worked with a hand-rolled solution in a few jobs ago, and when it came time to do a migration, the thing that really sold people was just counting the amount of tickets and hours that we had to spend on, you know, new features and maintenance on our internal library, as well as the amount of times where, you know, we’ve had an incident because something related to our instrumentation is just incorrect, which, you know, does happen a lot with your own hand-rolled stuff. -**Jacob:** So being in a split brain where some services have it and some services don't and they're emitting different metrics in different shapes, different variables, that can get pretty unwieldy pretty fast. And so if you're not very deliberate about planning how you're going to migrate your dashboards and alerts, then you're going to be stuck with both of them for some time, which if you're on call, and if you've been on call during a migration for this type of stuff, that gets really painful. If you get paged at 3:00 a.m., you have to wake up and go, "Oh, you know, which dashboard should I check? Is this the service on Otel or is the service on StatsD?" That's really frustrating. That's the type of thing that you don't want to have an on-call engineer think about. +So that is maybe a good basis to go off of. Does anyone have any questions sort of before I go into more specifics about this section? -[00:10:15] **Jacob:** So the other benefit of doing this All or Nothing approach is that the issues are incredibly visible. If a dashboard breaks, if an alert pages or a service crashes, it's pretty obvious, right? Hopefully, you're looking at a dashboard or, you know, the person you take the pager when you're doing this migration, so that you experience that pain. And hopefully, you also page on no data. It's very important. +**Speaker 1:** I'm going to do like a five count, something I learned from a teacher of mine. I’m just going to, we’re going to do five seconds of silence until someone raises their hand or I’ll just keep going. -**Jacob:** And then service crashes—you should have an alert on that ideally. So all of these things, though, make it really clear that as soon as you do this All or Nothing thing, you know, if all of your services are rolling out within an hour with this new change and everything is looking good, that's a very good sign and that gives a lot of confidence. +**Jacob:** So no cool. Oh, is there something in the chat? Oh, that's just me moving it a little. Cool, no worries. Thank you! -**Jacob:** This does, though, you know, the clauses here. This requires a lot more thorough testing in a really good development environment. If you have a lot of environment drift where your staging environment is entirely different than your production environment, this might be really challenging because this means you might have production bugs that you're just unable to catch in staging. +### [00:07:55] All or Nothing approach -**Jacob:** And if you do, if the blast radius is all of your services, that can be really dangerous. Also, you know, you do have those compatibility problems I mentioned. So you have to be very deliberate about observing which dashboards and alerts break and fixing them proactively or even in advance if you know what the metrics are going to be. +So let's talk about the migration paths. The first one is what I call the All or Nothing. This method has you entirely rip out your existing instrumentation in favor of OTEL. This would be, you know, a lot of people talk about you know, replacing the engine of a running plane or running car. A lot of people will say that. This would be like, you know, just getting a new plane and having everyone hop to the new one while it's flying. -**Jacob:** The example I have here is definitely a common one where you could imagine a metric type changes but a metric name doesn't, which most vendors will just reject. Or the dashboard that you're looking at will look very strange. This also does mean that there's more upfront effort to migrate your services. It's going to take, you know, probably a group of people to help you monitor this, depending on how many services you have to migrate. If you're, you know, a company with maybe five to ten services, that's not too bad to do with one person. But if you're a company with hundreds of services, you're going to want a team of people to monitor this with you. +So it's difficult, but you know, maybe it has its benefits. One of the pros is that, you know, once you've pushed your code and you've confirmed things are working and you have a good enough CI/CD system, your work is really done, right? It just rolls out and everybody's happy. This reduces the time for split brain. Split brain is what happens when you're on two systems that may not be compatible together. You could imagine, you know, if you're on StatsD or if you're on Prometheus even. -[00:12:15] **Jacob:** So on to the next one, we have the slow tail. This method gives your application developers the ability to migrate themselves by usually flipping an environment variable on and off for whichever method they want. The benefit here is that there's less upfront work. You can confirm that it works for one service and push it out for that service only, and you can do that in all of your environments. So if you have that, you know, if you don't have that confidence that I mentioned earlier about your staging environment versus your production environment, this would be really helpful because you could actually just push a single service without the fear of, you know, all these other services rolling out to verify that your change worked as expected. +If most of your metrics come from Prometheus, they don't have periods in them; they have, you know, pretty strict requirements about their shape overall. You can't do things like up/down counters; like that's not a type that they have. They used to not have a proper histogram; they now do. They now have exponential histogram support, which is experimental, but you know, there are things that OTEL has that Prometheus or StatsD just does not have and doesn’t have the ability to do. -**Jacob:** And finally, this also allows you to have more time to develop dashboards and alerts to handle those compatibility issues. The problem is that this can be very slow. If you have more than 50 services in at least two environments and it takes an hour to migrate a single service because you're trying to be extra careful, then you're looking at a multiple weeks or months-long process. +Being in a split brain where some services have it and some services don’t and they're emitting different metrics and different shapes, different variables, that can get pretty unwieldy pretty fast. And so if you're not very deliberate about planning how you're going to migrate your dashboards and alerts, then you're going to be stuck with both of them for some time, which, if you're on call and if you've been on call during a migration for this type of stuff, that gets really painful. -**Jacob:** If you leave a migration to app developers without a strong why, also they'll never do it, so it's usually going to fall on your team to make that happen. Bugs also may not reveal themselves if your services are not uniform. Imagine you have something like a queue worker that works off of Kafka, whose topology looks entirely different than something like, you know, a classic API server. If you're only, you know, instrumenting your API servers and then you go to instrument one of those Kafka servers, if there's something that's significantly different in how you did your instrumentation, that might take some time to figure out what's going wrong. +If you get paged at 3:00 a.m., you have to wake up and go, "Oh, you know, which dashboard should I check? Is this service on OTEL or is the service on StatsD?" That's really frustrating. That's the type of thing that you don't want to have an on-call engineer think about. -**Jacob:** Ideally, Otel has figured out a lot of this, but doing any of these migrations, you always need to check. You know, we just cannot and honestly should not know how you instrument your services. You don't want to have to explain your whole observability backend to us; that doesn't really make much sense. So ultimately, it's important that you do some work to check that the migration is working as you expected. +So the other benefit of doing this All or Nothing approach is that the issues are incredibly visible. If a dashboard breaks, if an alert pages or a service crashes, it’s pretty obvious, right? Hopefully, you're looking at a dashboard or, you know, the person you take the pager when you're doing this migration so that you experience that pain and hopefully you also page on no data. It’s very important. And then service crashes, you should have an alert on that ideally. -**Jacob:** One thing that I did just think of is that you could do a combination of this slow tail and All-in-One, where you do a migration for, you know, say some good sample of your services. And then once you're pretty confident with those, you could move to just enabling for everyone at once. That's another good option. I'm trying to think about the drawbacks of that. Not sure there are any. I think that's probably the way to go unless you're really nervous about some of these compatibility problems or some of these like unknown unknowns; that might be the only time that that would be a little scary. +So all of these things, though, make it really clear that as soon as you do this All or Nothing thing, you know, if all of your services are rolling out within an hour with this new change and everything is looking good, that’s a very good sign and that gives a lot of confidence. This does though, you know, the clauses here, this requires a lot more thorough testing in a really good development environment. -[00:15:16] **Jacob:** So the last one is the bridge. OpenTelemetry for some migrations actually provides a bridge where you can go from, you know, instrumentation A to instrumentation B without having to do any real code changes other than implementing the bridge. There are a few issues here. Well, already you can see the pros. I mean, it's pretty obvious, right? You don't have to make many code changes. +If you have a lot of environment drift, where your staging environment is entirely different than your public environment, this might be really challenging because this means you might have production bugs that you're just unable to catch in staging. And if the blast radius is all of your services, that can be really dangerous. -**Jacob:** The problem is that you're going to have some worse performance in comparison to writing using the new method. It's going to be, you know, a fair—there's a conversion cost to anything that you're doing in your application. If another option is you could just send from, you know, instrumentation A and then convert it to instrumentation B with the Otel collector. So if you wanted to go from StatsD, the Otel collector has a StatsD receiver and an Otel exporter, so that's also fine. But both of these have the same drawback, which is that if you wanted to take advantage of some of the capabilities of the Otel SDKs, you know, then this migration—you're not really doing a migration. +Also, you know, you do have those compatibility problems I mentioned, so you have to be very deliberate about observing which dashboards and alerts break and fixing them proactively or even in advance if you know what the metrics are going to be. The example I have here is definitely a common one where you could imagine a metric type changes but a metric name doesn’t, which most vendors will just reject. -**Jacob:** It means that you can, you know, try and migrate stuff piecemeal and like for dashboards and alerts, which is great, but it does have this drawback of, like, you're not actually doing the migration. You're still, you know, just putting off the hard work later. And so this can also be confusing to your app developers where someone says, "Well, my code says open tracing, but this trace says Otel. Why can't I, you know, use X feature?" And that can be a little confusing. I think that's like not the end of the world, though. It's, you know, as long as you're communicating well, that should be all right. +### [00:12:10] Slow Tail migration method -**Jacob:** But yeah, so this is really, I think, using the collector is another really good path forward. Overall, though, doing the thing where you just send some traffic to the collector from one of your services, migrate your apps and dashboards, and then you have everything sent to the collector, and then you can migrate your apps to Otel with that All-in-One approach. And then your dashboards and alerts should just, you know, already be changed, and you should be all set. +Or the dashboard that you're looking at will look very strange. This also does mean that there's more upfront effort to migrate your services. It's going to take, you know, probably a group of people to help you monitor this, depending on how many services you have to migrate. If you're, you know, a company with maybe five to ten services, that's not too bad to do with one person, but if you're a company with hundreds of services, you're going to want a team of people to monitor this with you. -**Jacob:** Before I move to the next portion, do I have any questions? Any thoughts? I'm going to do a five-count again. +So on to the next one, we have the slow tail. This method gives your application developers the ability to migrate themselves by usually flipping an environment variable on and off for whichever method they want. The benefit here is that there's less upfront work. You can confirm that it works for one service and push it out for that service only, and you can do that in all of your environments. So if you have that, you know, if you don’t have that confidence that I mentioned earlier about your staging environment versus your production environment, this would be really helpful because you could actually just push a single service without the fear of, you know, all these other services rolling out to verify that your change has worked as expected. -[00:17:34] **Speaker 2:** Jacob, I'm actually living through this right now. One thing that I'm running into quite a bit is getting the various developer teams to shift their mindset from sort of our previous vendor that we were with into our newer one. And it's, you know, there's underlying things we're doing, like the StatsD into like Prometheus style type thing with it. And yeah, that's led to a couple of people who are like, "Oh, these things aren't apples to apples here anymore," and that's causing a bunch of stuff. Any tips and suggestions on winning the hearts and minds here? +Finally, this also allows you to have more time to develop dashboards and alerts to handle those compatibility issues. The problem is that this can be very slow. If you have more than 50 services in, you know, at least two environments and it takes an hour to migrate a single service because you're trying to be extra careful, then you're looking at a multiple weeks or months-long process. -**Jacob:** Yeah, so really, like the thing to do is sell on features when you can. So if you could say, you know, this improves our—well, the easiest one is cost, right? If you're using StatsD, you're probably coming from Datadog. You're going to say, "You know, our previous spend with Datadog was X thousands, maybe millions of dollars, and with Prometheus, it's like, you know, $100 a month or something," right? That's the easiest one. +If you leave a migration to app developers without a strong why, also, they'll never do it, so it's usually going to fall on your team to make that happen. Bugs also may not reveal themselves if your services are not uniform. Imagine you have something like a queue worker that works off of Kafka, whose topology looks entirely different than something like, you know, a classic API server. If you're only, you know, instrumenting your API servers and then you go to instrument one of those Kafka servers, if there's something that's significantly different in how you did your instrumentation, that might take some time to figure out what's going wrong. -**Jacob:** But the more valuable one is, you know, you sell in the ecosystem where, for me, the real benefit of Prometheus is that the local development experience for metrics is actually much simpler. You don't have to run, you know, a Datadog agent; you don't have to do anything else. You can just hit your metrics endpoint and verify that your metrics are doing what you expect them to. And that's a really good developer flow for testing that stuff. +You know, ideally, OTEL has figured out a lot of this, but doing any of these migrations, you always need to check. You know, we just cannot and honestly should not know how you instrument your services. You know, you don’t want to have to explain your whole observability backend to us; that doesn't really make much sense. -**Jacob:** The other thing that is—I mean, people from Datadog—I used to work for Datadog—and their dashboard product is great. But Grafana, if you're using Grafana, also has a really strong dashboards product. Showing something like, you know, maybe you use Redis in your Kubernetes cluster, you install a Redis service monitor, and then you install the off-the-shelf Grafana dashboard for it, right? Like that's a pretty great experience, and that's all done at, you know, zero cost, which is pretty incredible. +So ultimately, it's important that you do some work to check that the migration is working as you expected. One thing that I did just think of is that you could do a combination of this slow tail all-in-one where you do a migration for, you know, say some good sample of your services, and then once you're pretty confident with those, you could move to just enabling for everyone at once. That’s another good option. -**Jacob:** You know, Prometheus metric cost is very small, in cents on the dollar. So the last thing you can do is trainings. I have a storied history with trainings. I think that they never really achieve the thing that you want, which is for more people to be excited. Really, what they do is can cause more confusion if you're not careful with like your language. I think maybe a strategy that I've always wanted to do is build some local tooling around the stuff for developers, whether it's, you know, writing maybe an end-to-end test or a little UI around their application metrics so they can do something with them locally to be like, "Oh yes, like this thing is working as expected." +I’m trying to think about the drawbacks of that. Not sure there are any. I think that's probably the way to go unless you're really nervous about some of these compatibility problems or some of these like unknown unknowns; that might be the only time that that would be a little scary. -**Jacob:** I mean, I think that the local dev for Prometheus is just pretty fantastic, and that's probably what I would sell on. But again, you know, it's very company-specific in many ways, right? If people are really bought into the Datadog model of things, which is, you know, high cost, very low thinking, Prometheus and Grafana is not really that. It's low cost but much more thinking, and ultimately, like you don't want your developers to have to think too much about their instrumentation. +The last one is the bridge. OpenTelemetry for some migrations actually provides a bridge where you can go from, you know, instrumentation A to instrumentation B without having to do any real code changes other than implementing the bridge. There are a few issues here. Well, already you can see the pros; I mean it’s pretty obvious, right? You don’t have to make many code changes. The problem is that you're going to have some worse performance in comparison to writing using the new method. -**Jacob:** The thing that I tend to do is have a wrapper library before doing a migration so that people are used to the same signatures; it's just doing a different function under the hood. It also makes your migration easier. I didn't mention it here because most companies already have some type of wrapper because they have some needs that are specific. It's usually like a thin wrapper, but doing as much as you can to not change their workflow and make it very simple is always going to be good. +It's going to be, you know, a fair, you know, there's a conversion cost to anything that you're doing in your application. If another option is you could just send from, you know, instrumentation A, and then convert it to instrumentation B with OTEL collector. So if you wanted to go from StatsD, the OTEL collector has a StatsD receiver and an OTEL exporter, so that's also fine. But both of these have the same drawback, which is that if you wanted to take advantage of some of the capabilities of the OTEL SDKs, you know, then this migration, you're not really doing a migration. -**Jacob:** The thing with Prometheus you can do is check what metrics are available and then you can auto-generate dashboards from those metrics. And you can add into your Helm charts, you know, these are the metrics I care about, and then you could just generate alerts automatically from that as well. So there's a lot of that quality of life stuff that, again, like it's easy with the Datadog UI but is automatic with, you know, infrastructure as code. +It means that you can, you know, try and migrate stuff piece meal and like for dashboards and alerts, which is great, but it does have this drawback of like, you're not actually doing the migration; you're still, you know, just putting off the hard work later. And so this can also be confusing to your app developers, where someone says, "Well, my code says OpenTracing, but this trace says OTEL. Why can't I, you know, use X feature?" And that can be a little confusing. I think that's like not the end of the world though. It's, you know, as long as you're communicating well, that should be all right. -**Jacob:** So that's like another trade-off I would say because not a lot of people I don't think use Terraform at Datadog. +But yeah, so this is really, I think using the collector is another really good path forward overall though. Doing the thing where you just send some traffic to the collector from one of your services, migrate your apps and dashboards, and then you have everything sent to the collector and then you can migrate your apps to OTEL with that all-in-one approach. -**Speaker 2:** Thanks for that, Jacob. +And then your dashboards and alerts should just, you know, already be changed and you should be all set. Before I move to the next portion, do I have any questions? Any thoughts? I'm going to do a five count again. -**Jacob:** Yeah, no problem! Any more questions on this part? Cool. +**Speaker 2:** Jacob, I'm actually living through this right now. One thing that I'm running into quite a bit is like getting the various developer teams to shift their mindset from sort of our previous vendor that we were with into our newer one. And it's, you know, there's underlying things we're doing like the StatsD into like Prometheus style type thing with it, and yeah, that's led to a couple people who are like, "Oh, these things aren't apples to apples here anymore," and that's causing a bunch of stuff. Any tips and suggestions on like winning the hearts and minds here? -**Jacob:** Okay, so now I'll talk about what I did, what the team did for our migration from StatsD to Otel for metrics. So we began migrating using that slow tail path. The reasons were, I'd say, good motivating factors. As I mentioned earlier, the metrics API and SDK weren't declared stable at the time, which meant we would have had to deal with some signature changes. +**Jacob:** Yeah, so really like the thing to do is sell on features when you can. So if you could say, you know, this improves our—well, the easiest one is cost, right? If you're using StatsD, you're probably coming from Datadog, you're going to say, you know, "Our previous spend with Datadog was X thousands, maybe millions of dollars, and with Prometheus, it's like, you know, 100 a month," or something, right? That's the easiest one. -**Jacob:** And that would have been potentially a lot of signatures. We did have a wrapper library, but still doing those types of changes can be pretty frustrating. We also wanted to use some new metrics features that StatsD didn't support. This is like asynchronous instruments. The biggest one was exponential histograms, which Otel sort of pioneered. +But the more valuable one is, you know, you sell in the ecosystem where, for me, the real benefit of Prometheus is that the local development experience for metrics is actually much simpler. You don’t have to run, you know, a Datadog agent; you don’t have to do anything else; you can just hit your metrics endpoint and verify that your metrics are doing what you expect them to. And that’s a really good developer flow for testing that stuff. -**Jacob:** And then the last one is that we would, you know, because we are the group that helps write these libraries, we wanted to understand the performance and quality of life features to make it as easy as possible and as, you know, as quick a decision. It's just, you know, you don't have to worry about performance; you don't have to worry about all this other stuff. How do we make this simple? +The other thing that is—I mean, people from Datadog; I used to work for Datadog, and their dashboard product is great. But Grafana, if you're using Grafana, also has a really strong dashboards product. Showing something like, you know, maybe you use Redis in your Kubernetes cluster, you install a Redis service monitor, and then you install the off-the-shelf Grafana dashboard for it, right? Like that’s a pretty great experience, and that’s all done at, you know, zero cost, which is pretty incredible. -**Jacob:** So in migrating to Otel for metrics, we did find a few performance issues. The first one was, you know, because we were using that wrapper library I mentioned, our implementation was working off of OpenTracing tags, which at the time was our tracing instrumentation, which we then would need to convert into Otel tags anytime you wanted to make a metric, which if you think about the amount of times that you call, you know, metric.record, that's a lot of time. So that gets pretty expensive. +You know, Prometheus metric cost, which is very small in cents on the dollar. So the last thing you can do is like training. I have a storied history with training; I think that they never really achieve the thing that you want, which is for more people to be excited. Really what they do is can cause more confusion if you're not careful with like your language. -**Jacob:** And so while we waited for improved performance in the metric libraries, we just began our tracing migration because the Otel team needed time to investigate some of the stuff that we brought to them. And it also meant that we could fix, you know, the very company-specific problem of this conversion. +I think maybe a strategy that I've always wanted to do is build some local tooling around the stuff to developers, whether it's, you know, writing maybe an end-to-end test or a little UI around their applications' metrics so they can do something with them locally to be like, "Oh yes, like this thing is working as expected." I mean, I think that the local dev for Prometheus is just pretty fantastic, and that’s probably what I would sell on. -**Jacob:** And so then we began our tracing migration, going from OpenTracing and OpenCensus to Otel. And so we went for this one with an All or Nothing approach because Otel for tracing had been stable for some time. We weren't really concerned about compatibility. +But again, you know, it's very company-specific in many ways, right? If people are really bought into the Datadog model of things, which is, you know, high cost, very low thinking, Prometheus and Grafana is not really that. It’s low cost but much more thinking, and ultimately, like you don't want your developers to have to think too much about their instrumentation. -**Jacob:** Ultimately, at the time, there weren't a lot of guides written, but the process was relatively straightforward, and the compiler is really doing most of the work for you. The structure for it, we already supported Otel traces in our product, so there's actually, like, no—in theory, there were no real fun changes either for our own dashboards and alerts that we were going off of. +The thing that I tend to do is have a wrapper library before doing a migration, so that people are used to the same signatures; it’s just doing a different function under the hood. It also makes your migration easier. I didn't mention it here because most companies already have some type of wrapper because they have some needs that are like specific. It's usually like a thin wrapper, but doing as much as you can to not change their workflow and make it very simple is always going to be good. -**Jacob:** So what I did for this was write some—write the code for the migration and then push up, build, and push an image to our staging and production environments for a service that doesn't get much traffic but does get some. It's important that it gets some. +The thing with Prometheus you can do is check what metrics are available, and then you can auto-generate dashboards from those metrics. And you can add into your Helm charts, you know, these are the metrics I care about, and then you could just generate alerts automatically from that as well. So there's a lot of that quality of life stuff that again, like it's easy with the Datadog UI but is automatic with, you know, infrastructure as code. So that's like another trade-off I would say because not a lot of people, I don’t think, use Terraform at Datadog. -**Jacob:** And then it confirmed that each version in our product looked the same before and after. We have a view of sort of, you know, these red metrics per version, and it shows you the difference in those versions. And if any of those looked different, if the rate, for example, dropped off, then that's a sign that we did something totally incorrect. +**Speaker 2:** Thanks for that, Jacob! -**Jacob:** Doing that for one service is good, but then the reason that the, you know, the All or Nothing approach is really useful is that you get that integration test. So when I migrated all of our services to this new method, it was really clear that service-to-service trace propagation wasn't working. You know, you just did a spot check of a few different traces, and it's really—that's like such an obvious thing. +**Jacob:** Yeah, no problem! Any more questions on this part? -**Jacob:** So that was a pretty easy fix. I actually had to just go into the Otel community and implement something that had a TODO around it. +### [00:22:45] Discussion on performance issues -**Jacob:** Another issue that we had right before, you know, sort of in the last bit of this migration was that our sampler wasn't configured correctly. Otel provides a lot of new features for sampling, and I was under-sampling in one environment and then over-sampling in another environment because I misconfigured it. So I had to roll that back really quickly, fix it, and then push it out a day or two later. +**Speaker 1:** Cool, okay. So now I'll talk about what I did, what the team did for our migration from StatsD to OTEL for metrics. -**Jacob:** But overall, this whole process took maybe a month of work to migrate, I think it's like a hundred or so services in all of our environments, which is, if you've ever led a migration, that's a pretty good time. I was pretty happy with that one. +So, we began migrating using that slow tail path. The reasons were, I’d say, good motivating factors. As I mentioned earlier, the Metrics API and SDK weren’t declared stable at the time, which meant we would have had to deal with some signature changes, and that would have been potentially a lot of signatures. We did have a wrapper library, but still doing those types of changes can be pretty frustrating. -**Jacob:** It also meant that we could get back to our metrics migration because we had sort of achieved that goal with OpenTracing tags. We were able to use those—use the fact that we didn't need to do this conversion to continue our metrics migration. +We also wanted to use some new metrics features that StatsD didn’t support. This is like asynchronous instruments. The biggest one was exponential histograms, which OTEL sort of pioneered. And then the last one is that we would, you know, because we are the group that helps write these libraries, we wanted to understand the performance and quality of life features to make it as easy as possible and as quick a decision. You know, you don't have to worry about performance; you don't have to worry about all this other stuff. How do we make this simple? -**Jacob:** It also—we came back to the Otel team having shipped some real performance and quality of life improvements that really let us continue with this without the fear of performance problems. +In migrating to OTEL for metrics, we did find a few performance issues. The first one was, you know, because we were using that wrapper library I mentioned, our implementation was working off of OpenTracing tags, which at the time was our tracing instrumentation, which we then would need to convert into OTEL tags anytime you wanted to make a metric. If you think about the amount of times that you call, you know, metric.record, that’s a lot of time, so that gets pretty expensive. -**Jacob:** These changes also let us use this feature called metrics views, which let you create a new metric series to provide seamless compatibility with StatsD. StatsD emits their histogram—what's really what's called a Prometheus summary—but we wanted to emit exponential histograms, but that's really the only change that was happening here. +And so while we waited for improved performance in the metric libraries, we just began our tracing migration because the OTEL team needed time to investigate some of the stuff that we brought to them. And it also meant that we could fix, you know, the very company-specific problem of this conversion. -**Jacob:** All of our other metrics were able to stay about the same, so we just needed to dual write the old summary, which Otel didn't really support at the time and I think doesn't and shouldn't; it's a bad metric type. And we also wanted to dual write the exponential histogram so that we could migrate all of our dashboards and alerts from this old approach to this new approach. +So then we began our tracing migration, going from OpenTracing and OpenCensus to OTEL. And so we went for this one with All or Nothing approach because OTEL for tracing had been stable for some time. We weren't really concerned about compatibility, and ultimately at the time there weren’t a lot of guides written, but the process was relatively straightforward and the compiler is really doing most of the work for you. The structure for it, we already supported OTEL traces in our product, so there actually, like, in theory, there were no real fun changes either for our own dashboards and alerts that we were going off of. -**Jacob:** And then finally, you know, as I said, we put this library behind a feature flag, and then we could just flip that on and off whenever we wanted. And then we would roll it out pretty slowly over a two or three month period to all of our environments. This also let us like test the change really effectively as well. +So what I did for this was write the code for the migration and then push up, build, and push an image to our staging and production environments for a service that doesn't get much traffic. It does get some; it's important that it gets some. And then confirmed that each version in our product looked the same before and after. We have a view of sort of, you know, these red metrics per version, and it shows you the difference in those versions. And if any of those looked different, if the rate, for example, dropped off, then that’s a sign that we did something totally incorrect. -**Jacob:** I think what's next? Let me go back. +Doing that for one service is good, but then the reason that the, you know, the All or Nothing approach is really useful is that you get that integration test. So when we migrated all of our services to this new method, it was really clear that service-to-service trace propagation wasn't working. You know, you just did a spot check of a few different traces, and it's really—that's like such an obvious thing. So that was a pretty easy fix. I actually had to just go into the OTEL community and implement something that had a TODO around it. -**Speaker 3:** Do you have any questions about this migration process? Do a five count again. +Another issue that we had right before, you know, sort of in the last bit of this migration was that our sampler wasn't configured correctly. OTEL provides a lot of new features for sampling, and I was under-sampling in one environment and then over-sampling in another environment because I misconfigured it. So I had to roll that back really quickly, fix it, and then push it out a day or two later. -**Rahul:** Jacob, this is Rahul. I have one question around the Otel protocol. So I'm guessing you must have used the Otel receivers and exporters vastly during your metrics and trace migration. So what is the go-to protocol within Otel? It supports STD and gRPC, but from security and performance point of view, which is the go-to protocol for traces and metrics? +But overall, this whole process took maybe a month of work to migrate, I think it's like a hundred or so services in all of our environments, which is, if you've ever led a migration, that’s a pretty good time. I was pretty happy with that one. It also meant that we could get back to our metrics migration because we had sort of achieved that goal with OpenTracing tags. -**Jacob:** I'm not sure I follow. We actually just emitted Otel directly from our SDKs to our SaaS, so we didn't go through a collector for this one. We could have gone through a collector, but we didn't want the operational overhead at the time. It was enough migrations to handle it once. But that didn't really answer your question. Can you restate your question? +We were able to use those, use the fact that we didn’t need to do this conversion to continue our metrics migration. It also—we came back to the OTEL team having shipped some real performance and quality of life improvements that really let us continue with this without the fear of performance problems. -**Rahul:** Yeah, I mean, I wanted to know what is the best protocol to use under Otel? Is it STD or gRPC in terms of metrics and traces? +These changes also let us use this feature called metrics views, which let you create a new metric series to provide seamless compatibility with StatsD. StatsD emits their histogram as what's really called a Prometheus summary, but we wanted to emit exponential histograms, but that's the really the only change that was happening here. -**Jacob:** Yeah, in terms of performance and security. I'm not the best with security recommendations. I would say we use gRPC for everything, just because it's, I don't know, our sort of de facto internal standard. HTTP is more accepted for some, like depending on your company security requirements. I'm not sure of the real differences—like to compare and contrast them. I don't think I'd be the right person to speak to those. Maybe some of the other Otel folks have more of an opinion or more information on that, though. +All of our other metrics were able to stay about the same, so we just needed to dual-write the old summary, which OTEL didn’t really support at the time and I think doesn't and shouldn't; it's a bad metric type. And we also wanted to dual-write the exponential histogram so that we could migrate all of our dashboards and alerts from this old approach to this new approach. -**Rahul:** Okay, cool. Were there any learnings around managing access tokens? And did you use multiple access tokens, or does it, you know, if you're sending a lot of metrics or traces, or a single access token? +And then finally, you know, as I said, we put this library behind a feature flag and then we could just flip that on and off whenever we wanted. And then we would roll it out pretty slowly over a two or three-month period to all of our environments. This also let us test the change really effectively as well. + +I think what's next? Let me go back. So any questions about this migration process? + +**Speaker 3:** Do a five count again. + +**Jacob:** + +This is Rahul. I have one question around the OTLP protocol. So I'm guessing you must have used the OTLP receivers and exporters vastly during your metrics and trace migration. So what is the go-to protocol within OTLP? It supports gRPC and HTTP, but from a security and performance point of view, which is the go-to protocol for traces and metrics? + +**Jacob:** I'm not sure I follow. We actually just emitted OTLP directly from our SDKs to our SaaS, so we didn't go through a collector for this one. We could have gone through a collector, but we didn't want the operational overhead at the time. It was enough migrations to handle at once, but that didn't really answer your question. Can you restate your question? + +### [00:30:15] OpenTelemetry protocol discussion + +**Speaker 3:** Yeah, I mean, I wanted to know what is the best protocol to use under OTLP? Is it gRPC or HTTP in terms of metrics and traces? + +**Jacob:** Yeah, in terms of performance and security? Yeah, so I'm not the best with security recommendations. I would say we use gRPC for everything just because it's, I don't know, our sort of de facto internal standard. HTTP is more accepted for some, like depending on your company security requirements. I'm not sure of the real differences, like to compare and contrast them. I don't think I'd be the right person to speak to those. Maybe some of the other OTEL folks have more of an opinion or more information on that though. + +**Speaker 3:** Okay, cool. Were there any learnings around managing access tokens? And did you use multiple access tokens? Does it portal you know, if you're sending a lot of metrics or traces or a single access token? **Jacob:** Yeah, we just used the same access token approach that we did. I don't know if I can speak to that just because it's, you know, internal security stuff. -**Rahul:** Yeah, no worries. +**Speaker 3:** Yeah, no worries. + +**Jacob:** I’d say that, you know, the best thing you can do for security in like a cloud-native environment is use some sort of secrets provider. The Kubernetes external secrets operator is great. You can hook it up to something like GCP's KMS and decrypt your secrets to then load access tokens from, though. You can do the same thing with AWS or Vault or any of these other providers as well. If you're particularly like security inclined, it's also important to use things like mTLS as well. Something like Istio can help you with some of that. The OTEL operator actually provides some mTLS features in OpenShift. So, you know, there are a lot of security features out there to be used. Hopefully, that seeds some interesting investigation for you. + +**Speaker 3:** Yep, yep, thanks! + +**Jacob:** No problem! + +**Speaker 1:** So moving on, you might be wondering what's next? Are we going to do a logs migration? And right now we're under a different migration, which is changing our infrastructure metrics to use the OTEL first receivers, like the K8s cluster receiver, the C++ receiver, and so forth. Because we really want to start using some of these things the community is writing. + +After that, we should be able to begin migrating to use the new OTEL logging case, which should be looking pretty good in a few—they're looking good now, but I’m not sure what the state of it is for Go just yet, given that Go just released like a new standard logging library. -**Jacob:** I'd say that, you know, the best thing you can do for security in like a cloud-native environment is use some sort of secrets provider. The Kubernetes External Secrets Operator is great. You can hook it up to something like GCP's KMS and decrypt your secrets to then load access tokens from, though you can do the same thing with AWS or Vault or any of these other providers as well. +Any more questions? -**Jacob:** If you're particularly security-inclined, it's also important to use things like mTLS as well. Something like Istio can help you with some of that. The Otel operator actually provides some mTLS features in OpenShift. So, you know, there are a lot of security features out there to be used. Hopefully, that seeds some interesting investigation for you. +**Speaker 4:** Hi, this is Jay speaking. I have one general question regarding OpenTelemetry. So I’m pretty new to this, and the reason why I was investigating into OpenTelemetry was for its ability to be backend agnostic for generating metric data. The question is, however, I was interested in exploring pull-based metrics exporters, but so far, I don’t know if I’m right or wrong, but the Prometheus exporter within the OpenTelemetry SDK is the only one that is supporting the pull-based metrics approach. Is that correct? -**Rahul:** Yep, yep. Thanks. +**Jacob:** So, it sounds like you're going—so there are a few different types of exporters within OTEL. So there's an SDK exporter, which, yeah, there's a Prometheus exporter. I think there might even be a Datadog exporter. But one might be a better fit is to use the OTEL collector and use their exporters, which are numerous, and pretty much every backend has some type of exporter. -[00:32:50] **Jacob:** No problem. So moving on, you might be wondering what's next. Are we going to do a logs migration? Right now we're under a different migration, which is changing our infrastructure metrics to use the Otel first receivers like the CP cluster receiver, the CET receiver, and so forth. Because we really want to start using some of these things the community is writing. +So I think I would try and say you should—if you're using OTEL SDKs, you should export to OTEL collector and then export to, you know, your protocol of choice. -**Jacob:** After that, we should be able to begin migrating to use the new logging case, which should be looking pretty good in a few—they're looking good now, but I'm not sure what the state of it is for Go just yet, given that Go just released like a new standard logging library. +**Speaker 4:** Yeah, but when using this OTEL exporter, this would rather be a push-based mechanism, right, to send data to the OTEL collector? -**Speaker 4:** Any more questions? +**Jacob:** Yeah, that's correct. -**Jay:** Hi, this is Jay speaking. I have one general question regarding OpenTelemetry. I'm pretty new to OpenTelemetry, and the reason why I was investigating OpenTelemetry was for its ability to be backend agnostic for generating metric data. The question is, however, I was interested in exploring pull-based metrics exporters, but so far, I don't—if I'm right or wrong—but the Prometheus exporter within the OpenTelemetry SDK is the only one that is supporting the pull-based metrics approach, is that correct? +**Speaker 4:** Okay. -**Jacob:** So it sounds like you're going—so there are a few different types of exporters within Otel. So there's an SDK exporter, which yeah, there's a Prometheus exporter. I think there might even be a Datadog exporter. But one—what might be a better fit is to use the Otel collector and use their exporters, which are numerous, and pretty much every backend has some type of exporter. +**Jacob:** You can also, if your instrumentation is in Prometheus right now, you can have the OTEL collector scrape the Prometheus metrics and then export them as OTEL or export them as StatsD or, you know, whatever you want. -**Jacob:** So I think I would try and say you should, if you're using Otel SDKs, you should export an Otel to an Otel collector and then export to, you know, your protocol of choice. +**Speaker 4:** Good, thank you! -**Jay:** But when using this Otel exporter, this would rather be a push-based mechanism, right, to send data to the Otel collector? +**Speaker 1:** Any more questions? -**Jacob:** Yeah, that's correct. +**Jacob:** Do you use any solution to manage or, you know, update the YAML file of the fleet of agent fleet on the go? And is there any built-in solution that you guys are using to manage the collectors? -**Jay:** Okay, okay. You can also, if your instrumentation is in Prometheus right now, though, you can have the Otel collector scrape the Prometheus metrics and then export them as Otel or export them as StatsD or, you know, whatever you want. +**Jacob:** Yeah, so I am a maintainer for the OTEL operator, and internally and externally, I recommend using the OTEL operator with the OTEL collector CRDs. They're pretty easy to use, easy to set up, easy to manage, and we're always developing and thinking about new features as well. And so that'll be the place to get those now and in the future, and I’d continue recommending that. -**Jacob:** Good, thank you! +**Speaker 4:** Okay, thanks! -**Speaker 5:** Do you use any solution to manage or, you know, update the YAML file of the fleet of agent fleet on the go? And is there any built-in solution that you guys are using to manage the collectors? +**Jacob:** Could I ask a question following on from the last question? Would you recommend using something like OpAmp in terms of managing your fleet of collectors, or would you say it's preferable to do it in an operator basis, you know, kind the CRD pattern? -**Jacob:** Yeah, so I am a maintainer for the Otel operator and internally and externally I recommend using the Otel operator with the Otel collector CRDs. They're pretty easy to use, easy to set up, easy to manage, and we're always developing and thinking about new features as well. And so that'll be the place to get those now and in the future, and I'd continue recommending that. +**Jacob:** Yeah, so if you're in Kubernetes, OpAmp—by the way, is still pretty early Alpha right now. Well, the protocol itself is stable, but the actual implementations are in Alpha. So I'm going to answer this question with the assumption that the implementations are done, if that's all right? -**Speaker 5:** Okay, thanks. +**Speaker 4:** Sure! -**Jacob:** Could I ask a question following on from the last question? Would you recommend using something like OpAmp in terms of managing your fleet of collectors, or would you say it's preferable to do it in an operator basis, you know, kind of the C pattern? +**Jacob:** So if you're in Kubernetes, I would recommend using the OpAmp bridge that we're developing. The bridge is a component that can connect to your vendor and will be able to manage pools of collectors rather than just having an extension that works on or a supervisor and an extension that works on a single collector pod. -**Jacob:** Yeah, so if you're in Kubernetes—OpAmp, by the way, is still pretty early alpha right now. Well, the protocol itself is stable, but the actual implementations are in alpha. +The reason for that is usually you are running a collector, and what you—you’re running a collector in a pool, not as a monolith. And so whereas OpAmp would be useful for, you know, a VM or just running it as a binary, doing it in Kubernetes, if you're running it as a pool, it's not a great pattern in Kubernetes to have a supervisor update a single pod's configuration to make it not uniform with the other pods in its replica set. -**Jacob:** So I'm going to answer this question with the assumption that the implementations are done, if that's all right? +So what that means is, you know, if you're going to run it in Kubernetes, if you're going to run a replica set of pods in Kubernetes, you want those pods to be the same configuration. And if you're only running a supervisor on each of those pods, but you're only making the change to a single one, that's an anti-pattern and can get you into some trouble. -**Speaker 5:** Sure, yeah. +The bridge, however, can manage pools of collectors and is definitely what I would recommend to use again with this stuff being completed. That's what I would recommend for Kubernetes. -**Jacob:** So if you're in Kubernetes, I would recommend using the OpAmp bridge that we're developing. The bridge is a component that can connect to your vendor and will be able to manage pools of collectors rather than just having an extension that works on—or a supervisor and an extension that works on a single collector pod. +**Speaker 4:** Okay, thank you! -**Jacob:** The reason for that is usually you are running a collector, and what you're running a collector in a pool, not as a monolith. And so whereas OpAmp would be—OpAmp on a using a supervisor would be very useful for, you know, a VM or just running it as a binary, doing it in Kubernetes, if you're running it as a pool, it's not a great pattern in Kubernetes to have a supervisor update a single pod's configuration to make it not uniform with the other pods in its replica set. +**Jacob:** No problem! -**Jacob:** So what that means is, you know, if you're going to run it in Kubernetes—if you're going to run a replica set of pods in Kubernetes, you want those pods to be the same configuration. And if you're only running a supervisor, if you're running a supervisor on each of those pods but you're only making the change to a single one, that's an anti-pattern and can get you into some trouble. +### [00:40:20] Managing collectors in production -**Jacob:** So the bridge, however, can manage pools of collectors and is definitely what I would recommend to use. Again, with this stuff being completed, that's what I would recommend for Kubernetes. +**Speaker 5:** I’ve got a few questions related to the work. So I work at a large U.S. bank, and I'm trying to bring in OTEL and essentially have that as our main strategy to try and move away from vendor-specific solutions. The question that we’re running into just now are the challenges of vanilla versus vendor. And what I mean by that is, do we just pull down, if I take the collector for example, do we just pull down the collector, configure the collector the way that we want it with receivers and exporters, or with the specific strategic vendors that we work with? -**Speaker 5:** Okay, thank you. +Do we look to bring their vendor-wrapped collector and deploy that within our enterprise? There are obviously pros and cons in doing both. It doesn’t sound like, you know, from your side, Jacob, obviously you’re working from the vendor side. But I have spoken to other vendors who have given me an interesting range of opinions on what area or which of those to look at. It’d be quite interesting to see what your thoughts are. -**Jacob:** No problem. +**Jacob:** This is a great, really great question. It totally depends on your deployment model, I would say. So one option, especially, you know, if you're running thousands of pods and, you know, hundreds of clusters, the model that I would recommend you use is the gateway model, where, you know, let’s say you have a collector per group of apps, and then all of those collectors forward to a centralized pool of collectors that then forward to your vendor. -**Speaker 6:** I've got a few questions related to the work. I work at a large US bank, and I'm trying to bring in Otel and essentially have that as our main strategy to try and move away from beyond vendors specifically. The question that we're running into just now are the challenges like vanilla versus vendor. And what I mean by that is do we just pull down, if I take the collector for example, do we just pull down the collector, configure the collector the way that we want it with receivers and exporters, or with the specific strategic vendors that we work with? +This means that those pools for your applications are vendor-neutral because all they're really doing is gathering and forwarding stuff for your application teams. And then a centralized team would manage the gateway collectors. And then that's the place where you make the decision about, am I going to use my vendor-wrapped collector or am I going to use my, you know, vendor-neutral one? -**Speaker 6:** There are obviously pros and cons in doing both. It doesn't sound like, you know, from your side Jacob, obviously you're working from the vendor side, but I have spoken to other vendors who have given me an interesting range of opinions on what area or which of those to look at. It'd be quite interesting to see what your thoughts are. +The choice becomes really easy to, you know, if there’s some feature that your vendor provides that’s only available in their vendor-specific collector, you could choose to use the wrapped collector there, and then in the future, if you wanted to change vendors, you still have OTEL data sending to that vendor collector, and so you can just change that one out very, very easily. -**Jacob:** This is a great, really great question. It totally depends on your deployment model, I would say. So one option, especially, you know, if you're running thousands of pods and, you know, hundreds of clusters, the model that I would recommend you use is the Gateway model where, you know, let's say you have a collector per group of apps and then all of those collectors forward to a centralized pool of collectors that then forward to your vendor. +The reason that this is good is because you wouldn’t have to go to your application teams or any of these other, you know, orgs and say, “Hey, you know, you have to reconfigure your whole setup because we’re changing our underlying vendor here,” whereas you, as the centralized team, could be the one to just change a single pool to make it all consistent. -**Jacob:** This means that your—those pools for your applications are vendor neutral because all they're really doing is gathering and forwarding stuff for your application teams. And then a centralized team would manage the Gateway collectors, and then that's the place where you make the decision about, "Am I going to use my vendor-wrapped collector or am I going to use my, you know, vendor-neutral one?" +That’s probably what I—that’s like a pretty future-proofed approach. The configuration for what you’re going to do, no matter what, is going to be complicated, but that one is probably going to do you best if you really want to use the vendor collector. If not, still doing the gateway approach is a pretty good one. You can centralize things like officiation sampling rates or even like, you know, requirements for telemetry, like attribute requirements can be centralized before they can egress. -**Jacob:** The choice becomes really easy to make, you know, if there's some feature that your vendor provides that's only available in their vendor-specific collector, you could choose to use the wrapped collector there. And then in the future, if you wanted to change vendors, you still have Otel data sending to that vendor collector, and so you can just change that one out very, very easy. +It also means you can have, you know, set points of egress as well, which, you know, if you're in a pretty locked-down Kubernetes cluster, as you know, and is really important to have, you know, only a certain amount of applications that can egress from the cluster. -**Jacob:** The reason that this is good is because you wouldn't have to go to your application teams or any of these other, you know, orgs and say, "Hey, you know, you have to reconfigure your whole setup because we're changing our underlying vendor here," whereas you, as the centralized team, could be the one to just change a single pool to make it all consistent. +**Speaker 5:** Okay, I think I definitely follow in terms of the deployment patterns and layouts and having the multiple levels of collectors. I think for me, being in the enterprise, what we are worried about is, again, moving stuff like this to production. So if we have a theoretical issue in a vanilla collector, you know, again, just a theoretical example, if we've got RTO and we've got to fix issues within a one or two hours, for example, that's probably going to be the tipping point for us on the vendor versus vanilla question. -**Jacob:** That's probably what I—that's like a pretty future-proofed approach. The configuration for what you're going to do, no matter what, is going to be complicated, but that one is probably going to do you best if you really want to use the vendor collector. If not, still doing the Gateway approach is a pretty good one. +And because we would obviously be looking for some kind of support from a vendor, and typically we would have that, as most folks would have through a vendor. But then that certainly, in my mind, gives us another problem where instead of going from agent-proliferation where you are just now, it's almost like going to OTEL proliferation. You know, it's almost like you're solving one problem and creating another. -**Jacob:** You can centralize things like efficient sampling rates or even like, you know, requirements for telemetry things like attribute requirements can be centralized before they can egress. It also means you can have, you know, set points of egress as well, which, you know, if you're in a pretty locked down Kubernetes cluster, as you know, is really important to have, you know, only a certain amount of applications that can egress from the cluster. +So I think, like the others on the call, you know, we're relatively early in our journey, and we're just trying to go through the not necessarily the technical questions but the hardening questions. What would reality look like when we're in production with, you know, very high volumes of traffic coming through? -**Speaker 6:** Okay, I think I definitely follow in terms of the deployment patterns and layouts and having the multiple levels of collector. I think we're thinking and going. I think for me, being in the enterprise, what we are worried about is, again, moving stuff like this to production. So if we have a theoretical issue in a, sorry, a vanilla collector, you know, again, just a theoretical example, if we've got RTO and we've got to fix issues within a one or two hours, for example, that's probably going to be the tipping point for us on the vendor versus vanilla question. +**Jacob:** Yeah, I mean, that sounds like you're on the right path here overall as far as you're thinking going. -**Speaker 6:** And because we would obviously be looking for some kind of support from a vendor, and typically we would have that as most folks would have through a vendor. But then that certainly in my mind gives us another problem where instead of going from agent polyproliferation, where you are just now, it's almost like going to Otel proliferation. You know, it's almost like you're solving one problem and creating another. +There definitely is that worry of like collector proliferation. You can avoid that with, you know, multiple pools to gateways if you'd like. If you're going from—I mean, usually you're doing this if you're going for like a legacy protocol, something like StatsD, which doesn't like to go over the Internet because it's UDP or something like Prometheus where you have all of these targets and you don't want to worry about managing the scrapes for them, right? -**Speaker 6:** So I think, like the others on the call, we're relatively early in our journey, and we're just trying to go through the not necessarily the technical questions, but the hardening questions. What would reality look like when we're in production with, you know, very high volumes of traffic coming through? +Doing this at scale is going to be really environment and volume dependent. I think if your vendor provides their own collector, they should be able to give you some support for, you know, the vanilla collectors that you run. I, you know, with the people that I work with, do give support for, you know, whatever collectors their customers run. -**Jacob:** I mean, that sounds like you're on the right path here overall as far as your thinking going. There definitely is that worry of collector proliferation. You can avoid that, you know, with multiple pools to gateways if you'd like. If you're going from—I mean, usually you're doing this if you're going for like a legacy protocol, something like StatsD, which doesn't like to go over the Internet because it's UDP, or something like Prometheus, where you have all these targets and you don't want to worry about managing the scrapes for them, right? +And I mean, we don’t have a vendor-specific collector like our company just doesn’t give out a vendor-specific one. But I provide support for any collectors that are customers run. So if that's the fear, I would check with your vendor to see if they also will give you that type of support. -**Jacob:** Doing this at scale is going to be really environment and volume dependent. I think if your vendor provides their own collector, they should be able to give you some support for, you know, the vanilla collectors that you run. I, you know, with the people that I work with, do give support for, you know, whatever collectors their customers run. +I also found that like the steady state of these things is pretty—once you tune it with resource and auto-scaling, it's pretty hands-off, I found. I actually was just working yesterday in a cluster that I touch every six months or so to do some like Helm chart testing, and it's been running for six months without issue and with like a huge varying scale of traffic because of auto-scaling and sort of just because of how simple I keep those collectors. -**Jacob:** And I mean, we don't have a vendor-specific collector—like, our company just doesn't give out a vendor-specific one. But I, you know, I provide support for any collectors that customers run. So if that's the fear, I would check with your vendor to see if they also will give you that type of support. +This is for both metrics and traces, too, for infrastructure and application. So, I mean, it's a much smaller example than what you're talking about for sure. But the point remains where it's like once you reach a good steady state, especially with like your balance cycle, if that's what you have, if you're auto-scaling the setup correctly and if your configuration is pretty, you know, nailed down, you should be—it should be pretty hands-off, knock on wood. But that’s definitely the hope. -**Jacob:** I also found that the steady state of these things is pretty—once you tune it with resourcing and autoscaling, it's pretty hands-off, I found. I actually was just working yesterday in a cluster that I touch every six months or so to do some Helm chart testing, and it's been running for six months without issue, and with like a huge varying scale of traffic because of autoscaling and sort of just because of how simple we keep—I keep those collectors. +**Speaker 1:** Thank you! -**Jacob:** This is for both metrics and traces too, for infrastructure and application. So, I mean, it's a much smaller example than what you're talking about for sure, but the point remains where it's like once you reach a good steady state, especially with like your Bal cycle, if that's what you have, if you're autoscaling the setup correctly and if your configuration is pretty, you know, nailed down, you should be—it should be pretty hands-off, knock on wood. But that's definitely the hope. +**Jacob:** Yeah, no problem! -**Speaker 6:** Yeah, okay. Thanks, Jacob. Appreciate that. +**Speaker 1:** So folks, we are coming up on time, so we've got about five more minutes in case anyone has any more burning questions for Jacob. -**Jacob:** Yeah, no problem. Thank you! +**Speaker 6:** All righty, I will take that as a no. But thank you, Jacob, so much for joining today and sharing your migration story. I think this has resonated with a lot of folks, so we definitely appreciate you coming on and sharing this experience with everyone. -**Speaker 1:** So folks, we are coming up on time. So we've got about five more minutes in case anyone has any more burning questions for Jacob. +### [00:48:35] Future plans for log migration -**Speaker 1:** Alrighty, I will take that as a no, but thank you Jacob so much for joining today and sharing your migration story. I think this has resonated with a lot of folks, so we definitely appreciate you coming on and sharing this experience with everyone. +Like I said, this recording will be made available on the OTEL YouTube channel. Also, for anyone who has missed Jacob's OTEL Q&A session that we had last month, there is a video up on the OTEL channel, and Reyes, who works with Ren and me on the OTEL end-user working group, did a wonderful write-up of the Q&A in case video isn’t your jam. So definitely be sure to check that out. -**Speaker 1:** Like I said, this recording will be made available on the Otel YouTube channel also for anyone who has missed Jacob's Otel Q&A session that we had last month. There is a video up on the Otel channel, and Reyes, who works with Ren and me on the Otel end user working group, did a wonderful write-up of the Q&A in case video isn't your jam. So definitely be sure to check that out. +Okay, so if you go to this link here, you should be able to find our various Slack channels. We love—we encourage everyone to just ask questions, share use cases, we love hearing all that stuff. And also, if you or anyone you know has a really cool OTEL use case, you're just getting started, or you're a more advanced user, does not matter, we would love to hear from you. -**Speaker 1:** Okay, so if you go to this link here, you should be able to find our various Slack channels, and we love—we encourage everyone to just ask questions, share use cases. We love hearing all that stuff. And also, if you or anyone you know has a really cool Otel use case, you're just getting started or you're a more advanced user, does not matter, we would love to hear from you. We're always looking for folks for Otel in Practice, Otel Q&A, and we also have monthly Otel end user discussions, which we run those for three different time zones. So we have them for EMEA, APAC, and Americas. So be sure to join any one of those because there's always really amazing and thoughtful discussions coming out of these. +We're always looking for folks for OTEL in Practice, OTEL Q&A, and we also have monthly OTEL end-user discussions, which we run those for three different time zones. So we have them for EMEA, APAC, and Americas. So be sure to join any one of those because there are always really amazing and thoughtful discussions coming out of these. -**Speaker 1:** So yeah, everyone, thank you very much, and once again, Jacob, thank you so much for taking the time to chat with us twice. +So, yeah, everyone, thank you very much! And once again, Jacob, thank you so much for taking the time to chat with us twice! -**Jacob:** Yeah, thanks so much for having me. I appreciate it. +**Jacob:** Thanks so much for having me! I appreciate it! -**Speaker 1:** Thank you! Bye! Have a good rest of your day. Bye! +**Speaker 1:** Thank you, bye! Have a good rest of your day! ## Raw YouTube Transcript diff --git a/video-transcripts/transcripts/2023-10-11T20:52:28Z-the-evolution-of-observability-practices.md b/video-transcripts/transcripts/2023-10-11T20:52:28Z-the-evolution-of-observability-practices.md index cd48f2f..004b78f 100644 --- a/video-transcripts/transcripts/2023-10-11T20:52:28Z-the-evolution-of-observability-practices.md +++ b/video-transcripts/transcripts/2023-10-11T20:52:28Z-the-evolution-of-observability-practices.md @@ -10,276 +10,334 @@ URL: https://www.youtube.com/watch?v=zSeKL2-_sVg ## Summary -The video features a panel discussion on the evolution of observability practices, hosted by the OpenTelemetry and User Working Group. Panelists include David Win, Iris Dear Mishi, VJ Samuel, Austin Parker, and Noika Melera. They share their experiences and insights on the transition to OpenTelemetry, discussing the challenges and benefits of adopting this standard in their respective organizations. Key points include the importance of developer engagement with observability tools, the need for standardization across platforms, and the shift in focus from merely collecting data to making it actionable. The panel also highlights the ongoing improvements in OpenTelemetry, such as the Collector Builder tool, and anticipates future advancements that will further integrate observability into development workflows. Throughout the discussion, panelists emphasize that the real challenges are often people-related rather than purely technical. +In this panel discussion hosted by the OpenTelemetry User Working Group, experts in the field of observability shared their experiences and insights about the evolution of observability practices. The panelists included David Win (Edge Delta), Iris (Farfetch), Vijay Samuel (eBay), Austin Parker (OpenTelemetry), and Noika Melera (Signos). They discussed the challenges and transformations in observability, particularly focusing on the adoption of OpenTelemetry as a standard for emission of telemetry data. The conversation highlighted the importance of developer productivity, the need for standardized tools, and the shift towards a more integrated observability culture within organizations. Key topics included the initial hurdles faced during implementation, the benefits of standardization provided by OpenTelemetry, and the anticipation of future developments in the project, especially related to logging and dynamic configuration management. The panelists expressed optimism about the ongoing evolution of observability and the role OpenTelemetry will play in shaping industry practices. ## Chapters -00:00:00 Welcome and introductions -00:05:10 State of observability before OpenTelemetry -00:10:50 Challenges in observability practices -00:14:40 Evolution of observability tools -00:19:30 OpenTelemetry standardization benefits -00:24:00 Observability culture and leadership buy-in -00:29:00 Collector Builder introduction -00:33:56 Surprising challenges in implementation -00:40:06 Observability practice focus changes -00:45:00 Future of OpenTelemetry and industry impact +00:00:00 Introductions +00:01:01 Panelist introductions +00:05:05 Discussion on pre-OpenTelemetry challenges +00:10:10 Observability transformation experiences +00:15:15 Importance of standardization in observability +00:20:00 Developer productivity and OpenTelemetry +00:25:25 Migration to OpenTelemetry +00:30:00 Customization and security in OpenTelemetry +00:35:35 Discussion on OpenTelemetry features +00:50:50 Future expectations for OpenTelemetry -**Moderator:** All right, I think we can go ahead and get started. Thank you all so much for being here. The panelists we have today are David W., Austin Parker, VJ Samuel, Iris Dear, Mishi, and Noika Melera. I'm hoping I got all those correct. We'll do a quick run of introductions. This discussion is hosted by the OpenTelemetry and User Working Group, which I am part of, as is Adriana. I see her there. Today we're going to just have a casual conversation. Feel free to get as opinionated as possible about basically the evolution of observability practices. +## Transcript -**Moderator:** David, since you kind of inspired this, I would love for you to do a quick introduction, and after that, we'll just kind of go through the rest of the panelists. Yeah, let's hear a little bit more from David, whose brainchild this was. +### [00:00:00] Introductions -**David:** Hello everyone, my name is David W. I am Principal at Edge Delta, which is in the observability pipeline space. The thing about being a startup is you just sort of do whatever needs to be done, so I flex the title appropriately as such. Ree and I were chatting about different ideas that might be fun to discuss, and one of them that seemed very apropos to the group would be sort of the evolution of observability and where things are going, how people are tackling the challenge of shifting to OpenTelemetry, and what are some of the interesting lessons learned along the way. Not only from the 10,000-foot view of like we see where the mountains will go but also at the 10-foot view of boy, this grass is tall sometimes, trying to get a little bit of feedback on all different directions of it. +**Host:** All right, I think we can go ahead and get started. Thank you all so much for being here. The panelists we have today are David W., Austin Parker, VJ Samuel, Iris Dear, Mishi, and Noika Melera. I'm hoping I got all those correct, and we'll do a quick run of introductions. This discussion is hosted by the OpenTelemetry User Working Group, which I am part of, as is Adriana. I see her there. -**David:** To continue with introductions, I'll go ahead and do it popcorn style. Iris, why don't you introduce yourself next? +### [00:01:01] Panelist introductions -**Iris:** Hello everyone, my name is Iris, Iris Dear, depending on the country where I am. Currently, I'm based in Portugal. I'm a Platform Engineer, Observability Engineer at Farfetch. My day-to-day is building an observability platform, maintaining it, modernizing it, and offering this kind of service to the engineers in my company. I don't know, that's all about it. Go ahead and call someone else out. +Yeah, today we're gonna just have a casual conversation. Feel free to get as opinionated as possible about basically the evolution of observability practices. And actually, David, since you kind of inspired this, I would love for you to do a quick introduction, and after that, we'll just kind of go through the rest of the panelists. But yeah, let's hear a little bit more from David, whose brainchild this was. -**VJ:** Hi everyone, I'm VJ Samuel. I work at eBay. My day job predominantly revolves around doing architecture for the observability platform internally. Everything—logs, metrics, events, tracing—helping all our developers do alerting, visualization, anomaly detection, the whole shebang with regards to observability. That's pretty much what I do. +**David:** Hello everyone, my name is David W., I am a principal people machine something at Edge Delta, which is in the observability pipeline space. The thing about being a startup is you just sort of do whatever needs to be done, so I flex the title appropriately as such. Yes, Ree and I were chatting about different ideas that might be fun to discuss, and one of them that seemed very appropriate to the group would be sort of the evolution of observability and where things are going, how people are tackling the challenge of shifting to OpenTelemetry, and what are some of the interesting lessons learned along the way. Not only from the 10,000-foot view of like we see where the mountains will go, but also at the 10-foot view of "boy this grass is tall sometimes," and trying to get a little bit of feedback on all different directions of it. -**Austin:** Hi everybody, I'm Austin Parker, Community Maintainer for OpenTelemetry. Formerly at LightStep, part of ServiceNow, and currently—it's a surprise, and you'll find out very soon what I'm currently doing. I've been a part of OpenTelemetry since it was created. I was an OpenTracing maintainer. I've been working in observability for over five years now and got a lot of thoughts from seeing it kind of grow and evolve from what it was to what it is. +To continue with introductions, why don't I go ahead and do it popcorn style? Iris, why don't you introduce yourself next? -**Noika:** Hi everybody, I'm Noika. I'm at the open-source startup Signos and have been working with observability stuff from back when we called it Real's Performance Management. Mainly now working with OpenTelemetry and Kubernetes stuff. +**Iris:** Hello everyone, my name is Iris Dear, depending on the country where I am. Currently, I'm based in Portugal. I'm a platform engineer, observability engineer at Farfetch. My day-to-day is building an observability platform, maintaining it, modernizing it, and offering this kind of service to the engineers in my company. I don't know, that's all about it. Go ahead and call someone else out. -**Moderator:** Excellent, thank you all so much again. We have a list of questions that are kind of intended to help guide the conversation, but once everyone gets going, I expect it to become a lot more dynamic. I think we're all totally happy to see where this takes us. +**VJ:** Hi everyone, I'm VJ Samuel. I work at eBay. My day job predominantly revolves around doing architecture for the observability platform internally. So everything—logs, metrics, events, tracing—helping all our developers do alerting, visualization, anomaly detection, the whole shebang. That's pretty much what I do. -[00:05:10] **Moderator:** To get us going, we want to know about the state of the world before you all undertook your OpenTelemetry journey. What was working pretty well? What did not work, or what sucked? What was the moment that prompted you to change? Feel free to raise your hand. All the panelists at least are on camera, so if you need visual cues as to when you can step in, hopefully that helps, but feel free to raise your hand too. +**Austin:** Hi everybody, I'm Austin Parker, community maintainer for OpenTelemetry. Formerly, I worked at Lightstep, a part of ServiceNow, and currently, it's a surprise, and you'll find out very soon what I'm currently doing. I've been a part of OpenTelemetry since it was created. I was an OpenTracing maintainer. I've been working in observability for over five years now, and I got a lot of thoughts from seeing it kind of grow and evolve from what it was to what it is. -**David:** I'll actually start because I think I have what is probably not a very unique story but an interesting one. Before I got into observability as a field, I started out in software doing QA. I was a software developer in test, and this was, you know, 2013, 2014, I guess is when I started really getting into technology as a career, or software as a career, I should say. The cloud was a thing, but Cloud Native wasn't quite a word yet. We didn't have this concept of like, oh, we're just building all these things with all these cool APIs and this idea of infrastructure on demand or whatever. +**Noika:** Hi everybody, I'm Noika Melera. I'm at the open-source startup Signos, and I've been working with observability stuff from back when we called it Real's Performance Management. Mainly, I'm now working with OpenTelemetry and Kubernetes stuff. -**David:** I saw the company I was at go through these various transformations, and one of them was a DevOps transformation where we went from, okay, when you build your code and you deploy your CI, you write, you pull a ticket, you write some code, it works on your machine, great, you push it, and then that night someone else gets to deploy it and see if it actually worked. One of the things we wanted to do was really tighten up the feedback loop here. We wanted to get from 24, 48 hours before changes got into test to minutes or hours. +**Host:** Excellent, thank you all so much again. We have a list of questions that are kind of intended to help guide the conversation, but once everyone gets going, I expect it to become a lot more dynamic. So, I think we're all totally happy to see where this takes us. -**David:** A big part of that was getting on-demand infrastructure rather than sort of static infrastructure. We're going through this, and we're building all this out, getting stuff into the cloud, and it's great. What we started to see, though, was it wasn't actually fixing a lot of the problems we had. There was kind of this ground truth that everyone had agreed on beforehand: the problem is that we have bad infrastructure. These servers are not properly cared for. We're just wiping stuff and recreating it rather than actually getting fresh images every time. +### [00:05:05] Discussion on pre-OpenTelemetry challenges -**David:** So it must be some config thing; it's probably not the code. Then something would go into production, and we would make it into a patch, and then the customer would come back and say, "Hey, this is actually broken," and we missed it because we thought this was because of our testing infrastructure. When we started going into the cloud, we had fresh images, all this stuff, and we're finding all these problems that we really didn't even know about before. +To get us going, we want to know about the state of the world before you all undertook your OpenTelemetry journey. What was working pretty well? What did not work or, slash, what sucked? And kind of what was the moment that prompted you to change? Feel free to raise your hand. I think all the panelists at least are on camera, so if you need visual cues as to when you can step in, hopefully that helps. But feel free to raise your hand too, and we'll do it that way as long as Austin doesn't thumbs up. -**David:** The question came back, "How do we know what's going on? How do we know what's breaking our product?" It was a platform as a service, so we had hundreds and hundreds of nodes, various logs in all sorts of different places. It was Windows servers, it was Linux servers, we had all these different databases, and it was very difficult; it was kind of big. It was hard to keep your head around. Someone, one of the engineers, actually came back and said, "Okay, I made a topology service topology," and it looked like if you've seen one of those nail art things where someone will make a picture by putting a bunch of nails in a piece of wood and then tying string together—it was like that, where you have just lines everywhere and things connecting to each other. +**Austin:** I turned off the reaction thing. I'll actually start because I think I have what is probably not a very unique story but an interesting one. Before I got into observability as a field, I started out in software doing QA. I was a software developer in test, and this was, you know, 2013-2014, I guess, when I started really getting into technology as a career, or software as a career, I should say. The cloud was, you know, a thing, but Cloud native wasn't quite a word yet, right? -**David:** Nobody could keep this in their head. Nobody could understand how services actually talk to one another. You could look at a very small section and say, "Okay, I get this," but looking at it holistically was impossible. I brought in, at the time, we tried New Relic, we tried Datadog, we tried a couple of things. What I found was, ironically enough, that it didn't matter what tools I brought in; the developers weren't interested in using them just because it wasn't data, it wasn't information that was kind of like at their level. +We didn't have this concept of like, "Oh, we're just building all these things with all these cool APIs," and this idea of like infrastructure on demand or whatever. I saw, you know, the company I was at go through these various transformations, and one of them, and what I helped kind of lead there, was a DevOps transformation where we went from okay, when you build your code and you deploy your CI, you know, you pull a ticket, you write some code, it works on your machine, great, you push it, and then that night someone else gets to deploy it and see if it actually worked. -**David:** They didn't understand how do these things correlate, what does it mean when SQL Server spikes and memory usage increases, but there's no way to really tell what was happening, and that got me into observability—trying to answer that fundamental question. +One of the things we wanted to do was really tighten up the feedback loop here, right? We wanted to get from 24-48 hours before changes got into test to minutes or, you know, hours and minutes. A big part of that was getting on-demand infrastructure rather than sort of static infrastructure. So we're going through this, and we're building all this out, and we're getting stuff into the cloud, and it's great. -**Austin:** Right off of that, I remember working in New Relic during my time there, and there was this really fundamental thing of like, oh, this shows how this request hit all these spots, and it shows it as a trace with a bunch of time spans including individual function calls on all these different services. Pretty cool, but we get these questions back that were like, "Well, just show me where the request went, show me what services were hit, and also like in an interpretable version." +What we started to see, though, was it wasn't actually fixing a lot of the problems we had. There was kind of this ground truth that everyone had agreed on beforehand, like, "Well, the problem is that we have, you know, the infrastructure is bad, right? These servers are not properly cared for." We're just wiping stuff and recreating it rather than, you know, actually getting fresh images every time, so it must be some just config thing, it's probably not the code. -[00:10:50] **Austin:** It was an example where there was a lot of focus on getting a certain piece of information back, right? But the developers wanted information that was at a different level. This was exactly it: I just want to know where the request is going, which services are involved, and how the failure on the SQL Server might come back up and affect the front end in these ways. That was very hard to tease out, whatever this was seven years ago, but it's a similar theme, right? It's really about making sense of a pile of stuff again. +Then, you know, something would go into production, it would make it into a patch, and then the customer would come back and say, "Hey, this is actually broken," and we missed it because we thought this was because of the testing infrastructure. So when we started going into the cloud, we had fresh images, we had all this stuff, and we're finding all these problems that we really didn't even know about before. -**Austin:** Trying to zero in on that moment before we get into these things, there's the point where you have everything, and then you realize that having everything was the problem, and then it's not. Then you're like, "Wow, this computer doesn't know how to draw maps," and somehow it just looks like a bowl of spaghetti. Then you're like, "Cool, how do I zone this back down again?" +The question came back, "How do we know what's going on? How do we know what's breaking our product?" It was a platform as a service, right? So we had hundreds or, you know, hundreds and hundreds of nodes, various logs in all sorts of different places. It was Windows servers, it was Linux servers, you know, we had all these different databases, and it was very difficult. It was kind of big; it was hard to keep your head around. -**David:** It seems like sense-making is at least one of the unifying concepts we see there. VJ, Iris, whoever wants to take it— is that a similar feeling that you guys got when you were hitting this inflection point? +Someone, one of the engineers, actually came back and said, "Okay, I made a topology service." The topology looked like, you know, if you've seen one of those nail art things where someone will make a picture by putting a bunch of nails in a piece of wood and then tying string together. It was like that, right? You have just lines everywhere and things connecting to each other. Nobody could keep this in their head; nobody could understand how services actually talk to one another. You could look at a very small section and say, "Okay, I get this," but looking at it holistically was impossible. -**VJ:** For us, the problem space was a little bit different in the sense that pre-OpenTelemetry, we had a pre-cloud native era and during Cloud native as well. If you take the pre-cloud native era, we had something called the Centralized Application Logging Platform inside of eBay for more than 20 years, and it had the concept of transactional logging, where you have a root transaction, nested transaction events, very similar to what we have in the tracing world today. +I brought in, at the time, you know, we tried New Relic, we tried Datadog, we tried a couple of things, and what I found was, ironically enough, that it didn't matter what tools I brought in—the developers weren't interested in using them just because it wasn't data, it wasn't information that was kind of like at their level, right? They didn't understand, you know, how do these things correlate? How do these, you know, what does it mean when SQL Server spikes and memory usage increases, but there's no way to really tell like what was happening? -**VJ:** But the problem with the pre-cloud native era is always that developers come into eBay; they have to learn proprietary clients. We had clients only for a few languages, so if they are not using that or writing their own code, you cannot observe things inside the company or you're on your own to figure out—spin up your own ELK stack or anything that you can do to monitor the system. +That's what got me into observability, right? Trying to answer that fundamental question. -**VJ:** When Cloud native came in and we had the large-scale Kubernetes adoption, we had the Prometheus endpoints scraping from log files, a little more flexible, but you do not have standardized SDKs across the board. Even for metrics, it used to be that a few people used the official Prometheus client, some used Micrometer, and for Node.js, you didn't even have an official community-supported SDK. +### [00:10:10] Observability transformation experiences -**VJ:** I think that's where OpenTelemetry came in with the promise of standardization. It was like, "Okay, now we can just offer all our developers one standard, and it's community-managed. They can hop from any company into eBay, and they should be able to use the observability platform as long as we are OpenTelemetry compliant." +**VJ:** Someone was piggybacking. Yeah, so right off of that, I remember working in New Relic during my time there. There was this really fundamental thing of like, "Oh, this shows how this request hit all these spots," and it shows it as a trace, right, with a bunch of time spans including individual function calls on all these different services. Pretty cool, but we get these questions back that were like, "Well, just show me where the request went. Show me what services were hit," and also like in an interpretable version. -**Iris:** I would say that my viewpoint, my story, is kind of a bit like VJ. My observability experience first started with some companies that had zero observability in place, and that's how I got to know it. Where I'm currently working, I jumped into a completely different world that had a very nice observability platform, a very nice observability culture, which was a big shock. Our job was actually not to come up with ways to monitor but to just keep improving. +It was an example where there was a lot of focus on getting a certain piece of information back, right? But the developers wanted information that was at a different level. This was exactly it; it was like, "Hey, I just want to know where the request is going, what services are involved, and how the failure on the SQL Server might come back up and affect the front end in these ways." That was very, very hard to tease out, whatever this was, seven years ago. But it's a similar theme, right? It's really about making sense of a pile of stuff. -[00:14:40] **Iris:** Of course, we came to that bottleneck that we were using a lot of open source, we were using APM vendors, and to get the information, you have to go in 10 different places, which is not great. I would say that it is from a developer productivity viewpoint as well why we went the OpenTelemetry route, because everything is centralized, and we can move from vendor to vendor if we need. We are collecting everything, standardizing everything, so I would say we have kind of the same case here. +Again, trying to zero in on that moment before we get into these things, there's the point where you have everything, and then you realize that having everything was the problem, and then it's not. Then you're like, "No, now I have everything, and I still don't understand everything." Well, maybe I need to actually draw a map. Then you draw a map, and you're like, "Wow, this computer doesn't know how to draw maps," and somehow this just looks like a bowl of spaghetti. Then you're like, "Cool, how do I zone this back down again, in and out, in and out?" -**Austin:** I think to both of your points, like that to me is what is really transformative about OpenTelemetry. It offers, for the first time, this truly universal idea of how should I, as a developer, emit this telemetry information. I think your story, VJ, is not unique. Most especially in large distributed systems, companies that have large distributed systems have had some sort of transactional tracing, some sort of structured logs with transaction IDs that they can look at for a decade or more. +It seems like sense-making is at least one of the unifying concepts we see there. VJ, Iris, whoever wants to take it—does that have a similar feeling that you guys got when you were hitting this inflection point? -**Austin:** When we think about an OpenTelemetry trace, the actual data model is very similar to what Google produced in Dapper almost 25 years ago now. There’s nothing new under the sun. What's been missing is the idea that this is actually a core part of being a developer—a core tool in the toolbox, a core part of your trade is learning how to emit good telemetry about what your system is doing. OpenTelemetry provides a standards-based way to do this that also can be natively integrated and available through your service framework, your RPC library, or whatever else. +**VJ:** For us, the problem space was a little bit different in the sense that pre-OpenTelemetry, we had a pre-cloud-native era and during Cloud native as well. If you take the pre-cloud-native era, we had something called the centralized application logging platform inside of eBay for more than 20 years. It had the concept of transactional logging where you have a root transaction, nested transaction events—very similar to what we have in the tracing world today. -**Austin:** To me, we're at the beginning of the beginning almost of seeing how that actually impacts the industry. +But the problem with the pre-cloud-native era is that developers coming into eBay had to learn proprietary clients. We had clients only for a few languages, so if they were not using that or writing their own code, you could not observe things inside the company, or you're on your own to figure out, spin up your own ELK stack or anything that you can do to monitor the system. -**Iris:** I just want to add a little bit to what Austin was saying. It's absolutely true. The other good thing about OpenTelemetry is that you don't have to break your whole system to implement it as well. That is one of the great things because it's compatible with almost all technologies that we currently have, especially the open source ones. You don't have to cause downtime or have your developers blind for hours and days; you can just put it there flawlessly. +When Cloud native hit and we had large-scale Kubernetes adoption, we had the Prometheus endpoints scraping from log files. It was a little more flexible, but you did not have standardized SDKs across the board. Even for metrics, it used to be that a few people used the official Prometheus client. Some used Micrometer, and for Node.js, you didn't even have an official community-supported SDK. -**Austin:** I think there's really two fundamental shifts. One is we started adopting architectures that became much harder to diagnose with the stuff that you taught yourself when you taught yourself to code. Adding in some logging lines and such is not going to work on a large microservice architecture. +I think that's where, when OpenTelemetry came in with the promise of standardization, it was like, "Okay, now we can just offer all our developers one standard." It's community-managed; they can hop from any company into eBay, and they should be able to use the observability platform as long as we are OpenTelemetry compliant. I think our story is a lot more developer productivity-focused, at least in the beginning. -**Austin:** The other change—and this has been a while, but it's still there—is that when you have conversations about funding, when you have conversations with VCs, they're going to ask you about your observability setup. Those two changes combined have really changed where the conversation is with observability, where it went from an internal discussion about maybe developer velocity and rollback times to, “Yeah, we have to have this, and it has to be applicable across this stack.” +**Iris:** I would say that my viewpoint, my story is kind of a bit like VJ. My observability experience first started with some companies that had zero observability in place, and that's how I got to know it. I was like, "Okay, what is happening?" That was my job. Then, where I'm currently working, I jumped on a completely different world that had a very nice observability platform, a very nice observability culture, which was a big shock. Our job was actually not to come up with ways to monitor, but to just keep improving. -**David:** I think there is a fair bit of standardization that gets promoted on the consumer side of things as well, given that there is a standard in how you instrument tech metrics, for example, gauges, counters, exponential histograms, whatever. Earlier, it used to be that if you picked one time series database, you would get certain capabilities defined in a certain way. If you have to switch, then the likelihood of that capability existing may not be always possible. +Of course, we came to that bottleneck that we were using a lot of open source, we were using APM vendors, and to get the information, you have to go in 10 different places, which is not great. I would say that it is from a developer productivity viewpoint as well why we went the OpenTelemetry route, because everything is centralized and we can move from vendor to vendor if we need. We are collecting everything, standardizing everything. -**David:** But now since there is a standardization, eventually there will be a good amount of standardization on what technology the providers—either open source projects or vendors—provide. The net benefits are definitely there across the board, not just on the instrumentation side. +I would say we have kind of the same case here. -[00:19:30] **Austin:** You bring up a really interesting point, right? Historically, everything about telemetry has really been a pretty binding abstraction to the data store. If you were using StatsD, metrics worked in the way they did because of the way that StatsD, you know, stored and let you query that. Prometheus metrics work the way they do because of how you store and query them. Logging databases tend to, you know, if you're using Elasticsearch or something, the format mattered, and that influenced how the client libraries were designed and so on and so forth. +### [00:15:15] Importance of standardization in observability -**Austin:** With OpenTelemetry, we kind of break that, right? OpenTelemetry tells you, “Hey, this is what a histogram looks like; this is what a structured event of any kind looks like.” I think it's interesting that you also see this rise in popularity of column stores for storing observability data—like metrics, logs, traces, sessions, whatever. You throw a rock, and you'll hit a new column store-based or heck, new ClickHouse-based telemetry thing. +**Austin:** I think to both of your points, like that to me is what is really transformative about OpenTelemetry, is that it offers a, you know, for the first time, I think, this truly universal idea of like how should I as a developer emit this telemetry information, right? Like I think VJ, your story, you know, is not unique. Most especially large distributed systems, most companies that have large distributed systems have had some sort of transactional tracing, you know, some sort of structured logs with transaction IDs that they can, you know, look at for a decade or more. -**Austin:** It's great, and I think OpenTelemetry has actually been a huge factor in this because it used to be if you wanted to go and build an observability tool or build some sort of analysis tool, you would have to come up with all this stuff or no one would use it. You'd have to come up with your API, you'd have to come up with your SDKs, you'd have to build integrations for 40 billion things, and OpenTelemetry means, actually, that's all a commodity now. You just get that for free with OpenTelemetry, and you can build really interesting workflows on top of that data. +When we think about an OpenTelemetry trace, you know, the actual data model is very similar to what Google produced in Dapper, you know, almost 25 years ago now. There's nothing new under the sun. What's been missing is the idea that this is actually like a core part of being a developer, like a core tool in the toolbox, a core part of your trade is learning how to emit good telemetry about what your system is doing. -**Austin:** I think that's very friendly for developers, right? Going forward, we haven't even really started to see how do these tools become integral in development workflows, right? When do we get to IDE-level integration and all that? +OpenTelemetry provides a standards-based way to do this that can also be natively integrated and available through your service framework or your RPC library or whatever else. That to me is, you know, we're at the beginning of the beginning almost of seeing how that actually impacts the industry. -**Iris:** I know it's very easy to feel like OpenTelemetry is done or like that it's kind of hitting this plateau, but I think we're really, really still in the early days with what people can actually do with it. +**Iris:** I just want to add a little bit to what Austin was saying as well. The other good thing about OpenTelemetry is that you don't have to break your whole system to implement it. That is one of the great things because it's compatible with almost all technologies that we currently have, especially the open-source ones. You don't have to cause downtime, have your developers blind for hours and days; you can just put it there flawlessly. That's the best of all for me. -**David:** I have to agree on this being early days for OpenTelemetry. A little bit about my background is I was in observability before I hopped over to CL for a bunch of years, and now I'm back in the observability space. The conversations that I'm having with people today remind me a lot about conversations about Git, like 15 years ago or something like that, when it was sort of a perennial joke that everyone's like, “Oh yeah, Git looks really interesting, we're thinking about moving to Git, but we're still on, you know, insert system here.” +**VJ:** I think there's really two fundamental shifts, right? One is we started adopting architectures that became much harder to diagnose with the stuff that you taught yourself when you taught yourself to code, right? That like, hey, adding in some logging lines and such is not going to work on a large microservice architecture. The other change—and this has been a while, but it's still there—is that when you have conversations about funding, when you have conversations with VCs, they're going to ask you about your observability setup, right? -**David:** It feels almost every conversation I have—and again we're in the pipeline space—it's about routing and moving and transforming all the data to the right spot. But everything that comes out of us is OpenTelemetry. If they just need to bolt on something, they want to do that. But the first thing they talk to us about is something with some sort of data not being in the right place, and the second thing is, “Oh, also we're starting to look at OpenTelemetry. Is that good?” Being the second question, I think is a good sign of climbing the curve. +So those two changes combined have really changed where the conversation is with observability, where it went from an internal discussion about maybe developer velocity and rollback times and these other pieces to like, "Yeah, we have to have this, and it has to be applicable across this stack." -**Moderator:** Speaking of that, how did you first hear about the projects? I think it’s kind of clear like some of the things that have drawn you all to it. Something I think, too, that people might be interested in is how did you get leadership buy-in? +I think there is a fair bit of standardization that gets promoted on the consumer side of things as well, given that there is a standard in how you instrument metrics, for example, gauges, counters, exponential histograms. Earlier, it used to be that if you picked one time-series database, you would get certain capabilities defined in a certain way. If you had to switch, then the likelihood of that capability existing may not be always possible. -**Iris:** I can say something about this because for me, it’s easy. When I started working with observability, I became obsessed with tracing. I’m still fascinated by tracing, so I was like, “Oh, tracing is so amazing,” and I started just searching online about it. Then I came across OpenTelemetry. I started playing with it for a long time and not actually implementing anything. +But now, since there is a standardization in the instrument, eventually there will be a good amount of standardization on what technology the providers, either open source projects or vendors, provide. The net benefits are definitely there across the board, not just on the instrumentation side. -[00:24:00] **Iris:** When I joined Farfetch, we were like, “Okay, we need something new, something better,” and we thought about OpenTelemetry. It was very surprising because there was immediate support. I like to say and brag that we have a very good observability culture, but maybe everyone had heard about it. We just made a small presentation, and our leadership was on board. Of course, it has a lot of benefits, and we had to present what we were lacking, the issues that we were currently facing, and how MRI can help us with. It was immediate support, so every time I tell the story, some people don't believe me, but that's exactly how it happened. It was very interesting, very fast support. +### [00:20:00] Developer productivity and OpenTelemetry -**VJ:** I can maybe go next. At least in our case, I first came to know about it as part of the OpenTelemetry will basically be the standard, and we would eventually retire OpenTracing and OpenCensus. I think we were passively trying out OpenTracing around that time, and this announcement came, and it was like, “Hey, interesting, this is something that we need to watch out for.” +**Austin:** I think that's actually a really interesting point, right? Like historically, everything about telemetry has really been a pretty binding abstraction to the data store. If you were using, you know, StatsD, metrics worked in the way they did because of the way that StatsD, you know, stored and let you query that. Prometheus metrics work the way they do because of how you store and query them. Logging databases tend to, you know, if you're using ElasticSearch or something, the format mattered, and that influenced how the client libraries were designed, and so on and so on and so forth. -**VJ:** We were doing passive experiments with the OpenTelemetry SDK and the Collector for a few months, and once it hit stability, that’s when we made the informed call that, “Okay, we’ll start tracing with OpenTelemetry as an offering inside the company.” So now we had the OpenTelemetry SDK for traces and the OpenTelemetry Collector for accepting all the spans, but on the other hand, we had Metricbeat and Filebeat for collecting logs, events, and metrics. +With OpenTelemetry, you know, we kind of break that, right? Like OpenTelemetry tells you like, "Hey, this is what a histogram looks like. This is what a structured event of any kind looks like." I think it's interesting that you also see this rise in popularity of column stores for storing observability data, like metrics, logs, traces, sessions, whatever. You throw a rock and you'll hit a new column store-based or heck, new ClickHouse-based telemetry thing. It's great, and I think OpenTelemetry has actually been a huge factor in this because it used to be if you wanted to go and build an observability tool or build some sort of analysis tool, you would have to come up with all this stuff or no one would use it. -**VJ:** At that point, we were at a crossroads of if we should support multiple agents or just standardize across OpenTelemetry Collector for everything. I wrote a very big memo internally on what it would take to migrate out of Beats for metrics, logs, and events, presented it to leadership, and I think we felt that being on a vendor-neutral industry standard would be the better thing for us in the long run. It’s best to do it now than later on, so that’s how we did the migration. +You'd have to come up with your API, you'd have to come up with your SDK, you'd have to build integrations for 40 billion things, and OpenTelemetry means like actually that's just all a commodity now. You just get that for free with OpenTelemetry, and you can build really interesting workflows on top of that data. I think that's very friendly for developers, right? Going forward, we haven't even really started to see like how do these tools become integral in development workflows, right? When do we get to IDE-level integration and all that? -**VJ:** The metric migration was a massive one because we, at that time, if I'm not wrong, we were already at 32 million samples per second across more than 100 Kubernetes clusters—million-plus Prometheus endpoints and whatnot. It was a very, very involved migration that we did, but the benefits are definitely there now. +I know it's very easy to feel like OpenTelemetry is done or like that it's kind of like hitting this plateau, but I think we're really, really still in early days with what people can actually do with it. -**Austin:** Were there any other challenges when you're coming from a culture that has a strong kind of build vibe? What I'm trying to get is, do you guys run your own Collector or are you taking something? How do you manage that in terms of the particular flavor that you decided to land on? +**Iris:** I have to agree on this being early days for OpenTelemetry. A little bit about my background is I was in observability before I hopped over to CI for a bunch of years, and now I'm back in the observability space. The conversations that I'm having with people today remind me a lot about conversations about Git like 15 years ago or something like that, when it was sort of a perennial joke that everyone's like, "Oh yeah, Git looks really interesting. We're thinking about moving to Git. I mean, we're still on, you know, insert system here." -**VJ:** We do run our own Collector internally, but I think the number of custom processors that we run is very less. It's mostly that we don't need all the exporters for all the various vendors that are packaged in, so we create only the ones that we absolutely need. For now, we export using either OpenTelemetry Protocol or Prometheus Remote Write, which are pretty much open standards right now, so it's about being lean. +But it feels almost every conversation I have—and again, we're in the pipeline space, so it's about routing and moving and transforming all the data to the right spot—but everything that comes out of us is OpenTelemetry. If they just need to bolt on something and they want to do that, they can do that. The first thing they talk to us about is something with some sort of data not being in the right place, and the second thing is, "Oh, also, we're starting to look at OpenTelemetry. Is that good?" -**Iris:** Are you all using the Collector Builder to make your images? +Being the second question, I think is a good sign of climbing the curve. But I think there's still plenty of room to grow. -**VJ:** Right now, no. We just craft our own Go mod and then just do it ourselves. +**Host:** Speaking of that, you know, how did you first hear about the projects? I think it's kind of clear, like some of the things have drawn you all to it. Something I think too that people might be interested in is how did you get leadership buy-in? -**Austin:** Y’all thinking about using the Collector Builder? +**Iris:** I can say something about this because for me, it's easy. When I started working with observability, I became obsessed with tracing. I'm still fascinated by tracing. I was like, "Oh, tracing is so amazing," and I started just searching online about it. Then I came across OpenTelemetry. I started playing with it for a long time and not actually implementing anything. + +So when I joined Farfetch, we were like, "Okay, we need something new, something better. How about OpenTelemetry?" It was very surprising because there was immediate support. I like to say and brag that we have a very good observability culture, but maybe everyone had heard about it. We just made a small presentation, and our leadership was on board. Of course, it has a lot of benefits, and we had to present what we're lacking, the issues that we're currently facing, and how OpenTelemetry can help us with that. It was immediate support. + +Every time I tell the story, some people don't believe me, but that's exactly how it happened. It was very interesting, very fast support. + +**VJ:** In my case, I can maybe go next. At least in our case, I first came to know about it as part of the announcement that OpenTelemetry will basically be the standard, and we would eventually retire OpenTracing and OpenCensus. I think that we were passively trying out OpenTracing around that time, and this announcement came, and it was like, "Hey, interesting. This is something that we need to watch out for." + +### [00:25:25] Migration to OpenTelemetry + +We were doing passive experiments with the OpenTelemetry SDK and the collector for a few months, and once they hit stability, that's when we made the informed call that, okay, we'll start tracing with OpenTelemetry as an offering inside the company. + +So now we had the OpenTelemetry SDK for traces and the OpenTelemetry collector for accepting all the spans. But on the other hand, we had Metricbeat and Filebeat for collecting logs, events, and metrics. At that point, we were at a crossroads of if we should support multiple agents or just standardize across the OpenTelemetry collector for everything. I wrote a very big memo internally on what it would take to migrate out of Beats for metrics, logs, and events. + +I presented it to leadership, and I think we felt that being on a vendor-neutral industry standard would be the better thing for us in the long run, and it's best to do it now than later on. That's how we did the migration. + +**Austin:** And on our end, the metric migration was massive because at that time, if I'm not wrong, we were already at 32 million samples per second across more than 100 Kubernetes clusters, million-plus Prometheus endpoints, and whatnot. It was a very, very involved migration that we did, but the benefits are definitely there now. + +**VJ:** Were there any other challenges that you faced? + +**Austin:** When you're coming from a culture that has a strong build vibe, I'm what I'm trying to get is, do you guys run your own collector, or are you taking something like how are you managing that in terms of the particular flavor that you decided to land on? + +**VJ:** We do run our own collector internally, but I think the number of custom processors that we run are very less. It's mostly that we don't need all the exporters for all the various vendors that are being packaged in. We create only the ones that we absolutely need. For now, we export using either OTLP or Prometheus Remote Write, which are pretty much open standards right now. It's about being lean, actually. + +**Iris:** Are you all using the collector builder to make your images? + +**VJ:** Right now, no. We just craft our own Go mod and then do it ourselves. + +**Austin:** Y'all thinking about using the collector builder? **VJ:** I think we can. We haven't really explored it, but yeah, we should. -[00:45:00] **Austin:** I'm just trying to get more people aware of the Collector Builder, because I think it's really cool. The Collector Builder, or OCB, is a tool that we provide from the Collector repo, and what you can do is you can give it a manifest of Go modules, and it will create a custom image or a custom build of the OpenTelemetry Collector for you. So you, like eBay, can get something that has just the receivers, processors, exporters, connectors, and extensions that you want. +**Austin:** I'm just trying to get more people to use the collector builder because I think it's really cool. + +**Iris:** Could you say what the collector builder is? -[00:29:00] **Austin:** It’s also a really great way to extend the Collector because in this way, instead of having to fork contrib or fork whichever raw base you want and then bring in the code, you can simply have a Go module published in wherever GitHub and pull that in through the Collector Builder and bring in your custom extensions and things that way. +**Austin:** The collector builder, or OCB, is a tool that we provide from the collector repo, and what you can do is you can give it a manifest of Go modules, and it will create a custom image or a custom build of the OpenTelemetry collector for you. Like eBay can, you know, get something that has just the receivers, processors, exporters, connectors, and extensions that you want. It's also a really great way to extend the collector because in this way, instead of having to fork contrib or fork whichever raw base you want and then bringing in the code, you can simply have a Go module published in wherever on GitHub and pull that in through the collector builder and bring in your custom extensions and things that way. -**Austin:** It’s a really great tool. It’s also very friendly for your security teams because it gives them a manifest that they can kind of look at and say, “Hey, this is exactly what's in there,” and that way I know if you're at a larger security review can be extremely taxing, and if you're trying to use the contrib image, there's an awful lot of dependencies. +It's a really great tool. It's also very friendly for your security teams because it gives them a manifest that they can kind of look at and say, "Hey, this is exactly what's in there," and that way, I know if you're, you know, at a larger security review can be extremely taxing, and if you're trying to use the contrib image, there's an awful lot of dependencies. -**Austin:** Something to kind of cut that down to size is very friendly to your friends in security. +So something to kind of cut that down to size is very friendly to your friends in security. -**Iris:** There's a new user now for the Builder. I actually had missed this, so there’s actually some—there’s check out the website, OpenTelemetry.io does have a little tutorial on building with the Collector. +**Iris:** Wow, there's a new user now for the builder. I actually had missed this. -**Iris:** I actually recently used it for a personal project where I was making some custom processors to talk to an API and do some data, so it was really, really fun to use and makes it really easy to use the Collector, maybe the way it's sort of intended. +### [00:30:00] Customization and security in OpenTelemetry -**VJ:** As long as we—oh, go on. +**Austin:** Yeah, check out the website. OpenTelemetry.io does have a little tutorial on building with the collector. I actually recently used it for a personal project where I was making some custom processors to talk to an API and do some data, so it was really, really fun to use, and it makes it really easy to use the collector maybe the way it's sort of intended. -**Austin:** I was just going to point out that VJ and Seth very helpfully shared a couple of links about the custom Collector Builder. +I don't know necessarily know if our intention was for people to just be pulling this giant contrib image constantly. -**VJ:** Perfect. +**VJ:** As long as we... Oh, go on. + +**Austin:** Oh, I was just going to point out that VJ and Seth very helpfully shared a couple links about the custom collector builder. + +**VJ:** Ah, perfect. **Austin:** The other, sorry, go ahead. -**Iris:** Go on first; you were talking. +**VJ:** Go on first; you were talking. + +**Austin:** Sure. Yeah, I'm just going to mention we've found it very useful. We have forked just an exporter and are working on trying to get a whole request approved for it, but in the meantime, we continue to need to persist those features and newer versions of the collector. So this helps us to kind of drag our single exporter along the way and keep rebuilding it with new releases, so it's been really helpful. + +**Austin:** Yeah, that’s another really great use case. If you have custom stuff or if you have just modifications, I know that PRs can sit for a bit sometimes. + +**VJ:** So, yeah. + +**Austin:** But the other cool use case, and also one of my current hobby horses, is if you are using large language models to assist you in development—so things like Copilot or ChatGPT—then, you know, it's not perfect because of the data cutoffs on that, but over time, especially as things like P data and other structures in the collector repo stabilize, then it's actually pretty easy to ask, you can ask ChatGPT, like, "Hey, scaffold me a collector extension." + +It'll use the wrong APIs, but they're actually, but it'll do 80% of the work for you. Then you can ask it, like, "Hey, I need to transform this data or do whatever," and it's actually a pretty good use case for AI assistants, I think, in programming. + +**Austin:** So, especially that, once you get to the point of just wanting the rest to transform it in a certain way, it's a good moment for the copilot. + +**VJ:** Yeah, I don't know if it understands OTL, OpenTelemetry Transform Language yet. Probably not. But, you know, the models will get better, and I think the biggest thing is just staying, you know, once we get to the point where things are changing a little less frequently, and it can get into the model, so stay tuned. Maybe we'll have something to say about that as a project in the future. + +**Host:** Okay, so moving the conversation along since we've come up on time almost, what was most surprising as your observability journey got underway? What were the trickiest stakeholders in your case? I think David, you like to say in observability, there's no technical problems, only people problems. + +**David:** Yeah, I should stop saying that even though I'm probably going to keep saying that. But there's, you know, particularly in this space, the technical problems mostly are straightforward; it's all the people problems. + +So what to the whole panel, what were some of the surprising things that you found either that were sticky and a little bit tricky and had to navigate, or that were surprisingly smooth? Both of those are valid surprises because we've gone over some of the key selling points here that I think a lot of people would recognize as being useful with going in an OpenTelemetry direction, but what's the part we didn't say that was either good or that was almost good? + +**Iris:** I'm currently running a poll on LinkedIn where I ask people if they think OpenTelemetry is easy to use. I won't disclose the results of that poll quite yet. + +**Austin:** I think there's a... yeah, I'd be interested to hear from people that have gone through implementation journeys. I can say we're in a pilot for integration with applications, and the .NET library is not quite having the log exporter stable until just very recently has been a large deterrent for a very long time. -**VJ:** Yeah, I'm just going to mention we’ve found it very useful. We have forked just an exporter and are working on trying to get a whole request approved for it. But in the meantime, we continue to need to persist those features and newer versions of the Collector, so this helps us to kind of drag our single exporter along the way and keep rebuilding it with new releases, so it's been really helpful. +### [00:35:35] Discussion on OpenTelemetry features -**Austin:** That’s another really great use case. If you have custom stuff or modifications, I know that PRs can sit for a bit sometimes. +About two years ago, I started trying to convince the application teams to all integrate with OpenTelemetry, and there's been kind of an appetite for some of the more cutting-edge teams in our company to try out something that still doesn't have a stable label. But that was a big deterrent. Now that I think it's the 16 release, now that that's stable on the .NET instrumentation library, there's a better story there, and there's more acceptance now. -**Moderator:** I think it’s going to require several moving parts. +Outside of that, the collector itself, just getting lots of new features for free has been amazing. That's part of the reason we know about the builder, just because we have these custom features that we really want, and we keep seeing better, newer versions of the collector come out. It's like, "Oh wow, we better upgrade because we don't want to sit on this old stuff." -**Austin:** Moving the conversation along since we’re coming up on time almost, what was most surprising as your observability journey got underway? What were the trickiest stakeholders in your case? +All these other great people have been contributing a lot of good enhancements, so that's been an easy sell to say, "Here's why part of the why of helping teams move to this instrumentation challenge that will take some time for them to work through." -[00:33:56] **David:** I think I should stop saying that, even though I'm probably going to keep saying that, but there's particularly in this space, the technical problems mostly are straightforward. It's all the people problems. To the whole panel, what were some of the surprising things that you found either that were sticky and a little bit tricky and had to navigate or that were surprisingly smooth? Both of those are valid surprises because we've gone over some of the key selling points here that I think a lot of people would recognize as being useful with going in an OpenTelemetry direction, but what's the part we didn't say that was either good or that was almost good? +**Austin:** Actually, I have a... can we play a fun game? Can I get everyone to open up your Zoom chat and tell me what language do you use OpenTelemetry in, and do you think it's good? Do you think the SDK is good in that language? -**VJ:** I'm currently running a poll on LinkedIn where I ask people if they think OpenTelemetry is easy to use. I won't disclose the results of that poll quite yet. I think there's a—I’d be interested to hear from people that have gone through implementation journeys. +**David:** This data may be used by the project. -**VJ:** I can say we're in a pilot for integration with applications, and the .NET library is not quite having the log exporter stable until just very recently. That’s been a large deterrent for a very long time. About two years ago, I started trying to convince the application teams to all integrate with OpenTelemetry, and there’s been kind of an appetite for some of the more cutting-edge teams in our company to try out something that still doesn’t have a stable label. +**Austin:** See, what you don't know is that the part of this request that's hard for me is that the Zoom client on my machine is very temperamental in terms of opening the chat window, so that's the hard part. -**VJ:** That was a big deterrent, but now, with I think it’s the 16 release now that that’s stable on the .NET instrumentation library, there’s a better story there, and there's more acceptance now. Outside of that, the Collector itself, just getting lots of new features for free has been amazing. So that’s part of the reason we know about the Builder, just because we have these custom features that we really want, and we keep seeing better, newer versions of the Collector come out. It's like, “Oh wow, we better upgrade because we don't want to sit on this old stuff,” and all these other great people have been contributing a lot of good enhancements, so that's been an easy sell to say here's why part of the why of helping teams move to this instrumentation challenge that will take some time for them to work through. +**VJ:** Okay, well I can see the chat node, yes, it's good. I think Node's got... I think JS has gotten a lot better. I think most of the problems in JS right now have a lot more to do with like ESM and CJS and various JavaScript things—ecosystem things—rather than any problem with the language SDK. -**Austin:** Actually, I have a—can we play a fun game? Can I get everyone to open up your Zoom chat and tell me what language you use OpenTelemetry in, and do you think it's good? Do you think the SDK is good in that language? This data may be used by the project. +**Iris:** Go, a lot of changes for metrics. -**David:** You don't know is that the part of this request that's hard for me is that the Zoom client on my machine is very temperamental in terms of opening the chat window, so that's the hard part. +**Austin:** Yeah, Go needs some love. I feel like Go was written by several dear friends of mine who know Go in and out, left, right, and indifferent, and it's great, and it's actually pretty idiomatic Go, I think. But it's kind of hard to use because it's idiomatic Go—personal opinion. -**Austin:** Okay, well, I can see the chat node. +**VJ:** Exemplars not getting implemented in which language? -**David:** Yes, it’s good. I think JS has gotten a lot better. I think most of the problems in JS right now have a lot more to do with like ESM and CJS and various JavaScript ecosystem things rather than any problem with the language SDK. +**Austin:** All of them, some of them, most of them. I think the only one where it's actually implemented is Java, and Java and .NET, and then from what I've asked around, not really anywhere else. I've seen a bunch of open PRs around that, but no actual implementations. -**Austin:** Go needs some love. I feel like Go was written by several dear friends of mine who know Go in and out left right and indifferent, and it's great, and it's actually pretty idiomatic Go, I think, but it's kind of hard to use because it's idiomatic Go, personal opinion. +**Iris:** I think we need to get better about closing PRs. -**Iris:** Exemplars not getting implemented in what? Which language? All of them? Some of them? Most of them? +**VJ:** .NET, Python, Java, pretty good. -**Austin:** I think the only one where it’s actually implemented is Java, and then from what I've asked around, not really anywhere else. I've seen a bunch of open PRs around that, but no actual implementations. I think we need to get better about closing PRs. +**Austin:** Yeah, I'd say Java, Kotlin, multiplatform, nonexistent. -**Iris:** .NET, Python, Java—pretty good. +**VJ:** Have you looked at the... to Derek, there's, I don't know if it actually is in the repos yet, but I know Splunk donated their Android client SDK thing that I think targets Kotlin. I know they want—they either—either it has been donated or is being donated. -**Austin:** Yeah, I’d say Java—Kotlin, multiplatform, nonexistent. +**Austin:** Yeah, I've been watching that donation process. -**VJ:** Have you looked at the—Derek, there's—I don't know if it actually is in the repos yet, but I know Splunk donated their Android client SDK thing that I think targets Kotlin. I know they want—it either has been donated or is being donated. +**VJ:** Okay, yeah, for us, we were looking for an SDK that could be used by Kotlin multiplatform to go both on iOS and Android clients. I know some people are using the Java SDK, you know, purely for Android, but we have to just implement that separately. -**David:** I’ve been watching that donation process. +**Austin:** That is interesting. Will you be at KubeCon by any chance? -**Austin:** Okay, yeah. I've been looking for an SDK that could be used by Kotlin multiplatform to go both on iOS and Android clients. I know some people are using the Java SDK, you know, purely for Android, but we have to just implement that separately. +**VJ:** I will. -**VJ:** That is interesting. Will you be at KubeCon, by any chance? +**Austin:** Look me up when you're there; we'll talk about it. -**Austin:** I will. +**VJ:** Cool. -**VJ:** Look me up when you're there. We’ll talk about it. +**Austin:** Go, terrible. -**Austin:** Cool. +**VJ:** Yeah, Go is right now a little bit of a hobby horse for me. I think it is high on my list of languages that feels like it needs some developer experience love. -**VJ:** Go—terrible. +**Austin:** I'm sorry for taking over your panel, Reese. -**Austin:** Yeah, Go's right now a little bit of a hobby horse for me. I think it's high on my list of languages that feels like it needs some developer experience love. +**Host:** Good. I kind of like that. It's hard to get a big group of people online to participate, so... -**Iris:** I'm sorry for taking over your panel, Reese. +**Austin:** Down API. -**Moderator:** Good. I kind of like it. It's always—I feel like it's hard to get a big group of people online to participate. +**VJ:** Kubernetes is also an interesting one. When I went to Kubernetes years ago, if you're not aware, Kubernetes has started to feature gate an alpha and beta, native OTEL traces from things like CUE and a few other components in the API server. -**Austin:** So down API. +So there is more support for OTEL coming in. The biggest thing is Kubernetes metrics are all super Prometheus out, and that is probably not going to change anytime soon. But there is some interesting stuff with Prometheus talking about like, "Should Prometheus just use OTEL libraries and stuff like that?" So we might see some changes there, and hopefully, that can lead to a more unified sort of telemetry story for Kubernetes. -[00:40:06] **VJ:** Kubernetes is also an interesting one. When I went to Kubernetes years ago, if you're not aware, Kubernetes has started to feature gate an alpha and beta native OpenTelemetry traces from things like CUE and a few other components, and the API server. So there is more support for OpenTelemetry coming in. +**Austin:** Functions as a service—I'm sure the majority of people are using Lambda, and that seems to be the main focus of the community at this point. -**VJ:** The biggest thing is Kubernetes metrics are all super Prometheus out, and that is probably not going to change anytime soon. But there is some interesting stuff with Prometheus talking about should Prometheus just use OpenTelemetry libraries and stuff like that, so we might see some changes there. +**VJ:** Yeah, I think we really just need people to... I actually, I mean, I will also plead ignorance. I don't know if they have an equivalent for Lambda layers or extension layers. Do you know? -**David:** Hopefully that can lead to a more unified telemetry story for Kubernetes. +**Austin:** I’ve looked across the project, and while at the top of the just the FaaS offering of the OpenTelemetry, if you look at the documentation, it says—for example, Azure Functions, AWS Lambda—there's really not much integration, if any, with Azure itself. -**Austin:** Your functions as a service—I'm sure the majority of people are using Lambda, and that seems to be the main focus of the community at this point. +So my experience on the pipeline side with Azure Functions, I believe is the official name—spending a lot on that naming convention—is there's a dual writer that you can enable, but then you essentially have to set up something that collects all of that information. So we recommend throwing up a small cache cluster, depending on how much traffic you want to throw at it. -**David:** I think we really just need people to—I actually—I mean, I will also plead ignorance. I don't know if they have an equivalent for Lambda layers or extension layers. Do you know? +But you could be running anything in there, and it's basically a pretty good dual writing mechanism, whereas the Lambda layer stuff tends to have startup and cooldown times that I think is actually... Azure Functions is probably a more scalable approach, particularly if you get really... -**Austin:** I've looked across the project, and while at the top of the OpenTelemetry site, if you look at the documentation, it says, for example, for Azure Functions, AWS Lambda, there's really not much integration, if any, with Azure itself. +**VJ:** Is it function to App Insights and then App Insights split out into OTEL? -**David:** So my experience on the pipeline side with Azure Functions, I believe is the official name—spending a lot on that naming convention—is there's a dual writer that you can enable, but then you essentially have to set up something that collects all of that information. +**Austin:** It's not. It doesn't bounce off of App Insights; it's from the function itself, I believe. -**David:** We recommend throwing up a small Cassandra cluster depending on how much traffic you want to throw at it. But you could be running anything in there, and it's basically a pretty good dual writing mechanism, whereas the Lambda layer stuff tends to have startup and cooldown times that I think—I think actually Azure Functions is probably a more scalable approach, particularly if you get really— +**VJ:** Yeah, a writer SDK that Microsoft provides to write out to wherever. Part of the challenge was like, "Well, is there a way from Azure Monitor to monitor the actual use of it rather than incurring extra cycles within the function itself?" -**Austin:** Is it function to App Insights, and then App Insights split out into OpenTelemetry? +**Austin:** Yeah, okay. I think the only tricky part is if you definitely want to use an alternative source, you have to essentially restrict the IM permissions of the function to not write to App Insights, and that's the only way to actually cut it off. -**David:** It doesn't bounce off of App Insights; it's from the function itself, I believe. +**Host:** Okay, I'm just going to step in real quick to bring around. Sorry, this is a very interesting discussion. Feel free to ping me on the CNCF Slack if you'd like to continue it, but I think it's going to require several moving parts. -**Austin:** Yeah, a writer SDK that Microsoft provides to write out to wherever. Part of the challenge was like, "Well, is there a way from Azure Monitor to monitor the actual use of it rather than incurring extra cycles within the function itself?" +**Austin:** Yeah, okay, thanks. We can definitely host this another conversation around this another time. -**Moderator:** Sorry, very interesting discussion. Feel free to ping me on the CNCF Slack if you'd like to continue it, but I think it's going to require several moving parts. +But to bring it back to this panel, I would like to know how has the focus of your observability practice changed or not changed after implementing OpenTelemetry, and what are you looking forward to next, either from the project or as a result of your observability efforts? -**Austin:** Thank you for that. We can definitely host this in another conversation around this another time. +**David:** I think there's two things we're seeing. One, of course, we're seeing enterprises really see that the amount of control they have with running OpenTelemetry sort of compensates for some of the grit that exists in the gearbox as they try to make it meet their needs. -**Moderator:** To bring it back to this panel, I would like to know how has the focus of your observability practice changed or not changed after implementing OpenTelemetry, and what are you looking forward to next, either from the project or as a result of your observability efforts? +On the other side, we're seeing new developers realize that part of being a competent new developer—and I think the Git comparison is very apt—is, you know, like, "Yeah, of course, you know source control if you're going to write code of any kind for production," right? -**David:** I think there are two things we're seeing. One, of course, is we're seeing enterprises really see that the amount of control they have with running OpenTelemetry sort of compensates for some of the grit that exists in the gearbox as they try to make it meet their needs. And then on the other side, we're seeing new developers realize that part of being a competent new developer—and I think the Git comparison is very apt—is, you know, like, "Yeah, of course you know source control if you're going to write code of any kind for production," right? +And so the same thing's happening with observability where previously maybe, hey, you were just entering a few log lines, and then you were ready to go. It's like, "No, we need to know how to actually implement one of these tools from the start and implement tracing from the start." -**David:** The same thing's happening with observability where previously, maybe, "Hey, you were just entering a few log lines, and then you were ready to go." It's like, "No, we need to know how to actually implement one of these tools from the start and implement tracing from the start." So, sort of from two directions, I think OpenTelemetry is getting a lot of momentum in the next year. +So sort of from two directions, I think OpenTelemetry is getting a lot of momentum in the next year. -**Austin:** I am looking forward to the day when you have to have a reason to be off OpenTelemetry instead of to be on OpenTelemetry, and to see how that affects the vendor landscape. I’m here on behalf of a vendor, but because we’re in pipelines, if I could get rid of all of our integrations that weren't OpenTelemetry, it would just be so much easier instead of having to do something really bespoke for everything that's out there. +### [00:50:50] Future expectations for OpenTelemetry -**Austin:** There was a great article not that long ago that you wrote, right, Noika? Isn't that right? About how OpenTelemetry, at the moment, is being treated in various tools differently? +**Austin:** I am looking forward to the day when you have to have a reason to be off OTEL instead of to be on OTEL. To see how that affects the vendor landscape—I'm here on behalf of a vendor, but because we're in pipelines, if I could get rid of all of our integrations that weren't OTEL, it would just be so much easier instead of having to do something really bespoke for everything that's out there. -**Noika:** We need to talk about that. +But there was a great article not that long ago that you wrote, right Noika? Isn't that right? About how OTEL at the moment is being treated in various tools differently? -**Austin:** I've got some great stuff coming out next week that is not quite so mean to my good friends at New Relic, but I think we all need a little bit of a kick in the pants sometimes. +**Noika:** We need to talk about that. I've got some great stuff coming out next week that is not quite so mean to my good friends at New Relic, but I think we all need a little bit of a kick in the pants sometimes. I think it's allowed. -**VJ:** One of the really fundamental things, like when I was working at New Relic on the integration with X-Ray data, right, is that it's like, "Hey, yes, we can get—we can shove these things together, and we can make it kind of work." But until we all adopt one open standard, this is just too complex a data type to just really fluidly chart in one view and see from one view and do things like what is the average amount of time consumed by this single function, by the single DB call, right? That is going to be very tough to do until you just say, "Look, we all have to be on these open standards." +**David:** Yeah, one of the really fundamental things—like when I was working at New Relic on the integration with X-Ray data, right—is that it's like, "Hey, yes, we can shove these things together, and we can make it kind of work." But until we all adopt one open standard, this is just too complex a data type to just really fluidly chart in one view and see from one view and do things like, "What is the average amount of time consumed by this single function, by this single DB call?" -**David:** Some really weird LLM training task, yes. Okay, it's going to use some super strange system for—for this doesn't really look like tracing or time spans at all, cool. But everything else, we need to adopt an open standard to really be able to have OpenTelemetry data be a first-class citizen, so I'm looking forward to OpenTelemetry winning. +That is going to be very tough to do until you just say, "Look, we all have to be on these open standards." -**Austin:** Same. I think it's already won in many ways, right? Just not to toot my own horn or to toot the project's horn, but in, you know, everyone likes to refer to the XKCD, like there's 14 competing standards. I'm going to make the unifying one now; there are 15 competing standards. +**Austin:** Some really weird LLM training task, yes. Okay, it's going to use some super strange system for... This doesn't really look like tracing or time spans at all—cool. But everything else, right? Yeah, we need to adopt an open CER to really be able to have OTEL data be a first-class citizen. -**Austin:** I think, whatever, you know, I’m under no delusion that this is the apex of innovation in the space. There will be future developments and other things, but I think in terms of consolidating the last two, three decades' worth of thinking about application and infrastructure telemetry data and how it can be used together, I think OpenTelemetry is going to be foundational for a generation of observability practitioners, for engineers, right? There will be something in the future, but I think in terms of where we are now, this is kind of the thing. +So I'm looking forward to OTEL winning—that's all. -**Austin:** Whatever comes next is going to be influenced by things that we don't even see. I've been doing a lot of research on AI and large language models and stuff, and if you look at what you're actually doing when you make a request to a large language model, you're making a trace. It's literally a trace. It is a Dapper-style trace where you have requests and responses. Even the most advanced thing we can think about in terms of software architectures right now, and whatever looks like stuff that we have a way to model in OpenTelemetry today. +**Iris:** Same. I think it's already won in many ways, right? Just not to toot my own horn or to toot the project's horn, but in, you know, everyone likes to refer to the XKCD like, "There's 14 competing standards; I'm going to make the unifying one. Now there are 15 competing standards." -**David:** So until that changes, I feel like OpenTelemetry is, you know, long term going to be what—at least until I am retired, so give it 30 years, but like it's going to be the thing. +I think, you know, I'm under no delusion that this is the apex of innovation in the space. There will be future developments and other things. But I think in terms of consolidating the last two or three decades' worth of thinking about application and infrastructure telemetry data and how it can be used together, I think OpenTelemetry is going to be foundational for a generation of observability practitioners, for engineers, right? -**Austin:** Justin in the chat asks any thought on Micrometer’s approach of blending the signals into a single observation API from which traces, logs, and metrics are all derived? +There will be something in the future, but I think in terms of where we are now, this is kind of the thing. Whatever comes next is going to be influenced by things that we don't even see. -**David:** I think that's fine. I think it's—I do think that the thing you actually need is we need as a project for people to commit to OTL as being an output format, and we need people to respect, you know, hey, this is trace data, and it should be interpreted differently than metric data or whatever. Even if you are going to like splay those out into a generic sort of event data structure, there are use cases for keeping those things separate in terms of observability pipelining. +**Austin:** Like I've been doing a lot of research on AI and large language models and stuff, and if you look at what, you know, what are you actually doing when you make a request to a large language model? You're making a trace. It's literally a trace. It is a Dapper-style trace where you have requests and responses. -**David:** Yeah, it's—there's maybe not a cut-and-dry answer to this. +You know, even the most advanced thing we can think about in terms of software architectures right now—and whatever looks like stuff that we have a way to model in OTEL today. So until that changes, I feel like OpenTelemetry is, you know, long-term going to be what, at least until I am retired, so, you know, give it 30 years, but like it's going to be the thing. -**Moderator:** I would like to point out that it is the top of the hour, so if you have to go, obviously feel free to jump. +**Justin:** In the chat, asks any thought on the Micrometer approach of blending the signals into a single observation API from which traces, logs, and metrics are all derived? -**Austin:** See you all. Have a good day, everyone. +**David:** I think that's fine. I think it's—I do think that the thing we actually need is we need, as a project, for people to commit to OTL as being an output format, and we need people to respect, you know, "Hey, this is trace data, and it should be interpreted differently than metric data or whatever," even if you are going to splay those out into a generic sort of event data structure. -**David:** Thank you all so much. +There are use cases for keeping those things separate in terms of observability pipelining. Yeah, it's... there's maybe not a cut-and-dry answer to this. -**Iris:** Bye-bye! +**Host:** I would like to point out that it is the top of the hour, so if you have to go, obviously feel free to jump. See you! Have a good day, everyone. Thank you all so much! Bye-bye! ## Raw YouTube Transcript diff --git a/video-transcripts/transcripts/2023-11-21T17:59:37Z-otel-in-practice-filtering-parsing-data-with-the-otel-collector.md b/video-transcripts/transcripts/2023-11-21T17:59:37Z-otel-in-practice-filtering-parsing-data-with-the-otel-collector.md index ca8c1e8..cf6bbda 100644 --- a/video-transcripts/transcripts/2023-11-21T17:59:37Z-otel-in-practice-filtering-parsing-data-with-the-otel-collector.md +++ b/video-transcripts/transcripts/2023-11-21T17:59:37Z-otel-in-practice-filtering-parsing-data-with-the-otel-collector.md @@ -10,298 +10,300 @@ URL: https://www.youtube.com/watch?v=y3AAerwIFfg ## Summary -In this YouTube video, the speaker, along with participants, discusses advanced filtering techniques for telemetry data, specifically focusing on OpenTelemetry (OTel) collectors. The main points covered include the need for filtering to manage noisy and sensitive data without altering the codebase, the configuration of filtering attributes, and the importance of transforming data to avoid sending unnecessary information. Various scenarios are explored, such as handling personally identifiable information (PII) and excessive spans from data collection. The speaker also addresses the limitations of OTel processors, including the inability to change span relationships and the nuances of head versus tail-based sampling. Throughout the presentation, practical examples and live demonstrations illustrate how to effectively implement these filtering techniques in a collector configuration. The video concludes with a discussion on best practices for testing configuration changes and managing observability tools. +In this YouTube video, the presenter discusses filtering techniques at the OpenTelemetry (OTel) collector level, primarily focusing on how to manage noisy and sensitive data in telemetry traces. The session features a live demonstration of filtering spans based on attributes, showcasing the importance of avoiding unnecessary data transmission without modifying the application code. Key topics include why filtering is necessary, the limitations of current filtering capabilities, and practical examples of how to implement filters in OTel configurations. The presenter interacts with viewers, answering questions and sharing insights on best practices, as well as discussing the implications of head vs. tail-based sampling. The session is led by a knowledgeable speaker who engages with the audience, ensuring clarity and understanding of the concepts presented. ## Chapters -00:00:00 Welcome and intro -00:02:01 Filtering for traces -00:04:05 Data collection challenges -00:05:20 Collector level filtering -00:08:00 Filtering by attributes demo -00:12:10 Filter processor details -00:14:06 Regex matching in config -00:19:00 Limitations of collector processors -00:21:35 Testing config changes -00:36:00 Closing remarks and Q&A +00:00:00 Introductions +00:01:34 Importance of filtering at OTel collector level +00:03:30 Discussion on noisy and sensitive data +00:05:36 Filtering spans based on attributes +00:08:34 Live demo: Adding filters in OTel configuration +00:12:36 Explanation of filter processor behavior +00:19:36 Head vs. tail-based sampling +00:24:30 Q&A on testing config changes +00:31:30 Limitations of collector processors +00:38:00 Closing remarks and future talks -**Speaker 1:** This is filtering the noise. I think I promoted as like better filtering for logging, but I'm mainly going to demonstrate filtering for traces. The principles are the same. I promise we'll get there, folks. +## Transcript -Let's talk about some level setting for considerations with—let me just turn off my age back here, sorry—considerations with why we would want filtering at the OPA Telemetry collector level. So just review these first principles. Everybody seeing the slideshow? Okay, it's got the little green line around. That's why I assume you're seeing this. Great. I'm gonna assume that's great. Give me a negative emoji react if you're not seeing it, but otherwise, we're good. +### [00:00:00] Introductions -So OpenTelemetry instrumentation, like any code-level instrumentation, unless you send a lot of data rather easily, some of that data is noisy, some of it is sensitive, and some of it is good but needs tweaking. Once we've done the work to do instrumentation, especially instrumentation at the code level, when we're faced with a data collection problem, we would ideally want to fix that without requiring changes to our code base. +**Speaker 1:** This is filtering the noise. I think I promoted as better filtering for logging, but I'm mainly going to demonstrate filtering for traces. The principles are the same. I promise we'll get there, folks. -For most people who are working operationally, the thing is like, “No, I cannot edit the code base.” I don't know where these hooks are added. I don't want to message the product team and say, “Hey, turns out we're sending, you know, people's Social Security numbers to our observability backend. Please go do this code commit tomorrow.” That's just not the workflow that makes sense for a lot of people. +Let's talk about some level setting for considerations with why we would want filtering at the OPA Telemetry collector level. Just review these first principles. Is everybody seeing the slideshow? Okay, it's got the little green line around. That's why I assume you're seeing this. Great! I'm gonna assume that's great. Give me a negative emoji react if you're not seeing it, but otherwise, we're good. -[00:02:01] We could talk all day and all night about how, well, you should have access to the code base; you should have a better connection. That's real DevOps, blah blah blah, great points. But let's talk about, like, you know, okay, it's one thing when, “Oh my God, we're sending a ton of people's PII into observability when we shouldn't.” What about the situation where it's like, “Hey, we're sending a little too much”? +OpenTelemetry instrumentation, like any code level instrumentation, unless you send a lot of data, is rather easily noisy. Some of that data is noisy, some of it is sensitive, and some of it is good but needs tweaking. Once we've done the work to do instrumentation, especially instrumentation at the code level, when we're faced with a data collection problem, we would ideally want to fix that without requiring changes to our codebase. -Hey, the traces, they, like, you know, every step of a 500-step for loop they're recording is a separate span, and they shouldn't. Is that really something where you want to bother the product people about it and ask them to, like, edit the code? You know, for sure, that's like, “It'd be really nice if we had operational tools to do that.” +### [00:01:34] Importance of filtering at OTel collector level -So there are a—let's take a look at this code where we can sort of imagine where we might have gone a little far here. So we had our developer. We said, “Hey, would you add some custom code points?” Maybe they got excited about it. We did our little brown bag. They added a bunch of points. They added, “Hey, let's send the user ID as a value for this span. Let's send, like, how many rolls are we doing? We'll use that.” +For most people who are working operationally, the thing is like, no, I cannot edit the codebase, right? I don't know where these hooks are added. I don't want to message the product team and say, "Hey, turns out we're sending people's Social Security numbers to our observability backend. Please go do this code commit tomorrow." That's just not the workflow that makes sense for a lot of people. -Or, right, I think we decorate the child span with that. Yeah. And then, um, let's also send their database key up as part of what we're sending. That probably was going a little far, right? We're like, “Oh no, we're shipping this data across the internet, and maybe we don't want to send their full database key.” +We could talk all day and all night about how you should have access to the codebase, you should have a better connection, that's real DevOps, blah blah blah, great points. But let's talk about, you know, it's one thing when, oh my God, we're sending a ton of people's PII into observability when we shouldn't. What about the situation where it's like, hey, we're sending a little too much? -By the way, key is a somewhat overloaded term here, but let's just say, like, this is not a critical security issue, but it is like, “Hey, that's more than we really want to be sending around and showing to everybody in the backend.” +Hey, the traces—every step of a 500-step for loop they're recording is a separate span, and they shouldn't. Is that really something where you want to bother the product people about it and ask them to edit the code? You know, for sure, that's like, "It'd be really nice if we had operational tools to do that." -There are a ton of—this actually is a diagram I drew for something else, but it works here. There are a ton of points where we could say, “Hey, I don't want that database key as a complete string to be stored.” +There are a ton of points where we could say, "Hey, I don't want that database key as a complete string to be stored." So going in and editing these code points would be editing it at the service level, which might be in a whole lot of places. Very often, we have these collector-to-collector architectures. -[00:04:05] Going in and editing these code points, right, would be editing it at the service level, which might be in a whole lot of places. We might be able to—we very often have these collector-to-collector architectures. So, what we're going to be talking about today is hitting it in one of these collector levels. +What we're going to be talking about today is hitting it in one of these collector levels, and then the last and relatively common is doing it at the data store level. This would be like, "Hey, maybe we're sending way too many spans on a single trace." -The last and relatively common is doing it at the data store level. This would be like, “Hey, maybe we're sending way too many spans on a single trace.” Maybe that example of, like, “Hey, we're sending 50 identical spans.” +Maybe that example of like, "Hey, we're sending 50 identical spans." Actually, this example, I don't think I—I don't have the code here. I'm not going to hop over to my IDE for this thing, but it is like it's saying, "Hey, if you roll seven dice, it's sending a span for every single dice roll." Like, I don't know, I don't want all those sub-spans, right? That might be an example where instead of trying to filter at the collector level, a pretty normal step would be like, "Oh, let's just go and edit the data store." -Actually, this example, I don't think—I don't have the code here. I'm not going to hop over to my IDE for this thing, but it is like, it's saying, “Hey, roll. If you roll seven dice, it's sending a span for every single dice roll.” +And say, "Hey, maybe older than this amount, we want to compress our traces in this way." That's a perfectly reasonable way to do it, depending on whether or not you have access to do that. I know with Chronosphere, generally, you do. Certainly, if you have a self-hosted data store for this, that should be very doable if you're using like Loki or Signoz to do that. -Like, “I don't know. I don't want all those sub spans,” right? That might be an example where instead of trying to filter at the collector level, a pretty normal step would be like, “Oh, let's just go and edit the data store and say, ‘Hey, maybe older than this amount, we want to compress our traces in this way.’” +But we're going to talk about filtering at the collector level. Any questions about why we're doing all this, why this matters, or is a tool that we would want to have? -That's a perfectly reasonable way to do it, depending on whether or not you have access to do that. I know with Chronosphere, generally, you do. Certainly, if you have a, you know, self-hosted data store for this, that should be very doable if you're using, like, Loki or Signoz to do that. +**Speaker 2:** Yes, why? -[00:05:20] But we're going to talk about filtering at the collector level. Any questions about, like, why we're doing all this, why this matters, or is a tool that we would want to have? +### [00:05:36] Filtering spans based on attributes -**Speaker 2:** Yes, why? +**Speaker 1:** Yeah, so in general, it's going to be sensitive data or it's overfit data, sort of the two main reasons. But there's also just going to be this general sort of vibe, which is, "Hey, I'm looking at this trace here, and I don't like what I see for some reason." -**Speaker 1:** Yeah, so in general, it's going to be sensitive data or it's be overfit data are sort of the two main reasons. But there's also just going to be this general sort of vibe, which is, “Hey, I'm looking at this trace here, and I don't like what I see for some reason.” +See, change the share here. I'm saying, "Hey, I'm looking at this trace, and this isn't helpful for a reason I will explain." That really could be anything, but you know when we work on the operational side, work on the SRE side, like here's an example where it's like, "Hey, you roll the dice three times, I don't need these little sub-spans." Once the request gets complex, it really starts to clog up the view. -So, see, change the share here. I'm saying, “Hey, I'm looking at this trace, and this isn't helpful for a reason.” I will explain, right? That really could be anything. But, you know, when we work on the operation side, work on the SRE side, like here's an example where it's like, “Hey, you roll the dice three times. I don't need these little sub spans.” +It's not so much about having an elegant view, that should be something that you can control on the UI side, but it's like, "Yeah, something's not looking right, and it's leading to distracting data." A perfect example is, "Hey, maybe you have a large request that is, in fact, asynchronous, but it's getting rolled into your database time." -Once the request gets complex, it really starts to clog up the view. It's not so much about, like, having an elegant view that should be something that you can control on the UI side, but it's like, “Yeah, something's not looking right, and it's leading to distracting data.” +### [00:03:30] Discussion on noisy and sensitive data -A perfect example is, um, “Hey, maybe you have a large request that is, in fact, asynchronous, but it's getting rolled into your database time.” So it's like, “Hey, that's very deceptive, and it's causing everybody to say, ‘Hey, the database is running really slowly,’ but that's not what's going on.” +It's like, "Hey, that's very deceptive, and it's causing everybody to say, 'Hey, the database is running really slowly,' but that's not what's going on." So that's a pretty good time when we would say, "Hey, we don't want to go and actually edit application code or custom code calls; we want to do something at the collector level." -And so that's a pretty good time when we would say, “Hey, we don't want to go and actually edit application code or custom code calls. We want to do something at the collector level.” +Just a reminder of what can happen inside the collector, right? We have receivers and exporters that do our transport, standard communication, can send stuff out to StatsD, can take stuff in from a million places. Then we have our processors in the middle, and we're going to talk about data scrubbing, data normalization, and a little bit about sampling. Batching, you know, it's batching—that is what it is—but those are the concepts that we care about. -So just a reminder of, like, what can happen inside the collector, right? Is like we have receivers and exporters, right, which do our transport standard communication, can send stuff out to StatsD, can take stuff in from a million places. +Okay, so let's talk about filtering by attributes. We have some attributes on this span that we say, "Hey, if we get these attributes, we really don't want to save this." So let's go ahead and demo that live. -Then we have our processors in the middle, and we're going to talk about data scrubbing, data normalization, and a little bit about sampling, batching. You know, it's batching; that is what it is. But that is the concepts that we care about. +Let's do a new share. Here's my OpenTelemetry collector configuration, and just stepping through what was necessary to add this. If a span has these attributes, I do not want to send it across. -[00:08:00] Okay, so let's talk about filtering by attributes. So we have some attributes on this span that we say, “Hey, if we get these attributes, we really don't want to save this.” So let's go ahead and demo that live. +In our application, right, we take a request attribute, and we add it to that span right down here. We say, "Hey, the user for this is this request attribute user ID that we received." Obviously, with a real application, we should have the user ID from a million other places, but we picked it up here. -Let's do a new share. Here, so here's my OpenTelemetry collector configuration. Just stepping through what was necessary to add this. Hey, if a span has these attributes, I do not want to send it across. +### [00:08:34] Live demo: Adding filters in OTel configuration -In our application, right, we take a request attribute, and we add it to that span right down here. We say, “Hey, the user for this is this request attribute user ID that we received.” Obviously, with a real application, we should have the user ID from a million other places, but we picked it up here. +It turns out that a couple of our users are like our internal test users, right? And so in fact, that's so consistent because it's like an automated testing suite. So we know they're always going to be named this. We say, "Hey, don't record any spans for these people because maybe they're testing stuff. Maybe those spans get crazy long; they introduce a bunch of fuzzy unnecessary data to our production environment." -It turns out that a couple of our users are like our internal test users, right? So in fact, that's so consistent because it's like an automated testing suite. So we know they're always going to be named this. We say, “Hey, don't record any spans for these people because maybe they're testing stuff. Maybe those spans get crazy long; they introduce a bunch of fuzzy unnecessary data to our production environment.” +So we can add a processor section which starts with filter and then has whatever name we want it to have. We say, "Hey, we can call this Dev users because that's descriptive." Then critically, and I think I have a slide about this too, we also add it to the pipeline down here, which is where I usually mess up. -So we can add a processor section, which starts with filter and then has whatever name we want it to have. So this is slash; we say, “Hey, we can call this Dev users because that's descriptive.” +I usually go ahead and create a very careful filter in the collector config and let me give a little stake of Zoom here, and then forget to add it to this thing. This pipeline is read left to right. If you're doing things like transform filters or transform processors, which we'll talk about, you want to have this in the correct order so that your transformations will have effect when you filter. You generally want batching at the end. -Then critically, and I think I have a slide about this too, we also add it to the pipeline down here, which is where I usually mess up. I usually go ahead and create a very careful filter in the collector config and let me give like one stke of zoom here, and then forget to add it to this thing. +So if we save this config and go and restart our collector—come on, bud—cool, cool, cool, cool. Then we send in a couple of requests. That was from David, and if I check my config, David's like an allowed user. But if I send one from Bob, so here's the service, right? It's the same dice roller that you've seen in the OpenTelemetry examples, right? -And this pipeline is read left to right, so, you know, if you're doing things like transform filters, transform processors, which we'll talk about, you know, you want to have just this in the correct order so that your transformations will have effect when you filter. You generally want batch at the end. +It returns an array of dice rolls, you know. Bob still gets his data because it's not like we're, you know, this is all observability side, so the application works exactly the same. But we should see that this span is dropped. -So if we save this config and go and restart our collector—come on, bud—cool, cool, cool, cool. Then we send in a couple of requests. So that was from David, and if I check my config, David's like an allowed user. But if I send one from Bob, so here's the service, right? +We can hop over to our UI, and I have gone so far as to this guy is just coming in as normal. I believe he has a user attached to it. Yeah, this is David; he's allowed. This is like the Juliet child chopped onions because no, I did not wait for this to show up in my observability backend. It would be there probably by now, but just so I didn't hit refresh and have it not show up. -It's the same dice roller that you've seen in the OpenTelemetry examples, right? It returns an array of dice rolls. You know, Bob still gets his data because it's not like we're, you know, this is all observability side, so the application works exactly the same. But we should see that this span is dropped. +But then if we did send it in with this guy who we didn't want, it will go ahead and drop that span. Now in this case, that span did have child spans, so it still shows up as, "Hey, there was time spent here," but we did drop the span. So we know it existed, but we're dropping all data from it, which in this case is what we wanted. -We can hop over to our UI, and I have gone so far as to—so this guy is just coming in as normal, and I believe has a user attached to it. Yeah, this is David. He's allowed. +What we wanted was to say, "Hey, just this span is what doesn't matter. This is like we're doing span-level metrics for execution time." -And this is like the Juliette child chopped onions because, no, I did not wait for this to show up in my observability backend. It would be there probably by now, but just so I didn't hit refresh and have it not show up. +Then we care about dropping that single span. Okay, I think I have screenshots of that back in the presentation. We'll find out. -But then if we did send it in with a—um, with this guy who we didn't want, it will go ahead and drop that span. Now, in this case, that span did have child spans, so it still shows up as, “Hey, there was time spent here,” but we did drop the span. +**Speaker 2:** New share, okay. -So we know it existed, but we're dropping all data from it, which in this case is what we wanted. What we wanted was we wanted to say, “Hey, just this span is what doesn't matter.” This is like we're doing like span-level metrics for execution time, and so then we care about dropping that single span. +**Speaker 1:** So we want to go ahead and drop out the span. Yep, there's our nice little thing, and then our regular one rolls on just as we'd expect. -Okay, I think I have like screenshots of that back in the presentation. We'll find out—new share. Okay, so we want to go ahead and drop out the span. Yep, there's our nice little thing, and then our regular one rolls on just as we'd expect. +Okay, so some stuff to remember about the filter processor. First of all, once it hits an attribute that'll filter, it's not going to hit the later conditions. So if there's an error on a later condition for some reason, it's not actually going to execute. -[00:12:10] Okay, so some stuff to remember about the filter processor. First of all, once it hits an attribute that'll filter, it's not going to hit the later conditions. So if there's an error on a later condition for some reason, it's not actually going to execute. +Any matching condition will work, so the config that you just saw where it's like, "Hey, user equal to Bob, user equal to David, user equal to Alice," filter, right? That is going to work. It doesn't have to meet all those conditions; it just needs to meet one. -And then, you know, any matching condition will work. So, you know, the config that you just saw where it's like, “Hey, user equal to Bob, user equal to David, user equal to Alice,” filter right, that is going to work. It doesn't have to meet all those conditions; it just needs to meet one. +### [00:12:36] Explanation of filter processor behavior -Then, if we drop out events, this is a confusion for some people, depending on the model that they're using for monitoring. The span is going to stay there. So if we say, “Hey, the type is a span event, and we want to drop every single span event,” we're still going to have that span. +If we drop out events, this is a confusion for some people depending on the model they're using for monitoring. The span is going to stay there, so if we say, "Hey, the type is a span event, and we want to drop every single span event," we're still going to have that span. -A metric works not the same way. If we drop data points from a metric until there are no data points, the metric itself will not be reported, which makes sense. You wouldn't generally report, right? Like, “Hey, this data point, but nil metric,” or sorry, “this metric, but nil data points, was not something you would normally report.” +A metric works not the same way. If we drop data points from a metric until there are no data points, the metric itself will not be reported, which makes sense. You wouldn't generally report, "Hey, this data point but nil metric," or sorry, "this metric but nil data points," was not something you would normally report. So yeah, this is my slide to remind myself and everyone else that you want to actually add these to the pipeline. -Okay, so now let's talk about this problematic data here where this is the key that we're getting through, which we really didn't want. So in this case, obviously, we could write something that just said, “Hey, by name, drop this attribute,” but we don't want to drop the attribute because we have this guest down in here that's useful to us. +Okay, so now let's talk about this problematic data here where this is the key that we're getting through which we really didn't want. In this case, obviously, we could write something that just said, "Hey, by name, drop this attribute." But we don't want to drop the attribute because we have this guest down in here that's useful to us. -We want to know if it's guest, admin, whatever else, whatever other role that's useful information to us. We just want to drop the full key. For that, this got me for a minute. You can have a filter that says, “Hey, it does it by match, and we'll take do star, say accept any attribute, but then it will not accept a full regex.” +We want to know if it's guest admin or whatever else. Whatever other role, that's useful information to us. We just want to drop the full key. For that, this got me for a minute. -[00:14:06] Then I read this portion of the documentation, which is, “Hey, here's some alternative config which could soon be deprecated,” and that includes using regex as your match type. I was like, “Wait, can I not? Am I not supposed to be able to use regex here? Am I supposed to just use this relatively simple direct string matching?” +You can have a filter that says, "Hey, it does it by match," and we'll take do star, say, "Accept any attribute," but then it will not accept a full regex. I read this portion of the documentation, which is, "Hey, here's some alternative config which could soon be deprecated." That includes using regex as your match type. -So you can. My thing is, write down this link because something that I've had on my to-dos for a few weeks is to get this document a little better linked into the other documentation because a complete list of the OpenTelemetry transform language functions is like not super well linked into the filter and transform processor. +I was like, "Wait, can I not? Am I not supposed to be able to use regex here? Am I supposed to just use this relatively simple direct string matching?" So you can, and my thing is write down this link because something that I've had on my to-dos for a few weeks is to get this doc a little better linked into the other documentation because a complete list of the OpenTelemetry transform language functions is not super well linked into the filter and transform processor. -Don't actually write this down; I will share this. I'll drop it into the chat; you don't have to write down a whole link. But, yeah, OpenTelemetry transform language, those functions include a pattern-based replace, which we'll demonstrate in a minute here. +Don't actually write this down; I will share this. I'll drop it into the chat. You don't have to write down a whole link, but yeah, OpenTelemetry transform language, those functions include a pattern-based replace, which we'll demonstrate in a minute here. -So some notes on using this regex in a collector config: you want to double your escapes like this, which you can set as a style thing on regex 101 or wherever other regex test tool that you're using. +Some notes on using this regex in a collector config: you want to double your escapes like this, which you can set as a style thing on regex 101 or whatever other regex test tool that you're using. You can use capture groups; they do work, but it will seem like they don't because you have to use this double dollar sign thing to do it. -You can use capture groups; they do work. But it will seem like they don't because you have to use this double dollar sign thing to do it. In this case, it's like, “Hey, whatever the actual—in this case, this person wants to change Cube to K8s and U. They want to change it to KES dot, but they want the ID. They want to preserve the ID.” +In this case, it's like, "Hey, whatever the actual—in this case, this person wants to change Cube to K8s, and U they want to change it to KES dot, and they want the ID; they want to preserve the ID." So they're saying, "Hey, match this key, which is Q, but then some string of letters and numbers, and then go ahead and replace it with K8, but that include what you matched in the parentheses." Apologies, that's a review for you all, but regex is my home and my joy, so I always like to talk about it. -So they're saying, “Hey, match this key, which is Q, but then some string of letters and numbers, and then go ahead and replace it with K8, but that include what you matched in the parenthesis.” Apologies, that's a review for you all, but regex is my home and my joy, so I always like to talk about it. +So yeah, these are just things you want to remember about your syntax. -But, yeah, so you need to escape the dollar sign for this capture group with a double dollar sign, and you have to use triple dollar sign to use a single dollar sign. Sorry, this is just one of those things you have to note because the beautiful thing about regex is that it always works, except when it doesn't. +In this case, what we'd want is a transform processor because we want to change our value to just be underscore guest or just be some kind of escape and underscore guest. In this case, right, we say, "Okay, go and find it." -These are just things you want to remember about your syntax. In this case, what we'd want is we'd want a transform processor because we want to change our value, right, to just be underscore guest or just be some kind of escape and underscore guest. +I've been a little bit greedy here where I've said, "Hey, look for this thing; it's a whole bunch of numbers, more than one number for this. Don't just take anything that is a user DB key," and then go ahead and overwrite the ID with replacement ID and then your whatever group you first matched. So then you'll get something like this: replacement ID or guest. -In this case, right, we say, “Okay, go and find it.” I've been a little bit greedy here where I've said, “Hey, look for this thing. It's a whole bunch of numbers, more than one number for this.” +I have a version of this trace in the Signal dashboard, but take my word on it; this works. Transform processors are a really effective way you can grab some other values if you need to. You can chain them together, but the big thing is just to do like really smart collapse of cardinality and collapse of PII. Really, really nice tool to do that. -Like, “Don't just take anything that is a user DB key, and then go ahead and overwrite the ID with replacement ID and then your whatever group you first matched.” So then you'll get something like this replacement ID. +What can't you do with collector processors? You know, we've seen this beautifully impressive demo here of the stuff we can do. A lot—there's a lot that you cannot do with collector processors. -I have a version of this trace in the Signal dashboard, but take my word on it, this works. So transform processors are a really effective way you can grab some other values if you need to. You can chain them together, but the big thing is just to do really smart collapse of cardinality and collapse of PII. +So here I would say is probably the big one: you cannot change the relationships that spans have. With the processor, you cannot just say, "Hey, you go be part of this trace now," nor can you create a parent-child relationship. Like, "Oh, you actually were kicked off, and we're inside this other one, and I'm going to know that from like your naming convention, and then I'm going to stick you onto this other one." -Really, really nice tool to do that. So what can't you do with collector processors? You know, we've seen this beautifully impressive demo here of the stuff we can do. A lot. There's a lot that you cannot do with collector processors. +You can do filtering and transformation, and you can touch Trace ID. I haven't experimented with this extensively, but apparently, it is not possible to push something into a trace after the fact. This makes some sense because of the way that sampling and other kinds of working with traces happen. -So here I would say is probably the big one: you cannot change the relationships that spans have with the processor. You cannot just say, “Hey, you go be part of this trace now,” nor can you create a parent-child relationship like, “Oh, you actually were kicked off, and we're inside this other one, and I'm going to know that from your naming convention, and then I'm going to stick you onto this other one.” +There's some computer science reasons why this is not a trivial thing at all. It would not be a trivial thing at all to implement, but you want to be aware of it. I learned this for the first time. It's sort of, you know, it's hard to notice an absence sometimes. You feel like, "Oh yeah, I'm fully in control of this. I can do whatever I want." -So you can do filtering and transformation, and you can touch trace ID. I haven't experimented with this extensively, but apparently, it is not possible to push something into a trace after the fact. This makes some sense because of the way that sampling and other kinds of working with traces happen. +But then Hazel's great talk for this same user group clued me into like, "Hey, this is a limitation here," and I haven't tried to hack it to every degree, but it's definitely—yeah, there's no built-in call to say, "Hey, add child relationship." Not at all. -There are some computer science reasons why this is not a trivial thing at all. It would not be a trivial thing at all to implement, but you want to be aware of it. I learned this for the first time—it's sort of, you know, it's hard to notice an absence sometimes. You feel like, “Oh yeah, I'm fully in control. I can do whatever I want.” +You also can't drop a whole trace, so you notice that I had this example where it's like, "Hey, don't look; the span is gone, no worries." But you would sort of like to say, "Hey, I can go in and say, 'Hey, this span looks bad; let me go ahead and just lose this trace.'" -[00:19:00] But then, Hazel's great talk for this same user group clued me into like, “Hey, this is a limitation here.” I haven't tried to hack it to every degree, but it's definitely—yeah, there's no built-in call to say, “Hey, like, add child relationship.” Not at all. +### [00:19:36] Head vs. tail-based sampling -You also can't drop a whole trace. You notice that I had this example where it's like, “Hey, don't look, the, you know, the span is gone. No worries.” But you would sort of like to say, “Hey, I can go in and say, ‘Hey, this span looks bad. Let me go ahead and just lose this trace.’” +And that is quite a new kettle of fish called tail-based sampling, where you're looking at all of the traces that you receive, and you're saying, "Hey, I only want these ones to come through based on some information about the trace." -That is quite a new kettle of fish called tail-based sampling, right? Where you're looking at all of the traces that you receive and you're saying, “Hey, I only want these ones to come through based on some information about the trace.” +This is the dichotomy between head and tail-based sampling. Head-based sampling is right before the trace even starts; you decide if you're going to run it or not, maybe based on like the tiniest amount of data, like what route it's in or whatever request information you have. But otherwise, basically, it's just probabilistic sampling. -So this is the dichotomy between head and tail-based sampling. Head-based sampling is right before the trace even starts. You decide if you're going to run it or not, maybe based on the tiniest amount of data, like what route it's in or whatever request information you have. +In that case, with tail-based sampling, this thing that we all want, whether or not we can have it, is you're saying, "Hey, now that I'm looking at the trace, I decide that you're interesting." You want to send it. -But otherwise, basically, it's just, you know, I want to say the word—I know it's the wrong word—iskoric has a completely different meaning, but you're doing probabilistic sampling in that case. +Tail-based sampling has a bunch of promise for the efficiency that it's going to add to your system. Only sending interesting traces—traces are relatively expensive to transmit, store, and process, so it's very promising but has its own caveats to make it work. -With tail-based sampling, this thing that we all want, whether or not we can have it, is you're saying, “Hey, now that I'm looking at the trace, I decide that you're interested.” You want to send you on. Tail-based sampling has a bunch of promise for the efficiency that it's going to add to your system, and only sending interesting traces—traces are relatively expensive to transmit, store, process—so it's very promising but has its own caveats to make work. +So this is, Mo, we did most of the presentation. I did not pause for questions a bunch of times, but do we have questions about that, about head and tail-based sampling, and why that is not supported in the simple filter transform system? -So this is most of the presentation. I did not pause for questions a bunch of times, but do we have questions about that, about head and tail-based sampling and why that is not supported in the simple filter transform system? +**Speaker 3:** Love it. I have to go give this same talk—not the same talk, but a version of this talk—to CGN next week, I think. Thank you so much for joining me on this journey because there's not a ton out there about these PII. I mean, the transform processor is in alpha, so I totally get it why there wouldn't be a ton out there about it. -**Speaker 3:** Love it. I have to go give this same talk—not the same talk, but a version of this talk—to CGL next week, I think. Thank you so much for joining me on this journey because there's not a ton out there about these PII, I mean, mildly. - -Like, the transform processor is in alpha, so I totally get why there wouldn't be a ton out there about it, but I'm glad we're getting to talk about it now. So do feel free to make comments or drop in the chat if you have other questions. There are 29 things in the chat, and I have not been looking at the chat at all. Sorry. - -[00:21:35] Most of it is our—but Paige is asking, “How do you recommend testing config changes before shipping them to prod?” Or monitoring to know if you messed up a filtering rule? +But yeah, I'm glad we're getting to talk about it now. So do feel free to make comments or drop in the chat if you have other questions. There are 29 things in the chat, and I have not been looking at the chat at all—sorry! Most of it is our—but Paige is asking, "How do you recommend testing config changes before shipping them to prod or monitoring to know if you messed up a filtering rule?" **Speaker 1:** Yeah, really good question. -**Speaker 3:** Sorry about that. I do think that a really good way to do kind of experimental stuff where you're saying, “Hey, I want to do a kind of a multi-level change to what we're doing. I want to do a complex transform.” +I'm gonna hit back a million times, sorry about that. I do think that a really good way to do kind of experimental stuff where you're saying, "Hey, I want to do a kind of a multi-level change to what we're doing. I want to do a complex transform." -The two things are, one, is to just have a test version—have some piece of your observability inside your test harness. Have your local version of a collector or somewhere else to test it. +The two things are, one, is to just have a test version—have some piece of your observability inside your test harness, and so have your local version of a collector or somewhere else to test it. The other is to this collector-to-collector architecture is extremely, extremely useful here. -The other is to—this collector to collector architecture is extremely, extremely useful here. So you can do like a canary deploy to say, “I'm just going to send it out to this one sub piece instead of just our master collector architecture.” +You can do like a canary deploy to say, "I'm just going to send it out to this one sub-piece instead of just our master collector architecture." Then the third is to do some further limitations because you can chain together an "and" here. -Then the third is to do some further limitations because you can chain together an AND here. You can use resource values, so you can say, “Hey, only do this to this service name.” That's super useful for doing this. +You can say, "Hey, only do this to this service name." So that's super useful for doing this. I didn't show that piece; I didn't do that here. But you can say, "Hey, I want this to have X service name and also check for this span value and then drop the span." -I didn't show that piece; I didn't do that here, but you can say, “Hey, I want this to have X service name and also check for this span value and then drop the span.” So you can start out as being more limited before it spread out to everybody else. Really good question. +So you can start out as being more limited before it spread out to everybody else. Really good question. -**Speaker 4:** Is a collector restart always required to pick up new config? +Is a collector restart always required to pick up new config? Great question from a long time ago. I would get kicked off of Twitch so fast because I'm not looking at the chat. How embarrassing! -**Speaker 1:** Great question from a long time ago. I would get kicked off of Twitch so fast because I'm not looking at chat. H—embarrassing. I used to know the answer to that, Paige, and now I do not. +I used to know the answer to that, Paige, and now I do not. I seem to recall that there was some issue that I ran about the feasibility of some kind of hot reloading of config, but essentially yes, you have to do a collector restart, certainly in your little test bed environment. -I seem to recall that there was some issue that I ran about the feasibility of some kind of hot reloading of config. But essentially, yes, you have to do a collector restart, certainly in your little test bed environment. +There's also not like—the collector has relatively limited remote abilities to pick up config. So yeah, that's something maybe one of the Honeycomb people will know something about, has fiddled with a bit, and I can check with the rest of Signoz team as well. -Yeah, there's also not like—the collector has relatively limited remote abilities to pick up config. So yeah, that's something maybe one of the Honeycomb people will know something about, has fiddled with a bit, and I can check with the rest of the Signoz team as well. +Okay, so we dropped this replacement ID. We did that talk about what we can and can't do. There is a tail sampling processor out there. It has its limitations, but you want to explore that separately as its own processor and its own service for dropping entire traces. -Okay, so we dropped this replacement ID. We did that talk about what we can and can't do. Yeah, there is a tail sampling processor out there. It has its limitations, but you want to explore that separately as its own processor and its own service for dropping entire traces. +### [00:24:30] Q&A on testing config changes -**Speaker 3:** Words of caution with metric transformations, so conversion between data types isn't supported by the metric model. But you can do it with a transformer, but don't do it. But you can do it, but don't do it. +Okay, so words of caution with metric transformations. Conversion between data types isn't supported by the metric model, but you can do it with a transformer, but don't do it. But you can do it, but don't do it. -It's exceedingly easy to create meaningless metrics, right? To like doing things like creating averages from max values, right? Which you shouldn't do. Yeah, I would say very much like you're in the realm of, you know, creating statistical transformations just with like Excel equations at that point. +It's exceedingly easy to create meaningless metrics, right? Like doing things like creating averages from max values, right? Which you shouldn't do. I would say very much like you're in the realm of, you know, creating statistical transformations just with Excel equations at that point. -You are in danger, certainly, of messing up what you're doing and getting stuff that looks okay but is, in fact, extremely bad signal. What have I seen? I've seen like somebody processing and creating a new average with a single new value. So they have an average of the last 10 values, and they get a single new value, and they say, “Okay, cool. I can just add that one to the average,” which was not quite right. +You are in danger, certainly, of messing up what you're doing and getting stuff that looks okay but is in fact extremely bad signal. What have I seen? I've seen like somebody processing and creating a new average with a single new value. So they have an average of the last 10 values, and they get a single new value, and they say, "Okay, cool, I can just add that one to the average," which was not quite right. -So yeah, things like that are worth a concern. A bunch of the transformations between measurement types are one-directional, and so you should not use the transform to try to go back or to try to move between these lanes because you will end up with bad data that is still data that still looks like data and will chart like data. +So yeah, things like that are worth a concern. A bunch of the transformations between measurement types are one-directional, and so you should not use the transform to try to go back and or to try to move between these lanes because you will end up with bad data that still looks like data and will chart like data. So that's a concern, very similar to going in and doing just queries directly into your data store to edit your data, right? You want to be exceedingly careful with that because, of course, there is always the possibility of destroying the information that you have. -**Speaker 1:** Slide two of cautions about metric transformations. So you can change the labels on metrics and traces. You can cause yourself data store problems by giving things the same names as two metrics, two time series. You give the same name, attributes, and scope. +Slide two of cautions about metric transformations. You can change the labels on metrics and traces, so you can cause yourself data store problems by giving things the same names as two metrics, two time series. You give the same name attributes and scope. I tried to break Signoz, the ClickHouse data store, doing this because I was very excited to see something break, but I couldn't quite do it. But presumably, you can get to the point where you're having your dashboards not load or other queries not working because you have double results when there should be unique values in a column. -So a more common outcome of that will be orphan values, will be spans that are not part of any trace and traces that are not connected to any service. +The more common outcome of that will be orphan values—spans that are not part of any trace and traces that are not connected to any service. -Oh yeah, Ren, I'll put it in the CNCF collector channel. I'll ask it in the CNCF collector channel, and the first thing I'm going to do is actually search the CNCF collector channel because it's interesting. +**Speaker 4:** Oh yeah, Ren, I'll put it in the CNCF collector channel. I'll ask it in the CNCF collector channel. The first thing I'm going to do is actually search the CNCF collector channel because it's interesting. The question was, "Do you have to do a collector restart always to pick up new collector config?" -The question was, “Do you have to do a collector restart always to get to pick up new collector config?” I think the answer is yes, but I—yes, yeah. I thought there was an issue about it. There was a talk about exploring some support for that, but I'm certain that by default right now the answer is no. +I think the answer is yes, but yes, yeah. I thought there was an issue about it. There was a talk about exploring some support for that, but I'm certain that by default right now, the answer is no. -That was the end of the slideshow, folks, which really just crept right up on me. Do we want—we saw live demo stuff. Oh yeah, let's go do one last live demo thing. That's right. That's what I wanted to do. +That was the end of the slideshow, folks, which really just crept right up on me. Do we want—we saw live demo stuff. Oh yeah, let's go do one last live demo thing. That's right; that's what I wanted to do. New share, okay? -New share, new share, share. Okay, so, man, there was totally some—oh right. We have the situation where we are getting way too many components. +So man, there was totally some—oh right, we have the situation where we are getting way too many components. This thing like, roll it, it sends you back an array of rolled dice. We had a situation where when we sent a request to say like, "Hey, roll, you know, 41 dice for me," we get back this nice array very, very quickly. -This thing, like, roll it, you know, it sends you back an array of roll dice. We had a situation where when we sent a request to say like, “Hey, roll, you know, 41 dice for me,” we get back this nice array very, very quickly. +But the problem is that when we go and look at the traces for these guys, they just have a ton, a ton of these individual single dice rolls, which, you know, in this version, right, is not really a problem. -But the problem is that when we go and look at the traces for these guys, they just have a ton—a ton of these individual single dice rolls which, you know, in this version, right, is not really a problem. Right, you know, your trace dashboard should be able to just hide those a whole bunch of rolls. +You know, your trace dashboard should be able to just hide those a whole bunch of n rules, but for whatever reason, in our example, we're saying, "Hey, this is an issue. We have hundreds of these; we have thousands of these. We know they're all identical; they always take the same amount of time. We want to get these out of here." -But for whatever reason, in our example, we're saying, “Hey, this is an issue. We have hundreds of these. We have thousands of these. We know they're all identical; they always take the same amount of time. We want to get these out of here.” +This is an older trace that didn't have 40; we just had six, but you get the point. So what we want to do is we want to say, "Hey, I want to drop these spans, these roll one spans." But some helpful and also sabotage-focused developer has named each of the spans uniquely, so we can't just say, "Hey, go ahead and drop any span that has this name," because it includes like this is the counter on that roll. -This is an older trace that didn't have 40; we just had six, but you get the point. What we want to do is we want to say, “Hey, I want to drop these spans, these roll one spans.” +So that is something that we want to handle. We don't need a reject for this; we just need the match tool. So if we go back into our IDE—let me—oh, I started typing the command, but I won't actually show this. -But some helpful and also sabotage-focused developer has named each of the spans uniquely, so we can't just say, “Hey, go ahead and drop any span that has this name,” because it includes—like, this is the counter on that role. +If we go back into our IDE, we go back into our config, we can say, "Hey span, if you have a match on your name that is roll ones followed by anything, let's go ahead and drop that span." -So that is something that we want to handle. We don't need a regex for this; we just need the match tool. So if we go back into our IDE, let me—oh, I started typing the command, but I won't actually show this. +If we save this and then we restart our collector, our app has been running this whole time. So now asking for 41 dice rolls, let's ask for 40. Who likes using a prime number? I certainly don't. -So if we go back into our IDE, we go back into our config, we can say, “Hey, span, if you have a match on your name that is roll ones followed by anything, let's go ahead and drop that span.” +While I politely wait for a second for these traces to hit my local collector, get to the batch moment, which is like every—I think every five minutes on my thing—and then get sent off to the backend, do we have other questions about this stuff? -If we save this and then we restart our collector, our app's been running this whole time. So now asking for 41 dice rolls—let's ask for 40. Who likes using a prime number? I certainly don't. +Oh, I didn't mention it, but you can—you have the same filter functions available on metrics and logs. So we demoed this all with spans, but it is totally doable with metrics and logs by the same principles. -While I politely wait for a second for these traces to hit my local collector, get to the batch moment, which is like every—I think every five minutes on my thing—and then get sent off to the backend, do we have other questions about this stuff? +But yeah, other questions feel free to drop them in chat or come off mute. I promise I am watching the chat now—no worries. -**Speaker 5:** Oh, I didn't mention it, but you can—you have the same filter functions available on metrics and logs. So we demoed this all with spans, but it is totally doable with metrics and logs by the same principles. +### [00:31:30] Limitations of collector processors -But yeah, other questions, feel free to drop them in chat or come off mute. I promise I am watching the chat now. No worries. +Okay, so yeah, now let me go ahead and start sharing this. This is just our last five minutes of requests. -Okay, so yeah, now let me go ahead and start sharing this. This is just our last five minutes of requests. If we look at—we were just looking at a previous one where it was like, “Hey, we have way too many of these.” +So we look at—we were just looking at a previous one where it was like, "Hey, we have way too many of these." Now if we go ahead and sort this and look at our most recent traces, we should sure enough be able to see—oh, I also gave it a name that's like, "Hey, drop this name." -Now, if we go ahead and sort this and look at our most recent traces, we should sure enough be able to see—oh, I also gave it the name that's like, “Hey, drop this name.” +So we're dropping two things here, but our actual dice is happening. It drops the routing span because again it has this name it doesn't like, and then it also drops the sub, like individual roll one spans. -So we're dropping two things here, but our actual R dice is happening. It drops the routing span because, again, it has this name it doesn't like, and then it also drops the sub-like individual roll ones spans. +Okay folks, that's been my stuff. Man, 38 minutes! Wow, usually when I'm doing something for like the first or second time, I time it out, talk it all through, I'm like, "45 minutes," and then when I actually get in front of people, "11 minutes." -Okay, folks, that's been my stuff. Man, 38 minutes, wow. Usually when I'm doing something for the first or second time, I time it out, talk it all through, I'm like 45 minutes. Then when I actually get in front of people, it's 11 minutes. Just zipping, zipping. +Just zipping, zipping! But this one, I guess there was a lot to say about this. -But this one, I guess there was a lot to say about this. Final stuff: the functions are all out, are all fully supported. The transform processor is still in alpha, and you're going to need your own build of the collector to make sure you include it, which, I mean, you need to do for other collector contrib stuff. +Final stuff: the functions are all fully supported. The transform processor is still in alpha, and you're going to need your own build of the collector to make sure you include it, which I mean you need to do for other collector contrib stuff. -So that's pretty standard, but just be aware of it. You wouldn't want to base your whole DIY observability stuff on transform processors. Do some work ahead of time to get your stuff labeled right and don't be completely relying on it because it is an alpha and it's going to shift a little bit. +So that's pretty standard, but just be aware of it. You wouldn't want to base your whole DIY observability stuff on transform processors. Do some work ahead of time to get your stuff labeled right, and don't be completely relying on it because it is in alpha and is going to shift a little bit. -There are some standards conversations happening, but it's definitely worth doing for stuff like this, for like, “Hey, you have this filtering or transformation task, and you don't want to bother your product team.” +There are some standards conversations happening, but it's definitely worth doing for stuff like this, for like, "Hey, you have this filtering or transformation task, and you don't want to bother your product team." -Okay, folks, that's been my time. I don't know, Adrian, if you want to give final stuff. +Okay folks, that's been my time. I don't know, Adrian, if you want to give final stuff. -**Speaker 6:** While you're filtering out these spans not to be shipped to Signoz, can you route them to something like S3? +**Speaker 5:** Oh, while you're filtering out these spans not to be shipped to Signoz, can you route them to something like S3? -**Speaker 1:** Oh, that's an interesting question. So not with this set of tooling. You cannot use like a filter processor to say, “Now go to a different exporter lane.” But you certainly can do that with, you know, with your import tools, right? +**Speaker 1:** Oh, that's an interesting question. Not with this set of tooling, you cannot use like a filter processor to say, "Now go to a different exporter lane." But you certainly can do that with your import tools, right? -So with your components where you're taking in data, you can obviously say on your receivers, “Oops, don't do that.” You can say on your receivers, “Hey, I want to pick up these values and send them to this other endpoint,” yeah, and then go into a separate pipeline. +So with your components where you're taking in data, you can obviously say on your receivers, "Oops, don't do that." You can say on your receivers, "Hey, I want to pick up these values and send them to this other endpoint." -I don't—yeah, filter is like drop; filter is like drop or remove data. It's not route this data to another endpoint. Great question. Pretty sure you can do that routing by span name, but also, should you do that? Should do— that's a good question. +Then go into a separate pipeline. I don't—yeah, filter is like drop; filter is like drop or remove data. It's not route this data to another endpoint. Great question! -That's a really good—I should write something about that. But that's really good. I'm going to look at a future blog post about that. That's a really good question. +Pretty sure you can do that routing by span name, but also, also should you do that? Should do that? That's a good question. -**Speaker 7:** Any other questions for Na? +That's a really good—I should write something about that. But that's really good. I'm gonna, I’m gonna look at a future blog post about that. That's a really good question. -[00:36:00] **Speaker 1:** Ren says in chat, “Yeah, there are a lot of tools out there that help with managing your pipeline, including like especially, you know, we got three, I think, observability team people here who can talk a lot about how, like, there are times when I think all of us want to be like, ‘Hey, we're your one-stop shop to look at your data.’” +**Speaker 6:** Any other questions for Na? -But there's going to be reasons that you're sending it to multiple places. Examples, of course, be like sending stuff to CloudWatch but also sending it to a New Relic dashboard, sending stuff to S3 as like, “Hey, this is—we're going to cold storage this, but we want to have it.” +**Speaker 7:** Ren says in chat, "Yeah, there are a lot of tools out there that help with managing your pipeline, especially, you know, we got three, I think, observability team people here who can talk a lot about how like there are times when I think all of us want to be like, 'Hey, we're your one-stop shop to look at your data,' but there are going to be reasons that you're sending it to multiple places." -Obviously, classically logs, right? Sending them to multiple places, to one place where the retention is 10 days and another place where the retention is 10 years. So, yeah, that's going to make a ton of sense. +Examples, of course, be like sending stuff to CloudWatch but also sending it to a New Relic dashboard, sending stuff to S3 as like, "Hey, this is—we're going to cold storage this, but we want to have it." -There's something written about that, Ren. Maybe I'll hit you up for like a—we'll do like a partner post or something on it because it's an interesting question and not one where there's a ton written about it. +Obviously, classically logs, right? Sending them to multiple places to one place where the retention is 10 days and another place where the retention is 10 years. So yeah, that's going to make a ton of sense. -**Speaker 6:** Sure, yeah, that sounds good. +So yeah, there's something written about that, Ren. Maybe I'll hit you up for like a—we'll do like a partner post or something on it because it's an interesting question and not one where there's a ton written about it. -**Speaker 1:** Paige is like, “In the last webinar I did, they asked us how many observability tools they had, and a significant portion of the groups had five plus.” +**Speaker 8:** Sure, yeah, that sounds good. -I think if we were being honest, everyone's going to say at least two, right? Every—well, no, at least three. They say, “Hey, I have an observability tool, which is the thing called observability that says observability on its SEO filing. Then I have some kind of logging tool, and then I have a debugging tool that I use, right? I have my ClickOps tool that I use to go in and look at what the heck's going on.” +**Speaker 1:** Paige is like, "In the last webinar I did, they asked us how many observability tools they had." A significant portion of the groups had five plus. I think if we were being honest, everyone's gonna say at least two, right? -And so that's at least three, right? If not, then, okay, well now what's—what do you use as like a pinger or other synthetic metrics, right? That can—can so, yeah, now we're at five, right, with just what I would say. +Well, no, at least three. They say, "Hey, I have an observability tool, which is the thing called observability that says observability on its SEO filing." Then I have some kind of logging too, and then I have a debugging tool that I use, right? -Oh, like you list all five, you're like, “Yeah, that's normal. That's normal. I need all this.” It's like me looking in my purse. I'm like, “Uh, I need to get some stuff out of here.” They're like, “No, I need all of this. I need two batteries for my phone.” +I have my ClickOps tool that I use to go in and look at what the heck's going on. And so that's at least three, right? If not, then okay, well now what's—what do you use as like a pinger or other synthetic metrics, right? That can—so yeah, now we're at five, right? -Okay, rid of these. Oh man, great chat. Thank you so much. And again, big shout out to R. You missed it right at the start when I was like really thanking you for doing the last-minute promo. +With just what I would say like, oh, like you list all five, you're like, "Yeah, that's normal. That's normal. I need all this." -I will be giving a similar but not identical version of this talk because we can all admit there were parts that dragged. I'll be giving a similar version of this talk at CGL next week, and then also we'll be doing it at another conference whose name escapes me in January. +It's like me looking in my purse. I'm like, "Uh, I need to get some stuff out of here." They're like, "No, I need all of this. I need two batteries for my phone." -We'll be doing Cod Smash, which is in January or something. I'm going to be giving the same talk, and also be at KubeCon. If you want to come say hi at KubeCon, come say hi. Also, thank you, thank you so much Na for joining us today. +Okay, rid of these. Oh man, great chat! Thank you so much! And again, big shout out to R. You missed it right at the start when I was really thanking you for doing the last-minute promo. -**Speaker 2:** And thank you everyone who was able to make it. This was actually a really good turnout for OTel in practice, so thank you for taking the time. +I will be giving a similar but not identical version of this talk because we can all admit there were parts that dragged. I'll be giving a similar version of this talk at CGN next week, and then also we'll be doing it at another conference whose name escapes me in January. -Also, for if you know anyone who wanted to attend but couldn't make it, let them know that we will be posting a recording of this on the OpenTelemetry YouTube channel. The handle is OTel Das Official. If you don't, haven't subscribed yet, we've got a bunch of previous videos from OTel in practice, OTel Q&A, and even some of it are in user discussion groups, so definitely check it out. +We'll be giving the same talk and also be at KubeCon, and if you want to come say hi at KubeCon, come say hi and also thank you, thank you so much, Na, for joining us today. -Please check out the Hazel Weekly one from six weeks ago now. It was really, really key stuff. So that's really worth—if you want a reason to go check it out, that's a really good one. +Thank you to everyone who was able to make it. This was actually a really good turnout for OpenTelemetry in practice, so thank you for taking the time. -It's next level. It's very, very good content. And we've got an OTel Q&A coming up on November the 16th with Jennifer Moore, where she talks about her experiences with OTel implementation at one of her previous companies. +### [00:38:00] Closing remarks and future talks -So that will be also a really great session to look forward to. It'll be in this time slot, same time slot. +Also, if you know anyone who wanted to attend but couldn't make it, let them know that we will be posting a recording of this on the OpenTelemetry YouTube channel. The handle is otel.official. -Does anyone else have anything else they want to share? +If you haven't subscribed yet, we've got a bunch of previous videos from OpenTelemetry in practice, OpenTelemetry Q&A, and even some of it are in user discussion groups. -**Speaker 5:** Ren? +So definitely check it out. Please check out the Hazel weekly one from six weeks ago now. It was really key stuff, so that's really worth—so if you want a reason to go check it out, that's a really good one. + +Yeah, it's next level. It's very, very good content. We've got an OpenTelemetry Q&A coming up on November the 16th with Jennifer Moore, where she talks about her experiences with OpenTelemetry implementation at one of her previous companies. + +So that will be also a really great session to look forward to. It'll be in this same time slot. + +Does anyone else have anything else they want to share? -**Speaker 6:** Nope, just a huge thank you to Na. +**Speaker 9:** Nope, just a huge thank you to Na! -**Speaker 1:** Yeah, this was awesome. +**Speaker 1:** Yeah, this was awesome! ## Raw YouTube Transcript diff --git a/video-transcripts/transcripts/2023-12-04T23:34:44Z-otel-q-a-feat-jennifer-moore.md b/video-transcripts/transcripts/2023-12-04T23:34:44Z-otel-q-a-feat-jennifer-moore.md index 4db1914..004b5eb 100644 --- a/video-transcripts/transcripts/2023-12-04T23:34:44Z-otel-q-a-feat-jennifer-moore.md +++ b/video-transcripts/transcripts/2023-12-04T23:34:44Z-otel-q-a-feat-jennifer-moore.md @@ -10,208 +10,210 @@ URL: https://www.youtube.com/watch?v=ztrknOWIUh8 ## Summary -In this YouTube Q&A session, Jennifer Moore shares her experiences with observability practices at her previous employer, Screencastify, where she served as the DevOps lead. The discussion delves into the company's application stack, consisting of TypeScript and Google Cloud services, and highlights the initial lack of effective observability tools. Jennifer explains her motivation to enhance the system's observability, which involved transitioning to OpenTelemetry for better tracing and diagnostics. She discusses the challenges faced, such as previous attempts at instrumentation and the complexities of integrating various observability tools. Despite initial neutrality from management regarding observability, the value became clear during critical incidents, leading to greater acceptance of the new practices. The conversation emphasizes the importance of observability in troubleshooting and operational efficiency, while also exploring the ongoing evolution of OpenTelemetry and its integration challenges. Jennifer's insights provide encouragement for teams looking to improve their observability strategies in a practical and impactful manner. +In this YouTube Q&A session, Jennifer Moore, a former DevOps lead at Screencastify, shares her experiences related to implementing observability using OpenTelemetry at the company. The discussion, led by the host, revolves around the company's transition from a screen recording application to a more complex video hosting and editing service, highlighting the technology stack involved, including TypeScript and Google Cloud. Jennifer discusses the initial lack of effective observability practices, her motivation to improve understanding of system behavior, and the challenges she faced, including the need to replace a proprietary SDK with OpenTelemetry. She emphasizes the importance of involving her team in the process to build collective knowledge and shares insights on management's gradual recognition of the value of observability during incidents. The conversation addresses the broader implications of observability, the learning curve associated with OpenTelemetry, and the need for a consistent interface for telemetry signals across different types. Jennifer's story serves as a compelling case for the importance of observability in software development. ## Chapters -00:00:00 Welcome and intro -00:00:20 Guest introduction: Jennifer Moore -00:01:21 Background on Screencastify -00:04:00 Role as DevOps lead -00:05:00 Observability needs at the company -00:06:30 Initial observability challenges -00:08:00 Implementing OpenTelemetry -00:10:00 Team involvement in observability -00:11:01 Management's perspective on observability -00:14:00 Transition to OpenTelemetry collector +00:00:00 Introductions +00:01:26 Background on Screencastify +00:05:01 Discussion on observability needs +00:07:10 Motivation for improved observability +00:09:19 Initial setup of OpenTelemetry +00:12:11 Team involvement in observability +00:13:30 Implementation of OpenTelemetry collector +00:19:21 Challenges with proprietary SDK +00:24:02 Management's recognition of observability value +00:36:23 Feedback on OpenTelemetry project -**Host:** Welcome everyone to Otel Q&A. We are super excited to have Jennifer Moore with us today. Welcome, Jennifer. +## Transcript -**Jennifer:** Hi, thanks. +### [00:00:00] Introductions -[00:00:20] **Host:** Okay, so I guess first things first. I asked Jennifer to come on because I had talked to her previously about some experiences that she had at a prior company around Otel, and I thought it was a really, really interesting and compelling story. So, I guess we'll start with, why don't you give us some background? I know this is not at your current employer; it's at a previous employer. Why don't you give us some background of, you know, for starters, like what their application stack was like, and then go from there. +**Host:** Welcome everyone to the Otel Q&A. We are super excited to have Jennifer Moore with us today. Welcome, Jennifer. -[00:01:21] **Jennifer:** Sure. This was a whole job ago, and all of this was happening like a year ago, so some of the details will have gotten fuzzy for me by this point, but that's cool. So the company is called Screencastify. They make a screen recording application. The kind of early days of a version two would have a video hosting and editing service, whereas before it had been just kind of a screen recorder, and it was saved to whatever storage you happened to have handy. And that was the whole thing. +**Jennifer:** Hi, thanks. -This stack was lots of TypeScript, and all running in GCP and containers. The main back end was a TypeScript, I think Next.js, I forget, some TypeScript backend server, running in Google Cloud Run, this sort of serverless containers product that they have. Everything is serverless, or many things are serverless, will be relevant. The back end handled all of the routine, you know, we're hosting a web service kind of stuff like account management and billing, etc., and also some on-demand video encoding. +**Host:** Okay, so I guess first things first. I asked Jennifer to come on because I had talked to her previously about some experiences that she had at a prior company around Otel, and I thought it was a really, really interesting and compelling story. So I guess we'll start with, why don't you give us some background? I know this is not at your current employer; it's at a previous employer. Why don't you give us some background of, you know, for starters, what their application stack was like, and then go from there? -Then there was what we called the task system, I think, or job system, something like that, which ran in Kubernetes. It was a bunch of Kubernetes microservices that would respond to videos being uploaded and edited and things to do video transcoding and processing work on that stuff. That all was powered by a BullMQ message queue, job queue, whatever, and yeah, I don't know, otherwise ran fairly self-contained. +### [00:01:26] Background on Screencastify -**Host:** All right. And what was your role then at this company? +**Jennifer:** Sure. This was a whole job ago, and all of this was happening like a year ago, so some of the details will have gotten fuzzy for me by this point, but that's cool. The company is called Screencastify. They make a screen recording application. The kind of early days of a version two would be a video hosting and editing service, whereas before, it had been just kind of a screen recorder and it was saved to whatever storage you happened to have handy. That was the whole thing. -[00:04:00] **Jennifer:** I joined as the kind of DevOps lead. A lot of what I did there was to kind of drive DevOps practices and then also do the chop wood, carry water parts of DevOps. So, a lot of working on the CI/CD pipelines, a lot of work on observability, internal tooling, and developer experience and things. You know, everything that wasn't primarily focused on building a feature in the application was probably something that came across my keyboard at some point. +This stack was lots of TypeScript, all running in GCP and containers. The main backend was a TypeScript, I think Next.js, I forget some TypeScript, backend server running in Google Cloud Run, this sort of serverless containers product that they have. Everything being serverless will be relevant. The backend handled all of the routine, you know, hosting a web service kind of stuff like account management and billing, etc. There was also what we called the task system or job system, something like that, which ran in Kubernetes. It was a bunch of Kubernetes microservices that would respond to videos being uploaded and edited to do video transcoding and processing work. That all was powered by a BullMQ message queue. -[00:05:00] **Host:** Cool. That's a lot of hats to wear then. Awesome. Okay, so now you mentioned observability, and we are here. So tell us to what extent, like when did you realize or actually let's take a step back. Did the company see a need, a big enough need for observability? Like had they seen that already when you had joined? +Yeah, and I don't know, otherwise, it ran fairly self-contained. -**Jennifer:** Yes, somewhat. I think they had encountered the concept and they had made a start at trying to do some observability things. There were definitely some people who recognized that it was a good idea, and like nobody was fighting them about it. But I think they didn't really know what they should expect out of it. They had kind of done some scattered starts at it and instrumented some things, but like you know some things in isolation and it wasn't in a very useful way. +**Host:** All right, and what was your role then at this company? -When I got there, they had tracing, but it was very disconnected and not really the things that people were interested in. A lot of anything that anybody actually needed to get out of it before I took that on came from logs. +**Jennifer:** I joined as the kind of DevOps lead. A lot of what I did there was to drive DevOps practices and then also do the chop wood, carry water parts of DevOps. A lot of working on the CI/CD pipelines, a lot of work on observability, internal tooling, and developer experience. Everything that wasn't primarily focused on building a feature in the application was probably something that came across my keyboard at some point. -[00:06:30] **Host:** Okay. So then you entered into the picture. What was at that point when you started implementing or making the systems more observable? What was sort of your motivation for that? +### [00:05:01] Discussion on observability needs -**Jennifer:** So my motivation throughout for making this system more observable was that I wanted to better understand what it was doing. One of the hats I was wearing was Sr. and I needed to support these things, and that's basically impossible to do if you don't know what it is or what it's doing, and certainly not how it's behaving. +**Host:** Cool. That's a lot of hats to wear then. Awesome. Okay, so now you mentioned observability, and we are here. Tell us to what extent, like when did you realize, or actually let's take a step back. Did the company see a big enough need for observability? Had they seen that already when you had joined? -After a little while, I grew very uncomfortable not having that understanding of the system and decided to just start removing the false start instrumentations that were already there and start instrumenting things more consistently. +### [00:12:11] Team involvement in observability + +**Jennifer:** Yes, somewhat. I think that they had encountered the concept, and they had made a start at trying to do some observability things. There were definitely some people who recognized that it was a good idea, and nobody was fighting them about it. But I think they didn't really know what they should expect out of it. They had done some scattered starts at it and instrumented some things, but like, you know, some things in isolation and it wasn't in a very useful way. When I got there, they had tracing, but very disconnected and not really the things that people were interested in. A lot of anything that anybody actually needed to get out of it before I took that on came from logs. + +**Host:** Okay, so then you entered into the picture. What was the point when you started implementing or making the systems more observable? What was sort of your motivation for that? + +### [00:07:10] Motivation for improved observability + +**Jennifer:** My motivation throughout for making this system more observable was that I wanted to better understand what it was doing. One of the hats I was wearing was senior, and I needed to support these things. That's basically impossible to do if you don't know what it is or what it's doing, and certainly not how it's behaving. After a little while, I grew very uncomfortable not having that understanding of the system and decided to just start removing the false start instrumentations that were already there and start instrumenting things more consistently. **Host:** So you actually went into the application code and started instrumenting things? -[00:08:00] **Jennifer:** Somewhat. So mostly what I did was remove the auto-instrument that were already there. There were, I don't know, DataDog's proprietary ones, and I'm sure they would work if they worked, but they didn't actually auto-instrument several of the libraries we were using, and so it wasn't helpful for us. I replaced those with OpenTelemetry auto-instrumentations, which did have auto-instrument for all of the things we wanted, and suddenly my flame graphs filled out and I could see what was going on from start to finish, more or less, and that was exactly what I'd been looking for—magic. +**Jennifer:** Somewhat. Mostly what I did was remove the auto-instrument that were already there, from, like, I don't know, Datadog's proprietary ones. I'm sure they would work if they worked, but they didn't actually auto-instrument several of the libraries we were using, and so it wasn't helpful for us. I replaced those with OpenTelemetry auto-instrumentations, which did have auto-instrument for all of the things we wanted. Suddenly, my flame graphs filled out, and I could see what was going on from start to finish more or less, and that was exactly what I've been looking for—magic. + +**Host:** And how was your learning curve in terms of OpenTelemetry? Is it something that you were familiar with initially, or is that something that you had to teach yourself? -**Host:** And how was your learning curve in terms of OpenTelemetry? Is it something that you were familiar with initially or is that something that you had to teach yourself? +**Jennifer:** Yeah, I mean, I was familiar with it conceptually initially. Back when I was on Twitter, I spent a lot of time talking to observability and resilience folks on Twitter. The goals and the concepts were all very familiar to me by that point, but the actual mechanics of how do you turn on and start using OpenTelemetry was not something I had done before. -**Jennifer:** So, yeah, I mean I was familiar with it conceptually initially. Back when I was on Twitter, I spent a lot of time talking to observability and resilience folks on Twitter. The goals and the concepts and everything were all very familiar to me by that point, but then the actual mechanics of how do you turn on and start using OpenTelemetry was not something I'd done before. +### [00:09:19] Initial setup of OpenTelemetry **Host:** Did that end up being like a single-handed effort on your part? Did you have anyone from your team assisting? -**Jennifer:** Yeah, like I didn't really want to be the only person who knew what it was doing or how it was working. I did do the initial setup myself, but then I made a point to kind of stop there and distribute follow-up work from that to the rest of the DevOps team, at least, so that there would be other people who knew what was going on and knew what they were looking at whenever things needed attention. +**Jennifer:** Yeah, I really did not want to be the only person who knew what it was doing or how it was working. I did do the initial setup myself, but then I made a point to kind of stop there and distribute follow-up work from that to the rest of the DevOps team, at least so that there would be other people who knew what was going on and knew what they were looking at whenever things needed attention. **Host:** And how was that received within your team? -[00:10:00] **Jennifer:** I mean, I think that went really well. Distributed tracing has some inherent complexity to that domain, and so there’s some sort of learning and ramp-up to get comfortable with the concepts there. Everybody had to go through that, but I think after having gone through that, everybody was very pleased to have done it and very happy to have it. - -**Host:** That's really great. So did you say then that you were starting to see favorable results? You mentioned that you yourself started seeing some favorable results, like seeing the things on the flame graphs. How would you say the rest of your team took it, and even a step further, how did management take that? Did they see that as a benefit? +**Jennifer:** I mean, I think that went really well. Distributed tracing has some inherent complexity to that domain, so there's some sort of learning and ramp-up to get comfortable with the concepts there. Everybody had to go through that, but I think after having gone through that, everybody was very pleased to have done it and very happy to have it. -**Jennifer:** I think, you know, at first management was kind of neutral, like neutral to positive about it. They recognized it was a good thing to do, but I don't think they really saw how much value it had at first. +**Host:** That's really great. In terms of, would you say then that you were starting to see favorable results? You mentioned that you started seeing some favorable results like seeing the things on the flame graphs. How would you say the rest of your team took it, and even a step further, how did management take that? Did they see that as a benefit? -But then we get to some stories with some incidents in them, and I think the value of it becomes a lot more obvious to them at that point. +**Jennifer:** I think at first management was kind of neutral, like neutral to positive about it. They recognized that it was a good thing to do, but I don't think they really saw how much value it had at first. But then we get to some stories with some incidents in them, and I think the value of it becomes a lot more obvious to them at that point. -**Host:** Awesome. Circling back a little bit, you did a lot of stuff around the auto-instrumentation side of things. Do you think this incentivized the developer of the actual application to go and do some additional work and add some manual tracing, for example? +**Host:** Just circling back a little bit, you mentioned you did a lot of stuff around the auto-instrumentation side of things. Do you think this incentivized the developers of the actual application to go and do some additional work and add some manual tracing, for example? -**Jennifer:** Yes. There were definitely some engineers who were very on board with the idea of having tracing and wanted to go all in on that. Then there were others who were pretty neutral about it. No one was fighting against it. A lot of it was, you know, like sure, that seems fine, you can do that. +**Jennifer:** Yes. There were definitely some engineers who were very on board with the idea of having tracing and wanted to go all in on that. Then there were others who were pretty neutral about it. No one was fighting against it. A lot of it was, you know, sure, that seems fine; you can do that, and a handful of boosters. -A handful of boosters. But were they okay with going in and instrumenting themselves as well to kind of enhance what the auto-instrumentation was giving? +**Host:** Were they okay with going in and instrumenting themselves as well to enhance what the auto-instrumentation was giving? **Jennifer:** Yes, I think so. It's a little bit hard to say because honestly, some things blew up organizationally, and so I didn't have as much opportunity for that to actually happen as I would have liked. But people were very eager to have done it. -**Host:** Right. At least they got to reap the rewards of what you had set up, right? +**Host:** Right, so at least they got to reap the rewards of what you had set up. -**Jennifer:** Yeah, which is always good. Now in terms of your setup, did you end up implementing an Otel collector? +**Jennifer:** Yes, which is always good. -[00:14:00] **Jennifer:** They did, yes. That was something that I had wanted to do. The company was sort of already on DataDog and kind of invested in the DataDog tooling ecosystem. Moving away from that proprietary stack and towards the OpenTelemetry libraries and collector was a process. But it's one that I tried to start, and it continued after I left. My understanding is that everything is OpenTelemetry there now, and I think they've even moved away from DataDog to another vendor. +### [00:13:30] Implementation of OpenTelemetry collector -**Host:** Oh wow, that's really cool to be able to move to a fully vendor-neutral sort of setup. It's super awesome, and I think especially like you had set the wheels in motion around that, and that it continued even after you left. +**Host:** Now in terms of your setup, did you end up implementing an Otel collector? -**Jennifer:** Yes. Now how's that continued? The knowledge that you gained at your previous workplace, is it something that you've been able to apply where you're at right now? +**Jennifer:** They did. That was something that I had wanted to do, but the company was sort of already on Datadog and kind of invested in the Datadog tooling ecosystem. Moving away from that proprietary stack and towards the OpenTelemetry libraries and collector was a process, but it's one that I tried to start and it continued after I left. My understanding is that everything is OpenTelemetry there now, and I think they've even moved away from Datadog to another vendor. -**Jennifer:** Yes and no. Right now, I am at Influx Data. At Influx, all of the engineering attention has been directed towards getting the 3.0 version of the database into 3.0, which is a thing that we have done now, and that's going pretty well. That frees up some people to think about other things again. Where that intersects with observability and OpenTelemetry has been more about getting the database itself to be better suited to work as a backend for collecting spans and telemetry data rather than instrumenting our own things for traces. +**Host:** Oh wow, that's really cool to be able to move to a fully vendor-neutral sort of setup. I think especially you had set the wheels in motion around that, and that it continued even after you left. Now how's that continued? The knowledge that you gained at your previous workplace, is it something that you've been able to apply where you're at right now? -We do have a lot of things that are very well instrumented and very detailed instrumentation, and a lot of that is very manual. That's done with more of a performance objective of understanding performance more than behavior or investigating errors or things like that. +**Jennifer:** Yes and no. Right now, I am at Influx Data. Our engineering attention has been directed towards getting the 3.0 version of the database out, which we have done now and that's going pretty well. That frees up some people to think about other things again. Where that intersects with observability and OpenTelemetry has been more about getting the database itself to be better suited to work as a backend for collecting spans and telemetry data rather than instrumenting our own things for traces. -Which is to say there's very deep small spans as opposed to wide spans, which I think are a little bit easier to work with if you're trying to ask something weird is going on and you have no idea what. +We do have a lot of things that are very well instrumented, and very detailed instrumentation, and a lot of that is very manual. That's done with more of a performance objective of understanding performance rather than behavior or investigating errors or things like that. -**Host:** Yeah, so you have a slightly different goal compared to the last place but still using OpenTelemetry to achieve that goal. +**Host:** So you have a slightly different goal compared to your previous place but still using OpenTelemetry to achieve that goal? -**Jennifer:** Yeah, cool. +**Jennifer:** Yeah. -**Host:** What would you say were the most challenging aspects, given all your experience around OpenTelemetry? What were the most challenging aspects in terms of implementing OpenTelemetry in your previous organization? +**Host:** What would you say were the most challenging aspects in terms of implementing OpenTelemetry, specifically in your previous organization? -**Jennifer:** Yeah, so like I think the most challenging was that there had already been a start with a vendor proprietary SDK. That confused a lot of things. I had tried to leave that in place while I instrumented with OpenTelemetry, but then these two libraries just started reporting on each other, and it made the whole traces more confusing than they had been without it. +**Jennifer:** I think the most challenging was that there had already been a start with a vendor proprietary SDK. That confused a lot of things. I had tried to leave that in place while I instrumented with OpenTelemetry, but these two libraries just started reporting on each other, and it made the whole traces more confusing than they had been without it. Removing that proprietary SDK and just having one mechanism for instrumentation was something that I wish I had done right away in retrospect. -Removing that proprietary SDK and just having the one mechanism for instrumentation in any given process was something that I wish I had done right away in retrospect. I think the other thing was in areas where there did not already exist auto-instrumentations. Like IBQ earlier, which didn't have one until I wrote it because I really needed it. - -I can say that's a challenge but also a benefit because nobody else had one either, and with OpenTelemetry, it was a viable thing that I could go and write my own auto-instrumentation package and be able to get that for myself with libraries that otherwise just didn't have it. +I think the other challenge was in areas where there did not already exist auto-instrumentations. For example, BigQuery didn't have one until I wrote it because I really needed it. I can say that's a challenge but also a benefit because nobody else had one either. With OpenTelemetry, it was a viable thing that I could go and write my own auto-instrumentation package and be able to get that for myself with libraries that otherwise just didn't have it. **Host:** How was that experience of writing the auto-instrumentation yourself? Was it a huge learning curve? -**Jennifer:** Yeah, that was a pretty substantial learning curve. It was a good thing that I started early because then we got into some incidents and I really needed it, and I was able to get it functional in a relatively short time span from there so that I could start using it and actually see what was going on. +### [00:19:21] Challenges with proprietary SDK -**Host:** That's so cool. You mentioned that your fellow DevOps engineers were leveled up in the ways of Otel. Is that something that you helped them out with? How is it that they gained the skills and leveled up so that they could also have that knowledge of OpenTelemetry that you had? +**Jennifer:** Yeah, that was a pretty substantial learning curve. It was a good thing that I started early because then we got into some incidents, and I really needed it. I was able to get it functional in a relatively short time span from there so that I could start using it and actually see what was going on. -**Jennifer:** I think the majority of that was just hands-on doing it. Again, after I had set up basically one auto-instrumentation and gotten that configured, I stopped there and made a point to have other people take that further. I wanted them to have that experience. +**Host:** You mentioned that your fellow DevOps engineers were leveled up in the ways of OpenTelemetry. Is that something that you helped them out with? How is it that they gained the skills and leveled up so they could also have that knowledge of OpenTelemetry that you had? -More broadly, we did also have a very strong learning culture, which was one of my favorite things about that company. We had a sort of technical book club, and we read the Observability Driven Development book for that book club. I think that was also very useful in terms of getting a lot of people familiar with the concepts and clarifying expectations about what this gets you from an office perspective and from an engineering perspective. +**Jennifer:** I think the majority of that was just hands-on doing it. After I had set up basically one auto-instrumentation and gotten that configured, I stopped there and made a point to have other people take that further because I wanted them to have that experience. More broadly, we did also have a very strong learning culture, which was one of my favorite things about that company. We had a sort of technical book club, and we read the Observability Driven Development book for that book club. I think that was also very useful in terms of getting a lot of people familiar with the concepts and clarifying expectations about what this gets you from an office perspective and from an engineering perspective. -**Host:** And hopefully got them stoked too about it, right? +**Host:** And hopefully got them stoked about it too, right? **Jennifer:** Yeah, I mean, the people who were inclined to be interested definitely came away from that more interested. -**Host:** It's really cool that this is a very operational-driven Otel undertaking. I guess it kind of makes sense because when things break, it's always going to be more on the operational side of things, and it's not necessarily the developers who are going to be the ones who are going to say why is this breaking, right? Because by then, it's like, I hate to say it, but less their concern, even though it is very much their concern. But unfortunately, I think our culture is still kind of shifting towards the not-my-problem because it's in production. +**Host:** It's really cool that this is a very operationally driven Otel undertaking. I guess it kind of makes sense because when things break, it's always going to be more on the operational side of things, and it's not necessarily the developers that are going to be the ones who are going to be, "Oh, why is this breaking?" Because by then, it's, I hate to say it, but less their concern, even though it is very much their concern. Unfortunately, I think our culture is still kind of shifting towards the "not my problem" because it's in production. -**Jennifer:** It is a little bit farther away from them, and I have some sympathy for that. I spent a lot of time doing app dev, but also only some sympathy because the whole time I was doing app dev, I was very frustrated by my inability to have any contact with production. That was also one of the things that I was trying to drive there, like beyond observability and OpenTelemetry, was to actually fulfill some of the promise of DevOps and have that not be such a bright line between the two. +**Jennifer:** It is a little bit farther away from them. I have some sympathy for that. I spent a lot of time doing app dev. But also only some sympathy because the whole time I was doing app dev, I was very frustrated by my inability to have any contact with production. That was also one of the things that I was trying to drive there, beyond observability and OpenTelemetry, was to actually fulfill some of the promise of DevOps and have that not be such a bright line between the two. **Host:** Do you think you got closer to achieving that goal? -**Jennifer:** I think so, yeah. One of the ways I did that was to really emphasize how the tools are used. Whenever it came up, like in doing incident retros, I made a point to ask people what they looked at that made them think that was the concern, and how did you execute those queries? Let's actually open up the tool and go through the UI and see how that's performed. - -[00:11:01] **Host:** I want to go back to our chat earlier about management's perspective on using observability. I believe you mentioned that it was kind of neutral until they realized this is useful. One of the things that I always find can be challenging in an organization is bringing something new to management, and you almost need that permission from them before executing. Is that something where you asked for permission, or did you end up asking for forgiveness at that stage? +**Jennifer:** I think so. One of the ways I did that was to really emphasize how the tools are used whenever it came up. In doing incident retros, I made a point to ask people, "What did you look at that made you think that was the concern?" and "How did you execute those queries? Let's actually open up the tool and go through the UI and see how that's performed." -**Jennifer:** Yeah, like neither, honestly. The way this worked out, they had already, before I joined, whatever conversations they had about getting better observability, and somebody signed off on doing something. They started some instrumentation, so that was clearly a thing that was okay. +### [00:24:02] Management's recognition of observability value -I didn't need to pitch them on having it, and as a matter of fixing it, that was more of a question for my fellow engineers. Somebody put this in, and somebody probably understands it, and I need to, but it's not satisfying the goals that we had for it. I need to do something about that, but management wasn't going to say no to that. +**Host:** I want to go back to our chat earlier about management's perspective on using observability. I believe you mentioned that it was kind of neutral until they realized this is useful. One of the things that I always find can be challenging in an organization is bringing something new to management and needing that permission from them. Is that something where you asked for permission, or did you end up asking for forgiveness at that stage? -**Host:** Right, because you already had some sort of an observability culture. In terms of investing further in it, we launched a big part of this launch, and a big part of the reason for going to the 2.0 version of the product was to enable some account and billing features that just weren't possible with the way those data schemas existed. +**Jennifer:** Neither, honestly. The way this worked out, they had already had whatever conversations about getting better observability, and somebody signed off on doing something. They started some instrumentation, so that was clearly a thing that was okay. I didn't need to pitch them on having it. As a matter of fixing it, that was more of a question for my fellow engineers. Somebody put this in, and somebody probably understands it, and I need to, but it's not satisfying the goals that we had for it. I needed to do something about that, but management wasn't going to say no to that. -They existed in Firebase, and they were moving to Postgres to have real schemas. That had some predictable problems, and the migration kind of stalled out, and that was real bad. Things just stopped working for hours at a time because the migration was kind of killing the Postgres server without ever completing, but we didn't know that. +**Host:** Right, because you already had some sort of an observability culture. -After some initial investigation, instrumenting all the things to figure out what is even going on was the next thing, and it wasn't something that I needed permission to do at that point because production is down, and it's going down for hours at a time repeatedly, so it needs to be fixed. +**Jennifer:** Yes, and in terms of investing further in it, we ran into an incident. A big part of this launch was to enable account and billing features that just weren't possible with the way those data schemas existed. They existed in Firebase, and they were moving to Postgres to have real schemas. That had some predictable problems, and the migration kind of stalled out. Things just stopped working for hours at a time because the migration was kind of killing the Postgres server without ever completing. After some initial investigation, instrumenting all the things to figure out what was even going on was the next thing. It wasn't something that I needed permission to do at that point because production was down for hours at a time repeatedly, so it needed to be fixed. -When the building's on fire, you don't need permission to install sprinklers anymore. +**Host:** When the building's on fire, you don't need permission to install sprinklers anymore. -**Host:** Absolutely. I guess at that point, even though you were using a different tool set for observability and you made the decision to use OpenTelemetry instead, that became less about putting in the tool that works for us. +**Jennifer:** Yes, absolutely. -**Jennifer:** Yes, and we had already started that by the time we got to that incident, and so that was very fortunate because we were getting a better experience with OpenTelemetry. There were some parts of the system that we could fairly well understand what they were doing, but the rest of it had not been instrumented, and that had to be done quickly. +**Host:** So I guess at that point, even though you were using a different tool set for observability, you made the decision to use OpenTelemetry instead because things were not working. -**Host:** That's your incentive when things are burning—OpenTelemetry to the rescue. Before I turn it over to the rest of the folks on this call, do you have any feedback for the OpenTelemetry project as a whole, good or bad, in terms of your experience? +**Jennifer:** Yes, and we had already started that by the time we got to that incident. That was very fortunate because we were getting a better experience with OpenTelemetry. There were some parts of the system that we could fairly well understand on what they were doing, but the rest of it had not been instrumented, and that had to be done quickly. -**Jennifer:** Yeah, I'm not sure that I don't know. I think that, and this was a year and a half ago that I was doing it, the early getting set up onboarding learning curve is pretty steep. I don't know exactly how to improve that, and I'm sure I'm not the first person to have said this either. +**Host:** That's your incentive—when things are burning, OpenTelemetry to the rescue. -If that can be improved, that would be great. Also, I think that metrics and logging got to stable since I was doing this, and it would have been really nice to have had that at the time, I think, because it would be very convenient to have a consistent sort of interface and SDK for all of those telemetry signals. +**Jennifer:** Exactly. -Then being able to send them to a collector and then have the collector send them to wherever they're going and not making those things application concerns would be great. I haven't had the chance to actually do that to see if it works or not. +**Host:** Before I turn it over to the rest of the folks on this call, do you have any feedback for the OpenTelemetry project as a whole, good or bad, in terms of your experience? One of the things we do as a group here is collect feedback from users to help make the OpenTelemetry experience better. -**Host:** Cool, thank you. Does anyone else on this call have any questions for Jennifer? +**Jennifer:** I'm not sure. I think that the early getting set up, onboarding, learning curve is pretty steep. I don't know exactly how to improve that, and I'm sure I'm not the first person to have said this either. If that can be improved, that would be great. Also, I think that metrics and logging got to stable since I was doing this, and it would have been really nice to have had that at the time because it would be very convenient to have a consistent interface and SDK for all of those telemetry signals. Being able to send them to a collector and then have the collector send them to wherever they're going, and not making those things application concerns would be great. -**Participant:** I was just curious on the last piece you just said about having a consistent interface in the SDK for the telemetry signals and not making them application concerns. I was just curious if you could elaborate a little bit more on that piece. +**Host:** Thank you. Does anyone else on this call have any questions for Jennifer? -**Jennifer:** Yeah, so you have OpenTelemetry for your traces, and then you have some sort of logging solution for your logs, and hopefully you're doing structured logs. Again, some solution for metrics, assuming you're doing metrics—probably Prometheus or whatever. But to get those logs and those metrics to go somewhere, you wind up having to configure some vendor's logging library so that you can ship your logs to your logging backend, or you log everything to the terminal or to a file or whatever, and have some sort of infrastructure magic pick it up and send it somewhere. +**Participant:** I was just curious about the last piece you just said about having a consistent interface in the SDK for the telemetry signals and not making them application concerns. Could you elaborate a little bit more on that piece? -With spans, you configure your exporter and you're done. Your exporter just sends them to the collector, and the collector address or whatever can come from config after you're instrumented. You don't have to maintain much there. At the time, I would have really liked to be able to send my logs and my metrics to the collector in the same way that I was doing with spans rather than having three solutions for three different kinds of telemetry. +**Jennifer:** Sure. You have OpenTelemetry for your traces, and then you're going to have some sort of logging solution for your logs. Hopefully, you're doing structured logs and some solution for metrics, probably Prometheus or whatever. To get those logs and metrics to go somewhere, you wind up having to configure some vendor's logging library to ship your logs to your logging backend or log everything to the terminal or a file or whatever and have some sort of infrastructure magic pick it up and send it somewhere. With Prometheus, you have to create your scraping endpoint and let Prometheus scrape it, so you have to have Prometheus configured and able to reach you, etc. -**Participant:** Gotcha. I would see we're getting closer to that, right? +Whereas with spans, you configure your exporter, and you're done. Your exporter just sends them to the collector, and that can—like the collector address or whatever—can come from config after you're instrumented. You don't have to maintain much there. At the time, I would have really liked to be able to send my logs and metrics to the collector in the same way that I was doing with spans rather than having three solutions for three different kinds of telemetry. -**Participant:** I personally don't know. I'm asking. +**Participant:** I see. We're getting closer to that, right? -**Participant:** I think we're getting closer to it. I think metrics have definitely matured a lot in the last year, and I know a lot of observability backends are now ingesting metrics as well, so in addition to traces, which is nice. +**Jennifer:** I think we're getting closer to it. -Then I think we're starting to see that with logs as well. Logs, I know, are a different beast because there’s no logging SDK. There’s a logs bridge SDK, so the idea is to use one of the common logging libraries that is supported by OpenTelemetry, and there's a bridge SDK for it, and then it'll convert that format to OTP so it can be ingested by whatever backend. +**Host:** I personally don't know; I'm asking. -So, which is kind of nice. I guess they didn't want to reinvent the wheel with logs because there's so many different things out there. +**Jennifer:** I think we're getting closer to it. Metrics have definitely matured a lot in the last year, and I know a lot of observability backends are now ingesting metrics as well, which is nice. We're starting to see that with logs too. Logs are a different beast because there's no logging SDK; there's a logs bridge SDK. The idea is to use one of the common logging libraries that is supported by OpenTelemetry, and there's a bridge SDK for it that will convert that format to OpenTelemetry so it can be ingested by whatever backend, which is kind of nice. I guess they didn't want to reinvent the wheel with logs because there are so many different things out there. -**Jennifer:** Yeah, the momentum on the way people do logs is enormous. It's not like that's not changing. But how they get collected and organized, you know... +**Jennifer:** Yes, which I have full sympathy for that strategy. The momentum on the way people do logs is enormous; that's not changing. -**Host:** Yeah, I think the game changer is being able to correlate the logs to the traces so that you have that fuller picture, which I'm super excited about because I understand a lot of us grew up on logs, but I think the traces will help tell the full story. So when you have that correlation, then you can get magic. +**Host:** I think the game changer is being able to correlate the logs to the traces so that you have that fuller picture, which I'm super excited about because a lot of us grew up on logs. Traces will help tell the full story, and when you have that correlation, then you can get magic. -**Jennifer:** I think logs correlated with traces is the best, most ideal outcome. I know that traces have events and can do point-in-time things, but logs are just better at that, and being able to see point-in-time events correlated to where they happened within a span and within a trace is exactly what I want. +**Jennifer:** I think logs correlated with traces is the best most ideal outcome. I know that traces have events and can do point-in-time things, but logs are just better at that. Being able to see point-in-time events correlated to where they happened within a span and within a trace is exactly what I want. **Host:** Anyone else have any other comments or questions for Jennifer? Shall we break early then? -**Host:** Cool. Awesome. Well, thank you, Jennifer, for sharing your story with us. I really love the story because I think it's the things that are broken that need observability. - -I think it's such a compelling case for anybody who's on the fence on whether or not it's a worthwhile investment. It's awesome that your company already kind of had its foot in the door for that, and that you were able to take it further to where it actually needed to be so that it was useful to you and your team for troubleshooting. +### [00:36:23] Feedback on OpenTelemetry project -I think it's a great story. I hope others will draw inspiration from that as well as a compelling reason as to why observability with OpenTelemetry is a good combination. +**Host:** Cool. Awesome. Well, thank you, Jennifer, for sharing your story with us. I really love the story because I think it's the things that are broken that need observability. I think it's such a compelling case for anybody who's on the fence about whether or not it's a worthwhile investment. It's awesome that your company already kind of had its foot in the door for that, and that you were able to take it further to where it actually needed to be so that it was useful to you and your team for troubleshooting. I think it's a great story. I hope others will draw inspiration from that as well as a compelling reason as to why observability with OpenTelemetry is a good combination. -Do we have... I think we've got some upcoming events? +Do we have, I think we've got some upcoming events? -**Host:** Yeah, on December the 7th, we have another Q&A with someone from HashiCorp who has attempted many times to instrument Nomad. I think that'll be a really interesting story as well. - -So anyone who is following along, stay tuned for that Q&A. Hopefully, we'll get to see you live, but if not, we will see you on YouTube. +**Host:** Yes, on December the 7th, we have another Q&A with someone from HashiCorp who has attempted many times to instrument Nomad. I think that'll be a really interesting story as well. Anyone who is following along, stay tuned for that Q&A. Hopefully, we'll get to see you live, but if not, we will see you on YouTube. **Jennifer:** Sounds good. It was great to meet you all. -**Host:** Yeah, thanks so much, Jennifer, and we'll let you know once the video is posted. +**Host:** Thanks so much, Jennifer, and we'll let you know once the video is posted. **Participant:** Yeah, I was going to say thank you, Jennifer, for your story. That's really, really inspiring because I feel like so many people want to adopt it, but then getting that buy-in from your leadership is always the problem. I mean, technically, it's not making their money until you show that this is causing problems, and then we really need to do this. -**Jennifer:** Yeah, I don't know why it's such a fight so much of the time. Problems are going to happen, and what this gets you is being able to point at where the problem is happening. I don't know why people don't want to be able to do that. +**Jennifer:** I don't know why it's such a fight so much of the time, because problems are going to happen. What this gets you is being able to point at where the problem is happening, and I don't know why people don't want to be able to do that. + +**Participant:** In my experience, it only works either in teams that are excited about progressive engineering practices or that are feeling so much pain from vendor lock-in that they're ready to set everything on fire. -**Participant:** It's so logical. In my experience, it only works either in teams that are excited about progressive engineering practices or that are feeling so much pain from vendor lock-in that they're ready to set everything on fire. +**Jennifer:** Yes, so true. So basically means you need a really nasty incident or really bad vendor lock-in as your compelling reason. -**Jennifer:** Yes, so true. Basically, it means you need a really nasty incident or really bad vendor lock-in as your compelling reason. +**Host:** Well, if you get that incident, don't let it go to waste. -**Participant:** Well, if you get that incident, don't let it go to waste. +**Jennifer:** Totally. -**Host:** Yeah, totally. Right, quote of the day. Awesome. Well, thank you, everyone. +**Host:** Right, quote of the day. Awesome. Well, thank you, everyone. ## Raw YouTube Transcript diff --git a/video-transcripts/transcripts/2023-12-18T23:04:41Z-otel-q-a-feat-luiz-aoqui-of-hashicorp.md b/video-transcripts/transcripts/2023-12-18T23:04:41Z-otel-q-a-feat-luiz-aoqui-of-hashicorp.md index d2e459b..40652ad 100644 --- a/video-transcripts/transcripts/2023-12-18T23:04:41Z-otel-q-a-feat-luiz-aoqui-of-hashicorp.md +++ b/video-transcripts/transcripts/2023-12-18T23:04:41Z-otel-q-a-feat-luiz-aoqui-of-hashicorp.md @@ -10,270 +10,232 @@ URL: https://www.youtube.com/watch?v=HRIx9gJtECU ## Summary -In this Otel Q&A session, Luiz Aoqui, a developer from HashiCorp working on the Nomad project, discusses his experiences with implementing OpenTelemetry (OTel) for distributed tracing in Nomad. The conversation covers his journey through three approaches to integrating OTel, beginning with an attempt to create extensive tracing across the entire Nomad workflow, which proved to be complex and generated significant overhead. Luiz highlights the importance of understanding Nomad's architecture, explaining its server-client structure and job scheduling process. He describes a shift in strategy to focus on more meaningful traces by using semantic conventions, allowing for easier filtering and visibility in the traces generated. The discussion emphasizes the challenges faced when working with older codebases and the need for thoughtful instrumentation. The session concludes with Luiz sharing insights and advice for those looking to implement OTel in their projects. The video will be available on the official OTel YouTube channel for further viewing. +In this Otel Q&A video, Luiz Aoqui, a developer at HashiCorp, discusses his experiences with integrating OpenTelemetry into HashiCorp's Nomad project. Throughout the session, Luiz shares insights from his attempts to implement distributed tracing in Nomad, a workload orchestrator similar to Kubernetes, highlighting the challenges and learning points from each approach. He initially aimed to create comprehensive traces for every function call but found that it led to excessive overhead and complexity. Instead, he shifted to instrumenting specific critical operations within Nomad, allowing for more meaningful traces while maintaining efficiency. The discussion emphasizes the importance of semantic conventions in OpenTelemetry for filtering traces effectively. Overall, Luiz's journey reflects the practical challenges faced in telemetry integration in legacy systems, and he concludes by encouraging a more focused, incremental approach to instrumentation. The video wraps up with viewers being informed about the availability of the recording on the OpenTelemetry YouTube channel. ## Chapters -00:00:00 Welcome and intro -00:02:30 Nomad overview -00:11:46 OpenTelemetry integration attempts -00:19:00 First approach: Distributed tracing -00:25:50 Challenges with first approach -00:32:37 Second approach: User-focused instrumentation -00:39:00 UX challenges in second approach -00:44:00 Guidance from OpenTelemetry community -00:45:19 Final approach: Instrumenting Nomad -00:54:00 Closing remarks and Q&A +00:00:00 Introductions +00:01:01 Overview of Nomad +00:12:12 First attempt at distributed tracing +00:24:24 Challenges with initial approach +00:30:30 Discussion on legacy code issues +00:32:32 Shift in approach to instrumentation +00:38:38 Adding metadata to spans +00:43:44 Meeting with OpenTelemetry community +00:47:07 New approach with focused instrumentation +00:55:04 Wrap-up and recording announcement -**Luiz:** Welcome everyone to Otel Q&A, and I am super excited to have Luiz Aoqui join me. He is a developer on HashiCorp Nomad. Welcome. +## Transcript -[00:19:00] **Luiz Aoqui:** Thank you. Really excited to be here. always fun to talk OpenTelemetry. I mean, I haven't used it yet, but I dabble a lot, and I'll share more details of my adventures with OpenTelemetry. I guess I'll introduce myself first. As I mentioned, I'm Luiz, I'm an engineer, working at HashiCorp specifically on the Nomad project. I've been working on Nomad for like four plus years now. At some point, I don't remember exactly when, I bumped into OpenTelemetry, Distributed Tracing more specifically. It felt super interesting to me as a developer in Nomad because Nomad can be so opaque to debug, but also to implement and deploy and use in multiple different ways. I was like, Oh, it might be cool to do some distributed tracing there to get a sense of what's going on internally. +### [00:00:00] Introductions -And then through the course of several sort of like after our release, we usually do like an internal hack week just to cool down a little bit after that stressful moment. By the way, we just released Nomad 1.7, so if you want to try that, it's out now, which means we'll probably have a hack week soon and maybe I'll do some extra OpenTelemetry work, but that's a tangent. Yeah, I've tried three different work streams with OpenTelemetry Nomad, and I'll go into more details there, but each of them had sort of their own challenges and also nice things, but also bad things. I'll explain those. +**Speaker 1:** So welcome everyone to Otel Q&A, and I am super excited to have Luiz Aoqui join me. He is a developer on HashiCorp Nomad. Welcome. -[00:02:30] But yeah, that's it about me. I don't know if we should do a quick Nomad intro, because a lot of the things, since the idea of distributed tracing is to expose the internals of Nomad, I think I kind of need to explain a little bit about those. +**Luiz:** Thank you. Really excited to be here. Always fun to talk OpenTelemetry. I mean, I haven't used it yet, but I dabble a lot, and I'll share more details of my adventures with OpenTelemetry. I guess I'll introduce myself first. So as I mentioned, I'm Luiz, I'm an engineer, working at HashiCorp specifically on the Nomad project. I've been working on Nomad for like four plus years now. And at some point, I don't remember exactly when, I bumped into OpenTelemetry, Distributed Tracing more specifically. -**Speaker:** Okay, yeah, that's helpful. Why don't you go ahead? +### [00:01:01] Overview of Nomad -**Luiz Aoqui:** Cool. Let's start there. I was going to do like PowerPoint, but Adrienne told me that's more like a Q&A and I also tend to overdo it in PowerPoint. So if you have something you can share, if not, that's cool. We are super chill. I'll just draw live, just because feel free to mute, ask questions. I don't have any specific agenda here, but I'll start there and then maybe that will help. The general idea is like, feel free to ask questions as I go. +And it felt super interesting to me as a developer in Nomad because Nomad can be so opaque to debug. But also to implement and deploy and use in multiple different ways. I was like, Oh, it might be cool to do some distributed tracing there to get a sense of what's going on internally. And then, like, through the course of several sort of like after our release, we usually do an internal hack week just to cool down a little bit after that stressful moment. By the way, we just released Nomad 1.7. So if you want to try that, it's out now, which means we'll probably have a hack week soon and maybe I'll do some extra OpenTelemetry work, but that's a tangent. -Cool. So first of all, what's Nomad? Nomad is a workload orchestrator. Think of it like Kubernetes, you give it a magic file, that file becomes containers or other things, right? But that's what the orchestration aspect of this is. Now internally, it's important to understand some details of Nomad to understand what I'm going to show a bit later, is that the general architecture is like, you have two components, very simple: like server and client. These two components have different roles. +And yeah, like I've tried three different work streams with OpenTelemetry Nomad, and you know, I'll go into more details there. But each of them had sort of like their own challenges and also nice things, but also bad things. And then sort of, I'll explain those. But yeah, that's it about me. I don't know if we should do like a quick Nomad intro because a lot of the things, since the idea of just tracing is to do the internals, exposing the internals of Nomad, I think I kind of need to explain a little bit about those. -So the server, and you usually have three or five servers, they maintain the core room, basically. They sort of are always talking to each other to maintain state. All of your jobs that you run, everything that you're running on your cluster is stored in the server. They have an internal database, you know, in the Kubernetes world, think of this as like etcd, but Nomad has its internal database. They all have the idea that they start a state of the cluster, and then they talk to each other to make sure that state is replicated and sort of available in case of a failure or something like that. +**Speaker 1:** Okay, yeah, that's helpful. Yeah, why don't you go ahead? -So maybe I'll write bullet points. They store state, that's their main goal. The second goal is that they also do the scheduling. The scheduling process happens inside the server. All servers have several, what we call workers, which is quite confusing because I know that in the Kubernetes world, workers is something completely different. But the workers here are the scheduler workers. What the workers do, and each instance normally has like one per core on your machine by default, I believe. +**Luiz:** Cool. Let's start there. I was going to do like PowerPoint, but like Adrienne told me that's more like a Q&A, and I also tend to overdo it in PowerPoint. So if you have something, you can share it; if not, that's cool. We are, it's super chill. Yeah. I'll just draw live, just because feel free to like mute, ask questions. I don't have any specific agenda here, but I'll start there and then maybe that will... But the general idea is like, feel free to ask questions as I go. Don't need to wait on anything. -What do workers do? They're like goroutines, you know, to go into a little bit. There's like a thread running on your servers and their goal is to, given a request to run something, so that would be like a job in a middle angle, given a job, where should I place things? So imagine that, I should move this. Imagine that I have a bunch of clients here. +Cool. So first of all, what's Nomad? Nomad is a workload orchestrator. Think of it like Kubernetes. You give it a magic file; that file becomes containers or other things, right? But that's what the orchestration aspect of this is. Now internally, you know, it's important to understand some details of Nomad to understand what I'm going to show a bit later, is that the general architecture is like, you have two components, very simple, like server and client, um, and these two components have different roles. -Oh, I didn't talk about what clients do, but clients, they run stuff. Clients are the machines that are actually running your containers. They're actually running your workloads. That's the point of the clients. Normally, you can have like from like one to like 10,000 clients. So like to infinity and beyond here. Some people really try to push Nomad to its limit, but you can have a lot of clients on your cluster. +So the server, and you usually have three or five servers, they maintain the core room, basically. So they sort of are always talking to each other to maintain state. So all of your jobs that you run, everything that you're running on your cluster is stored in the server. So, like, they have like an internal database. You know, in the Kubernetes world, think of this like etcd, but Nomad has like its internal database. But they all have the idea that they start a state of the cluster, and then they talk to each other to make sure that that state is replicated and sort of available in case of a failure or something like that. -The job of the scheduling is given a job or like, you know, like a YAML spec in the Kubernetes world, given this file, how do I translate this into specific work for each client? So let's say I want to run like three containers here on my file. The job of the scheduler is to say, okay, like I'm going to put two containers here on this client and then one container on this client. So that's what scheduling means. It's like, given the spec, how do I make it real? +So, maybe I'll write bullet points. So they store state; that's their main goal. The second goal is that they also do the scheduling. So the scheduling process happens inside the server. So all servers have several, what we call workers, which is quite confusing because I know that in the Kubernetes world, workers is something completely different. But the workers here are the scheduler workers. And what the workers do, and your each instance normally has like one per core on your machine by default, I believe. What do the workers do? They're like go routines, you know, to go into a little bit. -And this thing that sort of makes things real, this is called a plan. So the goal of the workers here, the scheduling workers, is to create a plan given, you know, what we call the job spec. So that's what the scheduling process does. Cool. So those are the two main things that servers do. They store state, they do scheduling, and then the clients run stuff. When the client receives the work, like after the plan is generated, it gets validated, gets stored in state. The server says, okay, like this client here is going to run two containers. This client is going to run one container. +So like, there's like a thread running on your servers, and their goal is to, given a request to run something, so that would be like a job in a middle angle, given a job, where should I place things? So imagine that, I should move this. So imagine that I have a bunch of clients here. Oh, I didn't talk about what clients do, but clients, they run stuff. So clients are the machines that are actually running your containers. They're actually running your workloads. That's the point of the clients. -So periodically, this client is going to go and ask the server like, Hey, give me, which allocations I should run. To translate a little bit, an allocation is like a pod in Kubernetes world. It's like the unit of workload that's going to run. Periodically, the client goes like, Hey, what should I be running? Since we just scheduled two containers, the server is going to reply, Hey, these are the two containers you should run. +And normally, you can have like, from one to like 10,000 clients. So like to infinity and beyond here. Some people really try to push Nomad to its limit. But like, you can have a lot of clients on your cluster. And so the job of the scheduling is, given a job or like, you know, like a YAML spec in the Kubernetes world, given this file, how do I translate this into specific work for each client? So let's say I want to run like three containers here on my file. The job of the scheduler is to say, okay, like I'm going to go put two containers here on this client and then one container on this client. So that's what scheduling means. It's like, given the spec, how do I make it real? And this thing that sort of makes things real, this is called a plan. So the goal of the workers here, the scheduling workers is to create a plan given, you know, what we call the job spec. What's the plan? So that's what the scheduling process does. -Then the client is going to start a few sort of internal components here. First, it's going to create an alloc runner, and the alloc runner is responsible for setting up the allocation environment. It's going to create folders in your file system, like folders inside that machine to store files. It's going to register the service, you know, either internally or register service in console. It's going to do things that prepare the way of the land, right? Call CNI plugins, all of that happens in the alloc runner. +Cool. So those are the two main things that servers do. They store state, they do scheduling, and then the clients run stuff. When the client receives the work, like after the plan is generated, it gets validated, gets stored in state. And then the server says, okay, like this client here is going to run two containers. This client's going to run one container. So periodically this client is going to go and ask the server, like, Hey, give me, which allocations I should run. And again, to translate a little bit, an allocation is like a pod in Kubernetes, so it's like the unit of workload that's going to run. -And then just like a pod can have multiple containers, an allocation can have multiple tasks. The alloc runner for each task inside your allocation is going to create a thing called a task runner. This task runner, kind of like the alloc runner, is going to set up the environment for the tasks, like for the specific container. It's going to create like environment variables, it's going to download files, download the Docker image, whatever. Everything for that specific workload to run. +So periodically the client goes like, Hey, what should I be running? Since we just scheduled two containers, the server is going to reply. Hey, these are the two containers you should run. And then the client is going to start a few sort of internal components here. So first, it's going to create an alloc runner, and the alloc runner is responsible for setting up the allocation environment. So it's going to create folders in your file system, like folders inside that machine to store files, is going to register service, you know, either internally or register service in console, is going to do things that like prepare the way of the land, right? Call CNI plugins, all of that happens in DialogRunner. -And then the task runner is going to start, like, so like the end result of the task runner is like a container or a binary or whatever you want to run here. It creates like a process on the machine. From that file that is specified, what you want to run, you give that to the server. +And then just like a pod can have multiple containers, an allocation can have multiple tasks. And so the alloc runner for each task inside your allocation is going to create a thing called task runner. And this task runner, kind of like the alloc runner, is going to set up the environment for the tasks, like for the specific container. So it's going to create like environment variables, it's going to download files, download the Docker image, whatever, everything for that specific workload to run. -The server is going to send it to the... Oh, I forgot to mention the process of how it gets to the worker. Okay. So normally you have three or five servers and one of them becomes the leader. The leader is the only one that can actually write state to disk and write the final, like the global, the single source of truth is stored in the leader. The leader also has a thing called the queue. That's called the eval queue. The eval queue is like work to be done. An eval is like something that has to happen in my cluster. +And then the task runner is going to start, like, so like the end result of the task runner is like a container or a binary or whatever you want to run here. It creates like a process on the machine. So that's like all things running on it. So from that file there that is specified, what you want to run, you give that to the server. The server is going to send it to the, oh, I forgot to mention the process of how it gets to the worker. Okay. So normally you have three or five servers, and one of them becomes the leader. So the leader is the only one that can actually write state to disk and write the final, like the global, the single source of truth is storing the leader. -That something that's described in an eval goes into the queue, and then the other servers sort of dequeue from there to find work to do. The workers are dequeuing work, processing that eval, generating a plan. That plan becomes state, so like gets written into the global cluster state, and then the clients are sort of like asking for updates on that state to figure out what needs to run, what needs to be stopped, and all of that. I think that's it. That was a lot. Let me know if there are any questions on this part. Hopefully, it all made sense. +And the leader also has a thing called the queue. So that's called the eval queue. And the eval queue is like work to be done. So an eval is like something that has to happen in my cluster. So that, that's something that's described into an eval, goes into the queue, and then the other servers sort of dequeue from there to find work to do. So the workers are dequeuing work, processing that eval, generating a plan. That plan becomes state, so like gets written into the global cluster state, and then the clients are sort of like asking for updates on that state to figure out what needs to run, what needs to be stopped, and all of that. -[00:11:46] So let's get to the actual stuff that I did. This is the Nomad repo. There are a few, so if you search for OTEL branches, you're going to find my previous adventures putting OpenTelemetry enrollment. I'm going to go sort of, and you can see, it's been a while since I've been trying to do this properly. I think I'll go probably chronologically here. +I think that's it. That was a lot. So let me know if there are any questions on this part. Hopefully, it all made sense. So let's get to the actual stuff that I did. So this is the Nomad repo. There are a few, so if you search for OTEL branches, you're going to find my previous adventures putting OpenTelemetry enrollment. And I'm going to go like sort of, and you can see, like, it's been a while since I've been trying to do this properly. -So the first attempt, when I learned about this, it was like, cool, let's put distributed traces everywhere. My goal here was for, given all of this process that happens, it should all like, in my mind, it was like, Oh, this is like one process. Everything here should be a single trace or like have a trace, a unique trace ID that, Oh, this could be a span. When it goes to the queue, that's another span. When it gets dequeued, that's like creating a bunch of spans in a huge tree. +### [00:12:12] First attempt at distributed tracing -I wanted to map this process of running something end to end, you know, as a user, submit a job, how do I get visibility of what's happening inside the cluster? That's the approach that I like to call boil the ocean because I wanted to do everything at once. First, let's run this branch, I guess, to see what that looks like. Let's hope that the two-year code works. I was testing this yesterday, but I think it will work. I'm also going to start the OpenTelemetry demo repo that I found. I need some sort of setup to run, like I need Jaeger, and I need a collector, and I need an application to test. +So I think I'll go probably chronologically here. So the first attempt, when I learned about this, it's like, cool, let's put distribute traces everywhere. And like, my goal here was for, given all of this process that happens is like, this should all, like, in my mind, it was like, Oh, this is like one process. Right? So like this, everything here should be a single trace or like have a trace, a unique trace ID that like, Oh, this could be a span. And then when it goes to the queue, that's another span. And then it gets dequeued, that's like create a bunch of spans in a huge tree. -So I'm just going to start the whole demo here just to have something to generate stuff, to generate data. Okay. Deprecations or warnings are not errors. So let's assume that this worked. If you look at the branch, let's start looking at the diff. One of the things that I did, or I had to do was to have a sim config, additional configuration for Nomad. We have it on, like when you start a Nomad agent, you need to pass a configuration file. +So I wanted to map this process of running something end to end, you know, as a user, submit a job, how do I get visibility of what's happening inside the cluster? So that's the approach that I like to call boil the ocean because I wanted to do everything at once. So first, let's run this branch, I guess, to see what that looks like. Let's hope that the two-year code, two-year code is two runs, and I was testing this yesterday, but I think it will work. I'm also going to start the, and the OpenTelemetry demo repo that I found was like, I need some sort of setup to run, like I need Jaeger and I need a collector and I need an application to test. So I'm just going to start the whole demo here, um, just to have something to generate stuff, uh, to generate data. -So I added like configuration for OpenTelemetry endpoint and just to explain to like, where to send all the OpenTelemetry data. I think I have it here. So like telemetry, OpenTelemetry endpoint, localhost for 3.7. That's where I'm running the OpenTelemetry collector from the demo and insecure because I'm not using TLS. I think that's all I need. +So, okay. Deprecations or warnings are not errors. So let's assume that this worked. So if you look at the branch, let's start looking at the diff. So one of the things that I did or I had to do was to have a sim config, uh, additional configuration for Nomad. So like we have it on, like when you start a Nomad agent, you need to pass a configuration file. So I added like configuration for OpenTelemetry endpoint and, yeah, just to like, explain to like, where to send all the OpenTelemetry data and I think I have it here. -If you haven't used Nomad before, Nomad Agent Dev is a quick way to start. It gives you like a fully functioning cluster locally. It starts a client, starts a server in the same process. It's a handy way to try stuff. It's also ephemeral. Once it goes down, it destroys everything created, which is nice for cleanups, but sometimes bad because sometimes you do want to keep stuff. But it's a good way to quickstart with Nomad. +Yeah. So like telemetry, OpenTelemetry, endpoint, localhost for 3.7. That's where I'm running the OpenTelemetry collector from the demo, and insecure because I'm not using TLS. So I think that's all I need. If you haven't used Nomad before, Nomad Agent Dev is a quick way to start. It gives you like a fully functioning cluster locally. So it starts a client, starts a server in the same process. And so like a very handy way to try stuff. It's also ephemeral. So once it goes down, it destroys everything created, which is nice for cleanups, but sometimes bad because sometimes you do want to keep stuff. -I'm going to read this config with the additional OpenTelemetry configuration. So let's start that. I'm actually also going to run a job. If you haven't seen it before, a Nomad job looks like this. It uses the HCL language, sort of like Terraform does. There's not a whole lot to go over this right now. I think the main thing is, it's kind of hard to map to Kubernetes concepts. +So, but it's a good way to quickstart with Nomad. And I'm going to read this config with the additional OpenTelemetry configuration. So let's start that. And I'm actually also going to run a job. So if you haven't seen it before, a Nomad job looks like this. It uses the HCL language, sort of like Terraform does. And there's not a whole lot to go over this right now. I think the main thing is like, and it's kind of hard to map to Kubernetes concepts. So I'll try my best, but like a group is kind of like a deployment tells you what to run, like how to run things. -I'll try my best, but like a group is kind of like a deployment. It tells you what to run, like how to run things. In the group, you can have a count, how many instances you want to run, stuff like that. The task is kind of like, or like the groups are kind of like the pod spec. It tells you what to run and how to configure stuff. The task is like a container definition inside a pod. You tell, Nomad doesn't only run containers, so you can do other stuff, and that's plugins, so those are test drivers. +So like in the group, you can have count, you know, like how many instances you want to run, stuff like that. The task kind of like, or like the groups kind of like the pod spec. So it tells you, it's a pod spec plus deployment. So like, it tells you like what to run and how to configure stuff. And then the task is like a container definition inside a pod. You tell, Nomad doesn't only run containers, so you can do other stuff. And that's plugins, so those are test drivers. And then the config is how you configure the test driver. So this is just like saying, hey, I want to run a Redis container, and it's going to expose this port and give it these resources to run. -The config is how you configure the test driver. This is just like saying, hey, I want to run a Redis container. It's going to expose this port and give it these resources to run. So fairly simple job here. When I run that, it creates evals. The evaluation because something changed in my cluster. That's how I communicate changes in Nomad, is through these evals. +So fairly simple job here. But when I run that, it will, so like running great. So like, as I mentioned, like it creates evals, like the evaluation because something changed in my cluster. That's how I communicate changes in Nomad; it is like through this evals, uh, that eval created an allocation. The allocation is like the pod. So it's like the thing that is actually running, you know, what is that, uh, and it works. It's all running. If I go to the Nomad UI, oh, this is a node version of the previously. There's like a special flag that I had to run, but let's hope it's actually running. -That eval created an allocation. The allocation is like the pod. It's the thing that is actually running. And it works. It's all running. If I go to the Nomad UI... Oh, this is a node version of the previous. There's a special flag that I had to run, but let's hope it's actually running. +Now if I go to Jaeger, which is where here, I will see a lot of services. So those are for all the demo stuff. But I also see two Nomad services here. So on this Boil the Ocean approach, I, as I mentioned, like, I wanted to get the whole flow end to end, from user to like the clients. So I started from the CLI. So you can see here, there's a CLI, Nomad CLI, what's it called? Service with a job run action here. And then, it's kind of small to see. -Now, if I go to Jaeger, which is where... here, I will see a lot of services. Those are for all the demo stuff. But I also see two Nomad services here. So on this boil the ocean approach, as I mentioned, I wanted to get the whole flow end to end, from user to like the clients. I started from the CLI. So you can see here, there's a CLI, Nomad CLI, what's it called? Service with a job run action here. +And so, like, I started putting traces everywhere I found stuff. Like, it's one huge trace with a bunch of different spans for like each individual action that is happening. So, like, you can see here, the Nomad CLI ran. The job run command. I think there's like, I don't remember how much extra data I put. Like, I was experimenting with all of this stuff. So I could put log entries here. So there's some Nomad CLI job run command ran that CLI made an HTTP request. So they put request. To this end point here, that put request became a job request in the FSM. -It's kind of small to see. I started putting traces everywhere I found stuff. It's one huge trace with a bunch of different spans for each individual action that is happening. You can see here, the Nomad CLI ran the job run command. I think there's like, I don't remember how much extra data I put. I was experimenting with all of this stuff. +The FSM for those who haven't heard this term before stands for finite state machine. That's how data replication works. Imagine, imagine you're trying to help your friend find your house. Well, normally you tell them to go to Google, but like, imagine you live in the middle of nowhere, there's no GPS. So you need to give them instructions on how to find your house. You know, drive two miles east, turn right, drive another, you know, how many miles, turn left. Like you give specific instructions. Now imagine you're not just telling one friend, but like multiple friends. And then you, you kind of give that instructions to all of them at the same time, so they can reach the same destination. -So I could put log entries here. There's some Nomad CLI job run command ran that CLI made an HTTP request. So they put request to this endpoint here. That put request became a job request in the FSM. The FSM, for those who haven't heard this term before, stands for finite state machine. That's how data replication works. +So that's what the FSM does. It's like give an instruction. It mutates the state in a consistent way across all servers. Hopefully, that made sense. I think I got, I confused myself, but let me know if that wasn't very clear. So it creates a job request in, so now we moved away from the CLI, so that's the HTTP request that was made. Now the Nomad agent is who received that HTTP request, uh, and turned it into like an internal request. For a register. So that's a job register, commands. I caused the job register, method internally. And you can see here, like I'm putting logs and stuff, trying to, you know, events and figuring out how the OpenTelemetry library works. -Imagine you're trying to help your friend find your house. Normally, you tell them to go to Google, but imagine you live in the middle of nowhere, there's no GPS. You need to give them instructions on how to find your house. You know, drive two miles east, turn right, drive another, how many miles, turn left. You give specific instructions. Now imagine you're not just telling one friend, but multiple friends. You kind of give that instructions to all of them at the same time, so they can reach the same destination. +After we registered that, we have these things called emission controllers, which there are two kinds of them. They're like the mutators and the validators. So the mutators are the, so there are things that we need to mutate. When the job comes in, so for example, we need to, uh, so for example, if you're using Vault in your job, we mutate your job to put a constraint to say, Hey, only run this on clients that have Vault. Otherwise, your job will never succeed. So that's what the mutator does. It mutates the input. And there are multiple of them. So one of them kind of, I hate this word, canonicalize the input. So like sets default values that have not been specified by the user just to make sure that the job is consistent. Console connect, expose checks, and like all of these are separate things that are mutating the job. -That's what the FSM does. It gives an instruction. It mutates the state in a consistent way across all servers. Hopefully, that made sense. I think I got confused myself, but let me know if that wasn't very clear. +And then we have validators. So they validate the job. And then it goes through, so like, Raft is the mechanism we use to propagate changes across our servers to make sure that those changes are consistent in all the servers. So it goes to Raft. There's like a bunch of methods that are called there. Then it goes to NQ. So that's the queue that I mentioned about the eval broker queue is like here is some piece of work that needs to be done. Now it's in the queue. And then you can see here that it got dequeued later on by another server. -So it creates a job request in... So now we moved away from the CLI. That's the HTTP request that was made. Now the Nomad agent is who received that HTTP request and turned it into an internal request for a register. That's a job register command. I caused the job register method internally. You can see here, I'm putting logs and stuff, trying to figure out how the OpenTelemetry library works. +And what's this? Oh yeah, I don't know what this is. Yeah, and then like got dequeued by somebody to process and create that plan that I mentioned. So like, and then, yeah, like there's a lot happening here, which is kind of neat, but also kind of overrun. But like after the CLI made the put request, it keeps monitoring the eval to get status updates. So it makes a bunch of HTTP GET requests, just to get the update for that eval. And then it starts monitoring the deployment and again, start making a bunch of HTTP requests. -After we registered that, we have these things called emission controllers, which there are two kinds of them. They're like the mutators and the validators. The mutators are the things that we need to mutate when the job comes in. For example, if you're using vault in your job, we mutate your job to put a constraint to say, "Hey, only run this on clients that have vault." Otherwise, your job will never succeed. +So sort of the gist of it is like that whole end to end approach. Like I wanted to create a span that showed me, uh, what happens after I run Nomad job run because like from the user perspective, all you see is just like this output, which is quite confusing. If you're not used to Nomad and quite tricky to debug. If you don't even know this, like if you're an ops person and your job runs in like a pipeline somewhere, you really don't have a lot of visibility of what happened or what's happening. -That's what the mutator does. It mutates the input. There are multiple of them. One of them kind of, I hate this word, canonicalizes the input. It sets default values that have not been specified by the user just to make sure that the job is consistent. Console connect, expose checks, and all of these are separate things that are mutating the job. +### [00:24:24] Challenges with initial approach -Then we have validators. They validate the job. Then it goes through… So like, Raft is the mechanism we use to propagate changes across our servers to make sure that those changes are consistent in all the servers. So it goes to Raft. There are a bunch of methods that are called there. Then it goes to NQ. +So that was my first attempt. It's like, how do I map this process end to end and give, give as much detail as possible to the cluster operators. Now why this was not a good approach is one, because it creates a bunch of overheads. Like if you look at all the different spans, they're like zero microseconds long or like 50 micro, like that's not very useful, I guess. Like it's nice to note about them, but like in a practical sense. It creates a lot of overhead, like each of these have a lot of data in them that needs to go through the network, needs to be stored somewhere. -That's the queue that I mentioned about the eval broker queue. It's like, here is some piece of work that needs to be done. Now it's in the queue. You can see here that it got dequeued later on by another server. And what's this? Oh yeah, I don't know what this is. +So like it creates a lot of overhead for every time a job runs, for example. What happens when you just drill down into one of the spans? What kind of information does it send you? I don't think I put a whole lot. It also depends on the span. Like sometimes they put data, sometimes like, I think you're. I did and where was it to Q maybe I put some Nomad specific data. Oh, here, like the eval ID, for example, I was like trying to experiment with adding metadata to different spans. But it wasn't very consistent. So like, some of them have additional data. Others, there's just the standard OpenTelemetry SDK data. -Yeah, and then like got dequeued by somebody to process and create that plan that I mentioned. So like... And then, yeah, there's a lot happening here, which is kind of neat, but also kind of overrun. After the CLI made the put request, it keeps monitoring the eval to get status updates. It makes a bunch of HTTP GET requests just to get the update for that eval. +Yeah. So the first problem was the overhead. The second problem is that if you look at the diff, there's a lot of code changes. For example, let's look at, it's like the HTTP handlers, you know, OpenTelemetry does have a nice wrapper. It does have a nice wrapper for like Go, HTTP, uh, but we don't quite use that. It's like I had to manually wrap each endpoint in a function that had to use reflect to figure out which RPC should be called. It's like, it's kind of messy, but not too bad. The actual bad part is that normally distributed tracing, I think it was like created with the idea of like microservices, so like there's like a metric boundary more or less between your services. -It starts monitoring the deployment and again, starts making a bunch of HTTP requests. The gist of it is like that whole end-to-end approach. I wanted to create a span that showed me what happens after I run Nomad job run because from the user perspective, all you see is just this output, which is quite confusing if you're not used to Nomad and quite tricky to debug. If you're an ops person and your job runs in a pipeline somewhere, you really don't have a lot of visibility of what happened or what's happening. +And since Nomad is more like a monolith, like each component of Nomad is a big monolith, a lot of this, there's like, there's boundaries are defined by function calls. And so I had to modify all the, like the functions that I needed to, to monitor, like that I wanted to instrument. Had to be modified to like, take a context, for example, as a new org, um, to keep propagating that span forward. And so like, using function as the boundary for your telemetry is not great because it just becomes, uh, like, it's not as transparent as monitoring the metric layer, right? -That was my first attempt. It was like, how do I map this process end to end and give as much detail as possible to the cluster operators? Now why this was not a good approach is one, because it creates a bunch of overheads. If you look at all the different spans, they're like zero microseconds long or like 50 micro. That's not very useful, I guess. It's nice to note about them, but in a practical sense, it creates a lot of overhead. Each of these has a lot of data in them that needs to go through the network, needs to be stored somewhere. +So there's no way to do it automatically, and it just requires so much code change that, yeah, like after I got, and I didn't even go through the whole process, if you notice, like I stopped at the dequeued, like after dequeued, the span sort of ends because I just couldn't keep going with all these code changes. So like most of this PR is just adding context as the first argument to a bunch of functions, uh, which is not bad by itself, like a lot of functions having a context there can be helpful, especially for async work. But a lot of times it was not, it was just to keep the span going. -It creates a lot of overhead for every time a job runs, for example. What happens when you drill down into one of the spans? What kind of information does it send you? +So there's a lot of code changes for not a whole lot of, not necessarily not a lot of benefit, but like a lot of code changes that were not the best changes to make. Oh, hi Matthew. Yes, I can send. I'll send you the list of all the branches, I guess. Oh yeah, thanks. Also, hi, long time no see. Hey, nice to see you. Yeah, so these are all the branches. And then, like, once you click on the branch, you can see GitHub will give you a link for like, one commit ahead, and that's the deal. -[00:25:50] I don't think I put a whole lot. It also depends on the span. Sometimes I put data. Sometimes I think I did and where was it to Q? Maybe I put some Nomad specific data. Oh, here, like the eval ID, for example. I was trying to experiment with adding metadata to different spans, but it wasn't very consistent. Some of them have additional data. Others, there's just the standard OpenTelemetry SDK data. +So, yeah, so this approach, you know, I think is very nice to have this view, and Jaeger has some nice things about giving you like how the systems interact. So like, I could make this bit like each service could be like a different server and like I'll have, you know, three servers, multiple clients and sort of have that view. But I don't think that was the right approach. Like it is, it feels, it felt like going against the principles of distributed tracing, which is like you instrument your microservice and then the network becomes sort of your boundary of when to move the span. -So the first problem was the overhead. The second problem is that if you look at the diff, there's a lot of code changes. For example, let's look at... It's like the HTTP handlers. OpenTelemetry does have a nice wrapper. It does have a nice wrapper for like Go HTTP, but we don't quite use that. I had to manually wrap each endpoint in a function that had to use reflect to figure out which RPC should be called. It's kind of messy, but not too bad. +And I was doing it at the functional level, which felt very, very fine grain to what I was doing. Just before you move on, you mentioned that you'd done some, you had to like create some wrappers yourself around some of the OTEL calls. -The actual bad part is that normally distributed tracing, I think it was like created with the idea of microservices. There are like a metric boundary more or less between your services. Since Nomad is more like a monolith, like each component of Nomad is a big monolith, a lot of this, there's like boundaries defined by function calls. I had to modify all the functions that I needed to monitor, like that I wanted to instrument. +**Speaker 1:** Yeah, all of these. -Had to be modified to take a context, for example, as a new argument to keep propagating that span forward. Using function as the boundary for your telemetry is not great because it just becomes like, it's not as transparent as monitoring the metric layer, right? There's no way to do it automatically, and it just requires so much code change. +**Luiz:** Yeah, so how come you had to do that? Sorry, I missed your reasoning on that. -Yeah, after I got... and I didn't even go through the whole process. If you notice, I stopped at the dequeues. After dequeues, the span sort of ends because I just couldn't keep going with all these code changes. Most of this PR is just adding context as the first argument to a bunch of functions, which is not bad by itself. A lot of functions having a context there can be helpful, especially for async work. But a lot of times it was not, it was just to keep the span going. +### [00:30:30] Discussion on legacy code issues -So there's a lot of code changes for not a whole lot of... not necessarily not a lot of benefit, but like a lot of code changes that were not the best changes to make. +**Speaker 1:** Oh yeah, so normally the SDK does it automatically for you if you're using a framework like net/http, or I think it does support like the Gorilla Mux. So normally you just import the library and it sort of does the magic for you. But the way we do it in Nomad, it's like another thing to keep in mind is like Nomad is a very old project. A lot of people don't know how old Nomad is, but it's as old as Kubernetes. I think Nomad is like one month younger than Kubernetes. Like the V1 came out sort of like around the same time. So that was like 2017, I want to say. -**Matthew:** Oh, hi, Matthew. Yes, I can send. I'll send you the list of all the branches, I guess. +Yeah. So like Nomad is a very old code base. And so a lot of the things we do like predates a lot of the new sort of code niceties. So like the whole HTTP layer, the whole RPC layer is something that we had to create because it wasn't a way like there wasn't a standard or like a more common way to do it. So you can leverage the OpenTelemetry conveniences as a result because you're working with some older. -**Luiz Aoqui:** Oh yeah, thanks. Also, hi, long time no see. +**Speaker 1:** Yeah, because it has so much custom code. -**Matthew:** Hey, nice to see you. +**Luiz:** Gotcha. Gotcha. That's very interesting. I think especially hits on an interesting point for folks who are looking to incorporate OpenTelemetry into their existing projects. Oftentimes, like we are dealing with very old code bases, and that's not something that we discuss enough. So I'm definitely happy that you're bringing this to light because I'm sure you're not the first and only person to encounter this. -**Luiz Aoqui:** Yeah, so these are all the branches. Once you click on the branch, you can see GitHub will give you a link for like one commit ahead, and that's the deal. +**Speaker 1:** Yeah, no worries. But like it did, as you can see, like the rep is fairly simple and does use the SDK quite a lot, but it's just like the way the project was was not the way that usually go HTTP handlers are written nowadays. So that was kind of why I had to do this. -So this approach, I think, is very nice to have this view. Jaeger has some nice things about giving you like how the systems interact. I could make this bit like each service could be like a different server and I'll have three servers, multiple clients, and sort of have that view. But I don't think that was the right approach. It feels like it felt like going against the principles of distributed tracing, which is like you instrument your microservice and then the network becomes sort of your boundary of when to move the span. +### [00:32:32] Shift in approach to instrumentation -I was doing it at the functional level, which felt very, very fine grain to what I was doing. Just before you move on, you mentioned that you'd done some, you had to like create some wrappers yourself around some of the OTEL calls. +**Luiz:** Cool. So that was the first approach, uh, kind of like gave up just getting too complex and the hack was done. So I didn't have a lot more time to work on. The second approach, I sort of shifted the perspective a little bit. So instead of like trying to instrument Nomad itself, the idea was to have it easier for people using OpenTelemetry and using them to instrument their own applications. -**Matthew:** Yeah, all of these. +So what this in this branch, what I did was, so here's the test runner. So, you know, going back to the task runner is the thing that runs your container, that actually runs your workload. You set up the environment to run your workload. And what I did here was I started adding. So if you use OpenTelemetry before, you know, this environment variable will tell resource attributes, which I think is part of the spec. And it defines, um, go here. -**Luiz Aoqui:** How come you had to do that? Sorry, I missed your reasoning on that. +So it defines values that are automatically added to your span context or like whatever you create with OpenTelemetry, this environment variable sort of creates standardized values to or like metadata to add to those resources that you create. And so what I did here, oops, this one is like, I say, okay, like if you're running, if you're already using OpenTelemetry and you're running, stopping Nomad, could be helpful to automatically get Nomad data for your OpenTelemetry stuff. -**Matthew:** Oh yeah, so normally the SDK does it automatically for you if you're using a framework like net/http, or I think it does support the Gorilla Mux. Normally you just import the library and it sort of does the magic for you. But the way we do it in Nomad, it's like another thing to keep in mind is that Nomad is a very old project. A lot of people don't know how old Nomad is, but it's as old as Kubernetes. I think Nomad is like one month younger than Kubernetes. The V1 came out sort of around the same time. +So having the alloc ID, the alloc name, the evals, like all of this data sort of gets injected automatically for you, um, by Nomad itself. So like this, but all of this whole code, what it's doing is just setting the environment variable with a bunch of Nomad data. So, let's take a look at how that works in practice. And I think what you're doing here is, is more or less like what Kubernetes does as well now when it emits like telemetry data that you have that kind of free with purchase, um, environment variables. Right. -So that was like 2017, I want to say. Nomad is a very old code base. A lot of the things we do predate a lot of the new sort of code niceties. The whole HTTP layer, the whole RPC layer is something that we had to create because there wasn't a standard or a more common way to do it. +**Luiz:** Which is quite nice. Yeah. Like I think that like information about the fault information about it, like that sort of just happens automatically for you, uh, when you're using the OpenTelemetry SDK and all the collector. -**Matthew:** Gotcha. Gotcha. That's very interesting. I think especially hits on an interesting point for folks who are looking to incorporate OpenTelemetry into their existing projects. Oftentimes, we are dealing with very old code bases, and that's not something that we discuss enough. So I'm definitely happy that you're bringing this to light because I'm sure you're not the first and only person to encounter this. +So let's run this. I don't think I need the config because I'm not sending any data. So I'm just starting Nomad and oh yeah, I remember now. So I need, so like I'm running Nomad. Nomad is going to automatically instrument my application. So I need to run something in Nomad that uses OpenTelemetry. -**Luiz Aoqui:** So yeah, no worries. But like it did, as you can see, the rep is fairly simple and does use the SDK quite a lot, but it's just like the way the project was was not the way that usually Go HTTP handlers are written nowadays. That was kind of why I had to do this. +So I found this project. I just Google like OpenTelemetry, simple job or docker image, so I found this OTEL gen project here that it just generates traces for you. And so I'm running that image and pointing to my docker container here that is running the collector and generating multiple traces. And there's a back job, so it's just going to run once. So when I run that. -[00:32:37] Cool. So that was the first approach. I kind of gave up because it was getting too complex and the hack was done, so I didn't have a lot more time to work on. The second approach, I sort of shifted the perspective a little bit. Instead of trying to instrument Nomad itself, the idea was to make it easier for people using OpenTelemetry and using them to instrument their own applications. +**Speaker 1:** Sorry, what does that batch job do? -What I did in this branch was, here's the test runner. Going back to the task runner, it's the thing that runs your container, that actually runs your workload. You set up the environment to run your workload. What I did here was I started adding... If you use OpenTelemetry before, you know this environment variable will tell resource attributes, which I think is part of the spec. +**Luiz:** It generates OpenTelemetry data. Let me find that repo here. So, yeah, so it's this project, which is very helpful. It just generates, you can tell that you generate metrics, traces, and then it just creates sample data for you in the OpenTelemetry format. -It defines values that are automatically added to your span context or whatever you create with OpenTelemetry. This environment variable sort of creates standardized values to or like metadata to add to those resources that you create. What I did here... +**Speaker 1:** And is it supposed to be like, it, as you said, it's just sample data. It's just like, you're not actually using it for anything other than for test purposes, kind of thing. -Oops, this one is like, I say, okay, if you're running, if you're already using OpenTelemetry and you're running Nomad, could be helpful to automatically get Nomad data for your OpenTelemetry stuff. So having the alloc ID, the alloc name, the evals, like all of this data sort of gets injected automatically for you by Nomad itself. +**Luiz:** Yeah, yeah. It's just like, imagine this is like your application that is generating traces. -So this whole code, what it's doing is just setting the environment variable with a bunch of Nomad data. Let's take a look at how that works in practice. I think what you're doing here is more or less like what Kubernetes does as well now when it emits telemetry data that you have that kind of free with purchase, environment variables. +**Speaker 1:** Okay. Got it. Got it. Cool. And that application is running. So let's look at the data it generated. So he created three traces here, three traces, say that fast, two times. So now, if you look at each trace, he now has all the Nomad stuff. So he has like, he has ultimate, like this was like a GitHub project. So I did not change any source code at all. -**Luiz Aoqui:** Right. Which is quite nice. Yeah. Like I think that information about the fault information about it, like that sort of just happens automatically for you when you're using the OpenTelemetry SDK and all the collector. So let's run this. I don't think I need the config because I'm not sending any data. +But just by the fact that is running Nomad, Nomad itself is instrumented like populating a lot of metadata for each trace for each span. So like it gets, the alloc ID, the alloc name, the eval, the group names, like if you are a user of Nomad and if you're already using OpenTelemetry, all the things that like with this change, all the things that you're running Nomad with OpenTelemetry, we will automatically have all of this additional context and data for you. Kind of like, I didn't know how much data to go. So I put like a lot, but this is like to help the data is not instrumenting Nomad itself was to help people that already instrumented their applications. When they're running them and they get additional benefit. -So I'm just starting Nomad and, oh yeah, I remember now. So I need... I'm running Nomad. Nomad is going to automatically instrument my application. So I need to run something in Nomad that uses OpenTelemetry. I found this project. I just Google like OpenTelemetry simple job or Docker image, so I found this OTEL gen project here that just generates traces for you. +### [00:38:38] Adding metadata to spans -It runs that image and points to my Docker container here that is running the collector and generating multiple traces. It has a batch job, so it's just going to run once. When I run that... +So that was a different approach to OpenTelemetry. So yeah, it generates this additional context here. Now the problem with this approach was one, I didn't know what to put. So I just put a lot of things. A lot of this is probably not useful, like job type and who cares. And then like, cause like, you know, metadata is good, but too much metadata is also tricky because you increase your, you know, your metric packet size. You need to store this somewhere. -**Matthew:** Sorry, what does that batch job do? +So it kind of balloons your store and store. And so the challenge with this approach was just to, how do we, it's more like a UX problem, like how do we allow people to control what information they want, or like, do we allow them to control or do we hard code some values? So it's like figuring that balance was a little tricky. And looking at the code, I think this is just like, yeah, I just hard coded a bunch of stuff because again, this is a HackyVic project. -**Luiz Aoqui:** It generates OpenTelemetry data. Let me find that repo here. Yeah, so it's this project, which is very helpful. It just generates... You can tell that you generate metrics, traces, and then it just creates sample data for you in the OpenTelemetry format. +But yeah, it's like, if you see, there's no knob for users to control what kind of data they want. I think the next work is kind of similar. I was also doing this, this environment variable here. I haven't actually opened up here for this. It's like, it's just like cleaning up code a little bit. This one is kind of, yeah, it's like the same as this one, but like cleaning up and writing better code with tests and everything. -**Matthew:** And is it supposed to be like, as you said, it's just sample data. You're not actually using it for anything other than for test purposes? +Oh yeah. One, one of the problems with the other approach is that if you are already using OpenTelemetry, you're probably doing something like this. You're already, you may already be adding your own values here, right? I know you probably, you may already have your own environment variable for your attributes out there. And so like, if I run this job or this version of the job, now that code that I initially wrote would kind of overwrite that. So you can see here, there's no key equals value because the way I did it, I just like overwrite that environment variable and you lose, stuff that you put yourself. -**Luiz Aoqui:** Kind of thing. Yeah, it's just like, imagine this is like your application that is generating traces. +So in this other approach, I, instead of overriding, I will merge the changes just like being nice for the user. What I think, oh yeah, I actually found a bug, the SDK here, um, which wasn't decoding the values as it was describing the spec. But then we actually figured out that there are like two versions of the spec in place, then I had to go and change this back. There's like a whole side quest that I had to go through to fix this. -**Matthew:** Got it. Got it. Cool. +And I think it hasn't been, let me do a check. So I changed this back, which means every SDK had to be updated. And there's the link. Yeah. And I think there's still some open ones. Like people are looking for, you know, contributing job and telemetry. This might be a good first issue. They're like sort of simple. It's just changing all the, that variable gets decoded, and there's still some languages that have not been fixed. -**Luiz Aoqui:** And that application is running. So let's look at the data it generated. So it created three traces here, three traces, say that fast two times. +So yeah, for contributions, this could be a good first issue to look into. Yeah, so that's the second approach. Good for users, not so much instrumenting Nomad itself, but could be useful. Just the UX paper cuts were the major drawbacks there. Nothing OpenTelemetry specific, just how do you do surface text. -Now, if you look at each trace, it now has all the Nomad stuff. It has like, this was like a GitHub project, so I did not change any source code at all. But just by the fact that it's running Nomad, Nomad itself is instrumenting, like populating a lot of metadata for each trace, for each span. It gets the alloc ID, the alloc name, the eval, the group names. +Oh, one thing, another downside of this approach is that since it's using environment variables, those values are rather static. Like you cannot modify an environment variable to a container that is already running. So if you want to put data that is not known ahead of time or some data that can change throughout the life cycle of that container, this approach doesn't quite work because the environment variable is already created. So it's not great for dynamic values or things like that. Those you usually have to do inside your application. -If you are a user of Nomad and if you're already using OpenTelemetry, all the things that you're running in Nomad with OpenTelemetry will automatically have all of this additional context and data for you. I didn't know how much data to go, so I put like a lot, but this is to help the data. It's not instrumenting Nomad itself, but to help people that already instrumented their applications. +### [00:43:44] Meeting with OpenTelemetry community -When they're running them, they get additional benefit. That was a different approach to OpenTelemetry. The problem with this approach was one, I didn't know what to put. So I just put a lot of things. A lot of this is probably not useful, like job type, who cares? Because, you know, metadata is good, but too meta, too much metadata. +Cool. So that was the second approach. And then each, each approach, like a year apart. And then after doing all of that, like that's, I think it's sort of when I met, right after this or like I met Adrienne and the other folks at the OpenTelemetry community. And I started to get some guidance of like, you know, Hey, I'm trying to do this, you know, going back to the original approach. It's not quite working because of all of these reasons. -[00:39:00] It's also tricky because you increase your metric packet size. You need to store this somewhere, so it kind of balloons your store. The challenge with this approach was just to how do we... It's more like a UX problem. +And speaking with Adrienne and I think Ted was also very helpful in giving feedback on this. It's like, don't try to boil the ocean. Like you don't need to instrument everything at once. You can start small. And like, you know, find your core, I think the exec suggested, like find your core business instrument that, and then you get a lot of value out of that already. And translating that into the Nomad word to me, it felt like, oh, okay, like I don't need to instrument everything end to end. -How do we allow people to control what information they want, or like, do we allow them to control or do we hard code some values? Figuring that balance was a little tricky. Looking at the code, I think this is just like, yeah, I just hard coded a bunch of stuff because again, this is a Hack Week project. +We already have this eval thing that is like the unit of work; by instrumenting that flow of the eval, maybe I can get, you know, enough information out of this to be useful. So let's in practice how that works. So let's go to this branch now, and I don't think I need a config for this one. So let's run this version. And what this version does is like, it's going back to the first implementation, like I want to instrument Nomad itself. -One of the problems with the other approach is that if you are already using OpenTelemetry, you're probably doing something like this. You may already be adding your own values here, right? You may already have your own environment variable for your attributes out there. If I run this job or this version of the job, now that code that I initially wrote would kind of overwrite that. +So this is adding traces and spans and all of that into the Nomad code itself to instrument Nomad for operators. And so going back to my traces here, nothing happened because I didn't run the job. Let's run a job to do some working. I don't need to wait. Yeah. So now I have a service called Nomad. And if you look at the traces now, instead of, you know, let's compare it to the previous implementation. -In this other approach, I instead of overriding, I merged the changes just to be nice for the user. What I think... Oh yeah, I actually found a bug in the SDK here, which wasn't decoding the values as it was describing the spec. +So before I had, uh, I think it, oh yeah, I think I might have deleted, but before I had like one huge trace of everything. Now, instead of going and like trying to keep the context around each function calls, what I did was more, and you can see here, like, there's a bunch of traces instead of a single one. So what I did was, I didn't try to connect all those function calls, but it was more about instrumenting, instrumenting the specific aspects like here in DialogRunner when DialogRunner starts. -But then we actually figured out that there are like two versions of the spec in place. Then I had to go and change this back. There's like a whole side quest that I had to go through to fix this. I think it hasn't been... Let me do a check. +### [00:47:07] New approach with focused instrumentation -So I changed this back, which means every SDK had to be updated. And there's the link. I think there's still some open ones. People are looking for, you know, contributing job and telemetry. This might be a good first issue. They're like sort of simple. It's just changing all the... +You create a new span like I, I wouldn't, I didn't worry about keeping the same span going. It's like, okay, this is a new action that is happening. It's going to be a new span. So I guess the approach was being more mindful of like, what you wanted as far as like what you wanted your traces to be. Because you went from, like, a mega trace to a bunch of smaller traces, where like, these smaller traces then have like, more meaningful information to you, is that? -That variable gets decoded, and there are still some languages that have not been fixed. So yeah, for contributions, this could be a good first issue to look into. +Yeah, yeah, exactly. It's like changing the perspective a bit instead of keeping the gigantic trace, be more like serious about like, okay, this function is important. So this function is going to create a trace, uh, and what this means is like, I don't need to change my, you know, my function signatures. I don't need to keep passing context around. It can just, whenever something important happens, I just create a trace there. -Yeah, so that's the second approach. Good for users, not so much instrumenting Nomad itself, but could be useful. Just the UX paper cuts were the major drawbacks there. Nothing OpenTelemetry specific, just how do you do surface text? +The downside, as you can see, is that there are a lot of traces here. But a good thing, like something that I learned later on is that, well, I knew this already, but I didn't know how to use it specifically, but traces have attributes. And I think that was the key to this approach is that, uh, and there's this notion of semantic conventions in the OpenTelemetry SDK. For those who don't know, it's like, it's that of predefined attributes for your, just like Docker has, or I guess they're called containers, like your container engine has predefined keys that it's supposed to use. -Oh, one thing, another downside of this approach is that since it's using environment variables, those values are rather static. You cannot modify an environment variable to a container that is already running. If you want to put data that is not known ahead of time or some data that can change throughout the lifecycle of that container, this approach doesn't quite work because the environment variable is already created. So it's not great for dynamic values or things like that. +Kubernetes has its own; AWS have their own attributes or like standardized attributes that they use. And so the key to this approach was like, okay, what does that look for Nomad? And so I think it was at the bottom. Yeah. So I created like semantic conventions for like, oh, what does a Nomad region key look like? Oh, it's Nomad.region. What does a Nomad space key? No, Nomad.space. And then this allows to each span, you know, these are all like different spans coming from all different parts of the code, but they have standardized attributes. -[00:44:00] That was the second approach. Each approach, like a year apart. After doing all of that, I think it's sort of when I met, right after this, or like I met Adrienne and the other folks at the OpenTelemetry community. I started to get some guidance of like, you know, Hey, I'm trying to do this, you know, going back to the original approach. It's not quite working because of all of these reasons. +And what this allows me to do is like, I could come here and say, Oh, I actually want job ID to be example. And so I can use these values to filter all this mess that nobody's ever done. It's not like a nice single huge trace; it's just a bunch of different disconnected traces. But by using the semantic conventions, I can help find and filter things better. So these are all the traces for, you know, this job here. -Speaking with Adrienne and I think Ted was also very helpful in giving feedback on this. It's like, don't try to boil the ocean. You don't need to instrument everything at once. You can start small. Find your core, I think the exec suggested, find your core business instrument that, and then you get a lot of value out of that already. +But let's say I go to the UI now. So this job created is an allocation, for example. So let me get the allocation ID here. And I said, I don't want the whole job. I just want this allocation. So now it's like all the traces for that allocation because, you know, even though they're different traces coming from different parts of the code, different parts of Nomad, they all share the same conventions. -Translating that into the Nomad world to me, it felt like, Oh, okay, like I don't need to instrument everything end to end. We already have this eval thing that is like the unit of work. By instrumenting that flow of the eval, maybe I can get enough information out of this to be useful. +And so that allows me as like a cluster operator to go and filter out all the quote unquote noise that gets generated, to be more meaningful to me. And you can see here the same things that we saw in the big trace, but now they're just breaking down. So like the alloc runner, the state stores, like the database. And I also started experimenting with like the log events as well. -[00:45:19] So let's in practice how that works. Go to this branch now, and I don't think I need a config for this one. Let's run this version. What this version does is like, it's going back to the first implementation, like I want to instrument Nomad itself. This is adding traces and spans and all of that into the Nomad code itself to instrument Nomad for operators. +So instead of having, uh, let me find it. I think that might be more interesting data there. So this is when I registered a job. Where'd I get the ID? Okay. Is this oops? Again, maybe this, I don't remember how I did it. Let's remove all traces and then I'll find manually. I was looking for... -So going back to my traces here, nothing happened because I didn't run the job. Let's run a job to do some work. I don't need to wait. +**Speaker 1:** Are you referring to just span events or actual like OTEL logs? I'm just curious. -**Matthew:** Yeah. +**Luiz:** Yeah. Yeah. Events, uh, data. Let me run the job again. And let me take a look at this. Yeah. So like, so for example, this is the, the trace, the sort of like the scheduler part, the worker scheduler, all of that. So like I started putting logs that like, as it was scheduling, it's like all the scheduling decisions are like part of the trace now. So you can see like why, why did this change happen? -**Luiz Aoqui:** So now I have a service called Nomad. If you look at the traces now, instead of, you know, let's compare it to the previous implementation. Before, I had... I think I... Oh yeah, I think I might have deleted... but before I had like one huge trace of everything. Now instead of going and trying to keep the context around each function calls, what I did was more... +And like you can see here as part of events in your span, instead of having, you know, the multiple tiny traces, those function calls sort of become more events instead of a new span. -You can see here, there's a bunch of traces instead of a single one. What I did was... I didn't try to connect all those function calls, but it was more about instrumenting the specific aspects. Like here in the alloc runner, when the alloc runner starts, you create a new span. I didn't worry about keeping the same span going. It's like, okay, this is a new action that is happening. It's going to be a new span. +**Speaker 1:** Yeah, that's a strange point. -So I guess the approach was being more mindful of what you wanted your traces to be. Because you went from a mega trace to a bunch of smaller traces, where these smaller traces then have more meaningful information to you, is that? +**Luiz:** That's awesome. We are at time, just FYI, an hour blew by really fast. Yeah, that, that, that's what I had showed. That was the last attempt that I made. And if you have questions, sorry, I missed the chat. You have to add attributes to each trace. -**Matthew:** Yeah, exactly. It's like changing the perspective a bit instead of keeping the gigantic trace, be more serious about like, okay, this function is important. This function is going to create a trace. +**Speaker 1:** Oh yeah. So the code to add the attributes, you do have to call for each trace. But you can kind of make it like helper functions to do like, let me find an example here. -What this means is I don't need to change my function signatures. I don't need to keep passing context around. I can just, whenever something important happens, I just create a trace there. The downside, as you can see, is that there are a lot of traces here. +**Luiz:** Yeah. So like you can pass in like helper functions, like, okay, given this allocation, you know, this pod, give me all the attributes for that. You do have for each trace, but you need to, to, you can create helper functions too. To ease the burden. -But a good thing, something that I learned later on is that, well, I knew this already, but I didn't know how to use it specifically, but traces have attributes. I think that was the key to this approach. There's this notion of semantic conventions in the OpenTelemetry SDK. For those who don't know, it's that of predefined attributes for your... just like Docker has, or I guess they're called containers, like your container engine has predefined keys that it's supposed to use. Kubernetes has its own, AWS has their own attributes or standardized attributes that they use. +### [00:55:04] Wrap-up and recording announcement -So the key to this approach was like, okay, what does that look for Nomad? I think it was at the bottom. Yeah. So I created semantic conventions for like, Oh, what does a Nomad region key look like? Oh, it's Nomad.region. What does a Nomad space key? Nomad.space. +**Speaker 1:** That is so awesome. Thanks for sharing your OTEL journey with us. Luiz, this has been super, super helpful. I hope everyone on this call got something out of it. I think it's so important to have these kinds of conversations because I think we're moving past the, you know, let's, you know, intro to OTEL. -This allows each span, you know, these are all like different spans coming from all different parts of the code, but they have standardized attributes. What this allows me to do is like, I could come here and say, Oh, I actually want job ID to be example. I can use these values to filter all this mess that nobody's ever done. +Like, we're now getting into like more people doing really digging deep into their OTEL implementations and, and hearing about, you know, the challenges that you've experienced and, and the different things that you've tried has been really awesome. So thank you so much for sharing this with us. -It's not like a nice single huge trace. It's just a bunch of different disconnected traces. But by using the semantic conventions, I can help find and filter things better. These are all the traces for, you know, this job here. But let's say I go to the UI now. +And, for those of you on the call, tell your friends who couldn't make it that we will be posting a recording of this on the OTEL YouTube channel. The YouTube channel is OTEL-official. So just keep an eye out. I'll post on socials and also on Slack once we have the recording up and running. -This job created an allocation, for example. Let me get the allocation ID here. I said, I don't want the whole job. I just want this allocation. So now it's like all the traces for that allocation because even though they're different traces coming from different parts of the code, different parts of Nomad, they all share the same conventions. +So thank you everyone for joining. Really appreciate it. -That allows me as a cluster operator to go and filter out all the "noise" that gets generated to be more meaningful to me. You can see here the same things that we saw in the big trace, but now they're just breaking down. +**Luiz:** Thank you everyone for coming and thank you for having me. -So like the alloc runner, the state stores, like the database. I also started experimenting with like the log events as well. Instead of having... Let me find it. I think that might be more interesting data there. +**Speaker 1:** Bye. -This is when I registered a job. Where did I get the ID? Okay. Is this... Oops. Again, maybe this... I don't remember how I did it. Let's remove all traces and then I'll find manually. - -I was looking for... Yeah. On... Sorry, the Zoom window is going to go. - -**Matthew:** See it? - -**Luiz Aoqui:** Yeah. Oh, maybe I'll find it. Share later. But the idea is that I started to experiment with logs a lot more. - -**Matthew:** Are you referring to just span events or actual OTEL logs? I'm just curious. - -**Luiz Aoqui:** Yeah. Yeah, events. Data. Let me run the job again. Let me take a look at this. Yeah. So like... So for example, this is the trace, the sort of like the scheduler part, the worker scheduler, all of that. - -I started putting logs that like, as it was scheduling, it's like all the scheduling decisions are like part of the trace now. So you can see like why did this change happen? You can see here as part of events in your span. Instead of having the multiple tiny traces, those function calls sort of become more events instead of a new span. - -**Matthew:** Yeah, that's a strange point. That's awesome. - -[00:54:00] **Luiz Aoqui:** We are at time, just FYI, an hour blew by really fast. - -**Matthew:** Yeah, that was what I showed. That was the last attempt that I made. If you have questions, sorry, I missed chat. - -**Luiz Aoqui:** You have to add attributes to each trace. - -Oh yeah. So the code to add the attributes, you do have to call for each trace. But you can kind of make it like helper functions to do like... Let me find an example here. Yeah. - -You can pass in like helper functions like, okay, given this allocation, this pod, give me all the attributes for that. You do have for each trace, but you can create helper functions too to ease the burden. - -**Matthew:** That is so awesome. Thanks for sharing your OTEL journey with us, Luiz. This has been super, super helpful. I hope everyone on this call got something out of it. - -I think it's so important to have these kinds of conversations because I think we're moving past the, you know, let's intro to OTEL. We're now getting into like more people doing really digging deep into their OTEL implementations, and hearing about the challenges that you've experienced and the different things that you've tried has been really awesome. - -So thank you so much for sharing this with us. For those of you on the call, tell your friends who couldn't make it that we will be posting a recording of this on the OTEL YouTube channel. The YouTube channel is OTEL-official, so just keep an eye out. I'll post on socials and also on Slack once we have the recording up and running. - -Thank you everyone for joining. Really appreciate it. - -**Luiz Aoqui:** Thank you everyone for coming and thank you for having me. - -**Matthew:** Bye. - -**Luiz Aoqui:** Thank you. +**Luiz:** Thank you. ## Raw YouTube Transcript diff --git a/video-transcripts/transcripts/2023-12-20T04:44:51Z-the-humans-of-otel-kubecon-na-2023.md b/video-transcripts/transcripts/2023-12-20T04:44:51Z-the-humans-of-otel-kubecon-na-2023.md index 1f9a986..0e36748 100644 --- a/video-transcripts/transcripts/2023-12-20T04:44:51Z-the-humans-of-otel-kubecon-na-2023.md +++ b/video-transcripts/transcripts/2023-12-20T04:44:51Z-the-humans-of-otel-kubecon-na-2023.md @@ -10,18 +10,24 @@ URL: https://www.youtube.com/watch?v=coPrhP_7lVU ## Summary -In this YouTube video, Tyler Yahn, a maintainer for the OpenTelemetry Go SIG, leads a discussion with several other contributors to OpenTelemetry, including Amy Tobey, Carter Socha, Juraci, and others, about the concept of observability in software systems. They explore the evolution of observability from siloed systems of logs, metrics, and traces to an integrated approach that allows for better data analysis and quicker problem resolution. The participants share their personal definitions of observability, emphasizing its role in understanding system behavior and improving application performance. They also discuss the significance of OpenTelemetry as a community-driven initiative that standardizes telemetry data collection across various platforms, minimizing vendor lock-in. Key points include the importance of integrating different telemetry signals, the role of OpenTelemetry in modern software development, and the collaborative nature of the project. Overall, the conversation highlights how effective observability can enhance operational efficiency and user experience. +In this YouTube video, Tyler Yahn, Amy Tobey, and other contributors to the OpenTelemetry project discuss their roles and experiences within the community, emphasizing the evolution and significance of Observability in software systems. The group includes maintainers like Carter Socha, Constance Caramanolis, and Juraci, who share insights on their contributions, from the OpenTelemetry CLI to security measures and the OpenTelemetry Collector. They explore the concept of Observability, defining it as the ability to monitor and understand system behaviors through integrated telemetry data, which encompasses logs, metrics, and traces. The conversation highlights the importance of collaboration across various vendors within the OpenTelemetry community and the transition towards standardized telemetry that enhances the capability to troubleshoot and analyze systems efficiently. The video concludes with a lighthearted debate over favorite telemetry signals, with many participants expressing a preference for traces due to their depth and utility in understanding operational behavior. ## Chapters -00:00:00 Welcome and introductions -00:04:20 Observability discussion -00:10:40 Observability definitions -00:12:30 OpenTelemetry involvement -00:15:00 OpenTelemetry definition -00:18:42 Community collaboration -00:20:30 Telemetry signal preferences -00:23:10 Traces vs. metrics discussion +00:00:00 Introductions +00:01:40 Guest introduction: Carter Socha +00:05:00 Discussion about Observability +00:06:40 Importance of integrated telemetry +00:09:10 Modern Observability explained +00:11:40 Observability as a spectrum +00:13:20 Personal experiences with OpenTelemetry +00:18:20 Community collaboration in OpenTelemetry +00:20:50 Discussion on telemetry signals +00:22:30 Lighthearted debate on favorite telemetry signals + +## Transcript + +### [00:00:00] Introductions **Tyler:** I'm Tyler Yahn. I am a maintainer for the OpenTelemetry Go SIG. We're working on some auto-instrumentation there, and specification. @@ -37,11 +43,13 @@ In this YouTube video, Tyler Yahn, a maintainer for the OpenTelemetry Go SIG, le **Tyler:** It would make sense. And I've talked to Austin about it. +### [00:01:40] Guest introduction: Carter Socha + **Carter:** Hey, I got Ted Young with me! Hello! I'm Carter Socha. I work on a couple different things. I'm one of the few product managers floating around, but I helped start the OpenTelemetry Demo, which I'm a maintainer of. I also work in the SIG security, which helps the project improve their security response process. **Bogdan:** My name is Bogdan. I took a break for parental leave so I'm just jumping back. -**Tyler:** Okay, what were you doing before? +**Carter:** Okay, what were you doing before? **Bogdan:** I done a lot of things, including member of TC, member of GC, maintainer of Collector. I was a former maintainer of Java, so I've done a lot. @@ -57,57 +65,59 @@ In this YouTube video, Tyler Yahn, a maintainer for the OpenTelemetry Go SIG, le **Jacob:** My name is Jacob Aronoff. I am a maintainer for the OpenTelemetry Operator project. -**Alex:** Hi, I'm Alex and I'm a contributor and maintainer in OpenTelemetry. I wrote a book about OpenTelemetry. I don't know what else. I do stuff with OTel. +**Alex:** Hi, I'm Alex and I'm a contributor and maintainer in OpenTelemetry. Wrote a book about OpenTelemetry. I don't know what else. I do stuff with OTel. **Jacob:** Cool. I am a contributor and maintainer of the OpenTelemetry Collector and the OpenTelemetry Collector Contrib repo and I have been spending a lot of time in various SIGs and specialty working groups around configuration and security. And previously I spent a bunch of time maintaining and contributing to Python. -**Purvi:** Hey, my name is Purvi. I am a senior software engineer. I worked over my career a lot with browsers and javascript. +**Purvi:** Hey, my name is Purvi. I am a senior software engineer. I worked over my career a lot with browsers and javascript. -[00:04:20] **Carter:** First question. We're here talking about Observability. So what does Observability mean to you? +**Carter:** First question. We're here talking about Observability. So what does Observability mean to you? **Purvi:** Yeah, that's a great question. Personally, I think Observability means that when you woke up at 02:00 a.m. to go fix a problem, you can fix it. And ideally, the next day you're able to look at that code again and find out a way to never have that problem exist. I think that's really what it means to me. It means being able to look at things coming out of the box and tell what's going on inside parts. Be very convenient. I like that. -**Jacob:** First of all, it's monitoring... But really, Observability is this nebulous term, but it did show up as part of a sort of shift in how we are thinking about monitoring our system. And I would say that shift is the way we used to do it was you had these different signals, you needed logs, so you had a logging system, you needed metrics, you made a metric system, you needed tracing, but you didn't know what that was, so you didn't do it. And instead of having these three separate, totally siloed systems, what we've been doing over the past couple of years, especially in the OpenTelemetry project, is trying to say it's really bad for these three things to be separate. Or the four things, if you include profiling. +### [00:05:00] Discussion about Observability -When you're using these tools, you use them together, you're moving back and forth between them, right? Like you get an alert based off of a metric that you set up. But when that alert goes off because errors or something spiked, the next thing you want is to look at the logs that are in the transactions that are causing these alerts. You want to look at the logs that are in a particular transaction. You really want to have a trace ID stapled to all those logs, so you can actually look them up. +**Constance:** First of all, it's monitoring... But really, Observability is this nebulous term, but it did show up as part of a sort of shift in how we are thinking about monitoring our system. And I would say that shift is the way we used to do it was you had these different signals, you needed logs, so you had a logging system, you needed metrics, you made a metric system, you needed tracing, but you didn't know what that was, so you didn't do it. And instead of having these three separate, totally siloed systems, what we've been doing over the past couple of years, especially in the OpenTelemetry project, is trying to say it's really bad for these three things to be separate. Or the four things, if you include profiling. -So we want to actually use all these tools together. And in order to use all of these tools together, you need to have the data coming in, the telemetry actually be integrated, so you can't have three separate streams of telemetry. And then on the back-end, be like, I want to cross-reference. All of that telemetry has to be organized into an actual graph. You need a graphical data structure that all these individual signals are a part of. For me, that is what modern Observability is all about. It's about having all this data connected into a graph in such a way that we can leverage the machine to do what they're good at, to reduce the amount of time we need to spend investigating issues. +### [00:06:40] Importance of integrated telemetry -Instead of being like, I wonder if this is the problem. Therefore I am going to collect all the logs and grep through them, try to whittle it down to something. I'm going to look at all the config files myself, try to figure out what's going on. You can just quickly get an answer to a lot of those questions and then move on to the next hypothesis. The amount of time you save with modern Observability, I think, changes how we actually practice, and that's an ongoing trend. +**Purvi:** When you're using these tools, you use them together, you're moving back and forth between them, right? Like you get an alert based off of a metric that you set up. But when that alert goes off because errors or something spiked, the next thing you want is to look at the logs that are in the transactions that are causing these alerts. You want to look at the logs that are in a particular transaction. You really want to have a trace ID stapled to all those logs, so you can actually look them up. So we want to actually use all these tools together. And in order to use all of these tools together, you need to have the data coming in, the telemetry actually be integrated, so you can't have three separate streams of telemetry. And then on the back-end, be like, I want to cross-reference. All of that telemetry has to be organized into an actual graph. You need a graphical data structure that all these individual signals are a part of. For me, that is what modern Observability is all about. It's about having all this data connected into a graph in such a way that we can leverage the machine to do what they're good at, to reduce the amount of time we need to spend investigating issues. Instead of being like, I wonder if this is the problem. Therefore I am going to collect all the logs and grep through them, try to whittle it down to something. I'm going to look at all the config files myself, try to figure out what's going on. You can just quickly get an answer to a lot of those questions and then move on to the next hypothesis. The amount of time you save with modern Observability, I think, changes how we actually practice, and that's an ongoing trend. But with OpenTelemetry going, effectively going GA this year, with tracing, metrics and logs now stable, yes, finally, only like two years late anyway. But the fact that we have that now, the fact that we now have telemetry that has all of these correlations baked into it, you're going to start seeing a new wave of analysis tools, all the existing ones out there, but also new ones being built, that leverage the fact that this data is available and that it's like a standard data format, kind of proprietary data format, stable data format. You can rely on it. So it's like okay to build your giant platform on top of this data or build some kind of like boutique analysis tool that just does one thing and does it really well. That's where I see it all going. And that's what Observability means to me. -But with OpenTelemetry going, effectively going GA this year, with tracing, metrics and logs now stable, yes, finally, only like two years late anyway. But the fact that we have that now, the fact that we now have telemetry that has all of these correlations baked into it, you're going to start seeing a new wave of analysis tools, all the existing ones out there, but also new ones being built, that leverage the fact that this data is available and that it's like a standard data format, kind of proprietary data format, stable data format. You can rely on it. +**Constance:** What does Observability mean to me... It means like, an application owner can see what's going on in their environment, and answer pertinent questions to them about their business and how they can improve their service. Observability is an overloaded term in our days, but it means the capability of monitoring and determining when something goes wrong in your production environment. -So it's like okay to build your giant platform on top of this data or build some kind of like boutique analysis tool that just does one thing and does it really well. That's where I see it all going. And that's what Observability means to me. +### [00:09:10] Modern Observability explained -**Purvi:** What does Observability mean to me... It means like, an application owner can see what's going on in their environment and answer pertinent questions to them about their business and how they can improve their service. Observability is an overloaded term in our days, but it means the capability of monitoring and determining when something goes wrong in your production environment. +**Purvi:** Observability means...what does it mean to me? I use it as a tool to kind of making sure that things are working the way you want. It's getting insight into black boxes or even white boxes. I view it more as you kind of see things, but you have a lot more questions from it, and then you use Observability to actually figure out what's going on. So I like to call it murder mystery, usually. -**Bogdan:** Observability means... what does it mean to me? I use it as a tool to kind of making sure that things are working the way you want. It's getting insight into black boxes or even white boxes. I view it more as you kind of see things, but you have a lot more questions from it, and then you use Observability to actually figure out what's going on. So I like to call it murder mystery, usually. - -**Constance:** I like that. +**Carter:** I like that. **Purvi:** Yeah, I want to use that on a slide. -**Jacob:** That's a good question. I think... not going to be strict on a definition, I think what this really means is it is a way for us to understand what a problem... we have a problem in our system... we should be able to answer or to determine what is going wrong or what's happening. And it doesn't matter if it comes from logs or metrics or tracing, as long as we can tell and understand what's going on. I think that's when we can say we have Observability. And it's not a yes or no. It is a spectrum. I don't expect to have Observability, perfect Observability from day one, but I am expected to have some sort of telemetry that helps me understand what's going on. So I think telemetry is like a path to getting perhaps utopic place where we understand everything about our systems. +**Carter:** You should. + +**Jacob:** That's a good question. I think... not going to be strict on a definition, I think what this really means is it is a way for us to understand what a problem...we have a problem in our system...we should be able to answer or to determine what is going wrong or what's happening. And it doesn't matter if it comes from logs or metrics or tracing, as long as we can tell and understand what's going on. I think that's when we can say we have Observability. And it's not a yes or no. It is a spectrum. I don't expect to have Observability, perfect Observability from day one, but I am expected to have some sort of telemetry that helps me understand what's going on. So I think telemetry is like a path to getting perhaps a utopic place where we understand everything about our systems. -[00:10:40] **Amy:** What does Observability mean to me... I think Observability is understanding what's happening inside of your applications. Maybe what's happening in the code you care about. +**Carter:** What does Observability mean to me... I think Observability is understanding what's happening inside of your applications. Maybe what's happening in the code you care about. Yeah. Oh my goodness. It means everything. Observability is life. -**Purvi:** Yeah. +**Amy:** I think Observability means that when something goes wrong, I can ask a question about my system and get a sense of what is happening without having to know ahead of time what to expect. Like I can just go and dig into my data and my services are instrumented well enough. Not like not perfectly, but well enough that I can just figure out what happened. And can I reproduce this thing that happened in probably production off in my own environment so that I can improve my code to manage it better next time. -**Jacob:** Oh my goodness. It means everything. Observability is life. I think Observability means that when something goes wrong, I can ask a question about my system and get a sense of what is happening without having to know ahead of time what to expect. Like I can just go and dig into my data and my services are instrumented well enough. Not like not perfectly, but well enough that I can just figure out what happened. And can I reproduce this thing that happened in probably production off in my own environment so that I can improve my code to manage it better next time. +### [00:11:40] Observability as a spectrum -**Amy:** I like what you said about not instrumented perfectly. There is no such thing as perfect instrumentation. That's a lie. Just like there's no such thing as done code, right? That's also a lie. Or that the network will never break. That's a lie. +**Jacob:** I like what you said about not instrumented perfectly. There is no such thing as perfect instrumentation. That's a lie. Just like there's no such thing as done code, right? That's also a lie. Or that the network will never break. That's a lie. -**Jacob:** Oh, that's such a good question. To me, Observability, it's really about being able to get curious with your data and be able to have a lot more confidence about your production system. So being able to kind of squash things before they arrive. Testing in production is the best way to test your system because no matter what people say, Prod is always its own different animal. And if you have really good Observability, you can test in Prod. It's a much better experience for your users and for your developers too. +**Purvi:** Oh, that's such a good question. To me, Observability it's really about being able to get curious with your data and be able to have a lot more confidence about your production system. So being able to kind of squash things before they arrive. Testing in production is the best way to test your system because no matter what people say, Prod is always its own different animal. And if you have really good Observability, you can test in Prod. It's a much better experience for your users and for your developers too. -[00:12:30] **Carter:** Absolutely. When did you get involved with OpenTelemetry? +**Carter:** Absolutely. When did you get involved with OpenTelemetry? **Purvi:** I got involved in 2019, I think. -**Carter:** Oh, so like early? +**Amy:** Oh, so like early? + +**Purvi:** Early, yeah. I was not at the original meeting, but yeah, I got in really early. I really love writing Go, and so that's where I started. But I was pretty quick into the specification and started working in that space and I think it was just coming from the pain point of using have to run systems. And being that person who has woke up at 2:00 a.m. I wanted a better software solution for this and I think that I saw the value in it and I jumped in. -**Purvi:** Early, yeah, I was not at the original meeting, but yeah, I got in really early. I really love writing Go, and so that's where I started. But I was pretty quick into the specification and started working in that space and I think it was just coming from the pain point of using have to run systems. And being that person who has woke up at 2:00 a.m. I wanted a better software solution for this and I think that I saw the value in it and I jumped in. +**Constance:** We work in pain and trauma, right? -**Jacob:** We work in pain and trauma, right? +### [00:13:20] Personal experiences with OpenTelemetry **Purvi:** Yeah, exactly. When I was hired into Equinix, they hired me to instrument their entire stack for the Equinix Metal product. So that's what I worked on for my first year. This was like three years ago, before all the fancy auto-instrumentation stuff was complete, adding instrumentation to all of our systems. @@ -115,61 +125,71 @@ So it's like okay to build your giant platform on top of this data or build some **Purvi:** A little bit. The team I was working on at Microsoft, at least my org at least, was already doing a lot in the OpenTelemetry space, and that seemed to be where all the cool things were happening. And so that kind of got my interest. And then I got switched to working with a development team that was focused solely on OpenTelemetry, both for external purposes and internal purposes, because Microsoft uses OpenTelemetry really heavily internally. And so that's what got me introduced. And when I started looking around, wondering where I could start, I realized there was no real good example of how to use OpenTelemetry in the wild. And so that was a problem that I thought every vendor might have. And something we could solve together as a community, and we have. -**Amy:** Cool. How long have you been working on OpenTelemetry? +**Carter:** Cool. How long have you been working on OpenTelemetry? **Purvi:** Since the beginning. So like 2019 or like, pre? -**Jacob:** Even pre. Did you start out with Ted in the... pre days? +**Carter:** Even pre. -**Purvi:** No. Actually, there were two competing projects that merged into OpenTelemetry. +**Purvi:** Did you start out with Ted in the...pre days? -**Jacob:** Right. So I was on the other project. +**Carter:** No. Actually, there were two competing projects that merged into OpenTelemetry. -**Purvi:** Which one, OpenCensus? +**Purvi:** Right. So I was on the other project. -**Jacob:** Yeah, I got involved with OpenTelemetry through working at Honeycomb. So I got involved with it, and I have a particular interest in OpenTelemetry JavaScript, and especially the browser side of OpenTelemetry JavaScript. It's really great to be involved with it. +**Carter:** Which one, OpenCensus? + +**Purvi:** Yeah, I got involved with OpenTelemetry through working at Honeycomb. So I got involved with it, and I have a particular interest in OpenTelemetry JavaScript, and especially the browser side of OpenTelemetry JavaScript. It's really great to be involved with it. **Carter:** What's your definition of OpenTelemetry? -[00:15:00] **Jacob:** I think OpenTelemetry is, I mean, it's a standard, I think it's a collaboration across the entire Observability space. And it is, I think, a path forward for all of instrumentation. The idea that you don't have any vendor lock-in, the idea that you can just take one code base and always have some way to look into a system, I think the future of how we're going to make software better in the long term. OpenTelemetry makes my life easier, because I can integrate it with open source components that I'm using, or proprietary components. And at the end of the day, all of the OpenTelemetry flows through to my Observability vendor, and I can see traces across all of my products that I use in one space. +**Purvi:** I think OpenTelemetry is, I mean, it's a standard, I think it's a collaboration across the entire Observability space. And it is, I think, a path forward for all of instrumentation. The idea that you don't have any vendor lock-in, the idea that you can just take one code base and always have some way to look into a system, I think the future of how we're going to make software better in the long term. OpenTelemetry makes my life easier, because I can integrate it with open source components that I'm using, or proprietary components. And at the end of the day, all of the OpenTelemetry flows through to my Observability vendor, and I can see traces across all of my products that I use in one space. It is my project...of soul. -**Purvi:** It is my project... of soul. I think OpenTelemetry gets really biased, but I feel like it's a really good combination of a lot of different views finally coming together, actually making the previously hard advancements easier, like gathering the data. The hard part is actually making sense out of it. And so they're finally coming together. It's worked out pretty well in terms of collaboration to get metrics, traces, and logs. +**Purvi:** I think OpenTelemetry gets really biased, but I feel like it's a really good combination of a lot of different views finally coming together, actually making the previously hard advancements easier, like gathering the data. The hard part is actually making sense out of it. And so they're finally coming together. It's worked out pretty well in terms of collaboration to get metrics, traces, and logs. **Carter:** Oh, that's a deep question... **Purvi:** Yeah, it is. On a technical side, it means OpenTelemetry for me is a set of tools that would help me get telemetry data out of my typical application. Sometimes also infra. But OpenTelemetry really is the tool that I can use in a vendor neutral way, get data out of my application so that I can get into that uptopic thing. If I had perfect instrumentation, then I can get into a uptopic place. But OpenTelemetry provides me the tools that I need to gradually get into that. Now, it does stop at a very specific place, in a sense, which is as soon as you send data out, that's where OpenTelemetry stops. That's where you get to the vendor or to the open source tools that provide the database, visualization tools, and so on. But a more deeper aspect, OpenTelemetry is where I have my colleagues, people that I work on a daily basis for a few years. -**Jacob:** Yeah. That's what OpenTelemetry is for me. What does it mean? What's the concept? OpenTelemetry is Observability backed by everybody. It's not a single vendor. It's letting you do the thing that is agnostic to where you send the data. In the same way that you don't have to relearn how to drive a car every time you step into a new car. You don't have to learn how to ride a bike based on the vendor of the bike that you buy from. You should be able to instrument your code no matter where you send that data. So that's how I sell it. That's how I think about it. +**Purvi:** Yeah. That's what OpenTelemetry is for me. + +**Carter:** What does it mean? What's the concept? + +**Purvi:** OpenTelemetry is Observability backed by everybody. It's not a single vendor. It's letting you do the thing that is agnostic to where you send the data. In the same way that you don't have to relearn how to drive a car every time you step into a new car. You don't have to learn how to ride a bike based on the vendor of the bike that you buy from. You should be able to instrument your code no matter where you send that data. So that's how I sell it. That's how I think about it. -**Carter:** Awesome. The other benefit is we as the maintainers, we have a lot of our maintainers here and approvers here, so we can collaborate and work together to figure out what's really needed in the next coming months. +### [00:18:20] Community collaboration in OpenTelemetry -[00:18:42] **Purvi:** I described it almost like summer camp. There are some people where, oh, I haven't seen you in a few months. How you been? It's catching up. Like, I mean, OTel has been amazing. The project itself has been wonderful. It's one of the first projects to take a bunch of standards and condense them down into less standards. We took OpenCensus, OpenTracing, we brought Prometheus to the table. The Elastic Cache format is there. OpenTelemetry just is a wonderful community, that's all. Trying to make things better in the Observability landscape by working across vendor boundaries, which has just been something that I've never done in the past. I've never worked in an open source project where so many vendors are involved and so many end user communities are involved, and it's been great. +**Jacob:** Awesome. The other benefit is we as the maintainers, we have a lot of our maintainers here and approvers here, so we can collaborate and work together to figure out what's really needed in the next coming months. I described it almost like summer camp. There are some people where, oh, I haven't seen you in a few months. How you been? It's catching up. Like, I mean, OTel has been amazing. The project itself has been wonderful. It's one of the first projects to take a bunch of standards and condense them down into less standards. We took OpenCensus, OpenTracing, we brought Prometheus to the table. The Elastic Cache format is there. OpenTelemetry just is a wonderful community, that's all. Trying to make things better in the Observability landscape by working across vendor boundaries, which has just been something that I've never done in the past. I've never worked in an open source project where so many vendors are involved and so many end user communities are involved, and it's been great. -**Carter:** Yeah. And that's what I like personally about OpenTelemetry, because everyone plays nice and I feel like it's a very deliberate, "No, we are not going to favor one vendor over another." And if a vendor tried to showboat, then it's pretty much shut down, which I think is great. We've just had a lot of really good folks at all the levels of the project trying to push everybody in the right direction, which I really appreciate. +**Carter:** Yeah. And that's what I like personally about OpenTelemetry, because everyone plays nice and I feel like it's a very deliberate, "No, we are not going to favor one vendor over another." And if a vendor tried to showboat, then it's pretty much shut down, which I think is great. We've just had a lot of really good folks at all the levels of the project trying to push everybody in the right direction, which I really appreciate. I'm going to give a shout out to Ted Young for, especially being one of those people that always just, he's over there somewhere. I can see him just like looking around, see, waving his head. He has no idea we're talking about him. -**Purvi:** I'm going to give a shout out to Ted Young for, especially being one of those people that always just, he's over there somewhere. I can see him just like looking around, see, waving his head. He has no idea we're talking about him. +**Purvi:** OpenTelemetry to me is really all about the community. Like communities being able to take ownership of their own telemetry data, because vendors should not be determining the type of telemetry data that gets sent to your systems. Because Observability about your system is so personal to your system. And when you have vendor lock in or lock in through, like, the instrumentation of vendors, it can be very limiting. -[00:20:30] **Jacob:** OpenTelemetry to me is really all about the community. Like communities being able to take ownership of their own telemetry data, because vendors should not be determining the type of telemetry data that gets sent to your systems. Because Observability about your system is so personal to your system. And when you have vendor lock in or lock in through, like, the instrumentation of vendors, it can be very limiting. +**Carter:** What is your favorite telemetry signal? -**Purvi:** What is your favorite telemetry signal? +### [00:20:50] Discussion on telemetry signals -**Carter:** That's a good question. I wish I had a good, nuanced answer there. Like, I don't know... metrics, I've known for the longest, I guess. But I think traces are probably a little closer because you get a lot more depth into operational behavior. So, yeah, I think I'd probably go with traces. It's also a little bit more automatic for you. You really have to understand what those metrics are and build them into something. Versus tracing, can show you based on just the structures they come with. So, yeah, I'll go traces. +**Purvi:** That's a good question. I wish I had a good, nuanced answer there. Like, I don't know...metrics, I've known for the longest, I guess. But I think traces are probably a little closer because you get a lot more depth into operational behavior. So, yeah, I think I'd probably go with traces. It's also a little bit more automatic for you. You really have to understand what those metrics are and build them into something. Versus tracing, can show you based on just the structures they come with. So, yeah, I'll go traces. **Amy:** Oh, it's traces, of course. **Purvi:** My favorite signal... Probably be the Bat Signal. If that thing could go on every time a system goes down, that would be. -**Jacob:** So I think I've heard this reference around... and I truly believe it. Traces are just the cooler version of logs. Like, it's like logs with a mustache, and maybe a top hat. Because essentially a span is just a log, but a trace-correlated log. So I'd probably say traces, but my backup answer is log. +**Constance:** So I think I've heard this reference around...and I truly believe it. Traces are just the cooler version of logs. Like, it's like logs with a mustache, and maybe a top hat. Because essentially a span is just a log, but a trace-correlated log. So I'd probably say traces, but my backup answer is log. + +**Carter:** Signal? I think the most... I like metrics. I think we did try to change the way how metrics were done before, and we may not have been the most successful yet, but we are getting there. But it was a necessary change, and I feel like it changed something in the way how things were done. For tracing, I mean, we didn't change too much from other Dapr paper or other things, but for metrics, I think we changed. + +### [00:22:30] Lighthearted debate on favorite telemetry signals -**Carter:** Signal? I think the most... I like metrics. I think we did try to change the way how metrics were done before, and we may not have been the most successful yet, but we are getting there. But it was a necessary change, and I feel like it changed something in the way how things were done. For tracing, I mean, we didn't change too much from other Dapr paper or other things, but for metrics, I think we changed. +**Purvi:** I love traces, especially because you could... My favorite example is when I used to do...when I was at Lyft and I would get paged in the middle of the night... One service, four deep.. everything was going... Everything between that and the front was getting paged. You were able to actually figure out, like, okay, this one's the cause. Instead of overly thinking about it. That's what I love about it. It's very different paradigm than what we're used to talking about. -**Purvi:** I love traces, especially because you could... My favorite example is when I used to do... when I was at Lyft and I would get paged in the middle of the night... One service, four deep... everything was going... Everything between that and the front was getting paged. You were able to actually figure out, like, okay, this one's the cause. Instead of overly thinking about it. That's what I love about it. It's very different paradigm than what we're used to talking about. +**Jacob:** Trace. Come on. They're beautiful. -[00:23:10] **Jacob:** Trace. Come on. They're beautiful. No, it is. Traces. Traces number one. They are the easiest to work with. They are so simple to get started, and they're just so much more useful than anything else. So traces all the way. +**Purvi:** No, it is. Traces. Traces number one. They are the easiest to work with. They are so simple to get started, and they're just so much more useful than anything else. So traces all the way. -**Purvi:** Traces, because it's clearly the elegant log, but also you can just get metrics out of it. It has, like, everything you need in a signal. It's metrics and logs correlated with context. It's beautiful. They're magic. +**Juraci:** Traces, because it's clearly the elegant log, but also you can just get metrics out of it. It has, like, everything you need in a signal. It's metrics and logs correlated with context. It's beautiful. They're magic. -**Carter:** Oh, that's easy. It's tracing. +**Purvi:** Oh, that's easy. It's tracing. ## Raw YouTube Transcript diff --git a/video-transcripts/transcripts/2024-01-19T04:34:21Z-observability-panel-with-charity-majors-amy-tobey-and-adriana-villela.md b/video-transcripts/transcripts/2024-01-19T04:34:21Z-observability-panel-with-charity-majors-amy-tobey-and-adriana-villela.md index 1fd547b..5c419e9 100644 --- a/video-transcripts/transcripts/2024-01-19T04:34:21Z-observability-panel-with-charity-majors-amy-tobey-and-adriana-villela.md +++ b/video-transcripts/transcripts/2024-01-19T04:34:21Z-observability-panel-with-charity-majors-amy-tobey-and-adriana-villela.md @@ -10,358 +10,318 @@ URL: https://www.youtube.com/watch?v=Fuy3W5bro9k ## Summary -The YouTube panel discussion focuses on the topic of observability, featuring experts Adriana Vela from ServiceNow, Charity Majors from Honeycomb, and Amy Toby from Equinix. The conversation delves into the definitions of observability, with Charity emphasizing the shift from a traditional metric- and log-based approach (observability 1.0) to a more integrated and explorative method (observability 2.0). Key points include the challenges of data overload, the need for better communication among teams regarding observability, and the importance of understanding the context of data collected. The panelists also discuss the role of AI in enhancing observability, stressing that while AI can assist in data correlation and reduce cognitive load, human insight remains crucial for addressing complex, novel issues. The discussion concludes with reflections on the future of observability and the importance of fostering a culture that values continuous feedback and understanding in software development. +In this YouTube panel discussion, titled "Observability: Understanding and Advancing the Community," host Adriana Vela, along with panelists Charity Majors and Amy Toby, delve into the intricacies of observability in software development. The conversation covers the evolution of observability from traditional metrics to the more advanced, integrated observability 2.0, emphasizing the importance of meaningful data and the continuous engagement of developers in monitoring their systems. Key points discussed include the challenges of adopting observability practices, the role of AI in augmenting human capabilities, and the need for organizations to understand the value of observability as an integral part of the software development lifecycle. The panelists express a shared wish for increased awareness and integration of observability principles across all roles in tech, aiming for a more holistic approach that enhances understanding and collaboration within engineering teams. ## Chapters -00:00:00 Welcome and introductions -00:03:00 Definition of observability -00:05:22 Observability 1.0 vs 2.0 -00:10:30 Meaningful data in observability -00:13:40 Challenges with open Telemetry -00:18:00 Transitioning to observability 2.0 -00:22:10 Importance of feedback loops -00:27:30 AI's role in observability -00:34:00 Current challenges in observability -00:40:00 Misconceptions about observability +00:00:00 Introductions +00:01:22 Guest introduction: Charity +00:01:58 Guest introduction: Amy +00:03:44 Discussion on definition of observability +00:06:12 Observability 1.0 vs 2.0 +00:14:00 Importance of alerting when users are impacted +00:19:36 Discussion on developer engagement +00:28:56 AI's role in observability +00:42:00 Challenges of observability in organizations +00:50:24 Future wishes for observability -**Host:** Here we are. Hello, hello, hello everyone! We're very excited to have a panel today around the topic of observability. Plan for the observability community. Here with us today we have Adriana, Amy, and Charity joining us. We hope to have a fun, billed conversation. Would you mind starting out, Adriana, and introducing yourself? Who you are and what you do? +## Transcript -**Adriana:** Oh yeah, so I am Adriana Vela. I work with Anna at ServiceNow Cloud Observability, formerly known as Lightstep. Say that three times fast. I, uh, yeah, I'm a senior staff developer advocate. I've been doing the dev rel thing for, what, almost two years? But before that, I was oscillating between, like, IC engineering and management roles. So yay! And I love observability. +### [00:00:00] Introductions -**Charity:** I knew you before you were a dev rel, before you were in observability! +**Speaker 1:** Here we are. Hello, hello, hello everyone! We're very excited to have a panel today around the topic of observability. A plan for the observability community. Here with us today we have Adriana, Amy, and Charity joining us. We hope to have a fun build conversation. Would you mind starting out, Adriana, and introducing yourself, who you are and what you do? -**Adriana:** Yeah! Yeah, when I was just, like, trying to figure out what the hell this stuff was all about. Time definitely has passed by. +**Adriana:** Oh yeah, so I am Adriana Vela. I work with Anna at ServiceNow Cloud Observability, formerly known as Lightstep. Say that three times fast. Um, I uh, yeah, I'm a senior staff developer advocate. Um, been doing the dev rel thing for what, almost two years? But before that was, you know, oscillating between like IC engineering and management roles. So yay! And I love observability. -**Charity:** What about you? +**Charity:** I knew you before you were a dev rel, before you were in observability. -**Charity:** I'm Charity Majors, co-founder and CTO of Honeycomb.io. I identify as an Ops engineer and probably always will, even though it's been a long time since I held the pager. But I feel like I started, I feel like I picked up my first pager rotation when I was 19, so I feel like I've got still like half my life on call. So I still, I can still call myself an operations engineer. +### [00:01:22] Guest introduction: Charity -**Adriana:** You paid your dues! +**Adriana:** Yeah, yeah, when I was just like trying to figure out what the hell this stuff was all about. Time definitely has passed by. Charity, what about you? -**Charity:** I think so. What about you, Amy? +**Charity:** Um, Charity Major, co-founder and CTO of honeycomb.io. I identify as an Ops engineer and probably always will, even though it's been a long time since I held the pager. But I feel like I started, I feel like I picked up my first page of rotation when I was 19. So I feel like I've got still like half my life on call. So I still, I can still call myself an operations engineer. -**Amy:** Hi, I'm Amy Toby. I am a senior principal engineer at Equinix, which is the largest data center company in the world. I've been doing observability since pretty much my first job as well, starting back on Solara Systems in, like, '98, '99. And then into NIO stuff. I'm responsible for some things in NIO, I'm sorry. And, like, I have been doing it all along. These days I do a little bit more kind of work in the soot technical space, but initial work at Equinix was bringing OpenTelemetry into our entire Equinix Metal operational stack and pumping those metrics out to Honeycomb. +**Adriana:** You paid your dues. -**Adriana:** Nice! Very exciting work that all of you are doing and creating community around observability and OpenTelemetry. And I get to call you all friends, which is super exciting to have you all today on the panel. +**Charity:** I think so. What about you, Amy? -[00:03:00] **Host:** Sweet! So, I think we're going to start with one of the spicy questions, and I'm going to kick it off with Charity. What is your definition of observability? The correct one, of course. +### [00:01:58] Guest introduction: Amy -**Charity:** We'll see. Yes, spice! You know, okay, so back in the days when nobody else gave a [__] about observability, the Halcyon days of... I'm saying that even though we almost went out of business many times. You know, I think we really tried to push this idea that there was a technical definition for observability. You know, that it was high cardinality, high dimensionality, explorability, and all these things. And nowadays, you know, like everyone and their cat does observability. It's like if you have any telemetry, you do observability. And you know what? Actually, I’ve come around to this. I'm all right with this. I feel like it's actually a great definition to say that observability is a property of a socio-technical system. So it exists in a continuum, right? +**Amy:** Hi, I'm Amy Toby. I am a senior principal engineer at Equinix, which is the largest data center company in the world. Um, and I've been doing observability since pretty much my first job as well, starting back on Solara Systems in like '98, '99. Um, S&M and then into NAIO stuff. I'm responsible for some things in NAIOs, I'm sorry. Um, and like it, I've been doing it all along. These days I do a little bit more kind of work in the soot technical space, but initial work at Equinix was bringing open telemetry into our entire Equinix metal operational stack and pumping those metrics out to Honeycomb. -I'm totally down with that because I think that it gives a lot of grace to people who are, you know, in the early days of their journey. Like, I don't want to be the person who, like, "Well, actually, you're not really doing observability." Nobody likes that guy, so I don't want to be that guy. +**Adriana:** Nice! Very exciting work that all of you are doing and creating community around observability and open telemetry. And I get to call you all friends, which is super exciting to have you all today on the panel. Sweet! So I think we're going to start with one of the spicy questions and I'm going to kick it off with Charity. What is your definition of observability? The correct one, of course. -That said, I do think that there's a real kind of step function difference in, let's call it observability 1.0, that built off the primitives of, you know, metrics, logs, and traces. Where every time you gather the data, you have to gather it again in a different format and pay for it again. So like you're gathering it once for metrics, again for logs, again for traces, again for APM, again for profiling, again for security. You know, like talk about a cost crisis in observability tooling. Like, no wonder, right? And you're very limited in each one of those by the format that you happen to gather in, and none of them can be connected to each other. The only thing that connects them is the poor human sitting in the middle guessing, right? Like guessing eyeballs, you know? +### [00:03:44] Discussion on definition of observability -So let's call that 1.0, and then observability 2.0 tooling, I think, is based on a single source of truth. Right? These arbitrarily wide structured data blobs you could, you can derive metrics, you can derive traces, you can derive all these things, but you've got one source of truth. It handles high cardinality, it handles, you know, it's a more explorable sort of interface. So you don't have these static dashboards; you have an interface where you can ask questions and follow breadcrumbs. Go, and then what? And then what? +**Charity:** We'll see. Yes, spice! You know, okay. So back in the days when nobody else gave a [__] about observability, the halcyon days of... I'm saying that even though we almost went out of business many times, you know, I think we really tried to push this idea that there was a technical definition for observability. You know, that it was high cardinality, high dimensionality, explorability, and all these things. And nowadays, you know, like everyone and their cat does observability. It's like if you have any telemetry, you do observability. And you know what? Actually, I've come around to this. I'm all right with this. I feel like it's actually a great definition to say that observability is a property of a sociotechnical system. So it exists in a continuum, right? -[00:05:22] So that's my very long-winded answer. I think that there's kind of two generations of observability tooling out there right now, and I think that they lead to two very different outcomes. Like one of the, I know I'm taking a very long time here, I'm almost done, I promise. I think that one of the most, one of the soot technical factors that you really associate with observability 1.0 is it's like a checklist thing. You know, you add it at the end before you put stuff in production. Like, cool, I can monitor this. +I'm totally down with that because I think that it gives a lot of grace to people who are, you know, in the early days of their journey. Like, I don't want to be the person who like, well actually, you're not really doing observability. Nobody likes that guy, so I don't want to be that guy. Um, that said, I do think that there's a real kind of step function difference in, let's call it observability 1.0, that built off the primitives of, you know, metrics, logs, and traces. Um, where every time you gather the data, you have to gather again in a different format and pay for it again. -And for observability 2.0, I think like the fundamental truth of it is it underlies the entire software development life cycle, and your ability to hook up these fast feedback loops rests on how well you can, you know, ask questions and explore your data and have this sort of constant conversation going with your code. +So like you're gathering it once for metrics, again for logs, again for traces, again for APM, again for profiling, again for security. You know, like talk about a cost crisis in observability tooling. Like no wonder, right? And you're very limited in each one of those by the format that you happen to gather in, and none of them can be connected to each other. The only thing that connects them is the poor human sitting in the middle guessing, right? Like guessing eyeball us, you know? So let's call that 1.0. And then, observability 2.0 tooling, I think, is based on a single source of truth. Right? These arbitrarily wide structured data blobs you could derive metrics from, you can derive traces, you can derive all these things, but you've got one source of truth. It handles high cardinality. It handles, you know, it's a more explorable sort of interface. -**Amy:** Awesome! What about you, Amy? +So you don't have these static dashboards. You have an interface where you can ask questions and follow breadcrumbs, go, "And then what? And then what?" Um, so that's my very long-winded answer. I think that there's kind of two generations of observability tooling out there right now, and I think that they lead to two very different outcomes. Like one of the... I know I'm taking a very long time here. I'm almost done, I promise. I think that one of the most... one of the soot technical factors that you really associate with observability 1.0 is it's for... it's like a checklist thing. You know, you add it at the end before you put stuff in production. Like, cool, I can monitor this. Um, and for observability 2.0, I think like the fundamental truth of it is it underlies the entire software development lifecycle and your ability to hook up these fast feedback loops rests on how well you can, you know, ask questions and explore your data and have this sort of constant conversation going with your code. -**Amy:** I think I agree with most of what Charity said, but I've been learning a lot from my work lately in kind of developing a new networking product, and one of the feature sets that we have to include in that today in a modern Network as a Service kind of product is observability features, and customers demand this. And we've been scratching at this, like, what, how much do they need? Like, what, how much should we build? How much should we hand off to either, you know, ServiceNow or Honeycomb and let those tools do what they're best at? +**Adriana:** Awesome! What about you, Amy? -And so what we've been finding with customers is, like, that operational part that Charity mentioned, right? That they are tired of this world where they have to go hire very expensive people who are often cranky, who can basically carry that mental burden of tying all the context together with the metrics and divining what the heck is going on with the network. And so what they're asking and what they're demanding from us and all of the other cloud vendors is how do we get the observability telemetry, which is mostly what they ask for? Because when we talk to customers, like, most of the time when folks we're talking to don't want to talk about cardinality. +### [00:06:12] Observability 1.0 vs 2.0 -So they're asking for, like, how do we get all the stuff we need so we can feed it to our AI Ops tools, feed it to our other observability systems that we already have, so that we can start to leverage the skills we already have to get those insights? But it always drives to how do I get insights about this hyper-complex system so that we can fix it and get back to business? And sometimes folks don't even want to do the heavy lifting. They're just like, do it for me! Nobody does, right? Because almost none of our customers, any of us here, their primary business is not telemetry. It's undifferentiated lifting. They want, they got a business to run; they don't want to be messing with this stuff. So they want me to provide really high-quality telemetry, and then they want you all to make that all super easy to consume. +**Amy:** I think I agree with most of what Charity said, but I've been learning a lot from my work lately in kind of developing a new networking product. And one of the feature sets that we have to include in that today in a modern network-as-a-service kind of product is observability features. And customers demand this. And we've been scratching at this, like, what, how much do they need? Like, how much should we build? How much should we hand off to either, you know, ServiceNow or Honeycomb and let those tools do what they're best at? And so what we've been finding with customers is like that operational part that Charity mentioned, right? That they are tired of this world where they have to go hire very expensive people who are often cranky, um, who can basically carry that mental burden of tying all the context together with the metrics and divining what the heck is going on with the network. -**Adriana:** Yeah, that's very true. I love that. It makes so much sense. What is your definition, Adriana? +And so what they're asking and what they're demanding from us and all of the other cloud vendors is how... give us the observability telemetry, which is mostly what they ask for, because when we talk to customers, like, most of the time when folks we’re talking to don't want to talk about cardinality. Um, so they're asking for like, how do we get all the stuff we need so we can feed it to our... my feelings about AIOps aside, but this is a common request: feed it to our AIOps tools, feed it to our other observability systems that we already have so that we can start to leverage the skills we already have to get those insights. -**Adriana:** So I think I started out with, you know, Charity's OG definition of observability, which carried me well throughout the years. And then I heard one recently from Hazel Weekly, which I quite like, which is, "Observability is the ability for you to ask questions and get meaningful answers," which I really like because it's, you know, it's up to you as to what is meaningful, right? So what data are you collecting that's meaningful to you? And I think that's really important in our world. +But it always drives to how do I get insights about this hyper-complex system so that we can fix it and get back to business. And some... sometimes folks don't even want to do the heavy lifting. They're just like, do it for... nobody does, right? Because like almost none of our customers, any of us here, their primary business is not telemetry. It's undifferentiated lift. They want... they got a business to run, they don't want to be messing with this stuff. So they want me to provide really high-quality telemetry, um, and then they want you all to make that all super easy to consume. -I will still say that I think trace-first for observability no matter what. But I think, like, we really need to focus on the meaningful data aspect of observability because, like, sure, you can, like, instrument the crap out of a system, and it's emitting a bunch of useless garbage, right? You know, garbage in, garbage out. So if you don't have good data that you're instrumenting well, good luck trying to figure out what the hell is going on. You can have the best observability tool out there; it won't do you any good. +**Adriana:** Yeah, that's very true. I love that. It makes so much sense. What is your definition, Ad? -So you're going to need, um, to have... It takes a bit of, you know, there's that learning curve of what's important to you, what should you be collecting to be able to do the thing. Observability is a design problem, like in a huge way. Like, once you've s... Like, I feel like we spent a long time, like, just sort of like figuring out the underlying plumbing, which is just a prerequisite to what really matters. +**Adriana:** So I think I started out with, you know, Charity's OG definition of observability, which carried me well throughout the years. And then I heard one recently from Hazel Weekly, which I quite like, which is observability is the ability for you to ask questions and get meaningful answers. Um, which I really like because it's, you know, it's up to you as to what is meaningful, right? So what data are you collecting that's meaningful to you? And I think that's really important, um, in our world. Um, I will still say that I think like trace-first for observability no matter what. -[00:10:30] This is a design problem because there's so much data, and I think Amy and I share very similar thoughts on AI Ops. But, like, there's two kinds of tools, right? There are tools that pave over complexity, and there are tools that try to help equip you to understand it. And I will always be in the second camp because at the end of the day, we are legally liable for the software that we put out into the world. Somebody somewhere is going to have to understand that [__], and the harder you've made it, the more magical you've made it, the harder that day of reckoning is going to be. +Um, but I think like we really need to focus on the meaningful data aspect of observability because like, sure, you can like instrument the crap out of a system, and it's emitting a bunch of useless garbage, right? You know, garbage in, garbage out. So you don't have good data that you're instrumenting, well, good luck trying to figure out what the hell is going on. You can have the best observability tool out there, um, it won't do you any good. So you're going to need, um, it takes a bit of, you know, there's that learning curve of what's important to you, what should you be collecting to be able to do the thing. -**Charity:** Definitely interestingly enough, I think we're seeing that a little bit with open telemetry, right? OpenTelemetry is awesome. It's, you know, this awesome standard that pretty much everyone has opted vendor-wise. But then now we're looking beyond, like, that initial adoption of OpenTelemetry and, like, people really using it out in the field and making sure. I think OpenTelemetry's biggest challenge, I think, in the near future is going to be making sure that it's something that is easy for onboarding because I think that can be really off-putting for organizations, and that can be really scary. +Observability is a design problem, like in a huge way. Like, once you've s... like I feel like we spent a long time like just sort of like figuring out the underlying plumbing, which is just a prerequisite to what really matters. This is a design problem because there's so much data. And I think Amy and I share very similar thoughts on AIOps and like, but like there's two kinds of tools, right? There are tools that, um, that pave over complexity, and there are tools that try to help equip you to understand it. And I will always be in the second camp because at the end of the day, we are legally liable for the software that we put out into the world. Somebody somewhere is going to have to understand that [__], and the harder you've made it, the more magical you've made it, um, the harder that day of reckoning is going to be. -And then that leads to the, you know, "You instrument code? I don't want to instrument my own code!" kind of mentality. I think that the place that the metrics go really matters, though. Because we talked about AI Ops, where I see folks really reaching for these tools. Charity called it paving over. So what we've done in the network world for decades now, and it's still the case at most organizations, is we have all these devices generating gobs of alerts and metrics. Nobody knows what any of it's doing. There's a sense that we should be able to correlate all these events. +Definitely interestingly enough, I think we're seeing that a little bit, um, with like open telemetry, right? Like open telemetry is awesome. It's, you know, this awesome standard, uh, that pretty much everyone has opted vendor-wise. Um, but then now we're looking beyond like that initial adoption of open telemetry and like people really using it out in the field and, and making sure. I think open telemetry's biggest challenge, I think in the near future, is going to be making sure that it's something that is easy for onboarding because I think that can be really off-putting, um, for organizations. And that can be really scary. And then that leads to the, you know, you instrument code, I don't want to instrument my own code kind of mentality. -So, like, some port fails on a core router, and all the stuff behind it disappears. We should be able to roll all that up, and I'm using the "should" word the way my therapist calls it, the "S" word, right? Like, we believe these things that should happen, but the real world is much harder than that. So, like, these alerts just flood through, and people fight these alerts. Sometimes they've been fighting their whole career trying to get on top of this alert stream, and so they reach for these tools to make sense of the morass that they built for themselves because they're not really empowered to go back to the telemetry and, like, dial back the cardinality or re-transform how they generate those metrics to get higher, you know, higher cardinality that they can process at the end. +I think that the place that the metrics go really matters though because we talked about AIOps, where I see folks really reaching for these tools. Um, Charity called it paving over. So what we've done in the network world for decades now, and it's still the case at most organizations, is we have all these devices generating gobs of alerts and metrics. Nobody knows what any of it's doing. There's a sense that we should be able to correlate all these events. So like some port fails on a core router, all the stuff behind it disappears. We should be able to roll all that up. And I'm using the "should" word the way my therapist calls it, the "s" word, right? -Often they don't even have choices about what to collect because they're buying network devices that offer you get your SNMP or your G, and that's what you get. Oh my God, it's a world out there of legions of people that are just fighting through the noise still. And it hasn't changed for them. Whereas in the software world, we keep advancing the state of the art, but it's going to be a challenge still to bring a lot of the classic sysadmin and network administrators who are used to this world along on that journey. +Like we believe these things that should happen, but the real world is much harder than that. So like these alerts just flood through, and people fight these alerts. Sometimes they've been fighting their whole career trying to get on top of this alert stream. And so they reach for these tools to make sense of the morass that they built for themselves because they're not really empowered to go back to the telemetry and like dial back the cardinality or re-transform how they generate those metrics to get higher, you know, higher cardinality that they can process at the end. -[00:13:40] I feel like one of the characteristics that I often think about when we're talking about, like, transitioning from the 1.0 world to the 2.0 world, which is going to take [__] forever, no doubt. But I feel like it's part of it is generating from a push world to a pull world, where we're used to, like, the only way you could make sense of, like, a NaaS-based system is by looking at the cluster of things that just alerted you and going, like, "Ah, it's probably that." +Often they don't even have choices about what telemetry they get because they're buying network devices that offer, you get your SNMP or your G, and that's what you get. Oh my God, yeah. Oh God, world out there of legions of people that are just fighting through the noise still. Um, and it hasn't changed for them. Whereas they're in the software world, we keep advancing the state-of-the-art, but it's going to be a challenge still to bring a lot of the classic sysadmins and network administrators who are used to this world along on that journey. -And so you rely on, you know, all of these alerts, but that's so noisy, and it does not scale. Right? But, like, as an ops person, I grew up thinking you only look at production when you get alerted. You don't have to look at it otherwise. And I feel like the trade-off that we have to learn to make is, okay, we're only going to alert you when users are impacted, right? We set SLOs, these are our agreements with ourselves, each other, and the world. This is the level of service we are going to give you, and we alert when the outage is bad enough that users are being affected. +I feel like one of the characteristics that I often think about when we're talking about transitioning from the 1.0 world to the 2.0 world, which is going to take [__] forever, no doubt. But I feel like it's part of it is generating from a push world to a pull world, where we're used to, like the only way you could make sense of like a Naous-based system is by looking at the cluster of things that just alerted you and go like, "Ah, it's probably that," right? And so you rely on, you know, all of these alerts, but that's so noisy and it does not scale, right? -But in order to get to that world, you have to agree that you're going to go look at your telemetry affirmatively because most problems will never rise to the level of waking you up! God, I mean, I hope, right? Like, most of the things, most of the codes you write have subtle bugs that affect a few people, or it's a small thing, right? +### [00:14:00] Importance of alerting when users are impacted -So, like, in order to have a system, in order to have a hygienic system that is not just like a hairball that the cat coughed up, you know, we can't just keep... like in the old days, we were shipping code every day that we didn't really understand these systems. We've never really understood, and then we wonder why it's a giant trash fire, right? Like, and the way that we start improving that is by, you know, not just making the telemetry richer and better, but also by committing to closing that loop. +But like, as an ops person, I grew up thinking you only look at production when you get alerted. You don't have to look at it otherwise. And I feel like, you know, the trade-off that we have to learn to make is okay, we're only going to alert you when users are impacted, right? We set SLOs. These are our agreements with ourselves, each other, and the world. This is the level of service we are going to give you, and we alert when user... when the outage is bad enough that users are being affected. But in order to get to that world, you have to agree that you're going to go look at your telemetry affirmatively because most problems will never rise to the level of waking you up. -As a software engineer, my job is not done when the tests pass. My job is done when I have looked at my telemetry in production and gone, "It's doing what I expect it to do. Nothing else looks weird," right? So, like, that's asking... it's like I like the technical debt metaphor here because it's asking you to, like, put in a little bit of effort upfront in order to avoid the... you know, there's this great graph that shows that the cost of finding and fixing bugs goes up exponentially from the moment that you write them. +God, I mean, I hope, right? Like most of the things, most of the codes you write has subtle bugs that affect a few people, or it's a small thing, right? So like in order to have a system, in order to have a hygienic system that is not just like a hairball that the cat coughed up, you know, we can't just keep like in the old days, like we were shipping code every day that we didn't really understand these systems. We've never really understood, and then we wonder why it's a giant trash fire, right? -So, like, you backspace? Good for you, good job, right? You find it in your test? Good for you! The next best time to find it is right after you deployed that [__]. After that, God knows how long it's going to take: days, months, years, who knows? But, like, the accumulation of that [__] is what makes this job a nightmare for so many people. +Like, and the way that we start improving that is by, you know, not just making, you know, the telemetry richer and better, but also by committing to closing that loop. As a software engineer, my job is not done when the tests pass. My job is done when I have looked at my telemetry in production and gone, "It's doing what I expect it to do. Nothing else looks weird," right? -**Charity:** Yeah, it's so true. Like, I think back to, like, you know, my old days of having to hand off deployment instructions to, like, our ops folks who, they had no [__] clue what was, you know, what it was they were deploying. And, you know, I had to pray that my instructions were correct, and then I had to pray that they read my instructions correctly, and then they would deploy it to prod. And these quote-unquote successful deployment-like things went correctly, was considered a successful project. The machine, we're done, right? +So like, that's asking... it's like I like the technical debt metaphor here because it's asking you to like put in a little bit of effort up front in order to avoid the, you know, there's this great graph that shows that the cost of finding and fixing bugs goes up exponentially from the moment that you write them. So like you backspace, good for you, good job, right? You find it in your test, good for you. The next best time to find it is right after you deployed that [__]. After that, God knows how long it's going to take—days, months, years, who knows? But like the accumulation of that [__] is what makes this job a nightmare for so many people. -**Adriana:** Right, yeah. +**Amy:** Yeah, it's so true. Like, I think back to like, you know, my old days of having to hand off deployment instructions to like our ops folks, who then they had no [__] clue what was, you know, like what it was they were deploying. And, you know, I had to pray that my instructions were correct, and then I had to pray that they read my instructions correctly, and then they would deploy it to prod. And these quote unquote successful deployment-like things went correctly and were considered a successful project. The machine, we're done! -**Charity:** And so, I love the idea of, like, no! Like, because otherwise it becomes a hit-and-run deployment, right? You know, or drive-by deployment, rather, is a better way of putting it, where, like, you know, it's like, "Deployed! All right, great! Let's just leave it to fate." But I think, like, you know, paying attention to what's going on as soon as you deploy, like, there's something to be said because otherwise, because you, as the person writing the code, you have context that nobody else in the world has or will ever have. +**Charity:** Right, right! -You know exactly what you're trying to do, why you're doing it, what you tried that worked, what didn't work, what the variables were named, what the [__]. You know all this stuff, and if you can close that loop before you have to concept switch another, it's so powerful. Like, these feedback loops are what socio-technical systems are all about. +**Amy:** Yeah, and so I love the idea of like, no, like because otherwise it becomes a hit-and-run deployment, right? You know, or drive-by deployment, rather, is a better way of putting it, where like, you know, it's like deployed, "Alright, great! Um, let's just leave it to fate!" -Like, this is the one job of technical leadership, in my opinion, whether you're an IC or a director or VP. One job is these feedback loops to make them as tight and short and functional as possible. I think that, like, nailed down of, like, engineering responsibility has been so important, and observability really brings it to a front. But a lot of people are just not paying enough attention or, like, not willing to do it. They're just like, "My job is to code, and that's it, and maybe I'm on call twice a year, but it's someone else's problem." +**Charity:** But I think like, you know, paying attention to what's going on as soon as you deploy, like there's something to be said because otherwise, because you, as the person writing the code, you have context that nobody else in the world has or will ever have. You know exactly what you're trying to do, why you're doing it, what you tried that worked, what didn't work, what the variables were named, what the [__]—you know all this stuff. And if you can close that loop before you have to context switch to another, it's so powerful. -[00:18:00] **Amy:** There's not much we can do about that, though, like, a lot of times, right? Like, if you are an engineering leader and you're trying to build a dynamic, powerful engineering organization, you have to either lead those people to care or lead them out, right? Like, because, like, the reality is, is like, there are the customers that y'all want that have that engineering leadership and culture of, like, giving a crap. +Like these feedback loops are what sociotechnical systems are all about. Like this is the one job of technical leadership, in my opinion, whether you're an IC or a director or VP. One job is these feedback loops—to make them as tight and short and functional as possible. I think that like nailed down of like engineering responsibility has been so important, and observability really brings it to the front. But a lot of people are just not paying enough attention or like not willing to do it. They're just like, "My job is to code and that's it, and maybe I'm on call twice a year, but it's someone else's problem." -And then there's the vast majority of organizations out there where most of the people who are writing the code that runs our world are trying to put bread on their table. And so as soon as they are done with the specification that they were given for their job, they e that code into Git, and they run because, like, why should they care? Why should they metrics? But this is a self-fulfilling prophecy because you know what makes people, like, check out, tune the [__] out? Not being connected to the results of their labor! +**Adriana:** There's not much we can do about that though, like a lot of times, right? Like, if you are an engineering leader and you're trying to build a dynamic, powerful engineering organization, you have to either lead those people to care or lead them out, right? -Like, there's something that's, like, what is it? Dan Pink says we all want in our job is autonomy, mastery, and purpose. The purpose of your work, if you're so disconnected from it, yeah, that's all there is to it. I'm like, I'm not trying to suggest that this is easy or can be done in a one-stop, you know, whatever, but I do think that every step we make along this path that tightens up the feedback loops that connects people more with the consequences of their code is good and valuable and pays off. +**Charity:** Like, because like the reality is, is like there's the customers that y'all want that have that engineering leadership and culture of like giving a crap, yeah? And then there's the vast majority of organizations out there where most of the people who are writing the code that runs our world are trying to put bread on their table. And so as soon as they are done with the specification that they were given for their job, they e that code into Git, and they run because like why should they care? Why should they metrics? -**Charity:** Interesting, oh, sorry, go ahead. +**Adriana:** But this is a self-fulfilling prophecy because you know what makes people like check out, tune the [__] out? Not being connected to the results of their labor. Like there's something that's like—what is it Dan Pink says? We all want in our job is autonomy, mastery, and purpose. The purpose of your work, if you're so disconnected from it... -**Adriana:** I was going to say it's even letting engineers understand customers. Like, I've worked at many shops and with so many engineers that, like, that customer impact doesn't matter to them or they're unaware of how the customers are properly using their software, or they themselves are not even using the software that they're building. That's why, like, dogfooding is so important. +**Charity:** Yeah, that's all there is to it. -I go in to help teams all the time where they're, you know, they're just churning. This is, like, the most common thing that I go work and help teams with, right? They just churn. They're not making progress. Like, usually I can guess that without even talking to anybody. I can just, like, look at their JIRA for two minutes and be like, "Yeah, they're churning, so let's go figure out why." +**Adriana:** I'm like I'm not trying to suggest that this is easy or can be done in a one stop, you know, whatever, but I do think that every step we make along this path that tightens up the feedback loops, that connects people more with the consequences of their code is good and valuable and pays off. -And then you go look, and I lost my train of thought. I'm sorry! +**Charity:** Interesting! Oh, sorry, go ahead. -**Charity:** People are not connected to their work; they're churning, right? +### [00:19:36] Discussion on developer engagement -**Adriana:** I still forgot where I was going to connect that. +**Adriana:** I was going to say it's even letting engineers understand customers. Like I've worked at many shops and with so many engineers that like that customer impact doesn't matter to them or they're unaware of how the customers are properly using their software or they themselves are not even using the software that they're building. That's why like dogfooding is so important. I go into help teams all the time where they're, you know, they're just churning. This is like the most common thing that I go work help teams with, right? They just churning, they're not making progress. -**Host:** It's okay! It's okay, we can move on. I wanted to add one thought to the mix because, you know, the overarching theme here is, you know, like they deploy, and then they go, or they write the code, and then that's it. And I mean, ultimately, we are failing the promise of DevOps at the end of the day, right? +Like usually I can guess that without even talking to anybody. I can just like look at their JIRA for two minutes and be like, "Yeah, they're churning." So let's go figure out why, right? And then you go look and I lost my train of thought. I'm sorry. People are not connected to their work, they're churning. -**Charity:** Exactly! +**Adriana:** Right! I still forgot where I was going to connect that. -**Host:** We did nothing to solve or to alleviate the problem because we are still continuing to do that. Yes, we've got CI/CD pipelines; that's awesome! We've got automation, but we still have not fulfilled the promise of DevOps at the end of the day. And I think it's interesting, though, because I think observability is kind of, you know, forcing us to face that once again, right? +**Charity:** It's okay! It's okay, we can move on. -Because now there... like, these are the consequences of... now we know. I think the opportunity here is now that we have the data to show the business how the consequences of bad engineering management are harming our customers. Like this, we didn't have back when we started DevOps, right? Like, we had no freaking idea how our failed deploys or any of this were impacting our business, right? +**Adriana:** I wanted to add one thought to the mix because, you know, we—the overarching theme here is, you know, like they deploy and then they go or they write the code and then that's it. And I mean ultimately we are failing the promise of DevOps at the end of the day, right? -We were just guessing. We were throwing dark birds. And, like, some of us were fairly accurate, like, "Yeah, this is what's happening," and then go talk to a customer, and they would confirm it. Great! Now we can go do DevOps. But there's tons of shops where they just never close that loop, right? +**Amy:** Exactly! -And then you could bring the data in, though, and this is one of the things I've been slowly doing, right? Is I go talk to a team and be like, "Well, the first thing you need to do is go talk to your product manager, and I'll help them define some freaking SLOs so you know what the acceptable abuse of your customers is," because most people don't even know. +**Adriana:** We did nothing to solve or to alleviate the problem because we are still continuing to do that. Yes, we've got CI/CD pipelines, that's awesome. We've got automation, but we still have not fulfilled the promise of DevOps at the end of the day. -[00:22:10] Yeah, let's start observing that! Let's put in SLOs. We don't have to alert or nothing. Just start looking, getting the data. Now I can go back to the business, and I can go tell your director of engineering or your VP or even your SVP and go yell at them, be like, "Look at this data! You need to go invest in this team and change how they work because this is what your customers are going through." And I have actual data to show it, and that's a game changer! That changes the discussion. +**Charity:** And I think it's interesting though because I think observability is kind of, you know, forcing us to face that once again, right? Um, because now there, like these are the consequences of... now we know. I think the opportunity here is now that we have the data to show the business how the consequences of bad engineering management are harming our customers. Like this—we didn't have back when we started DevOps, right? -**Adriana:** It really does! +**Adriana:** Like we had no freaking idea how our failed deploys or any of this were impacting our business, right? We were just guessing, we were throwing dark birds, and like some of us were fairly accurate, like, "Yeah, this is what's happening." And then go talk to a customer and they would confirm it, great! Now we can go do DevOps. But there's tons of shops where they just never close that loop, right? -**Charity:** I always get so much energy talking to Amy because, like, all these things, I feel like I'm just sitting here thinking about, like, she's out there in the field just doing like over and over and over, and I'm just like, "Oh my God, you're so cool!" +**Charity:** And then you could bring the data in though. And this is one of the things I've been slowly doing, right? Is I go talk to a team and be like, "Well, the first thing you need to do is go talk to your product manager." And I'll help them define some freaking SLOs so you know what the acceptable abuse of your customers is because most people don't even know. -**Amy:** Talk sounds cool, but the day-to-day is like meetings and waiting for the opportunity to go like, "Hey, here!" +**Adriana:** Yeah, let's start observing that. Let's start... let's put in SLOs on it. We don't have to alert or nothing, just start looking, getting the data. Now I can go back to the business and I can go tell your director of engineering or your VP or even your SVP and go yell at them, be like, "Look at this data! You need to go invest in this team and change how they work because this is what your customers are going through." And I have actually data to show it, and that's a game changer. That changes the discussion. -**Charity:** Yeah! +**Charity:** It really does! -**Amy:** But the impact! I mean, I just think it's so cool. But, like you say, we're failing the promise of DevOps. I actually feel like DevOps is like the Band-Aid that we have on a self-inflicted wound that never should have happened. +**Adriana:** I side note, I always get so much energy talking to Amy because like all these things, I feel like I'm just like sitting here thinking about like she's out there in the field just doing like over and over and over and I'm just like, "Oh my God, you're so cool!" -Like, we never should have separated them from Ops! Like there's no possible world in which having one set of people write the code and another set of people understand the code makes any [__] sense! And, like, I get it. We were doing the best we could at the time. You know, we're like, "This is too complex; we got to split this up somehow." +**Amy:** Talk sounds cool, but the day-to-day is like meetings and waiting for the opportunity to go like, "Hey, here!" -But, like, I feel like the entire idea of having different people do these jobs, and it's a little bit unfortunate. I think that we've kind of settled on going, "Oh, well, it's because Ops sucks, and we're all going to be developers." Like, whatever! +**Adriana:** Yeah, but the impact! I mean, I just think it's so cool. But like you say, we're failing the promise of DevOps. I actually feel like DevOps is like, it's like the Band-Aid that we have on self-inflicted wounds that never should have happened. -Like, okay, fine. I mean, most of the time when this happens, it's cultural inertia because most of the corporate world, they live in this world where they think about, "Okay, I have some people who make decisions about the product, and then I go and encapsulate that in some kind of requirements or design docs or whatever, and I ship it off overseas to be manufactured." +**Charity:** Like we never should have separated them from Ops. -So some other people do the labor, some stuff shows up in a warehouse I probably never even see, and it goes out to distribute, blah, blah, blah, blah, blah. Like, this is like most of the businesses in the world today are some kind of form of this kind of dis... breaking the labor away from the design of it. +**Adriana:** Like there's no possible world in which having one set of people write the code and another set of people understand the code makes any [__] sense. Like, and like I get it, we were doing the best we could at the time. You know, we're like, "This is too complex, we got to split this up somehow." -And this is actually, I think, changing radically out even outside of the software world where now product design is speeding up. You see kind of things coming faster, so now they're starting to look at what we're doing in the DevOps and going like, "Oh, maybe there was something there. Maybe expertise matters. Maybe the idea that there's kind of work that isn't knowledge work is [__]!" +**Charity:** But like, you know, I feel like the entire idea of having different people do these jobs and it's a little bit unfortunate. I think that we've kind of settled on going, "Oh well, it's because ops sucks and we're all going to be developers." Like whatever! -**Adriana:** Yeah, I like that! +**Adriana:** Like okay, fine! -**Charity:** I think one thing that is, is to some effect a consequence of this whole idea of separation of concerns, right? Which, you see especially in, like, large enterprises. Like, when I worked at a small to medium-sized startup, you know, I had access to, like, the freaking PR database. And then, like, I joined a bank right after I wrapped up that job, and they're like, "Oh yeah, we've got like a DBA for UAT and a DBA for prod." I'm like, "Huh? You don't even want you to log into your laptop if..." +**Charity:** I mean most of the time when this happens, it's like cultural inertia because most of the corporate world, they live in this world where they think about, "Okay, I have some people who make decisions about the product and then I go and I encapsulate that in some kind of requirements or design docs or whatever and I ship it off overseas to be manufactured so some other people do the labor. Some stuff shows up in a warehouse I probably never even see it and it goes out to distribute, blah, blah, blah, blah, blah." -**Amy:** Yeah, that's right! +**Adriana:** Like this is like most of the businesses in the world today are some kind of form of this kind of dis... breaking the labor away from the design of it. And this is actually, I think, changing radically out even outside of the software world, where now product design is speeding up. You see kind of things coming faster. So now they're starting to look at what we're doing in the DevOps and going like, "Oh, maybe there was something there. Maybe expertise matters. Maybe the idea that there's kind of work that isn't knowledge work is [__]." -**Charity:** ...to do your job! Yeah! That's changed a lot! You know, we borrowed so much of that stuff from the accounting world where things like, you know, the concept of you shouldn't have the same person file the expense report and approve the expense report totally makes sense. +**Charity:** Yeah, I like that. -You know, and I think that there are some concepts that, you know, translate nicely, and most of them really don't. I actually, the most recent talk that I wrote was called "Modern Engineering is Not Incompatible with Highly Secure or Compliance Environments" or something like that. Yeah, and it's just like, because it's been... and it was super... like, I've been wanting somebody to write this talk for like 10 years, and I finally was like, "All right, this is not my area. Just somebody's got to do this!" +**Adriana:** I think one thing that is also a, you know, is to some effect a consequence of this whole idea of separation of concerns, right? Which you see especially in like large enterprises. Like when I worked at like a small, like it was like small to medium-sized startup, um, you know, I had access to like the freaking PR database. And then like I joined a bank right after I wrapped up that job and then they're like, "Oh yeah, we've got like a DBA for UAT and a DBA for prod." -So I sat down and wrote it, and I did a bunch of research, and it was so interesting because, you know, this is all probably old news to y'all, but I learned a lot about how just, like, you know, it's really engineers should not have to think about security and compliance in their day-to-day work. +**Charity:** I'm like, "H, you don't even want you to log into your laptop if... yeah, that's right!" -Like, if you have to think about it all the time, something is very, very wrong. It should be baked into the architecture so that by default, you do the right thing, and you can use the accounting and, like, logging and, like, auditing principles of, like, your CI/CD system. And, like, engineers should just be doing the work. You really only when you need to, like, spin up a new data source or, like, build something new, then you pull your security folks in; you make those decisions from the get-go. +**Adriana:** To do your job. -But, like, you know, it's just like the entire frame and what's so unfortunate is I think that security has this really unfortunate, you know, the security through security they laugh about, but they all do it. But it's like they think that talking about how they do things well is going to be bad! +**Charity:** Yeah, that's changed a lot! You know, we borrowed so much of that stuff from the accounting world where things like, you know, the concept of you shouldn't have the same person file the expense report and approve the expense report totally makes sense. You know, and I think that there are some concepts that, you know, translate nicely, and most of them really don't. -**Adriana:** And it's not! +**Adriana:** I actually... the most recent talk that I wrote was called, um, "Modern engineering is not incompatible with highly secure or like or compliance environments" or something like that. And it's just like, because it's been, and it was super... like I've been wanting somebody to write this talk for like 10 years, and I finally was like, "Alright, this is not my area, Jes, somebody's got to do this." So I sat down and wrote it, and I did a bunch of research, and it was so interesting. -**Charity:** Like, I think that the only way we're going to make progress in this as an industry is if people start to realize that all their competitors are doing this well, and they're about to be outcompeted by the fact that everyone else in the world is so able to, like, write off for, like, a modern team without being bogged down in the security theater. +Um, because you know, this is all probably old news to y'all, but I learned a lot about how just like, you know, it's really engineers should not have to think about security and compliance in their day-to-day work. Like if you have to think about it all the time, something is very, very wrong. It should be baked into the architecture so that by default you do the right thing. And you can use the accounting and like logging and like auditing principles of like your CI/CD system, and like engineers should just be doing the work. You really only when you need to like spin up a new data source or like build something new, then you pull your security folks in. You make those decisions from the get-go. -[00:27:30] **Host:** I know you all touched about AI, and I wanted to ask a question about it. How do you think AI can help teams adopt observability? Or how is it that AI can actually have some benefits to the observability world? +But like, you know, it's just like the entire frame—and what's so unfortunate is I think that security has this really unfortunate, you know, the security through security they laugh about, but they all do it. But it's like they think that talking about how they do things well is going to be bad. Um, and it's not! Like I think that the only way we're going to make progress in this as an industry is if people start to realize that all their competitors are doing this well, and they're about to be outcompeted by the fact that everyone else in the world is so able to like write off for like a modern team without being bogged down in the security theater. -**Charity:** I think I've been saying about AI for a few years now, right? Like, I think the real opportunities lie in joint cognitive systems research, where the way we're using AI isn't necessarily to replace humans in the pipeline. It's to augment humans in the pipeline. And so, for observability, I really feel that the strongest opportunities for AI are in kind of automated attention on doing correlation, basically like auto-correlation, and surfacing that to humans. Not this idea that I'm going to eliminate all my cascading alerts through this because that's a fool's errand that we've been chasing for at least my entire career. +**Charity:** So definitely! I know you all touched about AI, and I wanted to ask a question about it. How do you think AI can help teams adopt observability? Or like how is it that AI can actually have some benefits to the observability world? -I don't think AI really changes that game, but if we start to think about it instead of this crazy, like, we're going to make all the alerts go away, but instead I'm going to help your people do more with less, right? Less context, less expertise, maybe less work. Obvious! Now it gets super-duper powerful because that's the hardest problem to solve, is how do I reason about this hyper-complex system and draw out insights from it? That's where I think AI can help by augmenting the human, not replacing it. +**Adriana:** I think I've been saying about AI for a few years now, right? Like I think the real opportunities lie in joint cognitive systems research where the way we're using AI isn't necessary to replace humans in the pipeline, it's to augment humans in the pipeline. And so for observability, I really feel that the strongest opportunities for AI are in kind of one kind of automated attention on doing correlation, basically like auto-correlation and surfacing that to humans. Not this idea that I'm going to eliminate all my cascading alerts through this because that—that's a fool's errand that we've been chasing for at least my entire career. -**Adriana:** I couldn't agree more! It's really about the inputs so much more than the outputs. Like, we are all familiar with, like, the problems of spam filters, right? Like false positives are brutal, you know, very high possibility for failure. But, like, there's so much we can do, you know, Honeycomb, we've been talking about bringing everyone up to the level of the best debugger, right? +Um, I don't think AI really changes that game, but if we start to think about it instead of this crazy, like we're going to make all the alerts go away, but instead I'm going to help your people do more with less, right? Less context, less expertise, um, maybe less work. Um, obvious, sudden, now it gets super duper powerful because that's the hardest problem to solve is how do I reason about this hyper-complex system and draw out insights from it? That's where I think AI can help by augmenting the human, not replacing it. -Increasingly, like, the hard part is getting detail. Like, these socio-technical systems are made of just as much of our brains as they are the data. Like, an example I often give is like The New York Times and The Washington Post. If you just switch their people overnight, nothing would work because so much of the system is in those people's heads. And the more we can bring it out of the people's heads and put it somewhere that other people can add to it and ask questions of it, you know, then it pools our knowledge and our resources. +### [00:28:56] AI's role in observability -Like, one of the biggest things that blew my mind after becoming a CTO was how many engineering executives out there trust their vendors more than they trust their employees. And they're constantly susceptible to vendors who come up and like, "So give me $10 million and your people will never have to understand your systems again. We will tell you what to look at and we'll tell you what it means." +**Amy:** I couldn't agree more. It's really about the inputs so much more than the outputs. Like we are all familiar with like the problems of spam filters, right? Like false positives are brutal, you know—very high possibility for failure. But like there's so much we can do. You know, Honeycomb, we've been talking about bringing everyone up to the level of the best debugger, right? Increasingly, like the hard part is getting detail. Like these sociotechnical systems are made of just as much of our brains as they are the data. -And that is... because people come and go and vendors last forever! But, like, come on! Like, that is just not going to work! But there are absolutely things that we can do to make your people more productive, you know, better, you know, to get better data in, to like pull your knowledge. +Like an example I often give is like the New York Times and the Washington Post, if you just switch their people overnight, nothing would work because so much of the system is in those people's heads. And the more we can bring it out of the people's heads and put it somewhere that other people can add to it and ask questions of it, you know, then it pools our knowledge and our resources. -**Charity:** I like I wrote that down: joint cognitive research because I think that that's so true. You know, at Honeycomb, we actually built a little AI thing, Bob, which is using ChatGPT to, when you deploy some code, you can ask questions about it using natural language. Just like, "What's slow?" You know, which is super useful because using a query explorer is hard for everyone. +Like one of the biggest things that blew my mind after becoming a CTO was how many engineering executives out there trust their vendors more than they trust their employees. And they're constantly susceptible to vendors who come up and like, "So give me $10 million and your people will never have to understand your systems again. We will tell you what to look at and we'll tell you what it means." And that is—and because people come and go and vendors last forever, but like come on, like that is just not going to work. -But, like, just being like, "What's slow? What changed?" is super valuable, and it helps those engineers that, like, just go around the room and it, like, ask a question to the person next to them. And it's like, you're now winning back time in a way! +But there are absolutely things that we can do to make your people more productive, uh, you know, better, you know, to get better data in, to like pull your knowledge. I like, I wrote that down—joint cognitive research, because I think that that's so true. You know, at Honeycomb, we actually, we built a little AI thing, Bob, which is using chat GPT to when you deploy some code, you can ask questions about it using natural language, just like, "What's slow?" You know, which is super useful because using a query like Explorer is hard for everyone. But like just being like, "What's slow? What changed?" is super valuable. -**Adriana:** So much time! +And it helps those engineers that like just go around the room and it like ask a question to the person next to them, and it's like you're now winning back time in a way. -**Amy:** Yeah, you know, I agree with everyone on the stance on AI because I think there's this overarching fear out there where they're like, "Everyone's like AI is gonna take our jobs!" No, I hope so! I don't think we're at Skynet levels yet, but also I think, like, we, like, personally, I like the idea of having, you know, something take away some of the cognitive load so I can focus on the cooler things and also take away some of the biases that I might have, assuming that the AI is trained without the biases, right? I guess that's the caveat. +**Adriana:** Yeah, you know, I agree with everyone on the stance on AI because I think there's this overarching fear out there where they're like, everyone's like, "AI is gonna take our jobs." No, I hope so! I don't think we're at Skynet levels yet, but also I think like we, like personally, I like the idea of having, you know, something take away some of the cognitive load so I can focus on the cooler things and also take away some of the biases that I might have, um, assuming that the AI is trained without the biases, right? I guess that's the caveat. -But, you know, I like the idea of like something saying to me, "Hey, this is where you look further. You might be interested in blah." So you look and you're like, "Sorry, AI, you're wrong, but thanks for pointing it out anyway!" And I think that that can be hugely helpful to just, I don't know, just take a load off, right? +But you know, I like the idea of like something saying to me, "Hey, this is where you look further. Um, you might be interested in blah," so you look and you're like, "Sorry AI, you're wrong, but thanks for pointing it out anyway!" Um, and I think that that can be hugely helpful to just, I don't know, just take a load off, right? -**Charity:** Definitely! Or like you're getting paged about something, it's a database you're not super familiar with, and it's like, "Hmm, something like this happened last year around this time, and here's what they did to resolve it." Would this be useful to you? Yes! Exactly! +**Charity:** Definitely! Or like you're getting paged about something, it's a database you're not super familiar with, and it's like, "Hmm, something like this happened last year around this time, and here's what they did to resolve it. Would this be useful to you?" Yes! -It's just your little help on your shoulder that's just telling you all the goodness that could be going on in your system anytime. +**Adriana:** Exactly! It's just your little help on your shoulder that's just telling you all the goodness that could be going on in your system anytime. -**Charity:** I think about something that I, or people, spend a lot of time trying to get information out, whether that's like combing through data or trying to find stuff in the history or something, that's an AI problem! Like, they're so good at that [__]. Let computers do what computers do best so that humans can do what we do best! +**Charity:** Exactly! Anytime I think about something that I or people spend a lot of time trying to get information out, whether that's like combing through data or trying to find stuff in the history or something, that's an AI problem. Like they're so good at that [__]. Let computers do what computers do best so that humans can do what we do best. -And I know that, like, there's this fear that we all have about, you know, change and stuff, and some jobs are absolutely going to be lost. But many more will be changed. And I think that the way that we make this good for people is not by resisting it, fearing it, but by being thoughtful about how we involve it and how we adopt it, and, you know, making a change gracefully. +And I know that like there's this fear that we all have about, um, you know, change and stuff, and some jobs are absolutely going to be lost. Um, and but many more will be changed. And I think that the way that we make this good for people is not by resisting it, fearing it, but by being thoughtful about how we adopt it and, you know, making a change. Grace, obviously now we're gonna start talking about government policies, which is... anyway. -**Charity:** Obviously, now we're gonna start talking about government policies, which is... anyway, but like, yeah, like Amy said, I hope part of my job gets automated away because I could be using the cycles on something else! +But like, yeah, like Amy said, I hope part of my job gets automated away because I could be using the cycles of something else. -**Host:** Well, there’s something else I want to maybe shift to. From when you were talking, Charity, one of your favorite topics, which is the unknown unknowns. Because I think AI can eat up most of the known knowns, right? Like, we go and ask the chatbot and be like, "Hey, this seems like it's happened before. Has this happened before? And, oh look, here's all incidents reports and what happened before." Those are known knowns. We kind of know what to do. +**Adriana:** Well, there's something else I want to maybe shift to from when you were talking, Charity. One of your favorite topics, which is the unknown unknowns. Because I think AI can eat up most of the known knowns, right? Like we go and ask the chatbot and be like, "Hey, this seems like it's happened before. Has this happened before?" And oh look, here's all incident reports and what happened before. Those are known knowns. We kind of know what to do. -[00:34:00] I think the place where today's efforts, anyway, we'll see if it changes, but I don't think it's going to change drastically in the next decade, um, are still not aimed at figuring out the unknown unknown, right? This is where I think humans are still going to have an edge for a very long time. +I think the place where today's efforts anyway— we'll see if it changes—but I don't think it's going to change drastically in the next decade—are still not aimed at figuring out the unknown unknown, right? This is where I think humans are still going to have an edge for a very long time, is basically like we'll use the AI to collate the data to do all the undifferentiated lifting, but it's still going to be people that peer into that space between the data and go, "I wonder if there's a packet loss somewhere in, you know, deep in this network that's never happened before. We've never seen it before, and now you have a novel insight." -Basically, like we'll use the AI to collate the data, to do all the undifferentiated lifting, but it's still going to be people that peer into that space between the data and go, "I wonder if there's a packet loss somewhere in, you know, deep in this network that's never happened before. We've never seen it before, and now you have a novel insight." +And that's the place where today's AI really... like it can kind of make things that seem like novel insights because it's approximating what a human would do, but those actual really tough novel insights I think are going to remain in our domain for a long time. They always come out with these demos that show these like leaps of things, you're like, "Oh, I never would have thought to look at that." But like those are impressive because they're rare, and at the end of the day, if it was the right thing, someone still has to go and like the thing that I keep coming back to that humans are good at is attaching meaning to things, right? -And that's the place where today's AI really, like, it can kind of make things that seem like novel insights because it's approximating what a human would do, but those actual really tough novel insights, I think are going to remain in our domain for a long time. +Like any computer can tell you if there's a spike or how big it was, but like only a person can come out and go, "This one really means something," because of that context that the computer won't necessarily always remember. -They always come out with these demos that show these, like leaps of things, you're like, "Oh, I never would have thought to look at that." But, like, those are impressive because they're rare. And at the end of the day, if it was the right thing, you someone still has to go and... +**Adriana:** What do you think is the current largest challenge in observability? -Like, the thing that I keep coming back to that humans are good at is attaching meaning to things, right? Like, any computer can tell you if there's a spike or how big it was, but like only a person can come out and go, "This one really means something," because of that context that the computer won't necessarily always remember. +**Amy:** I'll start with sampling. Ask me again in five years. It'll probably still be sampling. Like it's just, you know, we're generating a wealth of data. Um, yeah, you know, there's a few places where we could probably tune things up, and generally we're not sampling a lot today. But some of the systems that we want to turn on, we're going to have to because they do a bunch of... this is kind of circling back on like that tech debt under the covers. Like as soon as we turn on tracing in some of these systems, we see that they're doing a bunch of busy work that doesn't make any sense. -**Host:** What do you think is the current largest challenge in observability? +But the first thing that happens is it tips over our collectors and overloads our upstream account, and then we got to go turn it off. And so we don't actually get to figure out what the heck is going on. So I think like sampling and really just kind of getting control of all of the data we're generating that might not ever have any value is super important because it's by no means an easy problem to solve. It's probably the hardest problem in observability right now. It's like just how do we get this to a rational amount of data so that my cost is rational so I can do more observability, get deeper insights into my systems, but you got to pay the bill. And so like this is continually going to be our struggle. -**Amy:** I'll start with Amy. Sampling! Ask me again in five years; it'll probably still be sampling. Like, it's just, you know, there's a few places where we could probably tune things up, and generally we're not sampling a lot today, but some of the systems that we want to turn on, we're going to have to because they do a bunch of this is kind of circling back on like that tech debt under the covers. +That's the answer of a super user. -Like, as soon as we turn on tracing in some of these systems, we see that they're doing a bunch of busy work that doesn't make any sense. But the first thing that happens is it tips over our collectors and overloads our upstream account, and then we got to go turn it off. And so we don't actually get to figure out what the heck is going on. +**Charity:** True! What do you think is the largest challenge, Charity? -So I think, like, sampling and really just kind of getting control of all of the data we're generating that might not ever have any value is super important because it's by no means an easy problem to solve. It's probably the hardest problem in observability right now. It's like just how do we get this to a rational amount of data so that my cost is rational so I can do more observability, get deeper insights into my systems? But you got to pay the bill, and so, like, this is continually gonna be our struggle. +**Charity:** Um, the people don't understand how badly they have it now and how much better it could be. People who have spent their careers like slugging it out in the salt mines, using metrics, managing the overhead of metrics, not having high cardinality data, dancing around between metrics and logs and [__]. They genuinely have no idea that the way they're doing it is the hard way, and they don't... they don't actually... nobody's optimistic in computers because why would you be optimistic about computers? -**Charity:** That's the answer of a super user! +But like until people see their data in our world, it doesn't... they don't understand how much easier it could be, how much better it could be. And there's this mental model, you know, and observability 2.0 is dramatically easier than 1.0 because it's just interesting, shove it in, interesting, shove it in. There's no like custom metrics to measure or like manage and farm over time. There's none. But like the difficulty is that it's a mental model shift that people have to... and I am confident that 10 years from now, 15 years from now, uh, people will be, you know, using the systems that, you know, we kind of sketched out a few years ago for observability to understand their systems because the rate of complexity is just going up. -**Host:** True! What do you think is the largest challenge, Charity? +You know, too high, we might be out of business by then. Like we might have been way too early, but I am extremely confident that there will be no other way to understand your systems 10 years from now but using something like this. -**Charity:** Um, the people don't understand how badly they have it now and how much better it could be. People who have spent their careers, like, slugging it out in the salt mines using metrics, managing the overhead of metrics, not having high cardinality data, dancing around between metrics and logs and [__]. They genuinely have no idea that the way they're doing it is the hard way. +**Adriana:** That's fair! They are definitely not getting any simpler or not. And ironically, what that leads to is this increasingly... this increasing reliance on heroes and martyrs because if there's nothing knitting together all of your sources of the data except the people with the intuition and the scar tissue who can make good guesses, you have to... you don't have any other choice. -And they don't, they don't actually... nobody's optimistic in computers because why would you be optimistic about computers? But like until people see their data in our world, it doesn't... they don't understand how much easier it could be, how much better it could be. +I call them load-bearing humans in my [__]. Because like when I run into that, that's how I talk to the management tree about it. I'm like, "You have a load-bearing human. This one person is doing this task, and if they go on vacation or win the lottery, we are going to have a bad time," right? And that's the way to surface the risk is like you wouldn't put a single, you know, firewall. You would always do redundant, right? -And this, there's this mental model, you know, and observability 2.0 is dramatically easier than 1.0 because it's just interesting, shove it in, interesting, shove it in. There's no, like, custom metrics to measure or, like, manage and farm over time. There's none! But, like, the difficulty is that it's a mental model shift that people have to... and I am confident that 10 years from now, 15 years from now, people will be, you know, using the systems that, you know, we kind of sketched out a few years ago for observability to understand their systems because the rate of complexity is just going up. +So we need to make at least make that pun, and that's usually a good way to start getting over the line and get people realizing that that load-bearing humans are holding the world together. -You know, too high! We might be out of business by then! Like, we might have been way too early! But I am extremely confident that there will be no other way to understand your systems 10 years from now but using something like this. +**Adriana:** Yes, all over the world! And even as our rhetoric has gotten better about this and our understanding has gotten better about this, there's still these... it's like the finger in the dike, so to speak. You know, the little Amsterdam dude just like plugging the dike with his [__]. -**Adriana:** That's fair! They are definitely not getting any simpler or not. And, ironically, what that leads to is this increasingly, this increasing reliance on heroes and martyrs because if there's nothing knitting together all of your sources of the data except the people with the intuition and the scar tissue who can make good guesses, you have to... you don't have any other choice! +I got to stop using anecdotes, they're terrible. You say out loud, but like these people exist and then they're doing a hero's job, but they shouldn't have to and it's bad for everyone. It's also putting a lot of trust and retention... oh my God, very fragile. -I call them load-bearing humans in my God because, like, when I run into that, that's how I talk to the management tree about it. I'm like, "You have a load-bearing human. This one person is doing this task, and if they go on a vacation or win the lottery, we are going to have a bad time!" Right? +**Adriana:** Yeah, Ad, what about you? What do you think is the biggest challenge? -And that's the way to surface the risk is like, you wouldn't put a single, you know, firewall. You would always do redundant, right? Like, so we need to make at least make that pun, and that's usually a good way to start getting people realizing that load-bearing humans are holding the world together. +**Adriana:** Um, so I think the biggest challenges, uh, organizations getting in their own way, um, when it comes to observability. Um, either because they don't... they kind of get observability in much the same way that everyone kind of got like agile and DevOps is like important, but they kind of totally missed the point at the same time. And so we're, you know, we've made like a little bit of progress, but also not. -**Adriana:** Yes, all over the world! And even as our rhetoric has gotten better about this, and our understanding has gotten better about this, there's still these tech... it's like the finger in the dyke. So to speak, you know? +And, and on a similar vein, like with open telemetry, again, it's like, "Oh, I understand the benefits of observability. Let's get open telemetry into the org." And then again, like not understanding like what they should do with that, um, either because, you know, of vendors, um, who oversell, right? Where we have a tracing tool, we can totally take your traces and put them on a little island by themselves with no food or water, and nobody ever freaking looks at them because they're little islands. -**Charity:** The little Amsterdam! +**Charity:** Yeah, we'll totally do that for you! We'll take your money too, thank you! They'll be safe. -**Adriana:** Yeah, dude, just like plugging the dyke with his... +**Adriana:** True! Exactly! Never use them. Yeah, you know, we get, especially in large organizations, um, we get very tempted or drawn in by the shiny shiny new thing that seems too good to be true, and you're sold on the vaporware or like somebody reads an article about blah, and they speak to a salesperson about at blah company, and then all of a sudden they walk in all starry-eyed. -**Charity:** God, I got to stop using anecdotes! So terrible to say out loud! +**Charity:** Yeah, totally! -**Host:** But like these people exist, and then they're doing a hero's job, but they shouldn't have to, and it's bad for everyone! It's also putting a lot of trust and retention... +**Adriana:** So I feel like these are such huge barriers to, to actually getting [__] done for observability. -**Charity:** Oh my God! Very fragile! +**Charity:** True! What would you say is the biggest misconception around observability right now? -**Adriana:** Yeah! +**Adriana:** And I'll start with you, Ad. -**Host:** What about you? What do you think is the biggest challenge? +### [00:42:00] Challenges of observability in organizations -[00:40:00] **Adriana:** So I think the biggest challenges are organizations getting in their own way when it comes to observability. Either because they don't... they kind of get observability in much the same way that everyone kind of got, like, Agile and DevOps is, like, important but they kind of totally missed the point at the same time. +**Adriana:** Biggest misconception? Um, I think right now I would say like people thinking that it'll solve all their problems without putting in the work. I think there's a lot of like people assuming there's a lot of like magical fairy dust that comes in with observability, and then when they realized that it's extra work, it was like, "Oh crap!" -And so we're, you know, we've made like a little bit of progress, but also not. And on a similar vein, like with OpenTelemetry, again, it's like, "Oh, I understand the benefits of observability. Let's get OpenTelemetry into the org!" And then again, like not understanding, like, what they should do with that. +**Amy:** What about you, Amy? -Either because, you know, of vendors who oversell, right? Where we have a tracing tool, we can totally take your traces and put them on a little island by themselves with no food or water, and nobody ever freaking looks at them because they're a little island. +**Amy:** I think I'll extend what Adriana said, right? Like it's that extra work of like the need for understanding doesn't disappear. Ideally, as we get the data presentation into a better place, it gets easier to achieve, but the need for people to understand the world around them and be able to synthesize new information from the observability data, um, that doesn't go away. It just gets more efficient. -**Charity:** Yeah! +**Charity:** What do you think, Charity? -**Adriana:** Yeah, we'll totally do that for you. We'll take your money too. Thank you! They'll be safe! +**Charity:** I think the biggest misconception I see is just people thinking that observability is its own sort of standalone project that you can buy or do or check off when in fact it's so interwoven with the development of software, iterating on software, owning, deciding what features you're going to build, you know? I mean, deciding, figuring out how your product is working, figure out what your users are actually doing, why you should care about it, where you should spend your time. -**Host:** True! Exactly! Never use them! +Like it has its tentacles everywhere, and there are lots of things that we do that make observability better or worse have nothing to do with anything that's marked observability, and this is both I think it should be both encouraging and reassuring to people and also a little bit depressing and demoralizing because... but like it just, it sees for a more holistic view of like this is not—I don't think you could have like an observability project that you could just check off. -**Charity:** Yeah! +It requires us to get better at understanding the mission of building something, which I think we're doing bit by bit over the years. Like we're understanding more about how to build software that is better for our users, that is better for our engineers, that is more linked to the business, you know? And like you were talking earlier about, you know, there was this great discussion was kicked off by McKenzie of all people about measuring the work of software, whether or not it's measurable. You know, and of course they're completely wrong, and which meant that K Beck gave him a smackdown and Gade did gave him a smackdown, and there's a great discussion about how you measure software development. -**Host:** I, you know, we get, especially in large organizations, we get very tempted or drawn in by the shiny, shiny new thing that seems too good to be true, and you're sold on the vaporware, or like, some reads an article about blah, and they speak to a salesperson about at blah company, and then all of a sudden they walk in all starry-eyed. +But in one of the pieces, um, someone said that people have got to start realizing that building software is not like building construction. It's like, it's more like the design phase, and you would never say that a blueprint is better because it has more blue lines, right? Which gets to your point earlier, Amy, about how this all work is knowledge work, right? -So I feel like these are such huge barriers to actually getting [__] done for observability! +**Adriana:** And so I where was I going with all this? I don't know. I feel like it is indivisible from the act of getting better at delivering software. -**Host:** What would you say is the biggest misconception around observability right now? And I'll start with you, Adriana. +**Charity:** So if I... I think that a lot—God, I keep... I swear I've taken up 50% of the airtime in this and I really apologize! I made must have had too much coffee today, but I feel like so many software orgs out there have kind of given up on saying that they're trying to get better at delivering software. -**Adriana:** Biggest misconception? I think right now I would say, like, people thinking that it'll solve all their problems without putting in the work. I think there's a lot of, like, people assuming there's a lot of, like, magical fairy dust that comes in with observability, and then when they realized that it's extra work, it was like, "Oh crap!" +**Adriana:** Yes! -**Amy:** What about you, Amy? +**Charity:** And so instead they set these goals for themselves, kind of nibble around the edges: "We're going to get better at observability this year. We're going to get better at CI/CD. We're going to get better at blah, blah, blah, blah, blah." But like the great leaps forward that you make that you have to make in order to keep up with complexity, because you can't just hire your way linearly out of this, you have to make these big leaps. -**Amy:** I think I'll extend what Adriana said, right? Like, it's that extra work of, like, the need for understanding doesn't disappear. Ideally, as we get the data presentation into a better place, it gets easier to achieve, but the need for people to understand the world around them and be able to synthesize new information from the observability data, um, that doesn't go away. It just gets more efficient. +And that requires taking a much, you know, a much more bird-eye view and being like in order to deliver software better this year, it requires that we make these investments in observability, we make these investments in testing. And I feel like in conclusion, building software is an amazing job! I thank my lucky atheist stars every day that I get to be in this industry, not like picking weeds and getting dirt under my finger, but like sitting here at a computer solving puzzles and playing with my friends. -**Charity:** What do you think, Charity? +And I feel like I wish that everyone in software could have as magical of an experience in clear as I have had, and I think that observability is a big part of how we get there. -**Charity:** I think the biggest misconception I see is just people thinking that observability is its own sort of standalone project that you can buy or do or check off when, in fact, it's so interwoven with the development of software, iterating on software, owning, deciding what features you're going to build, you know? - -I mean, deciding, figuring out how your product is working, figure out what your users are actually doing, why you should care about it, where you should spend your time. Like, it has its tentacles everywhere, and there are lots of things that we do that make observability better or worse have nothing to do with anything that's marked observability. - -And this is both, I think it should be both encouraging and reassuring to people and also a little bit depressing and demoralizing because, but like it just... it sees for a more holistic view of like this is not... I don't think you could have, like, an observability project that you could just check off. - -It's... it requires us to get better at understanding the mission of building something, which I think we're doing bit by bit over the years. Like we're understanding more about how to build software that is better for our users, that is better for our engineers, that is more linked to the business. - -And, like, you were talking earlier about, you know, there was this great discussion that was kicked off by McKinsey, of all people, about measuring the work of software, whether or not it's measurable, you know? And of course, they're completely wrong, which meant that K Beck gave him a smackdown, and Gade gave him a smackdown, and there's a great discussion about how you measure software development. - -But in one of the pieces, someone said that people have got to start realizing that building software is not like building construction. It's like, it's more like the design phase, and you would never say that a blueprint is better because it has more blue lines, right? - -Which gets to your point earlier, Amy, about how this all work is knowledge work, right? And so I... where was I going with all this? I don't know! I feel like it is indivisible from the act of getting better at delivering software. - -**Host:** So, if I think that a lot... God, I keep... I swear I've taken up 50% of the airtime in this, and I really apologize! I may have just had too much coffee today, but I feel like so many software orgs out there have kind of given up on saying that they're trying to get better at delivering software. - -And so instead, they set these goals themselves, kind of nibble around the edges. We're going to get better at observability this year; we're going to get better at CI/CD; we're going to get better at blah, blah, blah, blah, blah. But, like, the great leaps forward that you make that you have to make in order to keep up with complexity because you can't just hire your way linearly out of this. You have to make these big leaps, and that requires taking a much, you know, a much more bird-eye view and being like, in order to deliver software better this year, it requires that we make these investments in observability, we make these investments in testing. - -And I feel like, in conclusion, in conclusion, building software is an amazing job! I thank my lucky atheist stars every day that I get to be in this industry, not like picking weeds and getting dirt under my fingernails, but like sitting here at a computer solving puzzles and playing with my friends. And I feel like I wish that everyone in software could have as magical of an experience and career as I have had. - -And I think that observability is a big part of how we get there. - -**Host:** That's true! That actually brings me to my next question as we're getting ready to close up. What do you think is in store for observability for this upcoming year? +**Adriana:** That's true! That actually brings me to my next question as we're getting ready to close up. What do you think is in store for observability for this upcoming year? **Charity:** The first person to jump in is... -**Adriana:** Okay, I'm not gonna pick on one of you! - -**Charity:** Oh, I haven't been keeping up on all the roadmaps and stuff, but I guess... So what is your wish? What is your wish that this year brings for observability? +**Adriana:** Okay! I'm not gonna pick on one of you! -**Adriana:** I guess the thing that I'm working on is a lot of folks still don't really get SLOs and SLIs as much as I think it will help both engineering management and engineers on the ground just communicate better about what they're trying to accomplish. +**Charity:** I haven't been keeping up on all the roadmaps and stuff, but I guess so, what is your wish? What is your wish that this year brings for observability? -Like, if I had a wish, like, it would be to drive kind of that understanding deeper into our industry just so now, so I can go into a room and talk about the power that's available here and just, like, having the metrics to understand and not just get stared at like I just nailed a fish to the wall, right? Like just, you know, it would just be nice to not have to start at zero over and over. +**Adriana:** I guess the thing that I'm working on is a lot of folks still don't really get SLOs and SLIs, um, as much as I think it will help both engineering management and engineers on the ground just communicate better about what they... we're all trying to accomplish. Like it's—if I had a wish, like it would be to drive kind of that understanding deeper in and just into our industry just so now so I can go into a room and talk about the power that's available here and just like having the metrics to understand and and not just get stared at like I just nailed a fish to the wall, right? -**Charity:** That's fair! +Like just, you know, it would just be nice to not have to start at zero over and over. -**Adriana:** I personally want to see more people, like, besides like, you know, we often associate, I guess, observability with more of the Y side of things, and it is so not just that, right? I mean, it's everybody's problem, and so I want to see more conversations where we get more developers giving a crap about observability and not just developers, like folks who are planning tech projects, like software releases, whatever. +**Amy:** That's fair! I personally want to see more people—like besides, like, you know, we often associate, I guess, observability with like more the Y side of things, and it is so not just that, right? I mean, it's everybody's problem. And so I want to see more conversations where like we get more developers giving a crap about observability and not just developers, like folks who are like planning tech projects, like software releases, whatever. -Like, make sure it's part of their language. Make sure that we empower our QAs to use... everybody misses that! Give it to your support team! Gosh! Like, what an opportunity! +Like make sure it's part of their language. Make sure that we empower our QAs to use—everybody misses that! Give it to your support team, gosh! Like what an opportunity! -**Amy:** Yeah! Well, even your QA, right? Like, they're testing your software, and they could use observability to figure out what the hell is wrong and tell developers, "Hey, this is why it's wrong!" +**Charity:** Yeah! Well, even your QA, right? Like they're testing your software and they could use observability to figure out what the hell is wrong and tell developers, "Hey, this is why it's wrong," you know? Um, so that I want to see. I want to see more of that. -You know? So I want to see more of that, and I think it puts us closer to, you know, Charity's observability 2.0 where it's like more part, more of a holistic part of the SDLC, right? +**Adriana:** And, and I think it, it, you know, basically it puts us closer to, you know, Charity's observability 2.0 where it's like more part, more of a holistic part of the SDLC, right? -**Charity:** Yeah! And I think that's where it needs to be so that it's, you know, part of... it's baked in, baked into how we deliver software and how we code software and all that good stuff! +**Charity:** I think that's where it needs to be so that it's, you know, part of... it's baked in, baked into how we deliver software and how we code software and all that good stuff. **Amy:** Definitely! Everybody does a better job when they can see where they're going and see what they're doing when they are getting feedback back from the system for sure. -**Host:** I had this sticker that says that SLOs are APIs for our engineering teams, and I completely agree! Like, if you don't have an agreement that you've all agreed upon, you're down in the weeds negotiating over every single decision you make about how to spend your time. - -Like, it's absurdly inex... it's absurdly expensive! - -**Charity:** I don't know! I expect that what's going to happen in 2024 is that everything's going to get even more muddled, and people are getting even more confused, and, um, it's fine! It's entropy! It's how it goes! - -I guess my wish for Christmas would be that... my wish would be that more people just have that feeling in their gut that you get when you see telemetry feedback to you about what you've done and just go... that feeling that you get, this, like, you feel more grounded, you feel more certain, you feel more confident about what you've done. - -Observability is the way that we can move swiftly and with confidence, and I feel like this is something we all know in our heads we need to learn in our bodies. And the only way that we can learn it in our body is by seeing it and experiencing it. - -**Adriana:** I would wish that everyone had that experience! - -**Host:** Most definitely! I love that! To keep that in store for 2024 as, like, find that fuzzy feeling when you start seeing your logs and metrics start showing up on your vendor. - -Definitely! Well, thank you all very much for the amazing insights you've been able to provide today on our panel around observability. This group is going to be coming back later in June to have a little other chat around observability. Stay tuned to hear about those details later! - -With that, we'd like to sign off and say thank you all for watching. Stay warm out there! +**Adriana:** Well, thank you all very much for the amazing insights you've been able to provide today on our panel around observability. This group is going to be coming back later in June to have a little other chat around observability. Stay tuned to hear about those details later. -**All:** Yay! +With that, we'd like to sign off and say thank you all for watching. Stay warm out there! Yay! ## Raw YouTube Transcript diff --git a/video-transcripts/transcripts/2024-02-23T19:09:30Z-otel-in-practice-opentelemetry-orientation-how-to-train-your-teams-on-observability.md b/video-transcripts/transcripts/2024-02-23T19:09:30Z-otel-in-practice-opentelemetry-orientation-how-to-train-your-teams-on-observability.md index f6fab4a..b4b7095 100644 --- a/video-transcripts/transcripts/2024-02-23T19:09:30Z-otel-in-practice-opentelemetry-orientation-how-to-train-your-teams-on-observability.md +++ b/video-transcripts/transcripts/2024-02-23T19:09:30Z-otel-in-practice-opentelemetry-orientation-how-to-train-your-teams-on-observability.md @@ -10,272 +10,296 @@ URL: https://www.youtube.com/watch?v=o-1UE67l9q4 ## Summary -In this YouTube video, Paige Cruz, a Developer Advocate at Chronosphere, shares her insights on observability and the importance of effective training in this emerging field. With a background as an SRE at observability companies, Paige discusses the challenges her team faced in leading observability initiatives, particularly in migrating systems and re-instrumenting applications. She emphasizes the need for hands-on workshops tailored to specific tech stacks, noting that successful training should bridge knowledge gaps and engage developers. Paige also highlights the complexity of observability concepts, the significance of understanding OpenTelemetry, and the necessity for developers to take ownership of instrumentation. Throughout the talk, she provides practical advice on creating effective training programs, measuring their success, and fostering a culture of observability within organizations. The discussion includes audience engagement, where participants reflect on their training experiences and share strategies for improving ownership and responsibility in observability practices. +In this YouTube video, Paige Cruz, a developer advocate at Chronosphere, shares her extensive experience in observability from various perspectives, including her previous roles as an SRE and in other observability companies. She emphasizes the importance of effective training for developers in implementing observability practices, particularly in tracing. Cruz discusses the challenges her team faced during migrations and how hands-on workshops tailored to specific tech stacks proved to be the most effective method for transferring knowledge. She highlights the complexity of observability as an emerging field and the need to bridge knowledge gaps among developers. The video also covers strategies for creating impactful training sessions, including understanding learners' backgrounds, scoping training content, and engaging participants through practical exercises. Cruz concludes by urging the community to democratize observability knowledge and shares various resources for further learning. ## Chapters -00:00:00 Welcome and introduction -00:01:47 Observability initiatives overview -00:02:38 Importance of hands-on workshops -00:03:50 Training development discussion -00:05:30 Observability complexity and knowledge gaps -00:08:00 Tracing as a valuable telemetry type -00:10:40 Effective training strategies -00:12:50 Identifying learner gaps -00:15:30 Tailoring training to specific needs -00:18:30 Creating effective training materials +00:00:00 Introductions +00:01:00 Observability initiatives overview +00:05:00 Importance of tailored training +00:10:00 Discussion on training effectiveness +00:12:40 Knowledge gaps in observability +00:18:00 Strategies for effective training +00:23:00 Scoping training content +00:30:20 Importance of understanding learners +00:39:00 Measuring training success +00:54:30 Audience Q&A and discussion -**Paige:** Thank you so much for having me, OTEL in Practice. I have been attending the last few sessions and excited to present a little bit about my world. I am Paige Cruz. I work at Chronosphere. It is my third observability company. When I wasn't working for observability companies, I was an SRE at customers of those observability companies working on observability. So if you can't tell, I have thought about this stuff for a very long time and from lots of different perspectives. +## Transcript -And just a kind of a quick tale from the field. I have retired from SRE today. I'm a dev advocate that really focuses on the open source side of observability, but in my day when I was practicing, I had to lead. My team was always tasked with leading big observability initiatives, whether that's migrating from one vendor to another, off of self-hosted open source onto maybe a managed service from one of our cloud providers, rolling out tracing. Oh, my God, the logging bill is so high, you got to get on it. There's just a lot of things that kind of fall under the observability umbrella that were kind of targeted for my team to be the stewards of. +### [00:00:00] Introductions -[00:01:47] And what I found when we had to do these initiatives that touched every team, like re-instrumenting either to add traces or away from a proprietary protocol onto beautiful open source hotel, is that yes, technically, if the dev teams were too busy, my team could do heads down work for like three or four quarters and yeah, we could deliver something and we could be migrated by the end of that session. However, then we were really seen as like the owners of observability and we were kind of treated as like Ask Jeeves. And it just really felt like the line of responsibility between ops and dev kind of went out of whack. And that's totally fair. We moved everything. It was a new platform, new UI, new libraries. Of course, there was going to be a lot of knowledge gaps to cover. +**Paige:** Thank you so much for having me, OTEL in Practice. I have been attending the last few sessions and I'm excited to present a little bit about my world. I am Paige Cruz. I work at Chronosphere. It is my third observability company. When I wasn't working for observability companies, I was an SRE at customers of those observability companies, working on observability. So if you can't tell, I have thought about this stuff for a very long time and from lots of different perspectives. -[00:02:38] And so I have steered quite a few migrations and I can say I've got to try lots of different tactics to be able to delegate that re-instrumentation work or to really pass that baton of ownership rightfully from solely on ops over to app devs as well. And the best tactics out of everything I tried, linking the docs, sending conference talks, the best was hands-on workshops that were specifically tailored to my org's tech stack, what we were trying to get developers to do, the value that it brings at the end of the day. +### [00:01:00] Observability initiatives overview -So this is why I wanted to bring train the trainer, specifically we're talking about tracing training today, and that just requires participation from devs across the stack up and down. And to quote the seminal work High School Musical, we are all in this together. So let's talk about why training matters. Yes, let's talk about how are you actually going to develop a training? These aha moments, unless you were a teacher in a previous career, you probably don't have a strong background in learning theory or delivering trainings and teaching. So let's just get into what you need to know to train folks. +And just a kind of a quick tale from the field. I have retired from SRE today. I'm a dev advocate that really focuses on the open source side of observability. But in my day when I was practicing, I had to lead. My team was always tasked with leading big observability initiatives, whether that's migrating from one vendor to another, off of self-hosted open source onto maybe a managed service from one of our cloud providers, rolling out tracing. Oh, my God, the logging bill is so high. You got to get on it. There's just a lot of things that kind of fall under the observability umbrella that were targeted for my team to be the stewards of. -And then I'll kind of walk through some really specific decisions that I made when developing the KubeCon workshop. And then probably the favorite part of anything that I put together is the further resources. So please, I will share these slides, books, talks, things that can help you along your journey. +What I found when we had to do these initiatives that touched every team, like re-instrumenting either to add traces or away from a proprietary protocol onto beautiful open source, was that yes, technically, if the dev teams were too busy, my team could do heads down work for like three or four quarters and yeah, we could deliver something and we could be migrated by the end of that session. However, then we were really seen as like the owners of observability and we were kind of treated as like Ask Jeeves. It just really felt like the line of responsibility between ops and dev kind of went out of whack. -[00:03:50] So why does training matter? Well, there are a few layers of complexity that we're talking about when it comes to, say, introducing tracing. The first thing is like, full stop, observability is still emerging. It is something that today we have learned basically on the job pretty informally. It kind of depends on what tech stack your company has, what vendors or protocols that your company is using, how many companies you've worked for, what you've seen. +That's totally fair. We moved everything. It was a new platform, new UI, new libraries. Of course, there was going to be a lot of knowledge gaps to cover. I have steered quite a few migrations and I can say I've got to try lots of different tactics to be able to delegate that re-instrumentation work or to really pass that baton of ownership rightfully from solely on ops over to app devs as well. -So even two developers with the same amount of years of experience could have totally different sets of knowledge about this space. And so, you know, a baby is not sitting there thinking about the milestones that we have been reaching. That is just not happening. We have to teach this stuff. And while it is done, and while I think the informal approach is important for developing operational skills and sort of how do you use this data in the heat of an incident to decide on mitigations, you still have to find some sort of way to bridge that gap between the formal patchwork knowledge and a strong foundation that you can build on. +The best tactics, out of everything I tried, linking the docs, sending conference talks, the best was hands-on workshops that were specifically tailored to my org's tech stack, what we were trying to get developers to do, the value that it brings at the end of the day. This is why I wanted to bring Train the Trainer, Trainer How to Train. Specifically, we're talking about tracing training today, and that just requires participation from devs across the stack up and down. To quote the Seminole work High School Musical, we are all in this together. -So that your developers can say, add tracing or maybe bring down the bill or understand how sampling affects like viewing aggregate trace data. There's like a lot of complicated stuff in this space. And so you really got to think about where your learners are, what they know, what they might have seen. +So let's talk about why training matters. Yes. Let's talk about how are you actually going to develop a training? These aha moments, unless you were a teacher in a previous career, you probably don't have a strong background in learning theory or delivering trainings and teaching. Let's just get into what you need to know to train folks. I'll kind of walk through some really specific decisions that I made when developing the KubeCon workshop. Probably the favorite part of anything that I put together is the further resources. Please, I will share these slides, books, talks, things that can help you along your journey. -[00:05:30] And specifically, so we've got observability, kind of new emerging domain, we're kind of bringing some things from monitoring, but we're also discovering a lot of new stuff. Within observability, we've got OpenTelemetry, which totally used to just be known for tracing. I'm so happy that OTEL has now, like, subsumed, like, I think the collector is the more popular, like, most well-known component at this point, which is kind of cool. But unless you have been really interested in observability or specifically tasked with bringing OTEL into your org, like a lot of my developer friends that I talked to don’t, they're like, "Oh, that's cool. OTEL exists, you know, we're on some vendor, it's not really something I'm looking into." +So why does training matter? Well, there are a few layers of complexity that we're talking about when it comes to, say, introducing tracing. The first thing is like, full stop, observability is still emerging. It is something that today we have learned basically on the job pretty informally. It kind of depends on what tech stack your company has, what vendors or protocols that your company is using, how many companies you've worked for, what you've seen. -So while we are like, "Oh my God, exponential histograms, logs are GA," like, all this stuff is happening, it's real. There's still a whole segment of our colleagues that maybe haven't even heard of OTEL. And if you were to just say, "Hey, add tracing to this service. Here's a link to the OTEL docs. It's got everything you need to know." If they don't know what a span is, if they don't know how to navigate or the different components of OTEL, like maybe y'all are using a vendor's collector versus the OTEL, there are just a lot of ways that you could get lost within the OTEL project. +### [00:05:00] Importance of tailored training -And so observability, new, OpenTelemetry, new and rapidly evolving. And finally, the nugget is for tracing, in particular, the problem that I have seen is that it really has this reputation as a tool for power users, the super set of engineers. And to be fair, like, if I reflect on how I got to my tracing knowledge, I literally worked building the tracing product at New Relic and then went to LightStep to go be an SRE and support the tracing infrastructure there. +Even two developers with the same amount of years of experience could have totally different sets of knowledge about this space. You know, yeah, like a baby is not sitting there thinking about the milestones that we have been reaching. Like, that is just not happening. We have to teach this stuff. While it is done, and while I think the informal approach is important for developing operational skills and sort of how do you use this data in the heat of an incident to decide on mitigations, you still have to find some sort of way to bridge that gap between the formal patchwork knowledge and a strong foundation that you can build on so that your developers can say, add tracing or maybe bring down the bill or understand how sampling affects like viewing aggregate trace data. -[00:08:00] And so I've had to think about tracing a lot, I've had to work on these systems, and that is just not a repeatable experience for everybody, nor should it be. We need all types of people to develop all sorts of things. And so this was the problem that I really honed in on for my workshop was like, I think tracing is really a super valuable type of telemetry. I want more people to use tracing. One of the barriers that I see is that yes, partial traces still can be helpful, but the best case is that you've got a full complete trace end to end throughout your system, which requires every service owner along the way to do their part in instrumenting and then using that data. +There's a lot of complicated stuff in this space. You really got to think about where your learners are, what they know, and what they might have seen. Specifically, we've got observability, kind of a new emerging domain. We're kind of bringing some things from monitoring, but we're also discovering a lot of new stuff. Within observability, we've got OpenTelemetry, which totally used to just be known for tracing. I'm so happy that OTEL has now, like, subsumed, like, I think the collector is the more popular, like, most well-known component at this point, which is kind of cool. -So I said, this is the problem that I want to solve, and I want to help people both know how to send trace data, oh my God, and how to use trace data. Because just having the information alone of how to instrument isn't enough if you don't know what to do when you get to a trace waterfall. So the counterpoints that I've heard, the doc site has everything they need to know. I wrote the exact instructions down. They just have to follow them. +But unless you have been really interested in observability or specifically tasked with bringing OTEL into your org, like a lot of my developer friends that I talked to don't, they're like, oh, that's cool. OTEL exists, you know, we're on some vendor, it's not really something I'm looking into. So while we are like, oh my god, exponential histograms, logs are GA, like, all this stuff is happening, it's real. There's still a whole segment of our colleagues that maybe haven't even heard of OTEL. If you were to just say, hey, add tracing to this service, here's a link to the OTEL docs. It's got everything you need to know. -But while I will say the OTEL documentation site is absolutely fabulous, I rely on it all the time, I still see a lot of questions like, is this library instrumented with OTEL? Is this, is the Ruby log spec, like, where is that at? And those are like very easily answered questions if you know where to look at the doc site. Just linking to OpenTelemetry.io is not going to necessarily help someone navigate through our set of projects. +If they don't know what a span is, if they don't know how to navigate or the different components of OTEL, like maybe y'all are using a vendor's collector versus the OTEL, there are just a lot of ways that you could get lost within the OTEL project. So, observability is new, OpenTelemetry is new and rapidly evolving. Finally, the nugget is for tracing, in particular, the problem that I have seen is that it really has this reputation as a tool for power users, the super set of engineers. -Yeah, so yes, plug for the OTEL doc site, it is fabulous, but people need more than just the knowledge, they need the skills. So very quickly, who here, you can maybe raise your hand or throw it in the chat, who here has taken a training course at work? You don't have to tell me where or what, but you've taken a training course that just felt like a total waste of time. You really could have used those extra two hours and actually done something productive. I will raise my hand. I have sat through a couple of those. It's okay. I know it's been out there. Even our mandatory trainings that we've got to take sometimes are like eye roll. +To be fair, like, if I reflect on how I got to my tracing knowledge, I literally worked building the tracing product at New Relic and then went to LightStep to go be an SRE and support the tracing infrastructure there. I've had to think about tracing a lot, I've had to work on these systems, and that is just not a repeatable experience for everybody, nor should it be. We need all types of people to develop all sorts of things. -So bad training is very different than good training. It could be that the training course you took, you were already an expert, and it was just covering stuff you already knew. It could be it wasn't tailored to your use case. Like, your company signed up for generic Kafka training, but actually you needed to know how to tune and specifically level up the operations for your cluster with your configurations and everything within the context of your org. +This was the problem that I really honed in on for my workshop: I think tracing is really a super valuable type of telemetry. I want more people to use tracing. One of the barriers that I see is that yes, partial traces still can be helpful, but the best case is that you've got a full complete trace end to end throughout your system, which requires every service owner along the way to do their part in instrumenting and then using that data. -So the generic 101 Kafka training wasn't going to do anything for you. It wasn't tailored to your use case. And then also it could just be the format and the delivery was totally mismatched. Like for me, I'm not a morning person. So anything that's before 9 am, good luck. My brain is not firing at full capacity. +I said, this is the problem that I want to solve, and I want to help people both know how to send trace data, oh my god, and how to use trace data. Because just having the information alone of how to instrument isn't enough if you don't know what to do when you get to a trace waterfall. The counterpoints that I've heard: the doc site has everything they need to know. I wrote the exact instructions down. They just have to follow them. -[00:10:40] And so there's a lot of if you've encountered bad training, this is not what I'm advising you to do. We're going to talk about how you avoid some of those things and have effective training. So to answer with kind of this section, why does it matter? Why does training matter? Well, I came across this quote that made me so sad, but like is totally exemplifies this. "I don't even know what I don't know, which makes it pretty hard to get started answering my own questions." And this was some anonymous user on Reddit who's trying to find out things about OpenTelemetry. +While I will say the OTEL documentation site is absolutely fabulous, I rely on it all the time. I still see things, a lot of questions like, is this library instrumented with OTEL? Is this, is the Ruby log spec, like, where is that at? Those are very easily answered questions if you know where to look at the doc site. Just linking to OpenTelemetry.io is not going to necessarily help someone navigate through our set of projects. -And that is the case. Like this is where we need to start. This is where our learners are. There are some people for whom OTEL is this mystical black box that they don't know anything about. And if you're on this call, I'm pretty sure you have, you know, your way around the projects, you know the architecture, you know what's up, but we've got to keep in mind that we are kind of a select group here and the observability and OTEL knowledge needs to be very widely distributed, democratized. +So yes, plug for the OTEL doc site, it is fabulous, but people need more than just the knowledge, they need the skills. Very quickly, who here, you can maybe raise your hand or throw it in the chat, who here has taken a training course at work? You don't have to tell me where or what, but you've taken a training course that just felt like a total waste of time. You really could have used those extra two hours and actually done something productive. I will raise my hand. I have sat through a couple of those. It's okay. I know it's been out there. -And that is why training is important because not everybody is here where we are yet. And we need them here for full traces, for better observability across companies, across industries. That's the why. Let's just talk about the how now. We don't want to replicate the bad, awful trainings that we have been through, but what I kind of shortened this as, you want to find these aha moments of discovery. You want to have these learners engaged, and really the whole point of training is that you are trying to change behavior or get learners to take actions. +### [00:10:00] Discussion on training effectiveness -So just giving them a rote, so for my workshop, if I had just given them, here's the step by steps of how to instrument specifically a Python Flask application with traces. Cool. They could leave and they would know, for that tech stack, like, where to go, where to find that information, and how to get traces out of that. Maybe they work at a place that has Ruby or works on Ruby on Rails. If I didn't do the work to explain the higher-level concepts or here's the OTEL registry, you can use it to check any language instrumentation library. If you don't include the actionable pieces in the takeaways, people can still get really lost. +Even our mandatory trainings that we've got to take sometimes are like eye roll. Bad training is very different than good training. It could be that the training course you took, you were already an expert and it was just covering stuff you already knew. It could be it wasn't tailored to your use case. Like, your company signed up for generic Kafka training, but actually you needed to know how to tune and specifically like level up the operations for your cluster with your configurations and everything within the context of your org. -[00:12:50] Even if your instructions were so beautiful and so detailed. Believe me, I have tried that route. So when really where you want to start with training is finding the gap, which is between where your developers are today, what they know, what they understand, what they've seen, and where you want them to be, what actions do they need to take? Do they need to re-instrument from proprietary format to OTEL metrics format? Do they need to add spans? Do they need to take away spans? Because maybe the auto instrumentation they turned on is like totally blowing up the bill. +So the generic 101 Kafka training wasn't going to do anything for you. It wasn't tailored to your use case. It could just be the format and the delivery was totally mismatched. Like for me, I'm not a morning person. So anything that's before 9 a.m., good luck. My brain is not firing at full capacity. If you've encountered bad training, this is not what I'm advising you to do. We're going to talk about how you avoid some of those things and have effective training. -Think really think concretely about what should happen after this training. What are you expecting to see the results and the change? And then that gap between where they are and where you want them to be, that is where your training needs to fill. And we're not going to get deep into learning theory, but really two things. What do they need to know? Knowledge, what info? And then skills. What do they need to be able to do with that knowledge? Together, that is the recipe for a successful training. +To answer with kind of this section: Why does it matter? Why does training matter? Well, I came across this quote that made me so sad, but like is totally exemplifies this: I don't even know what I don't know, which makes it pretty hard to get started answering my own questions. This was some anonymous user on Reddit who's trying to find out things about OpenTelemetry. That is the case. This is where we need to start. -And so when you're identifying the gap, get clear on the problem you're trying to solve. What is the main driver of the initiative? Better visibility? Are there a problem area in the profitable part of your system and you really need deeper level visibility there? Do you want to own your data? If you think about the motivations, that can help you really scope down. Because for us, observability experts, it is very tempting to include the whole world, be like, well, let's start with OpenTracing and OpenCensus and, you know, before OpenTelemetry, and like, you will lose people. I promise you. I promise you, you will lose people if you do that. +There are some people for whom OTEL is this mystical black box that they don't know anything about. If you're on this call, I'm pretty sure you have your way around the projects, you know the architecture, you know what's up, but we've got to keep in mind that we are kind of a select group here. The observability and OTEL knowledge needs to be very widely distributed, democratized, and that is why training is important because not everybody is here where we are yet. We need them here for full traces, for better observability across companies, across industries. That's the why. -So getting really crystal clear on these, answering these questions for yourself will set you up to really hone down the exact things you need to train on. And if you don't know your learners or you don't know what they already know or what they're exposed to, send a survey, do some informal coffee chats, and make sure that you're including a variety of roles. Grab front-end engineers, grab security engineers, back-end, edge, network, you know, because it does take everybody. You need a representative sample to really be able to understand your learner base. +Let's just talk about the how now. We don't want to replicate the bad, awful trainings that we have been through, but what I kind of shortened this as, you want to find these aha moments of discovery. You want to have these learners engaged, and really the whole point of training is that you are trying to change behavior or get learners to take actions. Just giving them a rote, so for my workshop, if I had just given them, here's the step by steps of how to instrument specifically a Python Flask application with traces. Cool. -And sometimes it helps to think about the flip side. So, like, where do I want them to be? If you're having trouble answering that, think about what is the consequence if they get this wrong? What is the consequence if we don't reduce the bill or add tracing in this next quarter in order to evaluate, you know, companies or whatever? Sometimes you can just flip the script because you can get a little bit lost in this wide world. +### [00:12:40] Knowledge gaps in observability -[00:15:30] So as a small, small exercise to think about who our learners are. And so I can make the point that learning and knowledge gaps are totally separate, understandable and expected, no matter your seniority when it comes to observability and monitoring. Let's maybe consider a few learners. We've got like a new grad, they are probably like maybe first job, probably were exposed to logging at some point in their education and maybe if they did an internship or something, but excited to learn doesn't know where to start. Needs both the knowledge and the skills. +They could leave and they would know, for that tech stack, where to go, where to find that information, and how to get traces out of that. Maybe they work at a place that has Ruby, or works on Ruby on Rails. If I didn't do the work to explain the higher-level concepts or here's the OTEL registry, you can use it to check any language instrumentation library. If you don't include the actionable pieces in the takeaways, people can still get really lost. Even if your instructions were so beautiful and so detailed. Believe me, I have tried that route. -Kind of best case to have somebody like excited to learn. Because you will encounter resistant adult learners sometimes. Even thinking about a senior developer thinking about the background, did they work at a mega corporation where there were so many teams that handled developer productivity and platform engineering that they are used to the amazing capabilities and benefits of observability, but going to a smaller org. Oh, I like I've got to add instrumentation. We don't have a wrapper library that covers that is always up to date and covers what I need to know. +Where you want to start with training is finding the gap, which is between where your developers are today, what they know, what they understand, what they've seen, and where you want them to be. What actions do they need to take? Do they need to re-instrument from proprietary format to OTEL metrics format? Do they need to add spans? Do they need to take away spans? Because maybe the auto instrumentation they turned on is totally blowing up the bill. -Even with senior engineering, it's tempting to say, oh, they should know what to do. Send them the docs, like, send them the ticket, let them do the work. But it really depends on where you've worked and what you've been exposed to. If you've worked at a small startup, you know that you've touched almost every part of an observability stack, from the bill to setting up collectors, to doing instrumentation within applications. +Think really concretely about what should happen after this training. What are you expecting to see the results and the change? That gap between where they are and where you want them to be, that is where your training needs to fill. We’re not going to get deep into learning theory, but really two things: What do they need to know? Knowledge, what info? And then skills. What do they need to be able to do with that knowledge? Together, that is the recipe for a successful training. -So really think through, yeah, what people's backgrounds are. That's why this is so individualized and contextualized. It's why a generic 101 basic training won't meet everybody's needs because everybody's kind of coming and bringing to the table like a different set of knowledge and, you know, by the time you're a staff engineer, I guess I would assume maybe you shouldn't assume you've seen a lot of systems. You've seen what works. You've seen what doesn't work. You've seen, you know, you've seen the rise of observability within monitoring, and you probably got a lot of opinions on how things should go. +When you're identifying the gap, get clear on the problem you're trying to solve. What is the main driver of the initiative? Better visibility? Is there a problem area in the profitable part of your system and you really need deeper level visibility there? Do you want to own your data? If you think about the motivations, that can help you really scope down. For us, observability experts, it is very tempting to include the whole world, be like, well, let's start with OpenTracing and OpenCensus and, you know, before OpenTelemetry, and like, you will lose people. I promise you. -That is awesome. And developing a training to meet both the staff engineer's needs and the new grad's needs is probably a stretch. So that is why we want to think about our learners. However, I will say one of the tips is to have once you've got a training kind of sketched out, you've got your first draft, having both a novice walk through it and an expert walk through it and kind of watching what they do, kind of seeing where the stumbling blocks are. That is a very powerful exercise. So tap into that expertise when it exists in your org. +Getting really crystal clear on these, answering these questions for yourself will set you up to really hone down the exact things you need to train on. If you don't know your learners or you don't know what they already know or what they're exposed to, send a survey, do some informal coffee chats, and make sure that you're including a variety of roles. Grab front-end engineers, grab security engineers, back-end, edge, network, you know, because it does take everybody. You need a representative sample to really be able to understand your learner base. -[00:18:30] Okay, so when it comes to actually, like, let's create our material, like, what is this course going to be on? Again, the reason I keep saying you've got to scope it to, like, what do they actually need to do with this information? Because what I was making the 101 instrumentation workshop, oh my God, all I had to cut so much out. I did not know how much I wanted to include in there. And so I had to get really crystal clear on at the end of this workshop. +Sometimes it helps to think about the flip side. Where do I want them to be? If you're having trouble answering that, think about what is the consequence if they get this wrong? What is the consequence if we don't reduce the bill or add tracing in this next quarter in order to evaluate companies or whatever? Sometimes you can just flip the script because you can get a little bit lost in this wide world. -I want attendees to leave and be able to bring OpenTelemetry tracing instrumentation into their own projects. I wanted them to be independent. I wanted them to be able to take this knowledge and use it in their daily life. So that meant that was my guiding star. Anytime I was like, should I put this in? This feels like it's too long. I went back to say, would this help them independently instrument traces in the future, yes or no? If no, cut it out. +As a small exercise to think about who our learners are, I can make the point that learning and knowledge gaps are totally separate, understandable and expected, no matter your seniority when it comes to observability and monitoring. Let's maybe consider a few learners. We've got a new grad. They are probably like maybe first job, probably were exposed to logging at some point in their education and maybe if they did an internship or something, but excited to learn, doesn't know where to start. They need both the knowledge and the skills. -So grounding experiences in the real-world context, that is where you will have a huge leg up if you're developing it for just your organization or business because you are sharing common organizational goals of selling the product or the service or whatever it is that you do. So you already have kind of a shared context and community. You're all working within the same tech stack. +It's kind of best case to have somebody like excited to learn because you will encounter resistant adult learners sometimes. Even thinking about a senior developer, thinking about their background, did they work at a mega corporation where there were so many teams that handled developer productivity and platform engineering that they were used to the amazing capabilities and benefits of observability, but going to a smaller org. Oh, I've got to add instrumentation. We don't have a wrapper library that covers that, is always up to date and covers what I need to know. -So while I had to make a choice of like, what is going to be the most easy to read language, minimal framework that I could kind of take any developer and walk them through instrumentation, you would get to say, "Oh, we do Go and Java, and these are our frameworks," and you can really create the dev environment and the exercises to mimic what your developers would be doing in the real world. +Even with senior engineering, it's tempting to say, oh, they should know what to do. Send them the docs, send them the ticket, let them do the work. But it really depends on where you've worked and what you've been exposed to. If you've worked at a small startup, you know that you've touched almost every part of an observability stack, from the bill to setting up collectors to doing instrumentation within applications. -That being said, if you're developing it for an org, I would say your best bet is to host an instrument-a-thon where you have folks actually bring their real service that you're asking them to instrument and working through it with them. Sandboxes and tutorials are great when you have sort of a wider, more generic use case, but always get as specific as you possibly can be and scope things down. +Think through what people's backgrounds are. That's why this is so individualized and contextualized. It's why a generic 101, like basic, basic training won't meet everybody's needs because everybody's kind of coming and bringing to the table a different set of knowledge. By the time you're a staff engineer, I guess I would assume. Maybe you shouldn't assume you've seen a lot of systems. You've seen what works. You've seen what doesn't work. -So, you have a leg up if you're working on one for your org. And then, again, like, what is the absolute minimum, minimum setup and dev environment? I mean, I think I talk about it later, but I really was torn on whether or not I should introduce the collector. Because you think about the collector is this big component of OTEL, surely people are going to be running into this in the future. Wouldn't it benefit them to know, kind of, really that full path that telemetry takes? +You've seen the rise of observability within monitoring, and you probably got a lot of opinions on how things should go. That is awesome. Developing a training to meet both the staff engineers' needs and the new grads' needs is probably a stretch. So that is why we want to think about our learners. -However, when I saw that the Jaeger all-in-one image had a native OTLP endpoint, I was like, oh my God, Collector, bye-bye. We only have two things to run. That is only two things to have to debug if something goes wrong during the workshop. That is much better than three, and it cut out a whole, you know, part of like configuring the collector, running the collector. And it wasn't, my workshop wasn't about production, production ready tracing system infrastructure. It was, how can you get traces today? And do you know what to do with them? +### [00:18:00] Strategies for effective training -So you, again, like, major benefits if you're working with a group of people with a shared mission, working on the same system and tech stack. And then when it comes to delivering, really tap into what already exists in your network. Like, yes, a standalone training can do the job. But if your organization has communities of practice or like sometimes they're called chapters, like chapter back-end or chapter front-end times and places to exchange ideas and learn on the job, tap into that and ask the organizers if you can host the next meeting and do the workshop or get their help in advertising. +However, I will say one of the tips is to have, once you've got a training kind of sketched out, you've got your first draft, having both a novice walk through it and an expert walk through it and kind of watching what they do, kind of seeing where the stumbling blocks are, that is a very powerful exercise. So tap into that expertise when it exists in your org. -Because with these organization-wide initiatives, just sending one Slack about a training is not going to get many people showing up. Maybe you've got people who can make it mandatory, that's cool, but forcing learning is also kind of comes with its own battles. So the second thing you want to consider is the modalities. Lots of people learn different ways. A lot of us work in distributed remote organizations. So you can't even count on being in the same time zone as somebody. +Okay, so when it comes to actually creating our material, what is this course going to be on? Again, the reason I keep saying you've got to scope it to like, what do they actually need to do with this information? Because when I was making the 101 instrumentation workshop, oh my God, all I had to cut so much out. I did not know how much I wanted to include in there. -So make sure that you provide both like a live guided session for people to, you know, that mimics more of a classroom, self-guided, but make sure that you've got office hours for people to check in with you. And then, you know, you can always record a live session and make that available as well. But having these multiple ways means you're going to have more luck getting more people actually getting through this content, learning the things they need to learn, and doing whatever you need them to do to increase system observability. +I had to get really crystal clear on at the end of this workshop, I want attendees to leave and be able to bring OpenTelemetry tracing instrumentation into their own projects. I wanted them to be independent. I wanted them to be able to take this knowledge and use it in their daily life. That meant that was my guiding star. Anytime I was like, should I put this in? This feels like it's too long. I went back to say, would this help them independently instrument traces in the future, yes or no? If no, cut it out. -So, I guess, like, to recap, you want to like, really think about what is the problem you're trying to solve with this training. You've got to learn, you've got to figure out where your learners are today, where you then you've got where you want them to go and then just really narrow that training gap and scope it as small as possible for the best success. +Grounding experiences in the real-world context, that is where you will have a huge leg up if you're developing it for just your organization or business because you are sharing common organizational goals of selling the product or the service or whatever it is that you do. You already have kind of a shared context and community. You're all working within the same tech stack. -So we'll take a quick look at some of the choices that I made for this workshop. And the first is do not reinvent the wheel. Like, I did not start with a blank page and I said, okay, instrumentation, let's go. Like, no, the beauty of working with open source and working with OTEL is that my friends, like, Reese has fabulous, like, I look at Reese's metrics workshops all the time. And the talks that she's done and Adriana, like, everybody across the vendors and our open source community members have already contributed a ton of interesting tutorials and knowledge, and it's really about shaping and scoping that down for your use case. +While I had to make a choice of like, what is going to be the most easy to read language, minimal framework that I could kind of take any developer and walk them through instrumentation, you would get to say, oh, we do Go and Java, and these are our frameworks and you can really create the dev environment and the exercises to mimic what your developers would be doing in the real world. -And yeah, making sure that it fits and so you're not starting from scratch. You weren't hired to be a teacher or instructional designer. Please borrow. Particularly the way that we, that folks organize information. You could take my Python Flask workshop and totally rewrite it in whatever language and framework you want, while keeping the same structure and sort of curriculum and lesson flow. Because that's where I spent a lot of my time. +That being said, if you're developing it for an org, I would say your best bet is to host an instrument-a-thon where you have folks actually bring their real service that you're asking them to instrument and work through it with them. Sandboxes and tutorials are great when you have a wider, more generic use case, but always get as specific as you possibly can be and scope things down. -So it doesn't need to be a cut, copy, pasta thing, but it's really like, use what exists. This is the beauty of having an open and sharing environment. The other thing is vendors often have training on their specific platform and products. What I will say is sometimes if you take those trainings, you're introduced to all of the features and all of the products on the platform that your org maybe doesn't even pay for or use or has interest in. +You have a leg up if you're working with a group of people with a shared mission, working on the same system and tech stack. When it comes to delivering, really tap into what already exists in your network. Yes, a standalone training can do the job. But if your organization has communities of practice or sometimes they're called chapters, like chapter back end or chapter front end, times and places to exchange ideas and learn on the job, tap into that and ask the organizers if you can host the next meeting and do the workshop or get their help in advertising. -And so that can contribute to learners being like, "Is this synthetics turned on or whatever?" And so I think borrowing the curriculum, borrowing the phrases, the links, the resources, how concepts are mapped out one by one, great. And if you're all in on one platform, go ahead and use the vendor training, but that is just my word of advice there. +With these organization-wide initiatives, just sending one Slack about a training is not going to get many people showing up. Maybe you've got people who can make it mandatory, that's cool, but forcing learning comes with its own battles. -So, what was the problem I was trying to solve? I believe tracing is an underutilized telemetry type. And I think a major barrier to implementation is purely getting useful data out, getting to that complete trace as a starting point. And so in my opinion, the best way to tackle that was to teach application developers, because a lot of us in ops and a lot of us in SRE have been talking about tracing forever. It's the app devs we've got to reach. They're the ones we need to do the instrumentation. +The second thing you want to consider is the modalities. Lots of people learn in different ways. A lot of us work in distributed remote organizations. You can't even count on being in the same time zone as somebody. Make sure that you provide both a live guided session for people to, you know, that mimics more of a classroom, self-guided, but make sure that you've got office hours for people to check in with you. -So that was my guiding light, getting app devs to instrument traces and actually use trace data. My learners were app devs interested in open source who are running cloud-native systems. I think tracing is useful. Always? Well, I shouldn't say always. I think tracing is very, very helpful no matter the size of your system. However, specifically working at KubeCon and CNCF, I was targeting cloud-native systems. +You can always record a live session and make that available as well. Having these multiple ways means you're going to have more luck getting more people actually getting through this content, learning the things they need to learn, and doing whatever you need them to do to increase system observability. -And what did they already know? I did not have the benefit of having access to my attendees ahead of time. And so, the assumptions I made were that they were likely familiar with metrics and logs. That they might have had exposure to APM, application performance monitoring, and that they were curious about tracing in OpenTelemetry. Because no one signs up for an OpenTelemetry workshop if they don't know what it already is, even at like the highest level. +### [00:23:00] Scoping training content -So that's what I started. How did I scope down? What was our minimum local dev environment? I chose Flask and Python. Python for its very welcoming community on just like as a language as a whole. Flask because it is a framework that doesn't do magic stuff behind the scenes. You don't really need to know the ins and outs of Flask in order to just look at it. It's a very basic web app. And if learners wanted to continue on and maybe like extend the workshop, the Python, specifically the Python OpenTelemetry cookbook section and the documentations for the Python Instrumentation Libraries or Tracing are excellent. +To recap, you want to really think about what is the problem you're trying to solve with this training. You've got to learn, you've got to figure out where your learners are today, where you want them to go, and then just really narrow that training gap and scope it as small as possible for the best success. -So I knew if people wanted to go further, that those resources would be available to them. Obviously, OTEL APIs and SDKs for instrumenting. And then, yeah, where did I want to store and visualize these traces? Well, this was a workshop focused on local development of traces. So, the Jaeger all-in-one where you're holding those traces in memory was a fine, totally acceptable use case for this. We were only going to be generating a few requests at once and kind of, it was mostly building up the feedback loop of adding instrumentation code, running the services, verifying that the data matched what we wanted, and like over and over again. +We'll take a quick look at some of the choices that I made for this workshop. The first is: do not reinvent the wheel. I did not start with a blank page and say, okay, instrumentation, let's go. The beauty of working with open source and working with OTEL is that my friends, like, Reese has fabulous metrics workshops all the time. -So, we didn't need to hold stuff long-term, so JaegerFIT. And because this was an open-source for open-source conferences, it needed to be all open source up and down the stack. So JaegerFIT, although I'm really excited to see some other, I think is it, hotel desktop viewer, I hope is getting revived. I see murmurs of that. I would like to have lots of options, but this is what I went with. +The talks that she's done and Adriana, like, everybody across the vendors and our open source community members have already contributed a ton of interesting tutorials and knowledge, and it's really about shaping and scoping that down for your use case. You're not starting from scratch. You weren't hired to be a teacher or instructional designer. Please borrow. -And then I kicked off, it feels silly, but you, I always kick off with a definition of observability because I think if you asked like five developers to define it, you would get 10 different answers. Actually, I think you'd get more than five different answers back. And so when you're conducting this training, it's important that you start with like, these are the terms we use, this is what it means for this org, or for this project, or this is what they mean here. You may have a different idea, let's talk about that, but like, for this session, here's where we're grounding. +Particularly the way that folks organize information. You could take my Python Flask workshop and totally rewrite it in whatever language and framework you want, while keeping the same structure and sort of curriculum and lesson flow. It doesn't need to be a cut, copy, pasta thing, but it's really like, use what exists. This is the beauty of having an open and sharing environment. -Doesn't need to be long, literally one sentence definitions, we don't want to overwhelm or bore people, especially at the beginning of a training. And then from there, I had talked about my big debate, do I include the collector, do I not? Here's what I decided to do. I thought it will serve people to know about it. I'm going to include the big OTEL diagram from the docs. Again, doc site showing up here. Love it. It's not a knock on the doc site. We just got to help people learn where to go to find the info. +The other thing is vendors often provide training on their specific platform and products. What I will say is sometimes if you take those trainings, you're introduced to all of the features and all of the products on the platform that your org maybe doesn't even pay for or use or has interest in. That can contribute to learners being like, is this synthetics turned on or whatever? -So I talked about the collector briefly. I kind of said, here's where it sits in the overall OTEL, like kind of architecture and the project landscape. Here's when you would use it. Today, we're going to be using a more simplified version where you don't, it had, the purpose of putting the slide is they know what it is. They can connect it to some other ideas within OTEL. And the next time they hear it in the wild in a conversation, they go, "Oh, okay. It was like the pro, you know, we can send telemetry data through that." That is a fine place for beginners to be, especially app devs who are likely not going to be like tuning collector configs. +I think borrowing the curriculum, borrowing the phrases, the links, the resources, how concepts are mapped out one by one, great. If you're all in on one platform, go ahead and use the vendor training, but that is just my word of advice there. -And so the hardest thing I think to do is to introduce new concepts and skills one at a time. Because if you've been working for this, with this stuff for a while, you have like the cursive knowledge where you just don't even know what it's like to be a beginner. You don't even know what would be a fire hose of information versus like this beautiful, nice breadcrumbs. +What was the problem I was trying to solve? I believe tracing is an underutilized telemetry type. I think a major barrier to implementation is purely getting useful data out, getting to that complete trace as a starting point. In my opinion, the best way to tackle that was to teach application developers because a lot of us in ops and a lot of us in SRE have been talking about tracing forever. It's the app devs we've got to reach. -So, like, if you think back to your first day on the job, you probably had HR benefits, then IT laptop setup, and then maybe like a session with the CEO about the history of the company. And then you're like with stuff to lunch. And then by the end of the afternoon, you finally get to your laptop, you get to your desk and you've got like a hundred emails and you're like missed meetings you didn't know about. And like, all of that is just so overwhelming when you're getting started. +They're the ones we need to do the instrumentation. That was my guiding light: getting app devs to instrument traces and actually use trace data. My learners were app devs interested in open source who are running cloud-native systems. I think tracing is useful always? Well, I shouldn't say always. I think tracing is very, very helpful no matter the size of your system. -So really, really focusing on just one little concept at a time should be a guiding principle. So the way I broke instrumentation up, I looked at a lot of different tutorials and I even got, I will admit I got a little bit confused because I would be looking at the same tech stack that they were trying to instrument and the code samples would, would kind of vary. And I looked at a bunch and I said, what is this missing concept here? What is this missing link? +However, specifically working at KubeCon and CNCF, I was targeting cloud-native systems. What did they already know? I did not have the benefit of having access to my attendees ahead of time. The assumptions I made were that they were likely familiar with metrics and logs, that they might have had exposure to APM, application performance monitoring, and that they were curious about tracing in OpenTelemetry. -And what it turned out to be was people were up front deciding whether they wanted to do automatic, programmatic, or manual instrumentation without explicitly saying, "Hey, this is what I'm doing and why." And so that meant if you were to just take one of those tutorials that maybe did all automatic, you might be really confused when someone asks you to manually add a span attribute because you were like, I, the OTEL agent is like wrapping my service call. Like it's doing everything. What do you mean? +Because no one signs up for an OpenTelemetry workshop if they don't know what it already is, even at the highest level. So that's what I started with. How did I scope down? What was our minimum local dev environment? I chose Flask and Python. Python for its very welcoming community as a language as a whole. Flask because it is a framework that doesn't do magic stuff behind the scenes. -And so that for me was the gap that I wanted to fill is to know when and where to use automatic, programmatic, and manual. And there's no like bearing the lead, like show everything up front. And this is in module one. I'm like, here's automatic, programmatic, manual. Here's the differences. No surprises. +You don't really need to know the ins and outs of Flask in order to just look at it. It's a very basic web app. If learners wanted to continue on and maybe extend the workshop, the Python, specifically the Python OpenTelemetry cookbook section and the documentation for the Python Instrumentation Libraries or Tracing are excellent. I knew if people wanted to go further, that those resources would be available to them. -And one note on the automatic, it is really nice to get a quick win in very early for learners. I decided to use for automatic instrumentation, just the console span exporter. Because for a lot of these folks, it was their first time even like, thinking of a span, knowing what a span is, and the textual representation I thought was kind of nice to look at and become familiar with what I could have done was like, put up an example span and like, blah, blah, blah, lectured through all the trace ID and parent ID. +Obviously, OTEL APIs and SDKs for instrumenting. Where did I want to store and visualize these traces? Well, this was a workshop focused on local development of traces. The Jaeger all-in-one where you're holding those traces in memory was a fine, totally acceptable use case for this. We were only going to be generating a few requests at once and kind of, it was mostly building up the feedback loop of adding instrumentation code, running the services, verifying that the data matched what we wanted, and like over and over again. -Or we could set up auto instrumentation, they could get their first span in front of them, and then I could say, "Hey, do you notice this thing?" And then they're looking at their laptop, they're engaged, it's their data. This is sort of the aha moment I'm talking about. We don't want to lecture at people, but we want them to discover on their own because that just goes so much further. We do learn by doing. +So we didn't need to hold stuff long term, so Jaeger fit. Because this was an open source for open source conferences, it needed to be all open source up and down the stack. Jaeger fit, although I'm really excited to see some other, I think is it, OTEL desktop viewer, I hope is getting revived. I see murmurs of that. I would like to have lots of options, but this is what I went with. -Programmatic manual. So the other thing I did was break up how and why. So after we do the different, we do automatic, we move on to programmatic. And once we have traces that have multiple spans, like the textual representation, like techno we got to start looking at this in a waterfall. So I took the time to do a quick tour of Jaeger, of each of the different visualizations and kind of talk through, here's how you analyze them, here's what you're looking for, here's how to search and filter, and I ended that how section with this slide on why because traces I think are very different from metrics and logs. +Then I kicked off, it feels silly, but I always kick off with a definition of observability because I think if you asked five developers to define it, you would get 10 different answers. Actually, I think you'd get more than five different answers back. When you're conducting this training, it's important that you start with like, these are the terms we use, this is what it means for this org, or for this project, or this is what they mean here. -And so I was like, "Why do we have so many visualizations? Why did I just show you like 10 slides of different ways that you can interact with and manipulate this data?" And it's like, well, the power of tracing, you know, the zoom out, the zoom in, the telescope, the microscope. But combining, separating, but following up the how to with the why was a really key component for people's light bulbs to go off. +You may have a different idea, let's talk about that, but for this session, here's where we’re grounding. It doesn't need to be long, literally one sentence definitions. We don't want to overwhelm or bore people, especially at the beginning of a training. -So with that, I wanted to share a few more resources and then kind of dive into hearing what training needs y'all have. These two books, highly recommend, "Design for How People Learn." It is hilarious, it is funny, it is a book on learning that, like, it is not boring at all. She talks, she walks the walk in this book as well as "Better Onboarding" from a book apart, which is also a fantastic publisher. +From there, I had talked about my big debate: do I include the collector or not? Here's what I decided to do. I thought it will serve people to know about it. I'm going to include the big OTEL diagram from the docs. Again, doc site showing up here. Love it. It's not a knock on the doc site. We just got to help people learn where to go to find the info. -When with all of the talk about internal development platforms and platform engineering and treating your platform like a product, like, let's just borrow the good books they're using already, like, that are onboarding and think about how to bring that into onboarding to observability. Open observability resources. I love that the OTEL Docs have a specific section that's like, "Are you a developer? Like, and you want to learn more? Start here." Versus, "Are you ops? Start here." Like, that really speaks to who, knowing who the audience is. +I talked about the collector briefly. I kind of said, here's where it sits in the overall OTEL architecture and the project landscape. Here's when you would use it. Today, we're going to be using a more simplified version where you don't, the purpose of putting the slide is they know what it is. They can connect it to some other ideas within OTEL. The next time they hear it in the wild in a conversation, they go, oh, okay, it was like the pro, you know, we can send telemetry data through that. -The CNCF Glossary. It's helpful to have bookmarks. Sometimes people ask you about acronyms and maybe you don't remember or you want sort of an official resource to turn to that one is nice, plain text, like small, like it is just very plainly defined. And then finally, while my workshop is not real-world, right? It was a very small Flask app. It had three endpoints. If you wanted something more realistic to look at, the OpenTelemetry demo is your choice. It's got, I think, I believe all the languages. +### [00:30:20] Importance of understanding learners -It's, it exercises almost every part of OTEL that is like GA now. Super helpful resource. That would be the one that I would use if I knew what hardware my learners had. Cause it was a little bit of a mouth breather last time I ran it. Okay, and then learning experiences. +That is a fine place for beginners to be, especially app devs who are likely not going to be tuning collector configs. The hardest thing I think to do is to introduce new concepts and skills one at a time. If you've been working with this stuff for a while, you have the cursive knowledge where you just don't even know what it's like to be a beginner. You don't even know what would be a fire hose of information versus like these beautiful, nice breadcrumbs. -So there is the tracing workshop that I gave at KubeCon is self-guided. It's up, it's on GitLab but you can, it's set up as like a slideshow, so you just kind of work through it at your own pace. There's a recorded session, which is sort of where I'm teaching the workshop, and I'm able to use more words than I put on, on the slides. And then sort of related, I developed a different learning experience for on-call onboarding when I was at LightStep, and I presented on how and why to do your own on-call onboarding at SREcon last year. +If you think back to your first day on the job, you probably had HR benefits, then IT laptop setup, and then maybe a session with the CEO about the history of the company, and then you're like with stuff to lunch. By the end of the afternoon, you finally get to your laptop, you get to your desk and you've got like a hundred emails and you're like missed meetings you didn't know about. All of that is just so overwhelming when you're getting started. -That is a great talk, and it is a practice that has evolved to multiple companies past LightStep, so it has kind of proven itself out in the world. So I believe we are, yes. Gotta do my attributions. Thank you to SlidesGo for the slide templates, FlatIcon, Freepik, and Storyset for all of the beautiful things. I am not an artist. I'm not a teacher, I'm just a developer advocate who wants people to trace more things. +Focusing on just one little concept at a time should be a guiding principle. The way I broke instrumentation up, I looked at a lot of different tutorials. I even got, I will admit I got a little bit confused because I would be looking at the same tech stack that they were trying to instrument and the code samples would kind of vary. -So with that, I'm really curious to learn. Does this resonate with folks? Do you have training needs today? Do you want to talk through some tactics or ways to learn your, about your learners? Let's talk. +I looked at a bunch and I said, what is this missing concept here? What it turned out to be was people were up front deciding whether they wanted to do automatic, programmatic, or manual instrumentation without explicitly saying, hey, this is what I'm doing and why. That meant if you were to just take one of those tutorials that maybe did all automatic, you might be really confused when someone asks you to manually add a span attribute because you were like, I, the OTEL agent is like wrapping my service call. It's doing everything. What do you mean? + +For me, that was the gap that I wanted to fill: to know when and where to use automatic, programmatic, and manual. There's no bearing the lead, like show everything up front. This is in module one. I'm like, here's automatic, programmatic, manual. Here's the differences. No surprises. + +One note on the automatic, it is really nice to get a quick win in very early for learners. I decided to use for automatic instrumentation just the console span exporter. For a lot of these folks, it was their first time even thinking of a span, knowing what a span is, and the textual representation I thought was kind of nice to look at and become familiar with. + +What I could have done was put up an example span and lectured through all the trace ID and parent ID, or we could set up auto instrumentation, they could get their first span in front of them, and then I could say, hey, do you notice this thing? They're looking at their laptop, they're engaged, it's their data. This is sort of the aha moment I'm talking about. We don't want to lecture at people, but we want them to discover on their own because that just goes so much further. + +We do learn by doing. After we do the different, we do automatic, we move on to programmatic. Once we have traces that have multiple spans, like the textual representation, we got to start looking at this in a waterfall. I took the time to do a quick tour of Jaeger, of each of the different visualizations, and kind of talk through, here's how you analyze them, here's what you're looking for, here's how to search and filter. + +I ended that how section with this slide on why because traces, I think, are very different from metrics and logs. I was like, why do we have so many visualizations? Why did I just show you like 10 slides of different ways that you can interact with and manipulate this data? The power of tracing, you know, the zoom out, the zoom in, the telescope, the microscope. + +Combining, separating, but following up the how to with the why was a really key component for people's light bulbs to go off. With that, I wanted to share a few more resources and then kind of dive into hearing what training needs y'all have. + +These two books, I highly recommend: Design for How People Learn. It is hilarious, it is funny, it is a book on learning that is not boring at all. She talks, she walks the walk in this book, as well as Better Onboarding from A Book Apart, which is also a fantastic publisher. With all of the talk about internal development platforms and platform engineering and treating your platform like a product, let’s just borrow the good books they’re using already that are onboarding and think about how to bring that into onboarding to observability. + +Open observability resources. I love that the OTEL Docs have a specific section that's like, are you a developer? And you want to learn more? Start here. Versus, are you ops? Start here. That really speaks to who, knowing who the audience is. The CNCF Glossary. It's helpful to have bookmarks. Sometimes people ask you about acronyms and maybe you don't remember or you want sort of an official resource to turn to that one is nice, plain text, like small, like it is just very plainly defined. + +Finally, while my workshop was not real-world, right? It was a very small Flask app. It had three endpoints. If you wanted something more realistic to look at, the OpenTelemetry demo is your choice. It's got, I believe, all the languages. It exercises almost every part of OTEL that is GA now. Super helpful resource. That would be the one that I would use if I knew what hardware my learners had. + +Learning experiences. The tracing workshop that I gave at KubeCon is self-guided. It's up, it's on GitLab, but you can, it's set up as a slideshow, so you just kind of work through it at your own pace. There's a recorded session, which is sort of where I'm teaching the workshop, and I'm able to use more words than I put on the slides. + +Sort of related, I developed a different learning experience for on-call onboarding when I was at LightStep, and I presented on how and why to do your own on-call onboarding at SREcon last year. That is a great talk, and it is a practice that has evolved to multiple companies past LightStep, so it has kind of proven itself out in the world. + +I believe we are, yes, gotta do my attributions. Thank you to Slidesgo for the slide templates, Flat Icon, Freepik, and Storyset for all of the beautiful things. I am not an artist. I'm not a teacher, I'm just a developer advocate who wants people to trace more things. + +With that, I'm really curious to learn. Does this resonate with folks? Do you have training needs today? Do you want to talk through some tactics or ways to learn about your learners? Let's talk. **Audience Member:** I actually have a couple of questions, Paige. The first is, are you able to share, do you have a shareable copy of your slide deck? **Paige:** Yes, let me. Our Google workspace is really locked down, so I will need 5 minutes after this meeting to get that over to you. But yeah, no worries. -**Audience Member:** And then I was also curious, how do you measure results or, like, what are some strategies to measure results from your workshops? - -**Paige:** Yeah, so my workshop in particular, I looked at the attendee feedback and I talked to folks afterwards and especially the folks that had raised hands and had questions or hit a stumbling block. I wanted to make sure because somebody had an issue PIP installing something in a virtual environment and it was like, not something I'd seen before and it was not something I could troubleshoot on the spot. So I walked them through. +**Audience Member:** I was also curious, how do you measure results or what are some strategies to measure results from your workshops? -Okay. This is what I was trying to teach you here. Here's another way that you can get at this. What else would you, what do you need to go forward? So there's a little bit of that personal one on one and then they send out a survey. And so I always look at survey results mostly not the numbers. Not a social scientist. I know people like the Likert scale, but I really care more about what people took the time to type and say. +**Paige:** My workshop in particular, I looked at the attendee feedback and I talked to folks afterwards, especially the folks that had raised hands and had questions or hit a stumbling block. I wanted to make sure, because somebody had an issue with pip installing something in a virtual environment and it was not something I'd seen before, and it was not something I could troubleshoot on the spot. -And so folks had said, like, this was a great introduction. This was my favorite tutorial. Like, thank you so much. And so I'm like, okay, we did good. If I was developing this for my company, so say you have an initiative that you need to re-instrument away from proprietary onto hotel metrics by the end of next quarter. Okay. So what you where you want your learners to be is to, well, what you want at the end of the day is maybe all of your services instrumented with OTEL metrics. Cool. +I walked them through, okay, this is what I was trying to teach you here. Here's another way that you can get at this. What else would you need to go forward? There's a little bit of that personal one-on-one. Then they send out a survey. I always look at survey results, mostly not the numbers. Not a social scientist. I know people like the Likert scale, but I really care more about what people took the time to type and say. -What's standing in the way? Where are your learners today? Well, half of them are using Vendor A, the other half are on Vendor B, and then maybe the ops team is, like, secretly running Prometheus somewhere, and you're like, oh my god. So, in that case, I would say, like, okay, well, for the Prometheus folks, there's like a bit of education to do with the interoperability between Prom and hotel metrics and kind of talking through which approach do we want. +### [00:39:00] Measuring training success -Do we want to use the pull-based or do we want to do the push-based? That is kind of the training and the knowledge and the skills that we need there. For each of the vendors, like I do think it is helpful to have this is how you would do it in vendor A and this is how it needs to be, this is how you need to change your behavior. Sort of a use this library, not that library, the contrasting from where they are in the context of what they know to here's the hotel metrics like cookbook for, oh my God, the cookbook for Python. +Folks had said, like, this was a great introduction. This was my favorite tutorial. Like, thank you so much. I'm like, okay, we did good. If I was developing this for my company, say you have an initiative that you need to re-instrument away from proprietary onto OTEL metrics by the end of next quarter. -Here's the, it'll go away. Here's. And even a really basic like primer on metrics again, because I, even with like, yes, observability is great when we talk about monitoring, really, we talk a lot about metrics and thresholds and alerts, and that is still an area that I see is like under serves as we like through the page at the developers and ran away, we like kind of forgot to teach them all the stuff we learned after years of like fighting with incidents and trying to understand our systems better and measure them. +What you want at the end of the day is maybe all of your services instrumented with OTEL metrics. Cool. What's standing in the way? Where are your learners today? Well, half of them are using Vendor A, the other half are on Vendor B, and then maybe the ops team is secretly running Prometheus somewhere, and you're like, oh my god. -So I think in that case, yeah. Yeah, I would just really get as specific as you can on what these groups would have in common, and then map from there to the goal. So I would measure the success of like, okay, two weeks after the workshop, or one week after the workshop, how many PRs did I see get through? How many services either started their re-instrumentation, finished their re-instrumentation? I would kind of use that to track the progress of the project, which is sort of the lagging indicator of like did they learn the thing or not. +In that case, I would say, okay, well, for the Prometheus folks, there's a bit of education to do with the interoperability between Prom and OTEL metrics and kind of talking through which approach do we want. Do we want to use the pull-based or do we want to do the push-based? That is kind of the training and the knowledge and the skills that we need there. -Yeah, but those are great questions. Thank you. +For each of the vendors, I do think it is helpful to have this is how you would do it in Vendor A and this is how it needs to be, this is how you need to change your behavior. Sort of a use this library, not that library, the contrasting from where they are in the context of what they know to here's the OTEL metrics cookbook for, oh my God, the cookbook for Python. -**Audience Member:** Do you have a favorite training that you've held or you've led? Do you want to share how, like, I love your metric stuff. So you want to talk a little bit about how you made choices there? +Here's the, it'll go away. Even a really basic primer on metrics again because I, even with like, yes, observability is great when we talk about monitoring. Really, we talk a lot about metrics and thresholds and alerts, and that is still an area that I see is under-served as we like through the page at the developers and ran away. We kind of forgot to teach them all the stuff we learned after years of fighting with incidents and trying to understand our systems better and measure them. -**Paige:** I mean, so it's interesting because I did the metrics. I did a metrics talk twice, and so I changed up the second, change it up the second time just based on like, you know questions that people had and stuff that I had wanted to kind of put in but didn't have time or hadn't quite figured out how to put into it. But, you know, I think what you're talking about is not applicable just for workshops, but like really anything that involves like a wider audience, because, you know, I think I frequently am like, oh, you know, what kind of level do I need to assume that people are at so that I know. +In that case, yeah, I would just really get as specific as you can on what these groups would have in common and then map from there to the goal. I would measure the success of like, okay, two weeks after the workshop, or one week after the workshop, how many PRs did I see get through? How many services either started their re-instrumentation, or finished their re-instrumentation? I would kind of use that to track the progress of the project, which is sort of the lagging indicator of did they learn the thing or not. -Yeah, and you know, you can make like general assumptions about, you know, okay, well, this event, you know, whatever, but sometimes it's. Yeah, I don't know. Sometimes you just guess and hope if it's the most people, you know, sometimes, yeah, there'll be like one or two people that were like, oh, this was like way too beginner or like two people that would say the opposite. Like, this is, you know, way over my head. +**Audience Member:** Do you have a favorite training that you've led or you've held? Do you want to share how, like, I love your metric stuff. So do you want to talk a little bit about how you made choices there? -And yeah, that's I think scoping like, does it need to be novice? Does it need to be mid-level where I assume some things or am I talking to the expert that probably wrote a system like this in the past that it is you can't stretch one training to fit all those groups for sure. +**Paige:** It's interesting because I did a metrics talk twice, and so I changed it up the second time just based on, you know, questions that people had and stuff that I had wanted to kind of put in but didn't have time or hadn't quite figured out how to put into it. I think what you're talking about is not applicable just for workshops but really anything that involves a wider audience because, you know, I think I frequently am like, oh, you know, what kind of level do I need to assume that people are at so that I know. -**Audience Member:** Yeah, I know. And yeah, I'm, I'm always like, I feel like on the verge of, oh, I want to, you know, because I always appreciate, you know, people who explain things very well. And so trying to, you know, fit that for the people that are there, but also like not wanting to piss them off, but like, you know, kind of have to be like, oh no, this is like, this is more basic than I wanted. +You can make general assumptions about, okay, well, this event, whatever, but sometimes it's, I don't know. Sometimes you just guess and hope if it's the most people, you know, sometimes there will be like one or two people that would say, oh, this was way too beginner or like two people that would say the opposite, like, this is, you know, way over my head. -**Paige:** Yes, absolutely. Yeah, it's hard. And I, when I write blog posts and stuff that people who edit are always like, you don't need to include the whole history of observability. You could kind of just like, start with your topic. I'm like, well, people still don't know. You know, I work a lot of booth duty and I can go, oh, you know, are you happy with, you know, observability at your company today? And they go, what's that? And I'm like, yeah. +That's why scoping, like, does it need to be novice? Does it need to be mid-level where I assume some things or am I talking to the expert that probably wrote a system like this in the past? You can't stretch one training to fit all those groups for sure. -Okay, all right. Let's start up. Let's start at not the beginning, but let's start at do a monitoring. Oh, okay. And yeah, just finding those ways to meet people where they are because we do use a lot of insider academic terms telemetry. There's just no shortage of them in this space. So yes, I and I do think the quality of the questions that you get asked maybe in your Slack help channels can really tell you if your training was a hit or a miss because if you're continuing to get, "Hey, is this random JavaScript library instrumented with OTEL?" It's like, I don't want to be rude and like point you to the registry, but also like, did you take the training? +I'm always like, I feel like on the verge of, oh, I want to, you know, because I always appreciate people who explain things very well. Trying to fit that for the people that are there, but also not wanting to piss them off, but like, you know, kind of have to be like, oh no, this is like, this is more basic than I wanted. -Making sure you track who takes the training is also huge because I have been on tiny teams that have kind of manpowered through migrations and we would get developers kind of treating us like we were the owners of their own CI pipelines. And we would say, "Oh, so and so we noticed you haven't taken the training yet. I totally get why, you know, this is a confusing thing. Why don't you check out lesson two? Here's the link." And then it's like, Okay, this is, we're negotiating the boundaries of responsibility between operators and developers. +**Paige:** Yes, absolutely. It's hard. When I write blog posts and stuff that people who edit are always like, you don't need to include the whole history of observability. You could kind of just start with your topic. I'm like, well, people still don't know. I work a lot of booth duty and I can go, oh, you know, are you happy with observability at your company today? -I think we're still, even though like DevOps movement has, we're like many years past that, it does still feel like we are negotiating these boundaries when it comes to monitoring and observability. No one team should be in charge of this. You should be the, as experts, you should be the advisors, the consultants, the guides, but not the like, total tyrannical, like, owners of the thing. It's exhausting to be responsible at that level. +They go, what's that? I'm like, okay, all right. Let's start up. Let's start at not the beginning, but let's start at do you do monitoring? Oh, okay. Finding those ways to meet people where they are because we do use a lot of insider academic terms telemetry. There's just no shortage of them in this space. -So, yeah. Or does anybody have good trainings that they've taken that they want to share or ideas for courses? I'm, yeah, like Rhys said, sometimes we can get in our head about, like, what do people even want to know? +I do think the quality of the questions that you get asked maybe in your Slack help channels can really tell you if your training was a hit or a miss because if you're continuing to get, hey, is this random JavaScript library instrumented with OTEL? It's like, I don't want to be rude and point you to the registry, but also like, did you take the training? -**Dan:** Go ahead, Dan. +Making sure you track who takes the training is also huge because I have been on tiny teams that have kind of man-powered through migrations. We would get developers kind of treating us like we were the owners of their own CI pipelines and we would say, oh, so and so, we noticed you haven't taken the training yet. I totally get why, you know, this is a confusing thing. Why don't you check out lesson two? Here's the link. -**Audience Member:** Hi, thanks. And thanks for the talk that resonated very well with me. Sort of just on your last point, specifically, I was going to ask. So, one of the things I think we struggle with is like this ownership boundary of, we provide some training or maybe not even training. I don't know if we do that well, but we teams are like, we need observability in our app or not even our app. +We’re negotiating the boundaries of responsibility between operators and developers. I think we're still, even though the DevOps movement has, we're many years past that, it does still feel like we are negotiating these boundaries when it comes to monitoring and observability. No one team should be in charge of this. You should be the experts, the advisors, the consultants, the guides, but not the total tyrannical owners of the thing. It's exhausting to be responsible at that level. -Like, I'm a team that owns a framework that dozens of teams that my org uses and we help them implement sort of, you know, tracing let's say at the root of that. So the app developer doesn't even need to know. But then like, you know, a month passes, three months passes and they, something's not working properly, they want to add something and they're always like looking to us to be the ones to like do the work or, or, or, you know, again, advising them is my role. +Does anybody have good trainings that they've taken that they want to share or ideas for courses? -So it's, it's like, that's okay, but like, I just feel like they don't really end up ever taking ownership. Like it's a, it's a common pattern. And I guess what I'm asking for, or opening a discussion on is just, like, how would you drive the connection between, like, the training and the knowledge transfers, et cetera, and then like, defining where that boundary lies because I always feel like people treat observability as like a third class citizen when I feel like it needs to be their first class citizen. +**Audience Member:** I'm, yeah, like Rhys said, sometimes we can get in our head about, like, what do people even want to know? -Like, we get all these incidents, the directors talk about it, et cetera, but they don't take the ownership at the end of the day. So I'm just curious about your experience in that relationship. +**Dan:** Hi, thanks. And thanks for the talk that resonated very well with me. Sort of just on your last point, I was going to ask. One of the things I think we struggle with is like this ownership boundary of, we provide some training or maybe not even training. I don't know if we do that well, but we teams are like, we need observability in our app, or not even our app. -**Paige:** I'm hoping somebody on the call has experience working at a midsize or larger companies because unfortunately I have been at startups where I didn't I have not had like a director of SRE or or really even like, I do think it comes down to engineering leadership needing to explicitly like, say, here are, here is what platform or ops or SRE provides. +I'm a team that owns a framework that dozens of teams in my org use, and we help them implement tracing, let's say at the root of that. So the app developer doesn't even need to know. But then like, you know, a month passes, three months pass, and they, something's not working properly. They want to add something and they're always looking to us to be the ones to like do the work or, or, you know, again, advising them is my role. -And here's what you're responsible for sort of like the, what is it? The cloud security, the shared responsibility model between the cloud provider and you like, and yes, there's like the fuzzy boundary in the middle, which is where you would do some consulting. But I have not been able to successfully win the owner, like get ownership successfully established because of the kind of the nature of the companies I worked at, but I'm really curious. +It’s like, that’s okay, but like, I just feel like they don’t really end up ever taking ownership. Like, it's a common pattern. I guess what I'm asking for, or opening a discussion on is just, how would you drive the connection between like the training and the knowledge transfers, et cetera, and then like defining where that boundary lies? -Like Adriana, I know you've worked at some big shops, you've seen a lot in the past. +Because I always feel like people treat observability as like a third class citizen when I feel like it needs to be their first-class citizen. We get all these incidents, the directors talk about it, et cetera, but they don't take the ownership at the end of the day. -**Adriana:** Yeah, yeah, I mean, I can say that like from personal experience, like I worked at an organization that was like, they hired me in to do observability, like to lead an observability practices team. And yet when it came time to like, okay, let's get the developers to instrument their code, they're like, huh? Why can't you do it? +**Adriana:** I'm just curious about your experience in that relationship. I'm hoping somebody on the call has experience working at midsize or larger companies because unfortunately I have been at startups where I didn't. I have not had a director of SRE or really even like, I do think it comes down to engineering leadership needing to explicitly say, here are, here is what platform or ops or SRE provides. -So it really, and then similarly, like or, or I should say an opposite story to that. We had last year Iris De Mirschi who she was at Farfetch. I forget where she's at now, but she was talking about how she came from an organization that was an observability first organization. +Here is what you're responsible for, sort of like the, what is it? The cloud security, the shared responsibility model between the cloud provider and you. There’s the fuzzy boundary in the middle, which is where you would do some consulting. I have not been able to successfully win the owner, like get ownership established because of the nature of the companies I worked at. -And they're like, because executives mandated observability and that, you know, their team wasn't responsible for instrumenting code, even though, like, she was, she was part of an observability team, like, they were helping with practices. They were like managing infrastructure, like the collectors, but application instrumentation was to be done by the application teams. +I'm really curious, like Adriana, I know you've worked at some big shops, you've seen a lot in the past. -And so it was a very different sort of outcome as a result, right? Because it's almost like, what are you going to say? Like, leadership has basically said that this is the way you're going to do it. So that's the way you're going to do it compared to where I was at before, where I was at before. I was trying to shape what observability for the org looked like, but unfortunately, leadership was like, they kind of half dipped their toes into the observability water. +**Adriana:** Yeah, I mean, I can say that from personal experience, I worked at an organization that was like, they hired me in to do observability, to lead an observability practices team. Yet when it came time to like, okay, let's get the developers to instrument their code, they're like, huh? Why can't you do it? -So they're like, eh, not really. +It really, and then similarly, like, or I should say an opposite story to that. We had last year Iris De Mirschi, who she was at Farfetch. I forget where she's at now, but she was talking about how she came from an organization that was an observability-first organization. -**Dan:** Oh no. Does this mean we all need to get promos to engineering leadership to make observability happen? Oh no. +They were like, because executives mandated observability and that, their team wasn't responsible for instrumenting code, even though she was part of an observability team. They were helping with practices. They were managing infrastructure, like the collectors. But application instrumentation was to be done by the application teams. -**Adriana:** Or be very persuasive. +It was a very different sort of outcome as a result, right? Because it's almost like, what are you going to say? Leadership has basically said that this is the way you're going to do it. So that's the way you're going to do it compared to where I was at before, where I was trying to shape what observability for the org looked like, but unfortunately leadership was like, they kind of half dipped their toes into the observability water. -**Dan:** Do you feel that like, I hear what you're saying, but like, I also feel like in the spirit of the goal of observability and the goal of like pushing these responsibilities downward. It's like this. These top level mandates are really like, sure, maybe the team will do it at the end of the day, but they're never really like, caring about it. +**Dan:** Does this mean we all need to get promos to engineering leadership to make observability happen? -Like, do you think those two are like competing? You have to have a convergence at the end of the day, right? Cause like you need the mandate from the top but like, you have to be, you have to have the folks willing to do it. +**Adriana:** Oh no. Or be very persuasive. Do you feel that like, I hear what you're saying, but I also feel like in the spirit of the goal of observability and the goal of pushing these responsibilities downward, it's like these top-level mandates are really like, sure, maybe the team will do it at the end of the day, but they're never really like caring about it. -And I think like to Paige's points in her talk, like, you know, having people who are excited about observability and, and having a proper training program in place as well to get the, you know, the individual contributors, the ones who are, who have an actual stake in this who are, who are going to be doing the work. +Do you think those two are competing? You have to have a convergence at the end of the day, right? Because like, you need the mandate from the top, but you have to be, you have to have the folks willing to do it. I think like to Paige's points in her talk, having people who are excited about observability and having a proper training program in place as well to get the individual contributors, the ones who have an actual stake in this, who are going to be doing the work. -Like because if you don't, if you don't have like that convergence from top and bottom, like it's just going to be a half-assed thing. +If you don't have that convergence from top and bottom, it's just going to be a half-assed thing. -**Adriana:** Yeah, go ahead. +**Audience Member:** Yeah, go ahead. -**Audience Member:** Yeah, I was just going to chime in quickly. So the one experience I had or the recent experience that when, when, when teams start facing issues, like when they are trying to figure out one particular issue, they are fixing like the incident. I guess like even in not in each day, but they are trying to find a particular issue with the application and then they need help, right? +**Audience Member:** Yeah, I was just going to chime in quickly. So the one experience I had or the recent experience that when teams start facing issues, like when they are trying to figure out one particular issue, they are fixing like the incident. I guess like even in not in each day, but they are trying to find a particular issue with the application and then they need help, right? -Like they don't know they haven't done the training or I don't know how to use the observability tool. I think that I kind of use that as a leverage whether then have you done your, have you instrumented your application correctly? Have you added the correct labels on your telemetry data? And then having to like having used that kind of work as well. +They don't know they haven't done the training or I don't know how to use the observability tool. I think that I kind of use that as leverage, whether then have you done your, have you instrumented your application correctly? Have you added the correct labels on your telemetry data? Having used that kind of work as well. -Like, okay, you want something that you want to have with identifying or figuring out this issue. I also want you to do your part as well. Make sure that you have done your part. Then it will make it easier. So I think that was one tactic used in the past. +Okay, you want something that you want to have with identifying or figuring out this issue. I also want you to do your part as well. Make sure that you have done your part. Then it will make it easier. I think that was one tactic I used in the past. -**Dan:** Yeah, I think, like, from what I'm hearing so far we I had similar experience in my previous org. We were building our brand new platform, and luckily it was all written in Java, which meant that, like, we just kind of utilized the Java Auto Instrumentation, built out the kind of generic dashboard and had all that kind of semantic convention baked into it which meant that like all of the telemetry was easily identified. +**Dan:** I think, like, from what I'm hearing so far we had similar experience in my previous org. We were building our brand new platform, and luckily it was all written in Java, which meant that we just kind of utilized the Java Auto Instrumentation, built out the kind of generic dashboard and had all that kind of semantic convention baked into it. -And we went just like we just went in all the application and then because it was not that much effort on our part to then make that change rather than like manually instrumenting everything and then we had other teams which are like all the services were returning lambdas and goes in that case, like, yeah, I had like, nine months project with them trying to kind of pursue them and slowly, like kind of building set of all kind of POC and like, based on the water. +This meant that all of the telemetry was easily identified. We just went in all the application and then because it was not that much effort on our part to then make that change rather than manually instrumenting everything. We had other teams, which were like all the services were returning lambdas and goes. In that case, I had a nine-month project with them trying to pursue them and slowly build a set of all kind of POC, based on the water. The extension layer building out a custom version of that all configured. -The extension layer building out like like custom version of that all configured. They like just add this layer. At least that part is sorted for you. But then you just have to do the instrumentation part on your. +They would just have to add this layer. At least that part is sorted for you. But then you just have to do the instrumentation part on your app. I guess it depends on the situation, but it's a mixed bag. -So, yeah, I guess it depends on the situation, but it's a mixed bag. +**Paige:** Yeah, Dan, it sounds like you're already working within sort of that entrenched pattern where you're already seen as the owners and that maybe you've been able to do these tickets for them in the past, but now you want to change. I remember two things I've tried: one, we'll do this first one together. We'll pair on this. Then you will know and this will be great. -**Paige:** Yeah, Dan, it sounds like you've already got you're working within sort of that entrenched pattern where you're already seen as the owners and that maybe like you've been able to be able to do these tickets for them in the past, but now you want to change. I remember two things I've tried. One, we'll do this first one together. We'll pair on this. Then you will know and this will be great. +Or I find an example PR that's like exactly like instrumenting the whatever the counter or the gauge you need. Like here, use this as a guide. It is hard to come out of an already established pattern like that, but DoorDash did a talk at KubeCon about how they rolled out service mesh and sort of the challenges with giving people out-of-the-box dashboards. -Or I find an example PR that's like exactly like instrumenting the, whatever the counter or the gauge you need. And like here, use this as a guide. So it is hard to come out of a already established pattern like that, but DoorDash did a talk at KubeCon about how they rolled out service mesh and sort of the challenges with giving people out of the box dashboards. +I can find that and put it in the slide deck; they would be great to talk to because they kind of had a similar problem of being the owners. -I can find that and put it in the slide deck they would be great to talk to because they kind of had a similar problem of being the owners. But we're out of time. Oh my gosh, that went by quick. If you have any more questions you can reach Paige on the socials that she shared, or you can just search Paige Cruz in CNCF Slack. +### [00:54:30] Audience Q&A and discussion -We will have this recording edited and up, I think, hopefully in the next couple of weeks it'll be on the OpenTelemetry YouTube channel. And yeah, it's called OTEL Official. Yes, so, you know, it's official real. Yes. Give it a follow. Give it a follow so that you can get all the latest updates. +**Paige:** But we're out of time. Oh my gosh, that went by quick. If you have any more questions, you can reach me on the socials that I shared, or you can just search Paige Cruz in CNCF Slack. We will have this recording edited and up, I think, hopefully in the next couple of weeks. It'll be on the OpenTelemetry YouTube channel. -And yeah, thank you all so much for being here Paige. Thank you so much. This was like really great. I love your slides. I screenshotted a couple. And it's all, I all work out in the open, so take, borrow whatever you need, adapt, extend. Let's bring more people to tracing. And please, yeah, email me and get in touch if you want to talk about this stuff more. I have spent a lot of time, a lot of time thinking about it. +Yes, it's called OTEL Official. Yes, so, you know, it's official, real. Yes. Give it a follow so that you can get all the latest updates. Thank you all so much for being here. Paige, thank you so much. This was really great. I love your slides. I screenshotted a couple. -Awesome. We will see you all next time. All right. Thank you. +It's all, I all work out in the open, so take, borrow whatever you need, adapt, extend. Let's bring more people to tracing. Please, yeah, email me and get in touch if you want to talk about this stuff more. I have spent a lot of time, a lot of time thinking about it. Awesome. We will see you all next time. All right. Thank you. ## Raw YouTube Transcript diff --git a/video-transcripts/transcripts/2024-03-05T19:49:24Z-otel-collector-user-feedback-panel.md b/video-transcripts/transcripts/2024-03-05T19:49:24Z-otel-collector-user-feedback-panel.md index 2d4e0db..e0f61df 100644 --- a/video-transcripts/transcripts/2024-03-05T19:49:24Z-otel-collector-user-feedback-panel.md +++ b/video-transcripts/transcripts/2024-03-05T19:49:24Z-otel-collector-user-feedback-panel.md @@ -10,148 +10,174 @@ URL: https://www.youtube.com/watch?v=LL8v_B417ok ## Summary -In this YouTube video, a panel discussion featuring four OpenTelemetry (OTEL) collector practitioners—Adriana Villela, Iris, Juraj Michalik, Greg Eales, and Joel—explores their experiences and challenges with the OTEL collector. The panelists discuss their roles in various companies, the usage of the collector for telemetry data collection, and the importance of community feedback in shaping its future. Key topics include deployment strategies, debugging difficulties, challenges in scaling the collector, and desired features for improvement, such as the need for profiling, better integration with Prometheus, and handling metrics more effectively. The session highlights the practitioners' collective learning experiences with the OTEL collector, emphasizing the balance between rapid development and stability in the ecosystem. The discussion concludes with an invitation to the audience to engage further with the community and participate in upcoming events at KubeCon in Paris. +In this YouTube video, a panel of OpenTelemetry (OTEL) collector practitioners discusses their experiences and challenges with the OTEL collector, moderated by Adriana Villela. The panel includes Iris, a senior observability engineer at Miro; Juraj Michalik from Swissery; Greg Eales from Ocado Technology; and Joel from Open Systems. The main topics covered include the usage of the collector in various companies, challenges in deployment, configuration, debugging, and scaling. The panelists shared insights on their preferences for using vanilla versus custom-built collectors, their wish lists for future features like profiling and better handling of cumulative metrics, and their experiences with scaling the collector using various methods such as horizontal pod autoscaling (HPA) and KEDA. The discussion highlights the importance of community feedback and ongoing development within the OTEL ecosystem, as well as the upcoming presence of the panelists at KubeCon in Paris. ## Chapters -00:00:00 Welcome and intro -00:01:40 Panelist introductions -00:05:00 Discussion on Collector usage -00:09:10 Metrics and traces handling -00:12:40 Collector distribution types -00:15:36 Contribution experiences -00:20:01 Debugging challenges -00:25:00 Collector scaling feedback -00:30:00 Wishlist for improvements -00:48:25 Closing remarks and KubeCon info +00:00:00 Introductions +00:01:00 Panelist introductions +00:05:30 Iris discusses Collector usage +00:09:30 Juraj discusses Collector usage +00:12:00 Greg discusses Collector usage +00:14:30 Joel discusses Collector usage +00:22:20 Challenges in debugging +00:30:00 Discussion on custom builds +00:37:10 Wishlist for future features +00:54:00 Upcoming presence at KubeCon -**Adriana:** Thank you everyone for joining. This is a really great turnout and we're excited to have assembled this panel of OTEL collector practitioners. One of the missions of the OTEL and User Working Group is to collect feedback on OTEL to deliver back to the SIGs. I think this started because Judah C, who works in the OTEL collector SIG, reached out to us to put together a survey to collect some information on the OTEL collector, to see how it's being used to sort of drive what the roadmap and vision is going to be for the collector in the next little while. Then we thought, well, okay. You know, let's extend this and also have a panel and have a nice lively discussion with a handful of OTEL collector practitioners. So here we are. +## Transcript -[00:01:40] So, why don't we introduce the panelists? I'll introduce myself first. My name is Adriana Villela. I work with Reese and Dan in the OTEL end user working group. Our groups are, our numbers are getting a little bit bigger, so I'm very excited to see that. Yeah, so we're hosting this event. Let's go over to our panelists. I'm going to pick names. Iris, how about you go first? +### [00:00:00] Introductions -**Iris:** Hello everyone. My name is Iris. I'm a senior observability engineer at Miro and I've been working with OpenTelemetry for a little bit more than a year now, in two companies. I have a lot of experience and use cases that I've worked because my two experiences are completely different from each other, and I'm happy to be here. Nice to meet you. Thank you. +**Adriana:** Thank you everyone for joining. This is a really great turnout and we're excited to have assembled this panel of OTEL collector practitioners. One of the missions of the OTEL and User Working Group is to collect feedback on OTEL to deliver back to the SIGs. I think this started because Judah C, who works in the OTEL collector SIG, reached out to us to put together a survey to collect some information on the OTEL collector, to see how it's being used to sort of drive what the roadmap and vision is going to be for the collector in the next little while. Then we thought, well, okay, let's extend this and also have a panel and have a nice lively discussion with a handful of OTEL collector practitioners. So here we are. -**Adriana:** Okay, next we have Juraj. Did I pronounce that correctly? Please correct me if I'm wrong. +Why don't we introduce the panelists? I'll introduce myself first. My name is Adriana Villela. I work with Reese and Dan in the OTEL end user working group. Our groups are getting a little bit bigger, so I'm very excited to see that. Yeah, so we're hosting this event. Let's go over to our panelists. I'm going to pick names. Iris, how about you go first? -**Juraj:** Hi, my name is Juraj Michalik. I'm based in Madrid, currently working very recently for a company called Swissery in the insurance space. Before I worked for like two different companies where we used, in the last one, very sparingly, and in the previous one, we basically fully migrated to OpenTelemetry for collection of logs, metrics, and traces. I'm a Senior Logging and Monitoring Engineer in this current role. +**Iris:** Hello everyone. My name is Iris. I'm a senior observability engineer at Miro and I've been working with OpenTelemetry for a little bit more than a year now, in two companies. I have a lot of experience and use cases that I've worked because my two experiences are completely different from each other and I'm happy to be here. Nice to meet you. Thank you. -**Adriana:** Awesome, thanks. Next we have Greg. +Okay, next we have Juraj. Did I pronounce that correctly? Please correct me if I'm wrong. + +**Juraj:** Hi, my name is Juraj Michalik. I'm based in Madrid, currently working very recently for a company called Swissery in the insurance space. Before I worked for like two different companies where we used, in the last one, although very sparingly, and in the previous one, we basically fully migrated to OpenTelemetry for the collection of logs, metrics, and traces. I'm a Senior Logging and Monitoring Engineer in this current role. **Greg:** Hi, can you hear me? My name is Greg Eales. I work for Ocado Technology, which started out as an online supermarket and now sells online supermarket technology to other supermarkets. We've been using the collector for a bit over a year, my team, but I've only been on the team for a bit less than a year. -**Adriana:** Oh, awesome. And finally we have Joel. +**Joel:** Hi, I'm Joel. I'm from a company called Open Systems. We're based in Switzerland, but we have a presence globally. We have offices in San Francisco, actually. I've never been there. Also in London and Germany. This is where we're based. We offer managed SASE, so managed connectivity and VPN for our customers. I'm on the observability team, so I'm responsible for making sure that we get telemetry from our devices, which we have 10,000 of, all across the world. We're now having to deal with data coming from Kubernetes, so Prometheus metrics, logs, all this nice telemetry, which we need to handle. We've been using the collector since about a year now, I think just under a year, I would say probably April last year was when we started. We've really decided very quickly that we want to fully migrate everything to it because at the moment we kind of had a big mass of different service teams doing things in their own way, so we're kind of consolidating everything now and we found that the collector and OpenTelemetry in general was a very nice way which we could do that. + +Awesome, well thank you again all of you for joining us. Let us get into the questions. First things first, and we'll start in the order in which folks were introduced. So, Iris, how about you start with telling us how you and your team are primarily using the collector? + +### [00:05:30] Iris discusses Collector usage + +**Iris:** Okay, so I could talk a little bit about how we're using it now and in my previous company as well, since I am very new at my current position. We are using the collector mostly in my current company, mostly as a transport layer at the moment. We already migrated traces. We're in the process of migrating logging and metrics. In my previous company, we were a bit more advanced than that. We were using the collector itself and the operator. Currently, we are only using it as a deployment. Basically, all it's doing is transporting from our legacy components and we're slowly moving to collect everything through OpenTelemetry Collector. In the past, we actually used the presets a lot, the presets that are offered by the collector itself. We were running it as a daemon set and the collectors were collecting everything, especially in our Kubernetes cluster. That is the goal right now, to have a collector take as much of the collecting of the information as possible and get rid of the legacy ones like Jaeger, for example, Prometheus. + +**Juraj:** Awesome, same question, Yudai, please. How is it that you and your company are using the collector? + +**Juraj:** In my current role, which I just joined this month, they're just starting with OpenTelemetry and traces in general. But in my previous role, we basically, it was a SaaS startup where early on, there was some concern about the costs. One of the things we started to work on was migrating from a vendor's proprietary collection pipeline and client libraries to OpenTelemetry, which then enabled us to do a POC of other alternatives, where we basically just used the concept of the pipelines to take the data we were already gathering and just send it to a different vendor and also a self-hosted solution. It was mostly, we ended up settling on the self-hosted solution, which was basically the Grafana looks good to me stack. So hosted in Kubernetes, the way we were on OpenTelemetry, there were a couple of modes. There was the typical Kubernetes deployment where we had the daemon set for collecting logs, metrics, and traces from everything running on the nodes. There were a couple of stateful sets, one dedicated to collecting Kubernetes related metadata. We also deployed it as sidecars to various things still running in ECS, which were being migrated to Kubernetes. We also had a couple of auto collector deployments with ingresses, for either interesting the data from the sidecars in the ECS into the Kubernetes, or we even had sort of a hybrid mode of our product where the execution layer would run in customers' VPCs. This was written in Java and had the Java agent instrument included in, and that would also send back metrics and traces to our ingress behind which we had the OpenTelemetry deployment and then in front of the looks good to me stack. We had another layer of OpenTelemetry collectors to do some unified transformation to ensure some labels were common, things like that. + +**Greg:** Awesome, and how about you, Greg? + +### [00:09:30] Juraj discusses Collector usage + +**Greg:** So we use the collector for traces. We had a sort of legacy kind of trace-like thing called request viewer, which was based on logs, which was very heavy. We're migrating now everybody, I think we've just started to delete the cluster for the old request viewer. Now everybody should be sending traces via the collector. We do some processing, do some sanitization, add some attributes, and do tail sampling as well, so that we can sample by whole traces. We don't do anything with metrics apart from our own metrics, so we gather the metrics from the collector and forward them, Prometheus remote write them out to Grafana. We use Grafana Cloud as the backend for all the telemetry. We deploy it in ECS, not Kubernetes. That's kind of been interesting. We have one central cluster. We don't have a sidecar. We have one central cluster that scales in and out with load of the collector. + +**Joel:** Okay. Interesting. Interesting. I definitely have some follow-up questions on that, which I'll save for a little bit later. Then finally, Joel. + +### [00:12:00] Greg discusses Collector usage + +**Joel:** So we have collectors deployed all over the place. We've run collectors on all of our hosts. They are kind of egress gateways for telemetry. Primarily, the data which we're shipping is log data. We have pipelines enabled for metrics and traces, but currently, it's just the collector's own metrics which we ship. There are very few applications which are using traces at the moment. So primarily we're using it as a kind of log telemetry mesh, and we're using that as a way to sort of lay the groundwork for the full migration in the future. We have the egress collectors running on our hosts. We have an ingress collector running in our central Kubernetes cluster. Then we have multiple layers of collectors behind that. So we have a routing collector, and then we have a collector which is specific for each backend which we're interested in. For example, for Loki, for Thanos, and for Tempo. Those are the main backends which we're shipping to at the moment. We self-host everything. The reason we've done it like that is to sort of try and keep the configuration overhead simple because those pipelines can get quite complicated when you try and stuff everything into one collector. So we have this sort of egress and ingress concept. The goal really that we have is we're trying to ensure that all telemetry in the future will have a uniform set of labels of metadata attached to it. We're starting with logs, and then we're going to roll that out also to metrics and traces in the future, so that you can really actually start to correlate between signals in the backends. We also, if I can quickly say, we also do some custom processing. We actually build our own collector, and so we, some of our teams, for example, have built processes that do things to proxy logs. For example, with proxy logs which come through, they need special processing. It's been quite well adopted also by the service teams when they see the power of building a custom processor. + +That's great. Next question is, are you, starting with Iris, are you using a vanilla collector, collector contributor, a custom built one, vendor distro? -**Joel:** Hi, I'm Joel. I'm from a company called Open Systems. We're based in Switzerland, but we have a presence globally. We have offices in San Francisco, actually. I've never been there. Also in London and Germany. This is where we're based. We offer managed SASE, so managed connectivity and VPN for our customers. I'm on the observability team, so I'm responsible for making sure that we get telemetry from our devices, which we have 10,000 of, all across the world. We're now having to deal with data coming from Kubernetes, so Prometheus metrics, logs, all this nice telemetry, which we need to handle. We've been using the collector since about a year now, I think just under a year, I would say probably April last year was when we started. We really decided very quickly that we want to fully migrate everything to it because at the moment we kind of had a big mass of different service teams doing things in their own way, so we're kind of consolidating everything now and we found that the collector and OpenTelemetry in general was a very nice way which we could do that. +**Iris:** We're using, sorry. Yeah, we're using the vanilla one. So far we haven't had the need to use any custom that is already not there, no new, nothing, or a vendor one. We're currently fully open source. So yeah, the vanilla one worked great in our case. I'm assuming that's like collector contrib as is kind of thing. -[00:05:00] **Adriana:** Awesome, well, thank you again all of you for joining us. So let us get into the questions. First things first, we'll start, let's start in the order in which folks were introduced. Iris, how about you start with telling us how you and your team are primarily using the Collector? +**Juraj:** Yeah, exactly. Of course, we have our own configurations internally, but yeah, it's the same base image. -**Iris:** Okay, so, I could talk a little bit about how we're using it now and in my previous company as well, since I am very new at my current position. We are using the Collector mostly in my current company mostly as a transport layer at the moment. We already migrated traces. We're in the process of migrating logging and metrics. In my previous company, we were a bit more advanced than that. We were using the collector itself and the operator. Currently, we are only using it as a deployment, basically all it's doing is transporting from our legacy components and we're slowly moving to collect everything through OpenTelemetry Collector. In the past, we actually used the presets a lot, the presets that are offered by the collector itself. So we were running it as a daemon set and the collectors were collecting everything, especially in our Kubernetes cluster. That is the goal right now, to have a collector take as much of the collecting of the information as possible and get rid of the legacy ones like Jaeger, for example, Prometheus. +### [00:14:30] Joel discusses Collector usage -**Adriana:** Awesome. Same question, Juraj, please. How is it that you and your company are using the collector? +**Juraj:** Same question, Yudai. I honestly do wonder if anybody's actually running the, like, not the contrib one because the main one just does only old DLP. I think that's kind of like, I have seen very few workloads that produce LPLP and can also interest LPLP on the other side. Anyway, we use a variety. We have multiple custom distributions built. One of the reasons behind that is being, for example, the OTEL collectors that are sort of exposed to the Internet. We wanted to minimize, well, first of all, we wanted to minimize the size of the binary, but also minimize the number of components just to have the only necessary ones, even from a security perspective, especially around exposing it to the Internet. We had another special distribution which contained the gold debugger in case we needed to debug some issues, like there was, for example, an issue with the Datadog exporter, which was not exporting properly certain labels and logs, things like that. All of it is based on the upstream OpenTelemetry collector contrib with just custom configuration. -**Juraj:** In my current role, which I just joined this month, they're just starting with OpenTelemetry and traces in general. But in my previous role, we basically, it was a SaaS startup where early on, there was some concern about the costs. One of the things we started to work on was migrating from a vendor's proprietary collection pipeline and client libraries to OpenTelemetry, which then enabled us to do a POC of other alternatives, where we basically just used the concept of the pipelines to take the data we were already gathering and just send it to a different vendor and also a self-hosted solution. It was mostly, we ended up settling on the self-hosted solution, which was basically the Grafana "looks good to me" stack. So hosted in Kubernetes, the way we are on OpenTelemetry. There were a couple of modes. There was the typical Kubernetes deployment where we had the daemon set for collecting logs, metrics, and traces from everything running on the nodes. There were a couple of stateful sets, one dedicated to collecting Kubernetes related metadata. We also had a couple of auto collector deployments with ingresses for either ingesting the data from the sidecars in the ECS into Kubernetes, or we even had a hybrid mode of our product where the execution layer would run in customers VPCs. This was written in Java and had the Java agent instrument included in, and that would also send back metrics and traces to our ingress behind which we had the OpenTelemetry deployment and then in front of the "looks good to me" stack, we had another layer of OpenTelemetry collectors to do some unified transformation to ensure some labels were common, things like that. +**Greg:** Awesome. And since you did, so since you've done contributions back to the collector and since you've used the collector builder, what have your experiences been around both of these things? I guess let's start with, like, has the OTEL Collector Builder, like, did you find it a useful, intuitive tool to use when you were building your own distribution? -**Adriana:** Awesome, and how about you, Greg? +**Juraj:** Yeah, I think the one, like, once we got the initial configuration going, then it's really easy to, that's why we have like six or seven different builds because it's just, you copy-paste the config once, edit it a bit, remove, add what you need, and then just run the builds in parallel basically and just give them different names. It was also quite easy to then just point at your fork of a specific processor to test it, so you can test things even before they get merged. Also with, the one thing I think the bit where there's that replaces at the bottom, that's something we ran into a couple of times where we needed to update that because the builds broke, but that was just looking at what the upstream was doing and updating that with that information. -[00:09:10] **Greg:** We use the collector for traces. We had a sort of legacy, kind of trace-like thing called request viewer, which was based on logs, which was very heavy. We're migrating now, I think we've just started to delete the cluster for the old request viewer. So now everybody should be sending traces via the collector. We do some processing, do some sanitization, add some attributes, and do tail sampling as well so that we can sample by whole traces. We don't do anything with metrics apart from our own metrics, so we gather the metrics from the collector and forward them, Prometheus remote write them out to Grafana. We use Grafana Cloud as the backend for all the telemetry. We deploy it in ECS, not Kubernetes, and that's kind of been interesting. We have one central cluster. We don't have a sidecar. We have one central cluster that scales in and out with the load of the collector. +Yeah, so the second question was on the contribution process, or? -**Adriana:** Okay. Interesting. I definitely have some follow-up questions on that, which we'll save for a little bit later. And then finally, Joel? +**Greg:** Yeah, yeah. How did you find the contribution process to be? -**Joel:** We have collectors deployed all over the place. We've run collectors on all of our hosts. They are kind of egress gateways for telemetry. Primarily, the data which we're shipping is actually fully, well not fully, but it's mainly log data. We have pipelines enabled for metrics and traces, but currently it's just the collector's own metrics, which we ship. There are very few applications using traces at the moment. So primarily we're using it as a kind of log telemetry mesh. We're using that as a way to sort of lay the groundwork for the full migration in the future. We have the egress collectors running on our hosts. We have an ingress collector running in our central Kubernetes cluster. Then we have multiple layers of collectors behind that. We have a routing collector. Then we have a collector which is specific for each backend which we're interested in. For example, for Loki, for Thanos, and for Tempo. Those are the main backends we're shipping to at the moment. We self-host everything. The reason we've done it like that is to sort of try and keep the configuration overhead simple because those pipelines can get quite complicated when you try and stuff everything into one collector. We have this sort of egress and ingress concept. The goal really that we have is we're trying to ensure that all telemetry in the future will have a uniform set of labels of metadata attached to it. So we're starting with logs, and then we're going to roll that out also to metrics and traces in the future, that you can really actually start to correlate between signals in the backends. We also, if I can quickly say, we also do some custom processing. We actually build our own collector. Some of our teams, for example, have built processes that do things to proxy logs. For example, with proxy logs, which come through, they need special processing. It's been quite well adopted also by the service teams when they see the power of building a custom processor. +**Juraj:** I think it's okay, like, it's kind of sort of nice that things get, because of the pretty frequent releases, you don't have to wait that long for your changes to go live after you. There's decent documentation around how you can contribute, like how to format your code, things like that. I think there's a world of pretty timely response to well-written issues where they tell you like, oh yeah, that's cool. If you have time, you can contribute and fix a certain thing. So that was, well, the only slight frustration I found was at times it can take a bit for like, there's a label on the pull request which is like ready to merge. Even after that gets assigned, it can take like up to a week or two for somebody to finally merge it, which can be frustrating because then the pipeline starts failing because something broke upstream and somebody merged upstream into the branch and it broke the pipeline. There are things like that, but other than that, it feels pretty good. -[00:12:40] **Adriana:** That's great. Next question is, are you... So starting with Iris, are you using a vanilla collector, collector contributor, a custom-built one, vendor distro? +**Greg:** All right, great. Thank you. Next, Greg, what collector distribution do you use? Do you use contrib? Do you build your own? -**Iris:** We're using the vanilla one. So far we haven't had the need to use any custom that is already not there, no new, or a vendor one. We're currently fully open source, so yeah, the vanilla one worked great in our case. I'm assuming that's like collector contrib as is kind of thing. +**Greg:** We, yeah, we build our own. We use the builder. We forked the contrib repo and, based off that, we do that because, well, initially we had a feature that we needed, which was to add ECS, well, AWS Cloud Map service discovery to the load balancer exporter, so we could get a list of targets for the load balancer. We forked that, did that locally, and we've actually contributed that back, or there's an open pull request at the moment that's taken quite a long time, it still hasn't been merged. I think it's getting there. I think it's near now. That's taken, that's been quite slow. We've discovered quite a few either missing features or bugs. It's been quite nice to both be able to debug the code locally and actually step through and find out is this how it's supposed to work? Also then to be able to go and fix it. Although we've had about five or six different bugs that we've got to the point of almost fixing them, and then we've seen that it's been fixed in the upstream repo. Somebody else has worked on it and has parachuted in and has fixed it. Which is obviously nicer because they presumably are doing it better than us. None of us are Go developers; I've got a Java background and we have various other developer backgrounds, but nobody's a Go developer, so we're kind of a bit nervous about contributing and a bit nervous about making changes and quite slow about that as well. But it's been a very good opportunity to learn more about Go. -**Iris:** Yeah, exactly. Of course, we have our own configurations internally, but yeah, it's the same base image. +**Joel:** And then same question around the builder. How did you find your experience in using the collector builder to build your own distros? -**Adriana:** Okay. Perfect. Same question, Juraj. +**Greg:** The builder's fine, no complaints. It's quite, quite, I mean, as I'm new to the Go ecosystem or, you know, I don't really know how any other Go project, large project works, but it seems to work quite well. My one complaint made is it about the builder or about the repo. There's like two or three components in the core project right, not the contrib, like the OTLP receiver, the batch processor, the memory limiter. I've been wanting to work on those, and it's really painful that they are not in the contrib repo. It's really painful to be able to make changes to them and get them even running locally. I don't even know how I'd manage to build another distribution of the main thing, and I don't know how that would work, but like, so with the builder, you just do the replace and point it to your local directory. As long as the component is in that repo that we're building from, it works fine. But for these other components, I would be much happier if they were in the contrib because it would make my life much simpler. -**Juraj:** I honestly do wonder if anybody's actually running the not the contrib one, because the main one just does only old DLP. I think I've seen very few workloads that produce LPLP and can also ingest LPLP on the other side. Anyway, we use a variety. We have multiple custom distributions built. One of the reasons behind that is being, for example, the OTEL collectors that are sort of exposed to the Internet. We wanted to minimize, well, first of all, we wanted to minimize the size of the binary, but also minimize the number of components just to have the only necessary ones, even from a security perspective, especially around exposing it to the Internet. We had another special distribution, which contained the gold debugger in case we needed to debug some issues, like for example, an issue with the Datadog exporter, which was not exporting properly certain labels and logs, things like that. All of it is based on the upstream OpenTelemetry collector contrib with just custom configuration. +**Joel:** That's definitely a good piece of information. We'll be sure to pass that on. And then finally, Joel. -**Adriana:** Since you're talking about having a collector with just the things that you need with the reduced binary, I'm assuming then you use the builder then? +### [00:22:20] Challenges in debugging -**Juraj:** Then we have like renovate bot to submit pull requests with updates. We also had some, I mean, we also contributed a bit back to the total collector. So we also used it, so we could test changes before they were merged, changes on processors, on other components before they were merged into upstream. +**Joel:** So, we are running our custom build. The reason is actually because we have, like I mentioned, service teams who built their own processes. That was the, actually one of the big draws for why we should adopt OpenTelemetry. I mean, that has been pretty, very, very well adopted, I would say. We use the builder; it was not a problem to get it running in our build pipelines. Very, very good experience there. It's always quite fun. We, you know, I think, Yudai, you mentioned, it's quite a fast release cadence, which is a good thing, but if it's very easy to let a few versions slip by, and when you're maintaining your own distro, sometimes you get some nasty surprises when, you know, if you were to just build and deploy some config with deprecated or completely removed without, you know, you have to chase down those errors and be very kind of on guard, even when you do a product deployment and you've checked everything in dev, it's always a little bit on your mind. Hey, something could have changed, which is going to blindside me. But I mean, that's just kind of, it comes to the territory because a lot of the processes and modules that we're using are in, you know, alpha or beta stability level. I think it's kind of expected, and it's one of the sort of things we're happy to pay for with the flexibility of the solution. So this is good. Yeah, but overall, really, really positive experience. -[00:15:36] **Adriana:** Awesome. And since you did, so since you've done contributions back to the Collector and since you've used the Collector Builder, what have your experiences been around both of these things? I guess let's start with, like, has the OTEL Collector Builder, like, did you find it a useful, intuitive tool to use when you were building your own distribution? +**Iris:** Awesome. Next question, starting with Iris, what's the biggest challenge or challenges that you've faced with the collector? And it can be stuff like deployment, configuration, monitoring it, debugging it. -**Juraj:** I think the one, like, once we got the initial configuration going, then it's really easy to, that's why we have like six or seven different builds, because it's just, you copy-paste the config once, edit it a bit, remove, add what you need, and then just run the builds in parallel, basically, and just give them different names. It was also quite easy to then just point at like your fork of a specific processor to test it, so we can test things even before they get merged. Also, with the one thing I think the bit where there's that like replaces at the bottom, that's something we ran into a couple of times where we needed to update that because the builds broke, but that was just looking at what the upstream was doing and updating that. +**Iris:** Debugging, for sure. Yeah, I think deploying it is pretty straightforward, but debugging, I had a situation recently. It's kind of embarrassing to say now after finding out what the issue was. It was the first time that I was actually using the collector with the ingress, enabling the ingress. Usually, it was all taken care of inside the Kubernetes cluster. Of course, having to pass the right paths and, yeah, I kept sending spans to try and see if it's going to work, but nothing. I just got an error message and nothing else. I did try something else, just not an error, but like a status code and that's it. It took me and my coworker more than two days to figure out what the issue was because, yeah, we tried to enable debug logs, debug exporter, everything, but there was just not enough information for this. In general, it has happened like this. After a lot of time spent, we figured it out, but yeah, it could have been a bit easier for us. -**Juraj:** With that information, yeah, so the second question was on the contribution process, or? +**Juraj:** What did you end up doing for troubleshooting? Did you have to go deep in the code or? -**Adriana:** Yeah, yeah, how did you find the contribution process to be? +**Iris:** Yeah, deep in the code. At this point, it became like a brainstorming session. Everything that we ever knew about IT, we're like, okay, this could be, this could be, this could be that, you know, and we finally got to it after a long time. But yeah, checking the code definitely helped. Then we realized, okay, you actually need the path in the receiver itself to explicitly add it. Yeah. That's why I say that after it's been solved, it's like such a silly thing, but at the moment of debugging, it can get pretty overwhelming. -**Juraj:** I think it's okay. It's kind of sort of nice that things get, because of the pretty frequent releases, you don't have to wait that long for your changes to go live after you submit them. I think there's decent documentation around like how you can contribute, like things out, like how to format your code, things like that. There’s a world of pretty timely response to like well-written issues where they tell you like, “Oh yeah, that’s cool. If you have time, you can contribute and fix a certain thing.” The only slight frustration I found was, at times it can take a bit for like, there’s a label on the pull request, which is like ready to merge. Even after that gets assigned, it can take up to like a week or two for somebody to finally merge it, which can be frustrating because then like the pipeline starts failing because something broke upstream and somebody merged upstream into the branch and it broke the pipeline. There are things like that, but other than that, it feels pretty good. +**Juraj:** Did you have like a Go background to aid in the troubleshooting or like, what? -**Adriana:** All right, great. Thank you. Next, Greg, what collector distribution do you use? Do you use contrib? Do you build your own? +**Iris:** I didn't work with Go before actually working with OpenTelemetry, but now I'm learning. That's the thing. OpenTelemetry and debugging and checking everything has become my experience in learning Go, and I'm also learning independently. So, yeah, it was a good excuse. -[00:20:01] **Greg:** We build our own. We use the builder. We forked the contrib repo and based off that. We do that because, well, initially we had a feature that we needed, which was to add ECS, well, AWS cloud map service discovery to the load balancer exporter, so we could get a list of targets for the load balancer. So we forked that, did that locally, and we've actually contributed that back, or there's an open pull request at the moment that's taken quite a long time, it still hasn't been merged. I think it's getting there. I think it's near now, but that's taken quite a bit of time. We've also discovered quite a few either missing features or bugs. It’s been quite nice to both be able to debug the code locally and actually step through and find out is this how it's supposed to work? And also then to be able to go and fix it. Although we've had about five or six different bugs that we've got to the point of almost fixing them and then we've seen that it's been fixed in the upstream repo. Somebody else has worked on it and has parachuted in and has fixed it, which is obviously nicer because presumably they’re doing it better than us. None of us are Go developers; I’ve got a Java background and we have various other developer backgrounds, but nobody's a Go developer. We’re kind of a bit nervous about contributing and a bit nervous about making changes and quite slow about that as well, but it’s been a very good opportunity to learn more about Go. +**Juraj:** Oh, that's great. How about you? What's your biggest challenge that you faced around the collector? -**Adriana:** And then same question around the builder. How did you find your experience in using the collector builder to build your own distros? +**Juraj:** I think when we ran into a couple of bugs, it was kind of hard to debug because they were sort of happening only with large loads of data at times. That's when we ended up having to build our own distribution to deploy with the gold debugger attached to it. At least a lot of decently used dev environment to actually see, okay, why is this happening? Then we only figured out, oh, this is like a bug in the like Datadog exporter, for example, where like it was not doing something properly. I think that like if you have a bit more complex pipeline, especially if you manipulate a lot with the research attributes, things like that, it's kind of hard to test locally or even, you know, like when you basically just sort of need to roll it out to some lower environments and see if it works properly. But then if you have some of your own local testing environment, we don't have that much data going through it, and it's probably maybe slightly different than what like the actual business applications produce. There would be, it might be nice to be able to have some sort of capability to test that bit better without having to actually deploy to full-blown environments. In the later stages of the migration, we had an incident, for example, where we stopped sending some data to Datadog, or like some of the labels changed. We didn't code it in the lower environments because we were no longer sending those to Datadog. We only realized that in production and then like SRE team started to like scream at us, like where's the data, where's the data? Oh, this label changed. Why did it change? Things like that. I mean, one other thing we ended up doing was because we had multiple sources of the data going, like different types of auto collectors doing different type of things, we ended up attaching the label to all the metrics logs and traces coming through it basically telling us like which auto collector type it came from. So like daemon set for the daemon set, something like Kubernetes for the one that collected Kubernetes metrics. Then like we had like internal/external interest, which was for some of it, the internal source for ECS, the external was for metrics and traces coming from the external customers, just so we could easily track down, okay, where is this data even coming from? -**Greg:** The builder’s fine, no complaints. It’s quite, quite, I mean, as I’m new to the Go ecosystem or, you know, I don’t really know how any other Go project, large project works, but it seems to work quite well. My one complaint made is it about the builder or about the repo. There’s like two or three components in the core project right now, not the contrib, like the OTLP receiver, the batch processor, the memory limiter. I’ve been wanting to work on those, and it’s really painful that they are not in the contrib repo. It’s really painful to be able to make changes to them and get them even running locally. I don’t even know how I’d manage to build another distribution of the main thing, and I don’t know how that would work, but like, so with the builder, you just do the replace and point it to your local directory. As long as the component is in that repo that we’re building from, it works fine. But for these other components, I would be much happier if they were in the contrib because it would make my life much simpler. +**Greg:** Right, right. And, I'm just out of curiosity. So, you mentioned debugging issues in the collector also was a pain point for you. Do you have a Go background? -**Adriana:** Okay, that's definitely a good piece of information. We'll be sure to pass that on. And then finally, Joel? +**Juraj:** I'm not really a software engineer. I'm more of an ops side, but I have been using things like Prometheus and Kubernetes for years before adapting OpenTelemetry. So, a bit. -**Joel:** We are running our custom build. The reason is actually because we have, like I mentioned, service teams who built their own processes. That was actually one of the big draws for why we should adopt OpenTelemetry. That has been pretty well adopted, I would say. We use the builder; it was not a problem to get it running in our build pipelines. Very, very good experience there. It’s always quite fun. We, you know, I think, Yudai, you’d mentioned, it’s quite a fast release cadence, which is a good thing, but if it’s very easy to let a few versions slip by, and when you’re maintaining your own distro, sometimes you get some nasty surprises when, you know, if you were to just build and deploy, some config with deprecated or completely removed without, you know, you have to chase down those errors and be very kind of on guard. Even when you do a product deployment and you’ve checked everything in dev, it’s always a little bit on your mind, “Hey, something could have changed which is going to blindside me.” But I mean, that’s just kind of, it comes to the territory because a lot of the processes and modules that we’re using are in, you know, alpha or beta stability level. I think it’s kind of expected, and it’s one of the sort of things we’re happy to pay for with the flexibility of the solution. So this is good. Yeah, but overall, really, really positive experience. +**Greg:** Gotcha. Okay. Same question, Greg. -**Adriana:** Awesome. Next question, starting with Iris, what's the biggest challenge or challenges that you've faced with the Collector? It can be stuff like deployment, configuration, monitoring it, or debugging it. +### [00:30:00] Discussion on custom builds -**Iris:** Debugging, for sure. Yeah, I think deploying it is pretty straightforward, but debugging, I had a situation recently. It’s kind of embarrassing to say now after finding out what the issue was. It was the first time that I was actually using the collector with the ingress, enabling the ingress. Usually, it was all taken care of inside the Kubernetes cluster. Of course, having to pass the right paths and, yeah, I kept sending spans to try and see if it’s going to work, but nothing. I just got an error message and nothing else. I did try something else, just not an error, but like a status code and that’s it. It took me and my coworker more than two days to figure out what the issue was because, yeah, we tried to enable debug logs, debug exporter, everything, but there was just not enough information for this. In general, it has happened like this. After a lot of time spent, we figured it out, but yeah, it could have been a bit easier for us. What did you end up doing for troubleshooting? Did you have to go deep in the code or? +**Greg:** I mean, I would, our biggest challenge has been like the quality of the platform we're providing to the rest of the organization, especially kind of coming from this other system, which kind of worked, to rolling out the collector and maybe our configuration being wrong or maybe there being bugs or whatever for some whatever reason, traces and spans not being available in the backend when our users were looking for them. Having gone out and said, hey, this is the future. This is the great new technology. It's much better than this thing we made in-house for them, their actual experience of them going and looking for the requests that they were trying to debug and it's not there. That probably initially was just naivety on our part, like we didn't know how to monitor the collector properly, we didn't know at which point along the pipeline from the trace or the span being created in the application to it arriving in Grafana where we should be looking. I think we've improved on that, but there are still kind of areas of doubt. We've got a couple of things which we're struggling to know how in principle we can even monitor it, or how we should do it. It just doesn't seem to be possible at the moment. That would be our kind of biggest thing that's caused us anxiety. The second thing would be just the sort of bugs or the things not working as expected in the collector. I joined the team a little bit later, a bit more senior than some of the other people, and I think when they adopted it, they kind of drank the Kool-Aid. They thought this is great, it's going to work, it's going to solve all of our problems. For a long time, they were struggling thinking that they were doing something wrong. Whereas as I'm a bit more senior, I'm very cynical. I'm like, oh, there's probably rubbish code. Let's go and have a look at the code. Oh, look, it doesn't work. They didn't think to ask those sorts of questions. Once you've been around for a while, you realize that all code is rubbish code and everything's got a bug somewhere. Some examples of that service graph. At the moment, retry. I don't know if you're aware, but I think the gRPC receiver will now return a retry code, but the HTTP receiver, OTLP receiver will not return a proper retryable code, it will just return a server, a 500, no matter what it is. Whereas a transient error should be like a 503 or something, so the client retries. We turned on retries in our agent, and we're like, hey, everything's going to retry, we're not going to lose these spans anymore. Like, we're still losing them. What's going on? It's just not implemented in the collector, right? Retriable, transient errors. Stuff like that, it's like, oh, right, okay, don't assume that it works. We've just switched from using the OpenCensus metrics, internal metrics, to the sort of native OTLP, or whatever you would call it. That all seemed great, but then some of them have disappeared for the tail sampler because the tail sampler has not yet been migrated to OTLP and it's only publishing OpenCensus metrics. Which is fine and we'll probably be quite happy to go and fix that because it's quite easy and submit a pull request for it, but it's just like somebody spent a week thinking that they've done something wrong, right? Actually, oh, it's just not. -[00:25:00] **Iris:** Yeah, deep in the code. At this point, it became like a brainstorming session. Everything that we ever knew about IT, we’re like, “Okay, this could be, this could be, this could be that,” you know, and we finally got to it after a long time. Checking the code definitely helped. We realized, okay, you actually need the path in the receiver itself to explicitly add it. That’s why I say that after it’s been solved, it’s such a silly thing, but at the moment of debugging, it can get pretty overwhelming. Now, did you have a Go background to aid in the troubleshooting or like what? +**Joel:** You mentioned, yeah, you could submit a pull request to fix it. Have you submitted any issues around these items that you've identified? -**Iris:** I didn’t work with Go before actually working with OpenTelemetry, but now I’m learning. That’s the thing. OpenTelemetry and debugging, checking everything has become my experience in learning Go, and I’m also learning independently. So, yeah, it was a good excuse. +**Greg:** It's a similar story to earlier that like I'm maybe not as good at my job as I should be. So what I do is I go and try and mess around with the code and fix it. Then I go and see if there's an open issue and then I find out there is an open issue and somebody's already working on it and I should have been doing something else with my time. -**Adriana:** Oh, that’s great. How about you? What’s your biggest challenge that you faced around the collector? +**Greg:** So pretty much, yeah, everything that we've seen so far. There was one thing, we use the count connector and that produces a stream of delta data points. We needed to export them to a Prometheus remote write, so I actually created a delta to cumulative processor and opened up a merger, an issue for that and got sponsored, but then somebody else had the same issue and now they're working on it. I haven't had time to actually, my, mine was a very much a proof of concept and we've got it running in production and it works, but it only meets our very particular use case. It's not like a generic and it's probably written wrong and everything. -**Juraj:** I think when we ran into a couple of bugs, it was kind of hard to debug because they were sort of like happening only with large loads of data at times. So, that’s when we ended up having to build our own distribution to deploy with the gold debugger attached to it. At least a lot of decently used dev environment to actually see, “Okay, why is this happening?” And then we only figured out, “Oh, this is like a bug in the Datadog exporter,” for example, where it was not doing something properly. I think that like, if you have a bit more complex pipeline, especially if you manipulate a lot with like the research attributes, it’s kind of hard to test locally or even, you know, like when you basically just sort of need to roll it out to some lower environments and see if it works properly. But then if you have your own local testing environment, we don’t have that much data going through it, and it’s probably maybe slightly different than what like the actual business applications produce. There’s a, it would be, it might be nice to be able to have some sort of capability to test that bit better without having to actually deploy to like full-blown environments. In the later stages of the migration, we had an incident, for example, where we stopped sending some data to Datadog, or like some of the labels changed. We didn’t code it in the lower environments because we were no longer sending those to Datadog. We realized that in production, and then like SRE team started to scream at us, like, “Where’s the data, where’s the data?” Oh, this label changed. Why did they change things like that? I mean, one other thing we ended up doing was because we had multiple sources of the data going, like different types of auto collectors doing different types of things. We ended up attaching the label to all the metrics, logs, and traces coming through it basically telling us like which auto collector type it came from. So like, daemon set for the daemon set, something like Kubernetes for the one that collected Kubernetes metrics. We had like internal slash external interest, which was for some of it, the internal force for ECS, the external was for metrics and traces coming from external customers, just so we could easily track down, okay, where is this data even coming from? +**Joel:** Right. So yeah, we opened up that issue and it got some attention, so that's how it should work. We're happy with that. -**Adriana:** Right, right. And just out of curiosity, you mentioned debugging issues in the collector also was a pain point for you. Do you have a Go background? +**Joel:** It's really interesting you mentioned that. I was going to be my point that I was going to make is that I think the handling of cumulative metrics is really missing. So that's something which we kind of miss with very sort of Prometheus focused. So all of our metrics are in Thanos. We want to ship them through. I think this is working, but, for example, like we ran into exactly the same issue with the count connector and we just figured out, yeah, that's going to be easy. Just plug it in and Bob's your uncle. You can ship that to Prometheus and it didn't work like that. It was sort of once you're so deep into the topic, you kind of know. Yeah, sure, that's not going to work. But you have to get quite deep into it to get to that point of understanding, which I think was disappointing to our users when we turned around and said to them, ah, yeah, by the way, that solution we proposed last week definitely isn't going to work. We sort of, we need to wait on some upstream development to go on there. So I would say, yeah, that's the hardest part at the moment actually is planning how we're going to bring metrics into our sort of unified pipeline. Because at the moment, it's just a tricky part. -**Juraj:** I’m not really a software engineer. I’m more on the ops side, but I have been using things like Prometheus and Kubernetes for years before adapting OpenTelemetry. So a bit. +**Joel:** We also have a sort of outstanding use case, I would say, which is a, you know, generic, I mean, we have, like I mentioned, a lot of logs. So we're very logs-based at the moment, kind of a very old school, which trying to change it internally, but that takes a long time. We have like a lot of use cases where we want to build metrics out of logs. Currently, there's no, you know, general way to do that. We're actually working on that as a sort of internal use case currently to build like a logs to metrics connector. But there's a few sort of, let's say functionalities I would have expected to be there, which aren't there yet, which, yeah, it's, it's also very good to see. I mean, I see Dan is here. Dan is sponsoring, I think, the cumulative to Delta or Delta to cumulative processor. It's very good to see that the, you know, the community is working on it upstream. It just can be a surprise sometimes when you think, ah, for sure, that feature must be there when actually, oh no, you can't generate metrics in the collector and remote write them to Prometheus. That's not there yet. -**Adriana:** Gotcha. Okay. Same question, Greg. +**Adriana:** Yeah, so a few surprises like that. Basically, that's, let's say the hardest thing. You can never be too sure when you're talking to other teams in your organization that what you're promising is actually achievable until you sort of dive in and get your hands messy with the code. -[00:30:00] **Greg:** I would say our biggest challenge has been like the quality of the platform we’re providing to the rest of the organization, especially kind of coming from this other system, which kind of worked to rolling out the collector and maybe our configuration being wrong or maybe there being bugs or whatever for some reason. Traces and spans not being available in the backend when our users were looking for them. Having gone out and said, “Hey, this is the future. This is the great new technology. It’s much better than this thing we made in house.” Their actual experience of them going and looking for the requests that they were trying to debug and it’s not there. That probably initially was just naivety on our part, like we didn’t know how to monitor the collector properly. We didn’t know at which point along the pipeline from the trace or the span being created in the application to it arriving in Grafana, where we should be looking. I think we’ve improved on that, but there are still kind of areas of doubt. We’ve got a couple of things which we’re struggling to know how, in principle, we can even monitor it, or how we should do it. It just doesn’t seem to be possible at the moment. That would be our biggest thing that’s caused us anxiety. The second thing would be just the sort of bugs or the things not working as expected in the collector. I joined the team a little bit later, a bit more senior than some of the other people. When they adopted it, they kind of drank the Kool-Aid. They thought this is great. It’s going to work. It’s going to solve all of our problems. For a long time, they were struggling thinking that they were doing something wrong. Whereas as I’m a bit more senior, I’m very cynical. I’m like, “Oh, there’s probably rubbish code. Let’s go and have a look at the code. Oh, look, it doesn’t work.” They didn’t like think to ask those sorts of questions. But you know, once you’ve been around for a while, you realize that all code is rubbish code and everything’s got a bug somewhere. Some examples of that, service graph. At the moment, retry. I don’t know if you’re aware, but I think the gRPC receiver will now return a retry code, but the HTTP receiver, OTLP receiver will not return a proper retryable code. It will just return a server, a 500, no matter what it is. Whereas a transient error should be like a 503 or something, so the client retries. We turned on retries in our agent, and we’re like, “Hey, everything’s going to retry, we’re not going to lose these spans anymore.” And like, we’re still losing them, what’s going on? It’s just not implemented in the collector, right? Retriable, transient errors. Stuff like that, it’s like, “Oh, right, okay, don’t assume that it works.” We’ve just switched from using the OpenCensus metrics, internal metrics, to the sort of native OTLP, or whatever you would call it. That all seemed great, but then some of them have disappeared for the tail sampler because the tail sampler has not yet been migrated to OTLP and it’s only publishing OpenCensus metrics. Which is fine, and we’ll probably be quite happy to go and fix that because it’s quite easy and submit a pull request for it, but it’s just like somebody spent a week thinking that they’ve done something wrong and actually it’s just not. +**Adriana:** All right. Well, thank you. I did have a final question for everyone, because I think especially as we see OpenTelemetry pick up more in the community. I want to talk just very briefly in the eight minutes that we have about scaling the collector. Any feedback pain points around scaling the collector, starting with Iris? -**Adriana:** And you mentioned like, yeah, you could submit a pull request to fix it. Have you submitted any issues around these items that you’ve identified? +**Iris:** So, so far about scaling, we have used at my current company, because we only are using for traces, we're using just normal horizontal auto scaler. It works great. In the past, we were also after KEDA, and I remember we even opened an issue in the OpenTelemetry, my colleague opened an issue to support it on the original chart, but I don't think it is supported right now. We did do it out of the box for our own obligation, and that worked very well for us, especially considering that the, okay, so HPA works because most of the queue is memory based, but yeah, using KEDA based on the queue size was a great move for us to do the auto scaling. But yeah, even the HPA works great depending on the load that you have. -**Greg:** It’s a similar story to earlier that like I’m maybe not as good at my job as I should be. What I do is I go and try and mess around with the code and fix it. Then I go and see if there’s an open issue. Then I find out there is an open issue and somebody’s already working on it. I should have been doing something else with my time. Pretty much everything we’ve seen so far. There was one thing; we use the count connector and that produces a stream of delta data points. We needed to export them to a Prometheus remote write, so I actually created a delta to cumulative processor and opened up a merger, an issue for that and got sponsored. But then somebody else had the same issue and now they’re working on it. I haven’t had time to actually, mine was very much a proof of concept and we’ve got it running in production and it works, but it only meets our very particular use case. It’s not like a generic and it’s probably written wrong and everything. So yeah, we opened up that issue and it got some attention. That’s how it should work. We’re happy with that. +**Juraj:** All right. Thanks. Juraj? -**Adriana:** Awesome. That’s great. And finally, Joel. +**Juraj:** For all the deployments, we were using HPA, also just based on memory and CPU. That worked very nicely, I think. Then for the daemons that we were looking at doing, VPA, just because if you have a single node with a bit higher load, you need to bump the resources and there's nothing you can do about that. -**Joel:** It’s really interesting you mentioned that. I was going to be my point that I was going to make is that I think the handling of cumulative metrics is really missing. That’s something which we kind of miss with very sort of Prometheus focused. All of our metrics are in Thanos. We want to ship them through, which I think is working. For example, like we ran into exactly the same issue with the count connector, and we just figured out, yeah, that’s going to be easy. Just plug it in and Bob’s your uncle, you can ship that to Prometheus, and it didn’t work like that. Once you’re so deep into the topic, you kind of know. Sure, that’s not going to work. But you have to get quite deep into it to get to that point of understanding, which I think was disappointing to our users when we turned around and said to them, “Ah, yeah, by the way, that solution we proposed last week definitely isn’t going to work. We sort of, we need to wait on some upstream development to go on there.” I would say, yeah, that’s the hardest part at the moment actually is planning how we’re going to bring metrics into our unified pipeline. Because at the moment, it’s just a tricky part. We also have an outstanding use case, I would say, which is, you know, generic, I mean, we have, like I mentioned, a lot of logs. So we’re very logs-based at the moment, kind of a very old school, which trying to change it internally, but that takes a long time. We have like a lot of use cases where we want to build metrics out of logs. Currently, there’s no, you know, general way to do that. We’re actually working on that as a sort of internal use case currently. To build like a logs to metrics connector. There are a few functionalities I would have expected to be there which aren’t there yet. It’s also very good to see, I mean, I see Dan is here. Dan is sponsoring, I think, the cumulative to Delta or Delta to cumulative processor. It’s very good to see that the community is working on it upstream. It just can be a surprise sometimes when you think, “Ah, for sure, that feature must be there,” when actually, oh, no, you can’t generate metrics in the collector and remote write them to Prometheus. That’s not there yet. A few surprises like that. Basically, that’s, let’s say the hardest thing. You can never be too sure when you’re talking to other teams in your organization that what you’re promising is actually achievable until you sort of dive in and get your hands messy with the code. +**Greg:** All right, great. And Greg? -**Adriana:** All right. Well, thank you. I did have a final question for everyone because I think especially as we see OpenTelemetry pick up more in the community, I want to talk just very briefly in the 8 minutes that we have about scaling the collector. Any feedback pain points around scaling the collector, starting with Iris? +**Greg:** So we're kind of like quite relatively early in our journey still with this, and we're at the moment happier to spend money than to lose data. So we've got like quite a, what's the word, conservative. We over-provisioned basically, and we've got a simple auto scaling policy based on memory because the biggest thing is the tail sampler, which holds all the memory. We do scale out quite far and then back down again, especially in our load test environment, obviously. That goes to almost nothing overnight and then during the load test, we get like 30, 40 instances sometimes. But we haven't put a lot of effort into trying to actually optimize that because we're much more worried about making sure that we've got, we're properly processing all that data. Because we're not sure what we're supposed to be monitoring, we're kind of very nervous about changing anything. But we're in AWS, so I would like to see exporting the system metrics that we've got, the internal metrics, to, you can write them to CloudWatch logs and then turn them into metrics somehow, it's called EMF or something, because you need the metrics to be within CloudWatch so that you can hit the AWS autoscaler. We can't use any Prometheus metrics, which is what we're normally doing. So there's quite a lot of stuff in my head that we could do there that I'm interested to look at. But apart from like getting it working on my machine, I haven't had anything deployed. But yeah, that's because we've been a bit burnt by losing data, so we're just very cautious about that at the moment. -**Iris:** So, so far about scaling, we have used, at my current company, because we only are using for traces, we’re using just normal horizontal auto scaler. It works great. In the past, we were also after KEDA, and I remember we even opened an issue in the OpenTelemetry, my colleague opened an issue to support it on the original chart, but I don’t think it is supported right now. But we did do it out of the box for our own obligation, and that worked very well for us, especially considering that the, okay, so HPA works because most of the queue is memory based. Using KEDA based on the queue size was a great move for us to do the auto scaling. But yeah, even the HPA works great and depending on the load that you have. +**Joel:** Fair enough. And finally, Joel? -**Adriana:** All right, thanks. Juraj? +**Joel:** So, essentially we run everything on one cluster per environment, and these are all deployments. We found that just HPA with the memory limited processor has been no problems at all. I think it's even just the out of the box config for the processor. I've never seen a collector OOM actually, which I find quite impressive. Yeah, that seems to work very, very nicely. Actually, I need to talk to Juraj after this call because I was suspicious of gRPC on Kubernetes because I have run into the issue where parts will scale out or deployment will scale out, but some of those parts just don't receive any telemetry. We are using gRPC on Kubernetes. -[00:48:25] **Juraj:** For all the deployments, we were using HPA, also just based on memory and CPU. That worked very nicely, I think. For the daemons that we were looking at doing, VPA, just because they’re like, yeah, if you have a single node with a bit higher load, you need to bump the resources and there’s nothing you can do about that. +**Juraj:** That sounds like an issue I need to dig into. -**Adriana:** All right, great. And Greg? +**Joel:** Definitely. If you can find me on the CNC Slack and if you ping me there, I can send you the link for the, I don't know if there's an open issue about it, but I discussed it already with another person. I can send you the link for the thread. -**Greg:** We’re kind of like quite relatively early in our journey still with this, and we’re at the moment happier to spend money than to lose data. So we’ve got like quite a conservative, we over-provisioned basically, and we’ve got a simple auto-scaling policy based on memory because the biggest thing is the tail sampler, which holds all the memory. We do scale out quite far and then back down again, especially in our load test environment, obviously. That goes to almost nothing overnight, and then during the load test, we get like 30, 40 instances sometimes. But we haven’t put a lot of effort into trying to optimize that because we’re much more worried about making sure that we’re properly processing all that data. Because we’re not sure what we’re supposed to be monitoring, we’re kind of very nervous about changing anything. But we’re in AWS, so I would like to see exporting the system metrics that we’ve got, the internal metrics, to you can write them to CloudWatch logs and then turn them into metrics somehow. It’s called EMF or something because you need the metrics to be within CloudWatch so that you can hit the AWS autoscaler. We can’t use any Prometheus metrics, which is what we’re normally doing. There’s quite a lot of stuff in my head that we could do there that I’m interested to look at, but apart from getting it working on my machine, I haven’t had anything deployed, anything actually running. But yeah, that’s because we’ve been a bit burnt by losing data, so we’re just like very cautious about that at the moment. +**Juraj:** Yeah, that's great. Thanks. -**Adriana:** Fair enough. And finally, Joel? +**Joel:** That's something very useful I take away from this panel at least. -**Joel:** Essentially, we run everything on one cluster per environment, and these are all deployments. We found that just HPA with the memory limited processor has been no problems at all. I think it’s even just the out of the box config for the processor. I’ve never seen a collector OOM actually, which I find quite impressive. That seems to work very, very nicely. Actually, I need to talk to Juraj after this call because I was suspicious of gRPC on Kubernetes because I have run into the issue where parts will scale out or deployment will scale out, but some of those parts just don’t receive any telemetry, and we are using gRPC on Kubernetes. So that sounds like an issue I need to dig into. +**Iris:** Can I mention, you mentioned that somebody mentioned the memory limiter. I was enthusiastic to tell my team another one of these things where like I've read the docs and I tell everybody, let's just do this. The memory limiter there is going to give you transient errors, right? If you get near when you're scaling out, it's fine because the client will just retry. The fact that retries don't work, we started losing this telemetry. That was like, that was a scaling problem. So now we never hit the memory limiter. We always scale well before we're going to get anywhere near that because it basically doesn't work, right? I mean, it's going to stop it out of memory, but it's not gonna, the client isn't going to retry and resend that span. That's a much bigger problem for us than spending money on memory. -**Juraj:** If you can find me on the CNC Slack and if you ping me there, I can send you the link for the, I don’t know if there’s an open issue about it, but I discussed it already with another person. I can send you the link for the thread. +### [00:54:00] Upcoming presence at KubeCon -**Joel:** Yeah, that’s great. Thanks. That’s something very useful I take away from this panel at least. Thanks for that. Can I mention, you mentioned that somebody mentioned the memory limiter. I was enthusiastic to my team. Another one of these things where like I’ve read the docs and I tell everybody, “Let’s just do this.” The memory limiter there is going to give you transient errors, right? If you get near when you’re scaling out, it’s fine because the client will just retry, right? The fact that retries don’t work, we started losing this telemetry. That was a scaling problem. Now we never hit the memory limiter. We always scale well before we’re going to get anywhere near that because it basically doesn’t work, right? I mean, it’s going to stop it out of memory, but it’s not going to, the client isn’t going to retry and resend that span. That’s a much bigger problem for us than spending money on memory. +**Adriana:** Yeah, that's a really great point. Well, I guess we are at time. We've got two minutes before we wrap up. Thank you to all of our panelists for showing up today and sharing their thoughts on the OTEL Collector and how you use it out in the wild. This is super valuable and thank you everyone else who joined to give this a listen. We definitely really appreciate it. Tell all your friends who were not able to make it today that we will have this recording up on the OTEL YouTube channel. Our channel, in case you're not aware, is called OTEL Official. Also, for anyone who is going to be at KubeCon in Paris next month, a number of us from the OTEL end user working group will be there, including Reese, Dan, Hope, and Rynn. I don't think Rynn is on this call, and I will be there as well. We're going to be hanging out in the OTEL Observatory, which, if you were at KubeCon North America last fall, it was the happening place for all things OTEL, and we've got a lot of cool things lined up at the OTEL observatory. We're actually going to be doing some more user feedback sessions. We're going to be recording Humans of OTEL. They're going to be SIG meetings, OTEL demos are going to be running. So, we're going to have signups also for some of these things. If you want to participate in another feedback session for another SIG or if you want to be interviewed for Humans of OTEL, we'll have the schedule posted up. I think there was a blog post on the OTEL blog that just came out with the signup links for various things. Now our feedback sessions are still TBD, but keep an eye out for that post in the OTEL blog as we finalize the details. So yeah, hope to see a number of you in Paris and thank you so much for joining us here today. Appreciate everyone taking the time today. -**Adriana:** Yeah, that’s a really great point. Well, I guess we are at time. We’ve got two minutes before we wrap up. Thank you to all of our panelists for showing up today and sharing their thoughts on the OTEL Collector and how you use it out in the wild. This is super valuable, and thank you everyone else who joined to give this a listen. We definitely really appreciate it. Tell all your friends who were not able to make it today that we will have this recording up on the OTEL YouTube channel. Our channel, in case you’re not aware, it’s called OTEL Official. Also, for anyone who is going to be at KubeCon in Paris next month, a number of us from the OTEL end user working group will be there, including Reese, Dan, Hope, and Rynn. I don’t think Rynn is on this call, and I will be there as well. We’re going to be hanging out in the OTEL Observatory, which, if you were at KubeCon North America last fall, it was the happening place for all things OTEL, and we’ve got a lot of cool things lined up at the OTEL observatory. We’re actually going to be doing some more user feedback sessions. We’re going to be recording Humans of OTEL. There are going to be SIG meetings, OTEL demos are going to be running. We’re going to have signups also for some of these things. If you want to participate in a different feedback session for another SIG, or if you want to be interviewed for Humans of OTEL, we’ll have the schedule posted up. I think there was a blog post on the OTEL blog, but just came out with the sign-up links for various things. Now our feedback sessions are still TBD, but keep an eye out for that post in the OTEL blog as we finalize the details. Hope to see a number of you in Paris, and thank you so much for joining us here today. Appreciate everyone taking the time today. +**Greg:** Yes. Thank you so much. -**Iris:** Yes. Thank you so much. And we will see you on Slack. +**Joel:** And we will see you on Slack. -**Juraj:** Yep. Thank you so much. +**Iris:** Yep. Thank you so much. -**Greg:** Bye. +**Juraj:** Bye. -**Joel:** Thanks everyone. Bye. +**Adriana:** Thanks everyone. Bye. ## Raw YouTube Transcript diff --git a/video-transcripts/transcripts/2024-05-01T19:33:21Z-otel-prometheus-interoperability-user-feedback-panel.md b/video-transcripts/transcripts/2024-05-01T19:33:21Z-otel-prometheus-interoperability-user-feedback-panel.md index e4e9076..71f2d86 100644 --- a/video-transcripts/transcripts/2024-05-01T19:33:21Z-otel-prometheus-interoperability-user-feedback-panel.md +++ b/video-transcripts/transcripts/2024-05-01T19:33:21Z-otel-prometheus-interoperability-user-feedback-panel.md @@ -10,154 +10,187 @@ URL: https://www.youtube.com/watch?v=9a3ctZhJj-o ## Summary -The YouTube video features a panel discussion about the interoperability between Prometheus and OpenTelemetry (OTEL), moderated by David Ashpole from Google. Panelists included Iris from Miro, Nikos Takas from Corelogs, Vijay Samuel, and Dan Bromitt from Home Depot. The discussion focused on how these technologies can work together effectively, sharing insights into their respective observability stacks and the challenges they face, such as configuration issues, performance concerns, and ensuring seamless integration. Key points included the desire for consistency in metrics collection across different environments, the advantages of OpenTelemetry SDKs for tracing, and the potential for future developments in Prometheus, such as native OTLP ingestion and improved remote write capabilities. Overall, the panel expressed a strong interest in aligning the two ecosystems to improve user experience and facilitate smoother transitions for large organizations. +The video features a panel discussion on Prometheus and OpenTelemetry (OTEL) interoperability, moderated by David Ashpole from Google. The panelists include Iris from Miro, Nikos Takas from Corelogs, Vijay Samuel, and Dan Bromitt from Home Depot. They discuss their experiences and challenges integrating Prometheus with OpenTelemetry in their observability stacks. Key topics include the architecture of their systems, difficulties with metrics collection and configuration, and the transition from Prometheus to OpenTelemetry for consistency across metrics, tracing, and logging. The panelists emphasize the importance of maintaining compatibility to avoid disruptions in large organizations, while expressing excitement for upcoming features like OTLP support and Prometheus 3.0. They also highlight the need for UX improvements and collaboration between the Prometheus and OpenTelemetry communities to enhance user experience. ## Chapters -00:00:00 Welcome and introductions -00:03:40 Observability stack overview -00:06:28 Challenges in store environments -00:09:40 Metric instrumentation discussion -00:12:20 OpenTelemetry and Prometheus integration -00:14:40 Challenges with performance and UX -00:18:00 Struggles with legacy systems -00:20:50 Scrape parity concerns -00:24:00 Future Prometheus features discussion -00:27:30 Feedback and closing remarks +00:00:00 Introductions +00:00:52 Guest introduction: David +00:03:24 Guest introduction: Iris +00:04:15 Guest introduction: Nikos +00:05:06 Guest introduction: Vijay +00:05:57 Guest introduction: Dan +00:07:39 Discussion about observability stacks +00:10:12 Challenges with metrics collection +00:22:06 Discussion about struggles and solutions +00:37:24 Discussion about upcoming Prometheus features +00:45:54 Audience feedback and closing remarks -**Speaker 1:** Everyone who's able to join for this Prometheus OTEL interoperability panel, which David reached out to us to see if we could get some end user feedback on making sure that Prometheus and OTEL play nice, which is always awesome. So we've got, I believe, 4 panelists, correct me if I'm wrong, who will be sharing their thoughts. And David is our moderator who will be asking questions, and why don't we get folks to introduce ourselves, introduce themselves, start with David, and then we'll have our panelists introduce themselves. +## Transcript -**David:** Sure. I'm David Ashpole. I work for Google, and right now I'm working with the Prometheus OTEL work group, and we're trying our best to make sure that the protocols, the exporters, all the pieces that integrate OpenTelemetry and Prometheus together work well and consistently and help users that are using both to have a good experience. Awesome. Next panelist. Anyone want to hop in? +### [00:00:00] Introductions -**Iris:** I can start. Cool. Hello everyone. I'm Iris. I work as a senior observability engineer at Miro. So my life is surrounded around Prometheus and OpenTelemetry recently. So yeah, it's nice to be here. +**Adriana:** Everyone who's able to join for this Prometheus OTEL interoperability panel, which David reached out to us to see if we could get some end user feedback on making sure that Prometheus and OTEL play nice, which is always awesome. So we've got, I believe, four panelists, correct me if I'm wrong, who will be sharing their thoughts. And David is our moderator who will be asking questions. Why don't we get folks to introduce themselves, starting with David, and then we'll have our panelists introduce themselves. -**Nikos:** Hello everyone. Pleasure to meet you. My name is Nikos Takas, and I'm an engineer at Corelogs, currently working on observability units. I'm a Prometheus Operator Maintainer and also a Persis Maintainer. I've been working with trying to get some codes into Prometheus to help with the Prometheus 3.0 and OTLP ingestion as well. My daily base is around Prometheus, OpenTelemetry, metric collectors, and so on. Pleasure to be here. +**David:** Sure. I'm David Ashpole. I work for Google. Right now I'm working with the Prometheus OTEL work group, and we're trying our best to make sure that the protocols, the exporters, all the pieces that integrate OpenTelemetry and Prometheus together work well and consistently and help users that are using both to have a good experience. -**Vijay:** Nice to have you. Let's see, my name is Vijay Samuel. I help with observability. I think I've met most of you, and nice to see you all again. +**Adriana:** Awesome. Next panelist. Anyone want to hop in? + +**Iris:** I can start. Cool. Hello everyone. I'm Iris. I work as a senior observability engineer at Miro. My life surrounds Prometheus and OpenTelemetry recently. So yeah, it's nice to be here. + +**Nikos:** Hello everyone. Pleasure to meet you. My name is Nikos Takas. I'm an engineer at Corelogs, currently working on observability units. I'm a Prometheus Operator Maintainer and also a Persis Maintainer. I've been working on trying to get some codes into Prometheus to help with the Prometheus 3.0 and OTLP ingestion as well. My daily focus is around Prometheus, OpenTelemetry, metric collectors, and so on. Pleasure to be here. + +**Vijay:** Nice to have you. Let's see. My name is Vijay Samuel. I help with observability. I think I've met most of you, and nice to see you all again. **Dan:** Yeah, good to see you. And then our last panelist, I think, is Dan. Is this Dan? I hope so. I don't think we've actually met. -**Dan:** Hi, yeah, there's so many Dans. Hey, I'm Dan Bromitt, Senior Manager of SRE at Home Depot for the Store and Payments Systems. Yeah, we use a lot of the OTEL and observability adjacent tools. Cool. Great to have you. +### [00:03:24] Guest introduction: Iris + +**Dan:** Hi, yeah. There are so many Dans. Hey, I'm Dan Bromitt, Senior Manager of SRE at Home Depot for the Store and Payments Systems. We use a lot of the OTEL and observability adjacent tools. + +**Adriana:** Cool. Great to have you. All right. So let's see. Why don't we go person by person and just give an overview of roughly what your observability stack looks like and how you're using Prometheus and OpenTelemetry in that stack. And why don't we start, we'll go in reverse order. So Dan, why don't we start with you? + +### [00:04:15] Guest introduction: Nikos + +**Dan:** Okay. So we observe services across a wide range of compute environments. That includes the data center, which are on-prem virtual machines, the cloud, which is GCP, primarily GKE, primarily Kubernetes, and also in our stores. Those are primarily small clusters inside of our stores where we are also running OpenTelemetry collectors. We are also running in parallel to them Prometheus installations as well. The usage of OpenTelemetry as regards to Prometheus is in some cases just the typical, hey, it's Kubernetes in the cloud, and we want to have multiple different projects that we can share our metrics across all these different GCP projects. But the really, and that's fine. I think that's largely the, I don't think that's any groundbreaking stuff happening there. For us, we have challenges in our store environments where these things are miniature data centers. Our stores have a miniature data center in the back of every single store with a lot of racks of network and server equipment. We want to be able to make sure that stuff is actually working and our customers are happy. The network connectivity is spotty. Some of them are offline every single day. Some percentage of them are offline. Being able to backfill metrics is a challenge. Hey, the metric was collected at the right time at the store level, but then shipping it up to a central Prometheus level, sometimes that backfill just doesn't work because of timestamps and stuff. I'm sure that you all are aware of. The collection of it is the configuration across a fleet of devices, a fleet of services. Being able to configure all that is also a challenge for us. We're getting better at that on the Kubernetes side, but it's still a challenge that exists. + +**Adriana:** Great. Thank you. Vijay, tell me about what your setup broadly looks like and which pieces of Prometheus and which pieces of OpenTelemetry you're using together. -[00:03:40] **David:** All right. So let's see. Why don't we go person by person and just give an overview of roughly what your observability stack looks like and how you're using Prometheus and OpenTelemetry in that stack. And why don't we start, we'll go in reverse order. So Dan, why don't we start with you? +### [00:07:39] Discussion about observability stacks -[00:06:28] **Dan:** Okay. So we observe services across a wide range of compute environments. That includes the data center, which are on-prem virtual machines, the cloud, which is GCP, primarily GKE, primarily Kubernetes. And also, in our stores, those are primarily small clusters inside of our stores where we are also running OpenTelemetry collectors. And we are also running in parallel to them, Prometheus installations as well. So the usage of OpenTelemetry as regards to Prometheus is in some cases just the typical, hey, it's Kubernetes in the cloud, and we want to have multiple different projects that we can share our metrics across all these different GCP projects. But the really, and that's fine. I think that's largely the, I don't think that's any groundbreaking stuff happening there. For us, we have challenges in our store environments where it's these things are miniature data centers. Our stores are, we have a miniature data center in the back of every single store with a lot of racks of network and server equipment. And we want to be able to make sure that stuff is actually working and our customers are happy. The network connectivity is spotty. Some of them are offline every single day. Some percentage of them are offline. Being able to backfill metrics, which is a challenge. Hey, the metric was collected at the right time at the store level, but then shipping it up to a central Prometheus level, sometimes that backfill just doesn't work because of timestamps and stuff. I'm sure that y'all are aware of. And the collection of it is the configuration across a fleet of devices, a fleet of services is, being able to configure all that is also a challenge for us. We're getting better at that on the Kubernetes side, but it's still, that's a challenge that exists. +**Vijay:** Sure. Starting with the instrumentation, right? We made a forum for the most part. We standardized on a mix of the Prometheus Java client and Micrometer for Java applications. We use the community supported Prometheus client for JavaScript, and basically these expose everything else that's not considered managed. So Java and JavaScript are managed languages, which means that there is a framework that people can build on top of. Everything else is the wild west in the sense for Go, people use the standard Go client, and there's Python and a few other languages that are there as well. All of these expose Prometheus endpoints today and are scraped by OpenTelemetry collectors that are deployed on every Kubernetes cluster that we host internally. This is the breadth of all our metric use cases that are there, more than 90-95%. There is a smaller group of metric use cases that send in directly into our gateway APIs, which can understand more proprietary formats, but they go through sanitization and whatnot before entering into our metric store. And finally, there's also the newer breed where we are trying to directly encourage some of the use cases to use the OpenTelemetry SDK to instrument and then use gRPC into the collector, enrich, and then write into the backend. The backend is predominantly Prometheus first, meaning we use the Prometheus SDK to build clustering and replication on top of the Prometheus storage engine. All the characteristics of Prometheus still apply because the same headlock and everything stay the same. We use an object store backed by Thanos for long-term storage. Over time, we want to get to a place where we know people from things like Micrometer and from this client to just use OpenTelemetry SDK because on tracing, we are 100% on OpenTelemetry today and we are starting to look at logs as well. So over time, we want metrics to be within the same fold. -**David:** Great. Thank you. Vijay, tell me about what your setup broadly looks like and which pieces of Prometheus and which pieces of OpenTelemetry you're using together. +### [00:10:12] Challenges with metrics collection -**Vijay:** Sure. Starting with the instrumentation, right? We made a form for the most part. We standardized on a mix of the Prometheus Java client and Micrometer for Java applications. We use the community supported Prometheus client for JavaScript, and basically these expose everything else that's not considered managed. So Java and JavaScript are managed languages, which means that there is a framework that people can build on top of. Everything else is the wild west in the sense for Go, people use the standard Go client, and there's Python and a few other languages that are there as well. All of these expose Prometheus endpoints today and are scraped by OpenTelemetry collectors that are deployed on every Kubernetes cluster that we host internally. This is the breadth of all our metric use cases that are there, more than 90-95%. There is a smaller group of metric use cases that send in directly into our gateway APIs, which can understand more proprietary formats, but they go through sanitization and whatnot before entering into our metric store. And finally, there's also the newer breed where we are trying to directly encourage some of the use cases to use the OpenTelemetry SDK to require to instrument and then use gRPC into the collector, enrich, and then write into the backend. The backend is predominantly Prometheus first, meaning we use the Prometheus SDK to build clustering and replication on top of the Prometheus storage engine. All the characteristics of Prometheus still apply because the same headlock and everything stay the same, and we use object store backed by Thanos for long term store. Over time, we want to get to a place where we know people from things like Micrometer and from this client to just use OpenTelemetry SDK because on tracing, we are a hundred percent on OpenTelemetry today, and we are starting to look at logs as well. So over time we want metrics to be within the same fold. +**Nicholas:** Our setup is quite simple. At the moment, we are leveraging OpenTelemetry Collector for traces and recently starting with logs. In metrics, we are evaluating because at the moment we are using a mix of Prometheus servers and Prometheus agents to write metrics from many different places. Our main idea is replacing the Prometheus agent by the OpenTelemetry collector and leveraging all the OpenTelemetry pipeline capabilities. But at the moment, we know it's a lot of OpenTelemetry collector consuming more resources than the Prometheus agent in different aspects, and this is what is blocking us a bit because we are collecting a huge amount of metrics. Replacing the collector by the agent at moments is blocking because of the resources, but we are pretty busy. Maybe this might be solved when we have Prometheus ingesting OTLP natively. You don't need more of the conversions between OpenTelemetry and the Prometheus format, at least during the remote writing process. And the same idea of replacing maybe Prometheus client by the OpenTelemetry SDK and implementing the OpenTelemetry from the instrumentation up to the storage layer. Mainly these at the moment. -[00:09:40] **Nikos:** Our setup is quite simple. At the moment, we are leveraging OpenTelemetry Collector for traces and recently starting with logs. In metrics, we are evaluating because at the moment we are using a mix of Prometheus servers and Prometheus agents to write metrics from many different places. And our main idea is replacing the Prometheus agent by the OpenTelemetry collector and leveraging all the OpenTelemetry pipeline capabilities. But at the moment, we know it's a lot of OpenTelemetry collector consuming more resources than the Prometheus agent in different aspects, and this is what is blocking us a bit because we are running a very large amount of metrics we are collecting. Replacing the collector by the agent at moments blocking because of the resources, but we are pretty busy. That maybe this might be solved when we have Prometheus ingesting OTLP natively; you don't need more of the conversions between OpenTelemetry and the Prometheus format, at least during the remote writing process. The same idea of replacing maybe Prometheus client by the OpenTelemetry SDK and implementing the OpenTelemetry from the instrumentation up to the storage layer is mainly this at the moment. +**Iris:** For us, it is pretty simple, the architecture as well. We're using mostly Prometheus clients. We are not collecting via Prometheus most of our metrics, especially custom metrics, but we're using Victoria Metrics. Everything is in Prometheus format because it is the open-source Victoria Metrics. We're also leveraging the Node Exporter, for example, that is our main exporter currently that we're using. We are at the moment experimenting with OpenTelemetry, especially with host metrics because it is more convenient for us to run OpenTelemetry collector and use all its capabilities for metrics collection, especially in the host level than to use a Node Exporter. We also are experimenting in some cases because we are also running a monolith, except for Kubernetes. So we do use the Prometheus receiver, adding script jobs to OpenTelemetry to reach in some places that are more difficult for us through Victoria Metrics. But, yeah, currently everything is Prometheus mostly, but we are thinking very hard to move towards OpenTelemetry as the case because tracing is already in OpenTelemetry. Partly it's still OpenTracing. But the goal is to move everything there. We want to have a uniform way of collecting all our data. -[00:12:20] **Iris:** For us, it is pretty simple, the architecture as well. We're using mostly Prometheus clients; we are not collecting via Prometheus most of our metrics, especially custom metrics, but we're using Victoria metrics. Everything is in Prometheus format because it is the open-source Victoria metrics. We're also leveraging the Node Exporter, for example, that is our main exporter currently that we're using. And we are at the moment experimenting with OpenTelemetry, especially with host metrics, because it is more convenient for us to run OpenTelemetry collector and use all its capabilities for metrics collection, especially in the host level than to use a node exporter. We also are experimenting in some cases because we are also running a monolith, except for Kubernetes. So we do use the Prometheus receiver, adding script jobs to OpenTelemetry to reach in some places that it is more difficult for us through Victoria Metrics. But yeah, currently everything is Prometheus mostly, but we are thinking very hard to move towards OpenTelemetry as the case, because tracing is already in OpenTelemetry, partly still open tracing. But the goal is to move everything there. So we want to have a uniform way of collecting all our data. +**Adriana:** Cool. Sounds like there's actually some similarity between most everyone. It sounds like I believe everyone's using primarily Prometheus libraries and instrumentation but is thinking of moving to OpenTelemetry. I think Iris, you touched on that a little bit. Is it just that you want consistency between the way you're doing tracing, which is OpenTelemetry, and the way you're doing metrics? Or is there something about the OpenTelemetry instrumentation that's drawing you to migrate? -**David:** Cool. Sounds like there's actually some similarity between most everyone. Sounds like I believe everyone's using primarily Prometheus libraries and instrumentation but is thinking of moving to OpenTelemetry. I think Iris, you touched on that a little bit. Is it just that you want consistency between the way you're doing tracing, which is OpenTelemetry, and the way you're doing metrics? Or is there something about the OpenTelemetry instrumentation that's drawing you to migrate? +**Iris:** We really like the OpenTelemetry SDKs for tracing. We've seen that it goes a long way better than any other SDKs before. Considering that we are using this for tracing, why not use it for the whole parts of the application for metrics in this case? And why not logging? So this is the main reason. Also, the infrastructure makes it very easy, the way that it is very flexible for us to use the OpenTelemetry infrastructure. So that really helps in this case. -**Iris:** We really like the OpenTelemetry SDKs for tracing. We've seen that it goes a long way better than any other SDKs before. Considering that we are using this for tracing, why not use it for the whole parts of the application for metrics in this case? And why not logging? So this is the main reason. Also, the infrastructure, not just as the case, but the infrastructure makes it very easy the way that it is very flexible for us to use the OpenTelemetry infrastructure. So that really helps in this case. +**Dan:** Hey, and David, I do want to call out that we are using the OpenTelemetry Java agent in production in stores, and some of our legacy stuff is still using different libraries for collecting metrics for emitting metrics, exposing metrics. But the majority of our new services are using the OpenTelemetry, either Go library or Java library to expose metrics. -**Dan:** Hey, and David, I do want to call out that we are using the OpenTelemetry Java agent in production, in stores, and some of our legacy stuff is still using different libraries for collecting metrics for emitting metrics, exposing metrics. But the majority of our new services are using the OpenTelemetry, either Go library or Java library to expose metrics. +**Adriana:** Very cool. All right. Let's talk about challenges. Nicholas, why don't you start us off? I know you mentioned performance issues with the Prometheus receiver and the collector. What else during your journey of adopting Prometheus and OpenTelemetry together has really been a struggle for you? Either that's still unsolved or you were able to work through. -[00:14:40] **David:** Very cool. All right. Let's talk about challenges. Nicholas, why don't you start us off? I know you mentioned performance issues with the Prometheus receiver and the collector. What else during your journey of adopting Prometheus and OpenTelemetry together has really been a struggle for you? Either that's still unsolved or you were able to work through. +**Nicholas:** That's a good point. I think that what I struggled with the most in the beginning was mainly UX, DX. I don't know how to properly call that, but it took me a little while to understand, okay, we are the metrics. We are not pushing the attributes. These are such roots and something like that to the metrics. Then we need to enable remote writing to send the metric info. So one, configuring the collector to expose these attributes as metric labels. This took my while to understand that. After that, since I didn't start using the operator, the OpenTelemetry operator to make the switching from the Prometheus receiver, to the... I just forgot the name of the service. The target allocator helps a lot on the problems of discovering targets using a lot of services. But if you're not using the OpenTelemetry collector, you need to figure out yourself how to run the target allocator with OpenTelemetry without the operator. This is like easy to do, but I did need a lot of research and reading docs, issues, and so on to make it work outside of the operator. I think that mainly the issues that I found were related to UX at the moment. UX in terms of getting it all set up? -**Nikos:** That's a good point. I think that what I struggled with the most in the beginning was mainly UX, DX. I don't know how to properly call that, but it took me a little while to understand. Okay, we are the metrics, we are not pushing the attributes, like these are such roots and something like that to the metrics. And then we need to enable on the remote writing to send the metric info, so one or configuring the collector to expose these attributes as metric labels. This took my while to understand that, and after that, since I didn't start using the operator, the OpenTelemetry operator, to make the like switching from the Prometheus receiver to the, I just forget the name of the service. The target allocator helps a lot on the problems to discovering targets using a lot of services. But if you're not using the OpenTelemetry collector, you need to figure out yourself how to run the target allocator with the OpenTelemetry without the operator. This is like easy to do, but I did need a lot of research and reading docs, issues, and so on to make it work outside of the operator. I think that mainly the issues that I found were related to UX at the moment. UX in terms of getting it all set up or UX after you have it all set up and you're trying to use the data that you've ingested. +**Nicholas:** Or UX after you have it all set up and you're trying to use the data that you've ingested? -**Dan:** Yeah. To make the thing set up, set up and configure the receivers and understanding the little needs regarding the writing informs or not, and then the issue that I mentioned regarding the setup of the Target Allocator, part of the OpenTelemetry collector. Also, you have everything there. It's quite okay to use this, like at least for us because we already know about the difference between the metrics would not happen the units if we decide to not have the reboots. Is there such reboots as metrics labels? We need to join with the info metric and so on, those kind of things we were aware of. So we're pretty easy to get used to these little niches at the moment. +**Nicholas:** Yeah. To make the thing set up, set up and configure the receivers and understanding the little needs regarding the remote writing, the informs or not. And then the issue that I mentioned regarding the setup of the Target Allocator part of the OpenTelemetry collector. Also, you have everything there. It's quite okay to use, at least for us, because we already know about the difference between the metrics. We were aware, so we're pretty easy to get used to these little niches at the moment. -**David:** That's helpful. Dan, what's been a struggle for you? Either was a struggle or still is a struggle using Prometheus and OpenTelemetry together. +**Adriana:** That's helpful. Dan, what's been a struggle for you? Either it was a struggle or still is a struggle, using Prometheus and OpenTelemetry together. -[00:18:00] **Dan:** So first of all, the agent, the Java agent, it only supports, so we tried to leverage it to add observability to legacy microservices that had no provisions for observability at all when they were created. They were running on a version of Java that the agent didn't support and a version of Tomcat that the agent didn't support. So I'll tell you, adoption was real rough for the first six months because of that. I think it was Java 7, which is frustrating. So I don't know if that's something OpenTelemetry fixes, but it was a problem. And the other big one, I think, just the massive challenge is the configuration of OpenTelemetry to send any sort of telemetry over a push like a pub/sub, like any sort of messaging framework like a pub/sub or something. Configuring that was a little bit challenging for our engineers to get it so that we could actually, hey, we have the apps that the telemetry is collected from the apps locally, and then we're going to push it over some sort of messaging framework to get it to a central location. That was a little bit rough. And then the last one is the timestamps, and I know that it's not an OpenTelemetry only challenge; I know it's related to Prometheus, but if we have a location that's offline for a while, it comes back online and tries to send out the metrics that it was unable to send out while it's offline, those timestamps get totally borked. +**Dan:** So first of all, the agent, the Java agent, it only supports... So we tried to leverage it to add observability to legacy microservices that had no provisions for observability at all when they were created. They were running on a version of Java that the agent didn't support and a version of Tomcat that the agent didn't support. So I'll tell you, adoption was real rough for the first six months because of that. I think it was Java 7, which is frustrating. So I don't know if that's something OpenTelemetry fixes, but it was a problem. And the other big one, I think just the massive challenge is the configuration of OpenTelemetry to send any sort of telemetry over a push, like a pub/sub, like any sort of messaging framework like a pub/sub or something. Configuring that was a little bit challenging for our engineers to get it so that we could actually... Hey, we have the apps that the telemetry is collected from the apps locally, and then we're going to push it over some sort of messaging framework to get it to a central location. That was a little bit rough. And then the last one is the timestamps, and I know that it's not an OpenTelemetry-only challenge. I know it's related to Prometheus, but if we have a location that's offline for a while, it comes back online and tries to send out the metrics that it was unable to send out while it's offline, those timestamps get totally borked. -[00:20:50] **Vijay:** Our biggest struggles were scrape parity, and a lot of those through PRs that we filed on the collector, we have had addressed. Some of them are like, what if a label name started with an underscore? What if we wanted to disable sanitization for whatever reason, if it started with the colon? So these are all scenarios where, initially, Prometheus greatly differed from how OpenTelemetry collector was handling it, specifically on the exporter, the remote write exporter. So those are all solved now, so we don't have a problem, but that's one of our biggest concerns as we try to marry these two, OpenMetrics and OpenTelemetry together. It's like we cannot afford any breakages. It should largely be rather than, if, going back to the document that was shared upfront, it should relax the requirement but still have ways to keep them as is so that when we try to move across versions, we don't have something break. And at the scale at which we operate, if something breaks, then it's a catastrophic breakage. Outside of that, I think forward looking on the collector itself, we are more of in wait for a few features to come out, being able to scrape native histograms and then report them into a Prometheus-compatible backend and things like that. But we don't have complaints per se right now. Certain things could be better, like Attribute processors could be a lot simpler to configure. They're very verbose right now, but those have nothing to do with the discussion we have right now. In summary, please don't break us. +**Adriana:** Thanks, Dan. Vijay, what were some of the biggest struggles you ever came and what's still left to do? -**Iris:** Our struggles come mostly from the size of the organization. Our organization is quite big. So we, just for, to envision it, we have at least 700 engineers that are using our observability platform. And we have to say most of them, of course, are front-end engineers, back-end engineers. So they're not, I'm not going to say fully proficient. They know their metrics and everything, but of course, if we make a big change and it's different, it could ruin their dashboard. It could ruin their alerting, and it is a very unpleasant experience. So that has been our challenge so far and why we are moving very slowly. Because if we change the way that we're collecting information and then the metrics, even if the formalities are different, it could be a big change. And 700 people to get adapted to this change needs time. So we are going very gradual with it. Prometheus is known territory for everyone that works on the team and for all the engineers that have had experience with their metrics at the moment. So it's safe. Of course, the OTEL collector needs a lot of digging into the documentation, finding out more, testing, and trials, so it makes it a bit slower, but mostly our biggest challenge that we're overcoming slowly, but still we are not there, is the experience of our engineers, so we don't ruin their dashboard, their observability performance in general. +### [00:22:06] Discussion about struggles and solutions -[00:24:00] **David:** Great. Thanks. So I know we just touched on this a little bit, but I think there was, I think it was actually a Prometheus blog that came out maybe a couple months ago about 2024 and some of the cool things that are coming. Some of the things I can think of are Prometheus remote write to OTLP support and the Prometheus receiver, Delta support for Prometheus, supporting dots, woohoo, stability, or even support for OTLP and some of the Prometheus exporter ecosystem that already exists today. Iris, we'll start with you again. Which of those are, or is there anything else that really stands out as something that you're really looking forward to having? +**Vijay:** Our biggest struggles was scrape parity, and a lot of those through PRs that we filed on the collector we have had addressed. Some of them are like, what if a label name started with an underscore? What if we wanted to disable sanitization for whatever reason if it started with the colon? So these are all scenarios where initially, Prometheus greatly differed from how the OpenTelemetry collector was handling it, specifically on the exporter, the remote write exporter. So those are all solved now, so we don't have a problem, but that's one of our biggest concerns as we try to marry these two, OpenMetrics and OpenTelemetry together. It's like we cannot afford any breakages. It should largely be rather than going back to the document that was shared upfront. It should relax the requirement but still have ways to keep them as is so that when we try to move across versions, we don't have something break. And at the scale at which we operate, if something breaks, then it's catastrophic breakage. Outside of that, I think forward-looking on the collector itself, we are more in wait for a few features to come out. Being able to scrape native histograms and then report them into a Prometheus compatible backend and things like that. But we don't have complaints per se right now. Certain things could be better, like attribute processors could be a lot simpler to configure. They're very verbose right now, but those have nothing to do with the discussion we have right now. In summary, please don't break us. + +**Iris:** Our struggles come mostly from the size of the organization. Our organization is quite big. To envision it, we have at least 700 engineers that are using our observability platform. Most of them, of course, are front-end engineers, back-end engineers. So they're not, I'm not going to say fully proficient. They know their metrics and everything, but of course, if we make a big change and it's different, it could ruin their dashboard. It could ruin their alerting, and it is a very unpleasant experience. So that has been our challenge so far and why we are moving very slowly. Because if we change the way that we're collecting information and then the metrics, even if the formalities are different, it could be a big change and 700 people to get adapted to this change needs time. So we are going very gradual with it. Prometheus is known territory for everyone that works on the team and for all the engineers that have had experience with their metrics at the moment. So it's safe. Of course, the OTEL collector needs a lot of digging into the documentation, finding out more, testing and trials, so it makes it a bit slower. But mostly our biggest challenge that we're overcoming slowly, but still we are not there, is the experience of our engineers, so we don't ruin their dashboard, their observability, performance in general. + +**Adriana:** Great. Thanks. So I know we just touched on this a little bit, but I think there was, I think it was actually a Prometheus blog that came out maybe a couple months ago about 2024 and some of the cool things that are coming. Some of the things I can think of are Prometheus remote write to OTLP support and the Prometheus receiver, Delta support for Prometheus supporting dots. Woohoo! Stability, or even support for OTLP and some of the Prometheus exporter ecosystem that already exists today. Iris, we'll start with you again. Which of those are, or is there anything else that really stands out as something that you're really looking forward to having? **Iris:** Can I say all of the above? I'm very excited for all of them, but if I had to distinguish maybe the exporter and receiver, the version 2.0, it's going to be very nice. We were using it heavily on testing heavily with it right now for our next move and the support of dots. Why not? Again, it comes from the organization and having things more uniform. It helps us a lot and the experience of our engineers. -**Nikos:** Cool. Wow. I guess it's a mix of the remote write 2.0, I guess it's going to be massive. People are working hard to have a better protocol consuming less resources. Since most of the people are using OpenTelemetry, they are emitting writing for some Prometheus-based system. I have in this remote writing 2.0 will help people saving resources, networking, CPU, and memory. Apart from that, I guess for the 3.0, of course, the OTLP ingestion on GA is going to be nice. Of course, there is a lot of work that people are doing on the Prometheus team, making out of order samples ingestion be enabled by default. This is something very important to supporting auto-allopinate. I guess the next one, just to be a little bit different from we, I guess it's the UTF-8 support and the metric label and name. I guess it's going to be nice. It's going to have some proximity between OpenTelemetry and Prometheus to use metric names and labels. I guess it's going to be useful for usability and joining the different communities. +**Adriana:** Yeah, that's actually really interesting to me. So you're really excited for Prometheus 2.0. When I talk to some people, they're like, there's Prometheus 2.0 and OTLP, but you're excited for Prometheus 2.0. Tell me a little bit more about that. + +**Iris:** I was talking mostly about the Prometheus exporter and receiver in OpenTelemetry Collector 2.0, right? That's what... + +**Adriana:** Yeah, sorry. Sorry. No, that's fine. That's fine. But yeah, I'm looking forward to Prometheus 3.0 as well. It's a great technology and of course, the support of OTLP will be awesome. -**Dan:** Yeah, so the OTLP, that is a very cool thing. It will simplify some of our config. It was we get to this, the central GMP, where everything's stored. I shouldn't use acronyms to the central Google Managed Prometheus that we run where all of our metrics are ultimately stored. One of the things that we're not really clear on is, hey, if we have data that is late. So if we have OTLP bit of telemetry that has a timestamp that is five hours late, is that still going to get stored? And we haven't really been able to find the answer to that question. So we're just assuming that we'll test it and see once it comes out. +**Nicholas:** Cool. Wow. I guess it's a mix of the remote write 2.0. I guess it's going to be massive. People are working hard to have a better protocol consuming less resources. Since most of the people are using OpenTelemetry, they are emitting writing for some Prometheus-based system. Having this remote write 2.0 will help people save resources, networking, CPU, and memory. Apart from that, I guess for the 3.0, of course, the OTLP ingestion on GA is going to be nice. Of course, there is a lot of work that people are doing on the Prometheus team, making out-of-order samples ingestion be enabled by default. This is something very important to support auto-allopinate. I guess the next one, just to be a little bit different from we, I guess it's the UTF-8 support and the metric label and name. I guess it's going to be nice. It's going to have some proximity between OpenTelemetry and Prometheus to use metric names and labels. I guess it's going to be very useful for usability and join the different communities. -**Vijay:** Yeah, we can think. Prometheus is going to support out of order writes. I'm not sure about the... +**Dan:** Yeah, so the OTLP, that is a very cool thing. It will simplify some of our config as we get to the central Google-managed Prometheus that we run where all of our metrics are ultimately stored. One of the things that we're not really clear on is, hey, if we have data that is late, so if we have an OTLP bit of telemetry that has a timestamp that is five hours late, is that still going to get stored? And we haven't really been able to find the answer to that question, so we're just assuming that we'll test it and see once it comes out. -**Dan:** It's already supporting, actually. It's already supporting. It's behind a feature flag and you can just enable it and configure how much behind you want in gesture samples. +**Vijay:** Yeah, we can think. Prometheus is going to support out-of-order writes. I'm not sure about the... -**Vijay:** Yeah, so the out of order writes thing, that's good. That's cool. It's the stuff that has a timestamp. That's super behind is the one where it's really unclear. +**Dan:** It's already supporting, actually. It's already supporting. It's behind a feature flag, and you can just enable it and configure how much behind you want in gesture samples. -**Dan:** Cool. Yeah, we can sync up afterwards. +**Vijay:** Yeah, so the out-of-order writes thing. That's good. That's cool. It's the stuff that has a timestamp that's super behind is the one where it's really unclear. -**Vijay:** And what are you excited about? It's coming soon. +**Dan:** Cool. Yeah, we can sync up afterwards. And Vijay, what are you excited about? It's coming soon. -**Nikos:** Actually, a lot of things. The ODLP ingestion is interesting. The part that we would really hope for is that when we support it, it should be in a way that, okay, I instrumented this way on my application and I see the same way on Prometheus. No addition of underscore, total or milliseconds or whatever and keep the dots as is and things like. But the part that's still somewhat unclear and we hope to see more clarity is how do the resource attributes and the metric attributes play along inside of Prometheus. What if there are name coalitions and things like that? That is somewhat unclear, but if the fundamental thing is that we want the instrumentation and what they query to perfectly match, especially when they are just using gRPC to send out from the client directly. Delta Temporality is something that some of the folks who started off directly on OpenTelemetry SDKs felt that, oh, this is nice. It is nice to have, but it's not supported on Prometheus because you drop it on the floor in the collector right now. That's there. But outside of this, this list, I know one of my wishlist items is like exemplars persistence. That's something that's not there yet on Prometheus. It's still the circular ring buffer that's there. Hopefully someone can get to it by 3.0. It's a nice thing to have, which we rely quite heavily on exemplars today. +**Vijay:** Actually, a lot of things. The OTLP ingestion is interesting. The part that we would really hope for is that when we support it, it should be in a way that, okay, I instrumented this way on my application, and I see the same way on Prometheus. No addition of underscore total or milliseconds or whatever and keep the dots as is and things like that. But the part that's still somewhat unclear and we hope to see more clarity is how do the resource attributes and the metric attributes play along inside of Prometheus? What if there are name coalitions and things like that? That is somewhat unclear, but the fundamental thing is that we want the instrumentation and what they query to perfectly match, especially when they are just using gRPC to send out from the client directly. Delta Temporality is something that some of the folks who started off directly on OpenTelemetry SDKs felt that, oh, this is nice. It is nice to have, but it's not supported on Prometheus because you drop it on the floor in the collector right now. That's there. But outside of this list, I know one of my wish list items is exemplars persistence. That's something that that's not there yet on Prometheus. It's still the circular ring buffer that's there. Hopefully, someone can get to it by 3.0. It's a nice thing to have, which we rely quite heavily on exemplars today. -**Nikos:** Oh, cool. Actually, I learned something. I didn't know exemplars weren't persistent. +**Nicholas:** Oh, cool. Actually, I learned something. I didn't know exemplars weren't persistent. -**Vijay:** Yeah, so they're not persistent and recording rules don't support it. For the latter, there's a PR which I never got to finishing, but someday it will be there. But the persistence is something that no one has picked up yet. +**Vijay:** Yeah, so they're not persistent, and recording rules don't support it. For the latter, there's a PR which I never got to finishing, but someday it will be there. But the persistence is something that no one has picked up yet. -**David:** Oh, cool. Thanks, Nicholas. Okay, so one of the things that we've talked about a little bit so far, it seems like most people are in agreement that breaking changes are really hard. You have big organizations, big deployments, and like when we mess with things in terms of how things are translated, that can really have a big impact. And at the same point, it seems like there's at least some agreement within this group that we are excited for UTF-8 support, and we really want to be able to preserve the original OpenTelemetry data that is coming from the application. VJ, I'll put you on the spot. What do you think the right way to strike that balance is for us? +**Adriana:** Oh, cool. Thanks, Nicholas. Okay, so one of the things that we've talked about a little bit so far, it seems like most people are in agreement that breaking changes are really hard. You have big organizations, big deployments, and like when we mess with things in terms of how things are translated, that can really have a big impact. And at the same point, it seems like there's at least some agreement within this group that we are excited for UTF-8 support, and we really want to be able to preserve the original OpenTelemetry data that is coming from the application. Vijay, I'll put you on the spot. What do you think the right way to strike that balance is for us? -**Vijay:** On your doc, I think I had a discussion with some of our leads internally as well. Like option one seems pretty appealing to us in the sense that at least today what we do is we separate installations between things that are going after Prometheus endpoints versus things that are emitting through gRPC directly. Okay. So it still gives us the opportunity to say that, hey, receiver, Prometheus receiver and remote write exporter, make sure that everything that Prometheus mandates today, enforce them on the scrape endpoints. But OTLP receiver and whatever is writing to our Prometheus backend, make sure that you are passing things the way that OpenTelemetry standards mandate. So it gives us a balance in keeping things as is, but things that we are moving to the next generation, we are moving it in the right way. So we have done this in quite a few evolutions where we used to run a modified OpenTSDB first before we moved into Prometheus. That's how we bridged the gap initially. This would be like a second generation where moving from, say, Prometheus semantics into OpenTelemetry semantics. I would hope everything goes behind either a configuration or a feature flag to keep the standard, the current standardization configurable to the specification that's there today on OpenMetrics and allow us to disable them as we need it. +**Vijay:** On your doc, I think I had a discussion with some of our leads internally as well. Like option one seems pretty appealing to us in the sense that at least today what we do is we separate installations between things that are going after Prometheus endpoints versus things that are emitting through gRPC directly. So it still gives us the opportunity to say that, hey, receiver, Prometheus receiver and remote write exporter. Make sure that everything that Prometheus mandates today enforces them on the scrape endpoints. But OTLP receiver and whatever is writing to our Prometheus backend, make sure that you are passing things the way that OpenTelemetry standards mandate. So it gives us a balance in keeping things as is, but things that we are moving to the next generation, we are moving it in the right way. So we have done this in quite a few evolutions where we used to run a modified OpenTSDB first before we moved into Prometheus. That's how we bridged the gap initially. And this would be like a second generation where moving from, say, Prometheus semantics into OpenTelemetry semantics. I would hope everything goes behind either a configuration or a feature flag to keep the standard, the current standardization configurable to the specification that's there today on OpenMetrics and allow us to disable them as we need it. -**David:** So to be clear, do you think there's a difference between push and pull? Or is it that it should keep or that we should make it preserve the names by default, but always give people an escape hatch? I think you said that you like that like OpenMetrics and Prometheus scrape endpoints still follow the Prometheus conventions, but for pushing Prometheus remote write and OTLP, you want to be able to just preserve it as is. Do you think there's a distinction there? +**Adriana:** So to be clear, do you think there's a difference between push and pull? Or is it that it should keep or that we should make it preserve the names by default, but always give people an escape hatch? I think you said that you like that OpenMetrics and Prometheus scrape endpoints still follow the Prometheus conventions, but for pushing Prometheus remote write and OTLP, you want to be able to just preserve it as is. Do you think there's a distinction there? -**Vijay:** Only for the OTLP. Prometheus remote write, however, we feel it needs to be, it can be done whichever way. But for things that are coming in through OTLP, it should pass through the pipeline exactly the way that the sender intended it. +**Vijay:** Only for the OTLP. Prometheus remote write, however, we feel it needs to be, it can be done whichever way. But for things that are coming in through OTLP, it should pass through the pipeline exactly the way that the sender intended it. That's what we're particular about. And on the scrapes, through either content negotiation or whatever, the behavior can be decided one way or the other, but we want to keep the scrapes as is because we have like more than 4 million scrapes that happen, and if something changes, then everything blows up at that point. -**David:** That's what we're particular about. And on the scrapes, through either content negotiation or whatever, the behavior can be decided one way or the other, but we want to keep the scrapes as is because we have like more than 4 million scrapes that happen, and if something changes, then everything blows up at that point. +**Adriana:** So don't break client side. -**Vijay:** So don't break client side. +**Vijay:** Yes, exactly. -**David:** Yes, exactly. Okay, that's very good feedback. Do you see these push and pull use cases being in the same pipeline and the exporting side needing to be automatic, needing to automatically decide whether it does that translation or not based on the source, or would you expect to have separate pipelines configured for those use cases? +**Adriana:** Okay, that's very good feedback. Do you see these push and pull use cases being in the same pipeline and the exporting side needing to be automatic, needing to automatically decide whether it does that translation or not based on the source, or would you expect to have separate pipelines configured for those use cases? -**Vijay:** I would vote for simplicity in the sense that we are okay to run two pipelines. At least the way that we are deployed right now, I don't see a way in which both push and pull can coexist, so we would anyway run them as two separate pipelines. So it's probably fine. So no, no additional complexity required on the exporter side is what I'm doing. +**Vijay:** I would vote for simplicity in the sense that we are okay to run two pipelines. Okay. And I don't, at least the way that we are deployed right now, I don't see a way in which both push and pull can coexist. So we would anyway run them as two separate pipelines. So it's probably fine. So no additional complexity required on the exporter side is what I'm doing. -**David:** Yeah. Okay. So just the configuration to decide which of those options you want and then you'll set up a pipeline for each of them. +**Adriana:** Yeah. Okay. So just the configuration to decide which of those options you want and then you'll set up a pipeline for each of them. **Vijay:** Correct. -**David:** Sounds good. Thank you. +**Adriana:** Sounds good. Thank you. Very cool. Thank you. Iris, I'll ask you a similar question. So you have a very large organization, right? You've said, 700 engineers. -**Iris:** Iris, I'll ask you a similar question. So you have a very large organization, right? You've said, 700 engineers. +**Iris:** Yeah. -**Iris:** Yeah. +### [00:37:24] Discussion about upcoming Prometheus features -**David:** Are you using PromQL? You said you mostly are using Prometheus and are familiar with Prometheus. Are you then using a lot of PromQL? +**Adriana:** Are you using PromQL? You said you mostly are using Prometheus and are familiar with Prometheus. Are you then using a lot of PromQL? -**Iris:** Yes. All PromQL for metric query. Have you seen the UTF-8 PromQL proposals read? Sorry to put you on the spot. +**Iris:** Yes, all PromQL for metric query. Have you seen the UTF-8 PromQL proposals? -**Iris:** No, I actually did skim through it right before this meeting. But I think that most of our issues, maybe I misunderstood, while I was going through the proposals is the first one to change open metrics to allow OpenTelemetry name would probably solve this. What makes me say that is that, yes, there are differences, but usually when you are making queries, especially if you are offering that to a wide organization, you are using an engine that is very intuitive when coming to finding the attributes, the labels, the metric names. So as long as we, for example, a Prometheus backend is able to support OpenTelemetry metrics, UTF-8, that should solve the problem of a common engineer that does not know a lot about observability to build a PromQL query, for example, so that is a very good first solution for this. +**Iris:** Sorry to put you on the spot. -**David:** I remembered my question now. Thank you. Thank you, Iris. Nicholas. +**Iris:** No, I actually did skim through it right before this meeting. But I think that most of our issues, maybe I misunderstood, while I was going through the proposals is the first one to change OpenMetrics to allow OpenTelemetry name would probably solve this. What makes me say that is that, yes, there are differences, but usually when you are making queries, especially if you are offering that to a wide organization, you are using an engine that is very intuitive when coming to finding the attributes, the labels, the metric names. So as long as we, for example, a Prometheus backend is able to support OpenTelemetry metrics, UTF-8, that should solve the problem of a common engineer that does not know a lot about observability to build a PromQL query, for example. So that is a very good first solution for this. -**Nikos:** Oh, shoot. I remember it again. Nicholas, my question for you. Part of what we're talking about here is a difference between Prometheus best practices that have been codified as the OpenMetrics standard and the OpenTelemetry semantic conventions that have come and given new ways of, say, writing metrics or naming things. Are you happy with the way that these two ecosystems have evolved? And what do you think the best approach is for users to try and give them a reasonable experience? Should we let them both be, or would you like to see either OpenTelemetry or OpenMetrics change such that they both work? They align in terms of how things should be named or that sort of thing. +**Adriana:** I remembered my question now. Thank you. Thank you, Iris. Nicholas. Oh, shoot. I remember it again. Nicholas, my question for you. Part of what we're talking about here is a difference between Prometheus best practices that have been codified as the OpenMetrics standard and the OpenTelemetry semantic conventions that have come and given new ways of, say, writing metrics or naming things. Are you happy with the way that these two ecosystems have evolved? And what do you think the best approach is for users to try and give them a reasonable experience? Should we let them both be, or would you like to see either OpenTelemetry or OpenMetrics change such that they both work? They align in terms of how things should be named or that sort of thing. -**Nikos:** That's a really good and tough question, actually. Personally, I feel Prometheus and OpenTelemetry are communities that stayed away for a little while, and we developed things very separately. Personally, what I would like to see, I think that maybe the OpenTelemetry entry, we should look more at how things are working when we have high adopted systems, like we have with Prometheus, different from the tracing and the logging, telemetry data type where we have many different data stores, like in the CNCF space. For the metrics, we actually only have Prometheus-based projects. Maybe we should look more at how Prometheus is doing things and avoiding changing too much. How this is working because most like in the end what users want to do is to use the new things without from needing to change or rewriting everything. You know if we look to how the metrics how we are trying to unite the metrics between OpenTelemetry and OpenMetrics, we are trying to get some, okay neutral moment here where we agree with each other. But in the end, I guess we're going to introduce some change to the end user. And I guess that we can unite efforts between the different communities to avoid these in the future. For example, oh, we think when we decide if when OpenTelemetry user groups start discussing how metrics, we would like the feeling that I had is like we didn't get close yet to the Prometheus ecosystem and so on and trying to, oh, let's try to solve most of the issues that we have in Prometheus, but without introducing a totally different metrics naming convention just to be easier with the end users. +**Nicholas:** That's a really good and tough question, actually. Personal feeling, Prometheus and OpenTelemetry are communities that stay away for a little while, and we develop the things very separately. Personally, what I would like to see, I think that maybe the OpenTelemetry entry, we should look more at how the things are working when we have high adopted systems like we have with Prometheus, different from the tracing and the logging telemetry data type where we have many different data stores, like in the CNCF space. For the metrics, we actually only have Prometheus-based projects. Maybe we should look more at how Prometheus is doing the things and avoid changing much. How this is working because most like in the end what users want to do is to use the new things without from need to changing or rewriting everything. You know, if we look to how we are trying to unite the metrics between OpenTelemetry and OpenMetrics, we are trying to get some neutral moment here where we agree with each other. But in the end, I guess we're going to introduce some changes to the end user. And I guess that we can unite efforts between the different communities to avoid this in the future. For example, when OpenTelemetry user groups start discussing how metrics, we would like the feeling that I had is that we didn't get close yet to the Prometheus ecosystem and so on and trying to solve most of the issues that we have in Prometheus, but without introducing a totally different metrics naming convention just to be easier for the end users. -**David:** Cool. Thank you. That makes sense. Cool. Thank you everyone so much for coming. Those are the questions I had prepared, and I really appreciate the feedback. I know we're ending a little early. I guess since we're done a little early, we could, if anyone here who's in the audience wants to share their feedback, we can definitely open up the floor. Any takers? +**Adriana:** Cool. Thank you. That makes sense. Cool. Thank you everyone so much for coming. Those are the questions I had prepared, and I really appreciate the feedback. I know we're ending a little early. I guess, since we're done a little early, we could, if anyone here in the audience wants to share their feedback, we can definitely open up the floor. Any takers? -**Iris:** Oh, I'd like to mention just one more thing if it's okay. On the querying side as well, if we can keep the conventions fairly alike, there are a few options that try to distinguish things with double quotes and whatnot. That would be new things that we need to teach folks. If we are just using, introducing UTF-8 conventions into label names and metric names by keeping the style exactly the same, that's still perfectly fine because people are just going to assume that we are just introducing new metric names and new label names rather than new conventions that they need to abide by. So if that can be taken into consideration, that would be great as well. +**Iris:** Oh, I'd like to mention just one more thing if it's okay. On the querying side as well, if we can keep the conventions fairly alike, there are a few options that try to distinguish things with double quotes and whatnot. That would be new things that we need to teach folks. If we are just introducing UTF-8 conventions into label names and metric names by keeping the style exactly the same. That's still perfectly fine because people are just going to assume that we are just introducing new metric names and new label names rather than new conventions that they need to abide by. So if that can be taken into consideration, that would be great as well. -**David:** You're talking about having to put UTF-8 names inside of quotes. +**Adriana:** You're talking about having to put UTF-8 names inside of quotes? **Iris:** Yeah, exactly. -**David:** Yeah, I think that was considered. I can give you pointers afterward if you want to know the details of like why it's a struggle. +**Adriana:** Yeah, I think that was considered. I can give you pointers afterward if you want to know the details of why it's a struggle. -**Iris:** Okay, sounds good. I was at the Prometheus Dev Summit earlier today, and one of the things they were discussing was introducing OTLP export support into the Prometheus client libraries. Is that something that would be useful to this group here? And if so, one of the things they weren't quite clear on where they wanted to land was whether that OTLP support needed to be part of each Prometheus client library, or if it would be okay if there was a third-party or separate repo bridge that had to be combined with the Prometheus client library and an OTLP SDK to make it function. Does anybody here have thoughts on that? +**Iris:** Okay, sounds good. I was at the Prometheus Dev Summit earlier today, and one of the things they were discussing was introducing OTLP export support into the Prometheus client libraries. Is that something that would be useful to this group here? And if so, one of the things they weren't quite clear on was whether that OTLP support needed to be part of each Prometheus client library, or if it would be okay if there was a third-party or separate repo bridge that had to be combined with the Prometheus client library and an OTLP SDK to make it function. Does anybody here have thoughts on that? -**Iris:** Can I chime in? Is that allowed? +**Vijay:** Can I chime in? Is that allowed? -**David:** Sure. +**Adriana:** Sure. -**Iris:** So when, in our environment, we use a lot of different metrics solutions, and we are trying to reach standardization when metrics are moving from one location to another, it's a mess when we have to have a bunch of different infrastructure set up to facilitate moving metric data for Datadog versus moving metrics for Prometheus, right? It's just a mess. So the as much as we can get into OTLP, so that way we can have everything moving through the same infrastructure for observability, that just simplifies our entire ecosystem. So the more we get into OTLP, the happier everybody on my teams will be. In our case, if this proposal came out a couple of years ago, we would have probably bounced on it because at least the problem that some of our use cases, especially things that run like cron jobs, folks use the Prometheus client and use the push gateway. Those use cases we would have ideally liked to replace it with the non-push gateway based implementation because the push gateway semantics don't necessarily work always for us. We just want to be able to send it into the gateway and put it on storage as quickly as possible. For those kinds, it's useful, but in today's world, if someone is trying to achieve such a use case, we'd rather recommend them to move to the OpenTelemetry SDK because there is a defined pattern for us to go after in that. +### [00:45:54] Audience feedback and closing remarks -**David:** Any other thoughts or feedback from anyone? +**Vijay:** So when, in our environment, we use a lot of different metrics solutions, and we are trying to reach standardization when metrics are moving from one location to another, it's a mess when we have to have a bunch of different infrastructure set up to facilitate moving metrics for Datadog versus moving metrics for Prometheus, right? It's just a mess. So the more we can get into OTLP, so that way we can have everything moving through the same infrastructure for observability, that just simplifies our entire ecosystem. So the more we get into OTLP, the happier everybody on my teams will be. In our case, if this proposal came out a couple of years ago, we would have probably bounced on it because at least the problem that some of our use cases, especially things that run like cron jobs, folks use the Prometheus client and use the push gateway. Those use cases, we would have ideally liked to replace it with the non-push gateway-based implementation because the push gateway semantics don't necessarily work always for us. We just want to be able to send it into the gateway and put it on storage as quickly as possible. For those kinds, it's useful, but in today's world, if someone is trying to achieve such a use case, we'd rather recommend them to move to the OpenTelemetry SDK because there is a defined pattern for us to go after in that. + +**Adriana:** Any other thoughts or feedback from anyone? **Iris:** Thank you everyone for joining. This was a really great discussion. Really appreciate it and looking forward to having more community discussions. -**All:** Great. Thanks, Adriana. Thank you. Thank you for the opportunity. Thank you very much. Bye-bye. +**Adriana:** Great. Thanks, Adriana. + +**Iris:** Thank you. + +**Vijay:** Thank you very much. + +**Dan:** Bye-bye. -**David:** Bye. +**Adriana:** Bye. ## Raw YouTube Transcript diff --git a/video-transcripts/transcripts/2024-06-05T21:36:59Z-the-humans-of-otel-kubecon-eu-2024.md b/video-transcripts/transcripts/2024-06-05T21:36:59Z-the-humans-of-otel-kubecon-eu-2024.md index b74019a..2753902 100644 --- a/video-transcripts/transcripts/2024-06-05T21:36:59Z-the-humans-of-otel-kubecon-eu-2024.md +++ b/video-transcripts/transcripts/2024-06-05T21:36:59Z-the-humans-of-otel-kubecon-eu-2024.md @@ -10,20 +10,24 @@ URL: https://www.youtube.com/watch?v=bsfMECwmsm0 ## Summary -In this YouTube video, titled "Observability and OpenTelemetry Insights," a group of professionals from various tech companies discuss their experiences and perspectives on observability and the role of OpenTelemetry in the industry. Key participants include Iris Durish from Meo, SE Nman from Cisco, Kayla Riel from New Relic, Morgan Mlan from OpenTelemetry, and others from companies like eBay, ServiceNow, and Honeycomb. They explore the evolution of observability, emphasizing its importance beyond traditional monitoring, and how OpenTelemetry serves as a standardized solution for collecting telemetry data across different systems. Key points include the significance of metrics, logs, traces, and profiling in understanding system behavior, the community aspect of OpenTelemetry, and its future potential in making observability accessible to all developers and organizations. The speakers express enthusiasm for the project’s growth and its impact on improving system insights and performance management. +In this YouTube video, various professionals in the observability field discuss their experiences and thoughts on observability and OpenTelemetry. Participants include Iris Durish from Meo, SE Nman from Cisco, Kayla Riel from New Relic, Morgan Mlan from Swun, Henrik Reet from Dan Race, and Vijay Samuel from eBay, among others. The discussion covers the importance of observability in monitoring systems, the evolution from traditional application performance monitoring (APM) to more comprehensive approaches, and the role of OpenTelemetry in standardizing observability practices across the industry. Key points include the need for deeper insights and understanding of systems beyond simple monitoring, the community effort behind OpenTelemetry, and its potential to empower teams by providing a unified framework for collecting telemetry data. Participants also share personal anecdotes about how they became involved in the OpenTelemetry project and express excitement about its growth and future impact on the observability landscape. ## Chapters -00:00:00 Welcome and introductions -00:03:00 Observability definitions -00:06:30 Observability evolution -00:09:40 OpenTelemetry significance -00:12:27 Community and collaboration -00:15:10 OpenTelemetry project involvement -00:19:30 Challenges in observability -00:22:41 Signals in observability -00:24:00 Metrics, logs, and traces -00:27:30 Favorite signals discussion +00:00:00 Introductions +00:01:36 Discussion on observability +00:03:44 Importance of OpenTelemetry +00:05:20 Observability vs Monitoring +00:08:00 Observability in complex systems +00:09:28 Community effort behind OpenTelemetry +00:12:16 OpenTelemetry as a standard +00:14:24 Personal anecdotes about OpenTelemetry involvement +00:20:16 Future of observability with OpenTelemetry +00:26:40 Favorite signals in observability + +## Transcript + +### [00:00:00] Introductions **Iris:** Well, I'm Iris Durish. I'm a senior observability engineer at Meo, and my professional life is all about observability. I build an observability platform that provides the tools for engineering teams at Meo to monitor, to observe, and get the best of their obligations. @@ -31,151 +35,121 @@ In this YouTube video, titled "Observability and OpenTelemetry Insights," a grou **Kayla:** My name is Kayla Riel. I work for New Relic, and I'm contributing to the open Telemetry Ruby project. -**Morgan:** My name is Morgan Mlan. I'm a director of product management at Swun. I've been with open Telemetry since day one. I'm one of the co-founders of the project. I'm on the governance committee. Wow, what do I work on with open Telemetry? A bit of everything. I mean, early on, it was literally everything. Myself, Ted, and various others were doing many, many jobs. More recently, I've been involved in the release of traces and then metrics 1.0, logs 1.0 last year. Right now, I'm working on profiling, as well as open Telemetry's expansion into mainframe computers. +**Morgan:** My name is Morgan Mlan. I'm a director of product management at Swun. I've been with open Telemetry since day one. I'm one of the co-founders of the project. I'm on the governance committee. Wow, what do I work on with open Telemetry? A bit of everything. I mean, early on, it was literally everything. I mean, myself, Ted, and various others were doing many, many jobs. More recently, I've been involved in the release of traces and then metrics 1.0, logs 1.0 last year. Right now, I'm working on profiling as well as open Telemetry expansion into Mainframe computers. + +### [00:01:36] Discussion on observability -**Henrik:** My name is Henrik Reet. I am a cloud-native advocate at Dan Race, and I'm passionate about observability, performance, and I'm trying to help the community by providing content on getting started on any solutions that help. +**Henrik:** My name is Henrik Reet. I am a cloud native advocate at Dan Race, and I'm passionate about observability performance. I'm trying to help the community by providing content on getting started on any solutions that out. **Vijay:** My name is Vijay Samuel, and I help do architecture for the observability platform at eBay. **Daniel:** I'm Daniel Gomez Blanco. I'm a principal engineer at Skyscanner and also a member of the open Telemetry community. -**Doug:** My name is Doug Odard. I'm a Senior Solutions Architect with ServiceNow Cloud Observability, which is also formerly LightStep. I'm also a previous customer of using open Telemetry for several years prior to that. +**Doug:** My name is Doug Odard. I'm a senior solutions architect with ServiceNow Cloud Observability, which is also formerly Lightstep. I'm also a previous customer of using open Telemetry for several years prior to that. **Adnan:** Hey, I am Adnan. I work at TraceTest as a developer advocate, which you can guess better than me what that is. I pretty much do a bunch of everything regarding open Telemetry. I'm one of the contributors for the documentation, for the blog, and the demo. **Ren:** My name is Ren Manuso. I work for Honeycomb IO, and I am one of the maintainers of the end user. -**Observability Discussion:** - -[00:03:00] **Ren:** What does observability mean to me? Observability to me is the biggest passion of my life and also my professional career. It is one of those areas that you are not very interested in when you start your career because you don't know anything about it. It's not taught in school. It's not preached by the tech communities a lot, but then you discover it, and you say, "Wow, this is amazing! We're actually making a change and helping the teams make the best of their products." - -**SE:** I think observability is a big game changer, right? It's an evolution from what we have done, especially APM, over the last few years. I worked for a very long time at EP Dynamics, and we sold APM agents to customers. We gave them a lot of the things that observability is promising today as well. But the big change I see with observability is that it's coming down, let's say, to everybody. This is making the things that we did there available for everybody and even more. We're moving away from this, "Hey, let's add a post-compilation agent into your application," to "Yeah, let's make native observability. Let's make this a thing that developers and operation teams are using across all organizations." - -**Morgan:** To me, observability means having peace of mind. It means having something that you can rely on in order to see what happened and what went wrong. I think observability is also a way to feel more technically connected to your customers and your users so that you can see the ways that they're interacting with your software instead of just the ways that you might interact with it. - -**Kayla:** I mean, observability to me transcends just the computing industry. It's the ability to peer into something and understand how it works, what it's doing right now, and thus, if it breaks, how to fix it more quickly. Certainly, when we think of open Telemetry in this industry, what observability classically means to me is visibility to back-end infrastructure and applications. - -**Doug:** Kind of excitingly, I think it's expanding right now, right? With open Telemetry, we're pushing into client applications. We're pushing into mainframes. - -**Adnan:** Usually, when people mention observability, they say, "Hey, it's about a replacement of the old name monitoring." But for me, it's more than monitoring. Monitoring is like you just look at something, and observability, for me, is like having enough information to understand a given situation. If you just look at metrics, then you have a guess that something is going wrong, but you don't understand. So having the options to get more information like logs, events, exceptions, traces, profiling, then at the end, combine all those dimensions together, then you say, "Okay, I got it. This is my problem, and I can resolve it." - -**Vijay:** What does observability mean to me? I belong to what is called the site engineering organization inside of eBay, and our goal is to make sure that we can observe everything that's going on in the site and ensure that we have high availability. So basically, observability means knowing if the site is running fine or not because that's why I'm there. - -[00:06:30] **Henrik:** What does observability mean to me? It's a way for us to understand what's happening within our systems because we run quite a complex system. We need to understand what goes on inside of them so we can deliver a good experience for our end users at the end of the day. - -**Adnan:** Oh my God, observability is to me—I've been a full-stack developer for years, and as we observed, I actually ended up on an incident response team doing tracking of incidents but also trying to figure out what was wrong. It pointed out to me how much we need this, how hard it was to look at so many different screens. Observability, for me, is the way to actually see what's happening in your system. It's the pinnacle of not being up the whole night trying to figure out what went wrong. With open Telemetry and with the rise of tracing in the last couple of years, it has hit an all-time high with regards to the possibilities that we have right now. I'm just really, really happy to be part of the project. I'm also really happy that it's growing at the pace it's growing right now, and I can't see how that's going to evolve within the next couple of years. - -**Kayla:** For me, observability is about being able to ask deeper questions of our systems, being able to demand more than just alerting on things that are emergencies, things we've seen before, but actually being able to go out into the unknown and understand how complex systems are performing. Open Telemetry is the tool that is making observability great again. - -**Morgan:** I would say that observability is seeing the surge now that open Telemetry is becoming so popular. It's allowing centralization of telemetry signals, it's allowing semantic conventions, and it's generally helping observability teams and the engineering teams take more attention to observability and building it and making it better. - -**Henrik:** What does open Telemetry mean to me? I think it's the vehicle for observability, right? I mean, it's enabling that. I joined the open Telemetry community a few years back because I was curious about this idea to bring observability to everybody. I think we are doing a really good job. - -**Adnan:** What it also means to me now is that it's an amazing community, right? We're at KubeCon here, and I meet so many people I just know from those digital conversations, and now I can talk to them in person. We talk a lot about open Telemetry, but we also talk a lot about other things than open Telemetry. We talk about observability, of course, about what we think about is going to happen in a few years. - -[00:09:40] **Kayla:** Open Telemetry, to me, seems like it's a community effort to take the best of what's already been out there for instrumentation and collect it in one group so that everyone can benefit from it. I think that we've learned so much as different engineers, but there's also so much to learn from users of products themselves, and open Telemetry does a great job of bringing both people who are experts in observability and experts in languages to make something really great and meaningful for everyone. - -**Morgan:** I mean, open Telemetry is my baby. I put so much effort into creating this project. What does it mean to me? I mean, there's the boring answer, which is it extracts signals, metrics, traces, logs, profiles, everything else from your infrastructure, from your services, from your clients, and makes those observable, processable on the back end. But I think to a lot of us who've been in this community so long, I mean, open Telemetry is also just a really nice open-source community to participate in. It's a thing I just enjoy working on. - -**Henrik:** Open Telemetry, for me, means the future. Because at the end, by having an open standard, we have the luxury, I would say, to have a common standard, a common format for all the solutions of the market. And having that common format for all the industry and all the vendors and all the solutions will just open use cases. +**Vijay:** What does observability mean to me? Observability to me is the biggest passion of my life and also my professional career. It is one of those areas that you are not very interested in when you start your career because you don't know anything about it. It's not taught in school; it's not preached by the tech communities a lot. But then you discover it, and you say, "Wow, this is amazing. We're actually making a change, and we're helping the teams make the best of their product." -**Vijay:** I think testing used to rely on feedback from users, and now with observable data, we can be so much more efficient in the way we test. We could be so much more efficient in replacing marketing tools, business analytics tools. I think it's the future. +### [00:03:44] Importance of OpenTelemetry -**Henrik:** One thing that also a lot of people talk about is AI everywhere, machine learning, blah blah blah, but I think it's the same thing as observability. I mean, Tesla, when you drive your car, it takes decisions based on the sensors that you measure. If you don't have those sensors and those measurements, then you can have the smartest systems, but without the data you cannot take the right decisions. +**Morgan:** I think observability is a big game changer, right? It's an evolution from what we have done, especially APM, over the last few years. I worked for a very long time at EP Dynamics, and we sold APM agents to customers. We gave them a lot of the things that observability is promising today as well. But the big change I see with observability is that it's coming down, let's say, to everybody. This is making the things that we did there available for everybody. Even more, we're moving away from this "Hey, let's add a post-compilation agent into your application" to "Yeah, let's make native observability. Let's make this a thing that developers and operations teams are using across all the organizations." -**Morgan:** I think it's an enabler also for the future implementations of modern applications. Open Telemetry is the standard for observability going forward, and it's very important because as we have gone through the journey of observability over the past few years, we have had to hunt for open standards in Prometheus and a few others. Now, at least with ingestion and collection, it's a single standard for everyone to adopt, and I think that's pretty powerful for the long run. +**Henrik:** To me, observability means having peace of mind. It means having something that you can rely on in order to see what happened and what went wrong. I think observability is also a way to feel more technically connected to your customers and your users so that you can see the ways that they're interacting with your software instead of just the ways that you might interact with it. -[00:12:27] **Doug:** What does open Telemetry mean to me? I think it's bringing people together, bringing everyone together under one single language and one single way of thinking about telemetry. I think human languages are difficult enough for us to understand each other, and I think open Telemetry is bringing the technology together under one single way of thinking about telemetry, thinking about how we observe our systems. +**SE:** I mean, observability to me transcends just the computing industry. It's the ability to peer into something and understand how it works, what it's doing right now, and thus if it breaks, how to fix it more quickly. Certainly, when we think of open Telemetry in this industry, what observability classically means to me is visibility to back-end infrastructure and applications. Kind of excitingly, I think it's expanding right now. With open Telemetry, we're pushing into client applications; we're pushing into mainframes. -**Daniel:** To me, open Telemetry is bringing the ability to have product teams, infrastructure teams helping their jobs make it easier, and also just improve the customer experience and just make it an overall better experience to do our jobs. +### [00:05:20] Observability vs Monitoring -**Adnan:** Open Telemetry is, I'm going to say, the future of observability. We've seen so many companies, so many vendors move to an open Telemetry first mindset. The way that you can use open Telemetry to generate, to actually gather all telemetry signals with one set of libraries, with one tool, is just the way it was supposed to be. You're not locked into one vendor or one cloud provider anymore. You can do basically whatever you want, and you can use both the metric logs and traces for basically anything you want to do. +**Doug:** Usually, when people mention observability, they say, "Hey, it's about... it's a replacement of the old name monitoring." But in fact, for me, it's more than monitoring because monitoring is like you just look at something. Observability, for me, is like having enough information to understand a given situation. If you just look at metrics, then okay, you have a guess that something is going wrong, but you don't understand. Having the options to get more information, like logs, events, exceptions, traces, profiling, then at the end combine all those dimensions together, then you say, "Okay, I got it. This is my problem, and I can resolve it." -**Morgan:** Open Telemetry is an instrumentation protocol that helps us ask more detailed questions about observability because it collects multiple signals from many flexible types of systems. Folks monitor everything from the control plane in Kubernetes all the way up to physical on-prem systems. It's a really flexible language, and it's a beautiful community of humans that came together over the pandemic to build something really special. +**Vijay:** What does observability mean to me? I belong to what is called the site engineering organization inside of eBay, and our goal is to make sure that we can observe everything going on in the site and ensure that we have high availability. So basically, observability means knowing if the site is running fine or not because that's why I'm there. -**Kayla:** I was working in a very fast-paced observability team, and we were maintaining a lot of tools. We really did not have conventions there. We did not have centralization, and we really were not flexible when it came to being vendor-agnostic in general. So we discovered this amazing tool called open Telemetry. We said, "Okay, let's give it a try." It worked great for us, and here I am today, more than a year later, and pushing the migration to open Telemetry in my second project. +**Adnan:** What does observability mean to me? It's a way for us to understand what's happening within our systems because we run quite a complex system. We need to understand what goes on inside of them so we can deliver a good experience for our end users at the end of the day. -[00:15:10] **Morgan:** How did I get involved in open Telemetry? I mentioned that I got curious a few years ago. I was at EP Dynamics working as a so-called domain architect. I was an expert for Node.js, Python, and a lot of those other languages, and there was always this conversation around, "Hey, there's this thing now called open Telemetry. Should we not integrate this into our product?" I was like, "Okay, I want to learn more." Then I was like, "What is a good way to learn something new about an open-source technology? Get involved in that." +**Kayla:** Oh my God, so observability is to me... I've been a full stack developer for years, and so as we observe, I actually ended up on an incident response team doing tracking of incidents, but also trying to figure out what was wrong. It pointed out to me how much we need this, how hard it was to look at so many different screens. Observability, for me, is the way to actually see what's happening in your system. It's the pinnacle of not being up the whole night trying to figure out what went wrong. With open Telemetry and with the rise of tracing in the last couple of years, it has hit an all-time high with regards to the possibilities that we have right now. I'm just really, really happy to be part of the project. I'm also really happy that it's growing at that pace that it's growing right now, and I can't see how that's going to evolve within the next couple of years. -**Kayla:** I got involved in open Telemetry last spring when New Relic asked me to take a look at what the current status was of the open Telemetry Ruby project. I also work as an engineer on the New Relic Ruby agent team, and that gave me an opportunity to start to contribute to the project. +### [00:08:00] Observability in complex systems -**Doug:** I was working at Google on Google's observability products like tracing, profiling, debugging, that whole thing. One of the challenges we had with tracing was getting data from people's applications. It was really, really hard. You needed integrations of hundreds of thousands of pieces of software. No one team, no one company is going to maintain that. It's just infeasible. +**Daniel:** For me, observability is about being able to ask deeper questions of our systems, being able to demand more than just alerting on things that are emergencies, things we've seen before, but actually being able to go out into the unknown and understand how complex systems are performing. -**Morgan:** We wanted to do something open source. There were other open source standards. There was one that had started, I think, roughly around the same time we were doing this called open tracing, and at some points, especially amongst the more socially media savvy members of the team, which I am not one of, there was some contention between those projects about where people who maintain databases and language front-end things should actually spend their integration efforts, and it was limiting the success of both projects. +**SE:** Open Telemetry is the tool that is making observability great again. I would say that observability is seeing the surge now that open Telemetry is becoming so popular. It's allowing centralization of telemetry signals; it's allowing semantic conventions, and it's generally helping observability teams and the engineering teams take more attention to observability and building it and making it better. -**Kayla:** I was leading open census. Ted and Van and others were leading open tracing, and in late 2018, early 2019, we finally sort of brought things to a head and decided to merge those into what is now called open Telemetry. I've been involved since then. I now work at a different company but still on the same things. +### [00:09:28] Community effort behind OpenTelemetry -**Adnan:** When I started the adventure in open Telemetry, of course, I joined a company that has their vendor agent called OneAgent. I saw this movement of open Telemetry and coming from the performance background, I looked at it and I said, "Wow, an open standard? That sounds quite exciting because I had a performance gig for a customer where I implemented collecting logs and processing it and putting machine learning." I told myself at that time, "It would be wonderful to have one common standard." +**Vijay:** What does open Telemetry mean to me? I think it's the vehicle for observability. I mean, it's enabling that. I joined the open Telemetry community a few years back because I was curious about this idea to bring observability to everybody. I think we are doing a really good job. What it also means to me now is that it's an amazing community. We're at KubeCon here, and I meet so many people I just know from those digital conversations, and now I can talk to them in person. We talk a lot about open Telemetry, but we also talk a lot about other things than open Telemetry. We talk about observability, of course, about what we think about what is going to happen in a few years, and all those other things. And that's what open Telemetry means to me. -**Vijay:** When I looked at the definition of the project and the things behind the project, I was so excited. I said, "Oh gosh, I want to be involved in a project." That's where I started to build content to help the community get started. I used to be a developer, but I'm a bad developer for sure, so that's why I'm trying to help the project in other ways and other directions. +**Adnan:** Open Telemetry to me seems like it's a community effort to take the best of what's already been out there for instrumentation and collect it in one group so that everyone can benefit from it. I think that we've learned so much as different agent engineers, but there's also so much to learn from users of products themselves, and open Telemetry does a great job of bringing both people who are experts in observability and experts in languages to make something that's really great and meaningful for everyone. -**Morgan:** My goal is to increase the adoption of the open standards, making sure that it's being adopted everywhere so that we can move forward by trying even more exciting implementations. I started a few years ago for two reasons: one, we were looking to introduce open standards inside the company, and at that time, open tracing and open census was converging into open Telemetry. +**Morgan:** Open Telemetry is my baby. I put so much effort into creating this project. What does it mean to me? I mean, there's the boring answer, which is it extracts signals, metrics, traces, logs, profiles, everything else from your infrastructure, from your services, from your clients, and makes those observable, processable on the back end. But I think to a lot of us who've been in this community so long, and a lot of us like yourself and Henrik here and others who participated in the community so much, I mean, open Telemetry is also just a really nice open source community to participate in. It's a thing I just enjoy working on. I know that's abstract and kind of like a sort of squishy thing to say, but I don't know. Open Telemetry has a lot of meaning to me in many different ways, all very positive. -**Doug:** We started evaluating open Telemetry for that, and given that we were moving into open Telemetry for tracing, I also went through the journey of migrating our metrics collection into open Telemetry. That's basically how I got involved. +**Henrik:** Open Telemetry, for me, means the future because at the end, by having an open standard, we have the luxury, I would say, to have a common standard for a common format for all the solutions in the market. Having that common format for all the industry, all the vendors, and all the solutions, it will just open use cases. I think testing used to rely on feedback from users, and now with observable data, we can be so much more efficient in the way we test. We could be so much more efficient in replacing marketing tools, business analytics tools. I think it's the future. One thing that also a lot of people talk about is AI everywhere, machine learning, blah, blah, blah, blah. But I think it's the same thing as testing. I mean, Tesla, when you drive your car, takes decisions based on the sensors that you measure. If you don't have those sensors and those measurements, then you can have a smart system, but without the data, you cannot take the right decisions. I think it's an enabler also for the future implementations of modern applications. -[00:19:30] **Kayla:** How did I get involved in open Telemetry? I got involved through working as a Skyscanner end user. I was driving adoption of open standards, open Telemetry. During COVID, there was a need for simplification in how we approach our infrastructure, how we collect, how we process, how we export data, and also basically to lead the adoption of open standards and that simplification. +### [00:12:16] OpenTelemetry as a standard -**Adnan:** I got involved in the community at Telemetry, decided to interact with all end users and meet people who want to solve the same problems. They want to find a solution that works. +**Kayla:** Open Telemetry is the standard for observability going forward, and it's very important because as we have gone through the journey of observability over the past few years, we have had to hunt for open standards in Prometheus and a few others. Now, at least with ingestion and collection, it's a single standard for everyone to adopt, and I think that's pretty powerful for the long run. -**Morgan:** I actually, for several years in my previous position, was hired to develop observability software. I was writing my own thing. We were doing a lot of alert management and various things. It was so much work, and I thought, "This has got to be easy." Plus, I wanted to make sure that it could be future-proof, but also extensible. +**Vijay:** What does open Telemetry mean to me? I think it's bringing people together, bringing everyone together under one single language and one single way of thinking about telemetry. I think human languages are difficult enough for us to understand each other, and I think open Telemetry is bringing the technology together under one single way of thinking about telemetry, thinking about how we observe our systems. -**Daniel:** When I discovered open Telemetry, I was just like, "Oh, thank you," because it's something that the company could carry forward, and we didn't have to worry about storing the data as much. +**Adnan:** To me, open Telemetry is bringing the ability to have product teams and infrastructure teams helping their jobs make it easier and also just improve the customer experience and just make it an overall better experience to do our jobs. -**Kayla:** It really provided an excellent platform so that we can focus on the task at hand versus how to do the job. +**Kayla:** Open Telemetry is the future of observability. We've seen so many companies, so many vendors move to an open Telemetry first mindset. The way that you can use open Telemetry to generate and actually gather all telemetry signals with one set of libraries, with one tool, is just the way it was supposed to be. You're not locked into one vendor, one cloud provider anymore. You can do basically whatever you want, and you can use both the metric logs and traces for basically anything you want to do. So I'm really happy to see it grow. -**Adnan:** How I got involved in the project was actually first as a customer. It was about three, close to four years ago, kind of the infancy of open Telemetry. I would go online, look at the documentation, or I would be in the code a lot, but I wanted to learn more. +### [00:14:24] Personal anecdotes about OpenTelemetry involvement -**Vijay:** I would go to a call, and there would be someone from Google and Microsoft and other companies, and then there was this guy from this small fintech in the US. At first, it was a little awkward, but they were so excited to have me in the call because I was an end user. It was a wonderful experience to begin that way and realize that I could contribute to this rather than simply be a consumer of it. +**Doug:** Open Telemetry is an instrumentation protocol that helps us ask more detailed questions about observability because it collects multiple signals from many flexible types of systems. Folks monitor everything from the control plane in Kubernetes all the way up to physical on-prem systems. It's a really flexible language, and it's a beautiful community of humans that came together over the pandemic to build something really special. -**Morgan:** It was great. Then I transitioned my career into working for a vendor, and we implement these systems now for customers like myself that I was years ago. It's kind of a pay-it-forward, give-back type of thing. +**Adnan:** I was working in a very fast-paced observability team, and we were maintaining a lot of tools. We really did not have conventions; we did not have centralizations, and we really were not flexible when it came to being vendor agnostic in general. So we discovered this amazing tool called open Telemetry. We said, "Okay, let's give it a try." It worked great for us, and here I am today, more than a year later, pushing the migration to open Telemetry in my second project. -**Adnan:** How did we get involved in the open project? We started contributing more to the blog with you guys, started contributing a bit to the docs as well. It's just been a wholehearted effort in the team to always dedicate a few minutes of each day to check out the open project and find a way to contribute. +**SE:** How did I get involved in open Telemetry? I mentioned that I got curious a few years ago. I was at EP Dynamics, working as a so-called domain architect. I was an expert for Node.js, Python, and a lot of those other languages. There was always this conversation around, "Hey, there's this thing now called open Telemetry. Should we not integrate this into our product?" I was like, "Okay, I want to learn more." Then I was like, "What is a good way to learn something new about an open source technology?" Well, get involved in that. I was involved in JavaScript at some point, and then at some point, I realized that if I really want to get a good view into open Telemetry, doing documentation is a good way into that. That's how I ended up being a maintainer for the documentation. -[00:22:41] **Doug:** I got involved in the open Telemetry project. Honestly, I was working at one observability company in marketing, and they didn't see the point. They didn't want me to get involved. I really believed in open source. I'd worked at Mozilla and Wikipedia and really believed that this was the way forward strategically. +**Kayla:** I got involved in open Telemetry last spring when New Relic asked me to take a look at what the current status was of the open Telemetry Ruby project. I also work as an engineer on the New Relic Ruby agent team, and that gave me an opportunity to start to contribute to the project. I noticed that a lot of the signals for Ruby weren't yet stable, so a lot of my work so far has been going into trying to bring logs and metrics to stability. -**Morgan:** So the second I could switch to a company that did let me get involved, that's what I did. Now I'm at Honeycomb, and I'm glad to say within the first three months, I became a project member and started working with the end-user working group and worked to grow it into a SIG and into all the programs that it has today together with others. +**Doug:** I was working at Google on Google's observability products, like tracing, profiling, debugging, that whole thing. One of the challenges we had with tracing was getting data from people's applications. It was really, really hard. You need integrations of hundreds of thousands of pieces of software. No one team, no one company is going to maintain that; it's just infeasible. So we wanted to do something open source. There were other open source standards. There was one that had started, I think, roughly around the same time we were doing this, called open tracing and open census at some points. Especially amongst the more social media savvy members of the team, which I am not one of, there was some contention between those projects about where people who maintain databases and language front-end things should actually spend their integration efforts, and it was limiting the success of both projects. I was leading open census; Ted and Van and others were leading open tracing. In late 2018, early 2019, we finally sort of brought things to a head and decided to merge those into what is now called open Telemetry. That's sort of how I've been involved since then. I now work at a different company but still on the same things. That's how my journey started, and it's just grown and grown and snowballed. -**Kayla:** Tracing is my favorite signal. My favorite signal now is profiling because I think this is really closing a big gap that was missing in observability. I come from the APM space, and now for me, APM and observability, it's very hard to make a difference here. +**Henrik:** When I started the adventure in open Telemetry, I joined Dynatrace. Dynatrace has their vendor agent called OneAgent. I saw this movement of open Telemetry, and coming from the performance background, I looked at it and I said, "Wow, an open standard? That sounds quite exciting." Because I had a performance gig for a customer where I implemented collecting logs and processing it and putting machine learning, and I told myself at that time, "It would be so wonderful to have one common standard." Instead of doing a custom implementation, I could have something for everyone. When I looked at the definition of the project and the things behind the project, I was so excited. I said, "Oh gosh, I want to be involved in a project." That's where I started to build content to help the community get started. I used to be a developer, but I'm a bad developer for sure, so that's why I'm trying to help the project in other ways and other directions. My goal is to increase the adoption of open standards, making sure that it's being adopted everywhere so then we can move forward by flying even more exciting implementations. -[00:24:00] **Henrik:** But one thing that when I talk with people using APM products right now is they're like, "Hey, where's the code-level visibility?" With open Telemetry, my commercial agent is giving me that line of code that is breaking something, and this is what we get with profiling, and that's why I'm really, really excited about it. +**Doug:** I started a few years ago for two reasons. One, we were looking to introduce open Telemetry inside the company. At that time, open tracing and open census were converging into open Telemetry. We started evaluating open Telemetry for that, and given that we were moving into open Telemetry for tracing, I also went through the journey of migrating our metrics collection into open Telemetry. That's basically how I got involved. -**Vijay:** To decide a favorite signal is kind of difficult for me. I really love the power of traces. I think that traces can tell stories in ways that are very meaningful. But on the other hand, I've been so immersed in logs and trying to allow logs to have more connections to spans and traces, but I definitely have a soft spot for logs as well. +**Vijay:** How did I get involved in open Telemetry? I got involved through working at Skyscanner as an end user. I was driving adoption of open standards, open Telemetry. During COVID, there was a need for simplification in how we approach infrastructure, how we collect, how we process, how we export data, and also basically to lead the adoption of open standards and that simplification. I got more involved in the community as Telemetry decided to interact with all end users and meet people that want to solve the same problems. -**Morgan:** I mean, I'm partial to distributed traces because that's where this project got its start. I think early on, that's where a lot of the value was. No one else was really doing standardized distributed trace collection. There were some open-source examples of it attached to like Zipkin and Jaeger, but I think the reason open Telemetry got so much traction so quickly is that it was providing that. +### [00:20:16] Future of observability with OpenTelemetry -**Kayla:** I'm also partial to logs, which we launched last year just because that's one where I've been involved in a lot of parts of open Telemetry. +**Adnan:** I actually, for several years in my previous position, was hired to develop observability software. I was writing my own thing; we were doing a lot of alert management and various things. It was so much work, and I thought this has got to be easy. Plus, I wanted to make sure that it could be future proof, there I use that term, but also extensible. When I discovered open Telemetry, I was just like, "Oh, thank you," because it's something that the company could carry forward, and also we didn't have to worry about storing the data as much. It's really provided a really excellent platform so that we can focus on the task at hand versus how to do the job. -**Henrik:** But that's one where I was involved in a lot of the core specifics early on and driving that, and so it was really exciting to see that ship. Also, logs are just a thing that throughout my career before working on any of this, I just got frustrated with because they're never standardized, they're slow to process, they're expensive. +**Ren:** How I got involved in the project was actually first as a customer. It was about three, close to four years ago, kind of the infancy of open Telemetry. I would go online; I would look at the documentation, or I would be in the code a lot, but I wanted to learn more. So I would go to a call, and there would be someone from Google and Microsoft and other companies. Then there was this guy from this small fintech in the US. At first, it was a little awkward, but they were so excited to have me in the call because I was an end user. It was a wonderful experience to begin that way, to realize that I could contribute to this rather than simply be a consumer of it. Then I transitioned my career into working for a vendor, and we implement these systems now for customers like myself that I was years ago. It's kind of a pay it forward, give back type of thing. -**Morgan:** Open Telemetry is going to bring a lot of changes there for the better for everyone who uses logs. And finally, I guess profiles, because I'm working on that now, and it's new. When I was at Google many years ago, I launched what I think was the world's first distributed continuous profiling product, at least publicly available, which was Google Cloud profiling stack profiling. They still support it. I still think it's free; it's very powerful. +**Daniel:** How did we get involved in the open project? We started contributing more to the blog, and we started contributing a bit to the docs as well. It's just been a wholehearted effort in the team to always dedicate a few minutes of each day to check out the open project and to find a way to contribute. -**Henrik:** Profiling has always been a bit of a niche thing. I know that Splunk and other companies support it, but it's not as well known as metrics and traces. I think with open Telemetry starting to later this year, we're going to launch full support for profiles. That's really going to change things. +**Kayla:** I got involved in the open Telemetry project. Honestly, I was working at one observability company in marketing, and they didn't see the point. They didn't want me to get involved. I really believed in open source. I'd worked at Mozilla and Wikipedia and really believed that this was the way forward from a strategic perspective. The second I could switch to a company that did let me get involved, that's what I did. Now I'm at Honeycomb, and I'm glad to say within the first three months, I became a project member and started working with the end user working group and worked to grow it into a SIG, into all the programs that it has today together with others. -**Adnan:** We had customers at Google who would spend an hour on our profiler and save 20-30% of their aggregate CPU because they found some really poorly optimized code really quickly. For more people to have that ability and speed to speed things up and for developers to actually get insight on how things work, that's super exciting. The tech has been there a long time, and open Telemetry bringing this mainstream is huge. +**Adnan:** Tracing is my favorite signal. My favorite signal now is profiling because I think this is really closing a big gap that was missing in observability. I mentioned before that I come from the APM space. Now, for me, APM and observability, it's very hard to make a difference here. But one thing that when I talk with people using APM products right now is they're like, "Hey, where's code-level visibility with open Telemetry?" My commercial agent is giving me that line of code that is breaking something, and this is what we get with profiling. That's why I'm really excited about it. -**Morgan:** When people ask me who is your favorite kid, usually I say I don't have a favorite kid. You know, all my kids are wonderful. They all have a great thing out of it. I think I love traces because sometimes they help you to understand where it's slowed down. I love metrics because as a performance engineer, I used to use metrics a lot. +**Kayla:** To decide a favorite signal is kind of difficult for me. I really love the power of traces. I think that traces can tell stories in ways that are very meaningful. But on the other hand, I've been so immersed in logs and trying to allow logs to have more connections to spans and traces, but I definitely have a soft spot for logs as well. -**Kayla:** I love logs because at the end, there's no single answer. If you just do analytics on logs, wow, you are so much more precise. I don't think I'll have a favorite signal. I'll just say that depending on what I need, I can pick and choose. +**Vijay:** I mean, I'm partial to distributed traces because that's where this project got its start. Early on, that's where a lot of the value was. No one else was really doing standardized distributed trace collection. There were some open-source examples of it attached to things like Zipkin and Jaeger, but I think the reason open Telemetry got so much traction so quickly is that it was providing that. -**Daniel:** There's clearly one signal that will help me more. There's one thing that I'm very eager and waiting for since Valencia: continuous profiling. I love profiling and I think traces are great, but if there is a problem somewhere, profiling would be so much helpful. +**Doug:** I'm also partial to logs, which we launched last year, just because I've been involved in a lot of parts of open Telemetry. But that's one where I was involved in a lot of the core specifically early on and driving that. It was really exciting to see that ship. Also, logs are just a thing that throughout my career, before working on any of this, I just get frustrated with because they're never standardized, they're slow to process, and they're expensive. Open Telemetry is going to bring a lot of changes there for the better for everyone who uses logs. -**Adnan:** So I think, yeah, I don't answer any questions, but I say, yeah, I love all the signals provided by open Telemetry. +**Morgan:** And finally, I guess profiles because I'm working on that now, and it's new. When I was at Google many years ago, I launched what I think was the world's first distributed, like continuous profiling product, at least publicly available one, which was Google Cloud profiling stack profiling. They still support it; I still think it's free. It's very powerful. But profiling has always been a bit of a niche thing. I know like Splunk and other companies support it, but it's not as well known as metrics and traces. I think with open Telemetry, starting later this year, we're going to launch full support for profiles. That's really going to change things. We had customers at Google who would spend an hour with our profiler and save like 20-30% of their aggregate CPU because they found some really poorly optimized code very quickly. For more people to have that ability and speed to speed things up and for developers to actually get insight on how things work, that's super exciting. The tech has been there a long time, and open Telemetry bringing this mainstream is huge. -[00:27:30] **Vijay:** I am thoroughly biased towards metrics. I feel metrics are the most powerful signal as long as you are thinking through your instrumentation and making sure that you have the right granularity and cardinality being sent into the platform. You can do powerful, powerful things with regards to anomaly detection, machine learning, and many other things. +### [00:26:40] Favorite signals in observability -**Kayla:** I have to say traces because they give you the content. Traces give you the backbone correlation for all the other signals. Right? But I do think that the current design of the API design of metrics is so powerful that I'm falling in love again with metrics because of the way that we decouple instrumentation and measurement from the aggregation of those metrics. +**Henrik:** When people ask me who is your favorite kid, usually I say I don't have a favorite kid. You know, all my kids are wonderful; they all have great qualities. I love traces because sometimes they help you to understand where it's slowed down. I love metrics because as a performance engineer I used to use metrics a lot. I love logs because logs at the end, there's no signal. If you just do analytics on logs, wow, you are so much more precise. I don't think I'll have a favorite signal. I'll just say that depending on what I need, I'll pick and choose. There's clearly one signal that will help me more. -**Daniel:** It's powerful and so much richness to basically give us a way to describe our system that I'm calling back again in love with metrics. +**Doug:** There's one thing that I'm very eager and waiting for since Valencia: continuous profiling. I love profiling, and I think traces are great, but if there is a problem somewhere, profiling would be so much helpful. -**Morgan:** My favorite signal, I have to say, I'm partial to traces because I've been doing software development for so long. That was the first thing that really turned me on to it was the ability to see that, especially because I know what it's like to debug. +**Vijay:** So I think, yeah, I don't answer any questions, but I say, "Yeah, I love all the signals provided by open Telemetry." -**Doug:** But I also know what it's like in an incident to have to focus in very quickly. So yes, traces are my favorite, but I do also like to send that Trace ID and Span ID into the logs. Now it's kind of becoming my next favorite. +**Kayla:** I am thoroughly biased towards metrics. I feel metrics are the most powerful signal as long as you're thinking through your instrumentation and making sure that you have the right granularity, cardinality being sent into the platform. You can do powerful, powerful things with regards to anomaly detection, machine learning, and many other things. -**Kayla:** My favorite signal is traces. I'm going to say traces definitely. +**Morgan:** I mean, I have to say traces because they give you the content. Traces give you the backbone correlation for all the other signals, right? But I do think that the current design of the API, the design of metrics is so powerful that I'm falling in love again with metrics because of the way that we decouple instrumentation and measurement from aggregation of those metrics. It's so powerful and so much richness to give us a way to describe our system that I'm calling back again in love. -**Adnan:** My favorite signal is Ed Sheeran. +**Henrik:** My favorite signal, I have to say, I'm partial to traces because I've been doing software development for so long. That was the first thing that really turned me on to it, especially because I know what it's like to debug. But I also know what it's like in an incident to have to focus in very quickly. So yes, traces are my favorite, but I do also like to send that trace ID and span ID into the logs now. It's kind of becoming my next favorite. -**Doug:** What is my favorite signal? I mean, I work for Honeycomb, so I am constitutionally obliged to say traces are my favorite signal. +**Adnan:** My favorite signal is traces. I'm going to say traces definitely. My favorite signal is Ed Sheeran. What is my favorite signal? I mean, I work for Honeycomb, so I am constitutionally obliged to say traces are my favorite signal. ## Raw YouTube Transcript diff --git a/video-transcripts/transcripts/2024-06-17T19:00:22Z-otel-q-a-feat-daniel-dias-and-oscar-reyes-of-tracetest.md b/video-transcripts/transcripts/2024-06-17T19:00:22Z-otel-q-a-feat-daniel-dias-and-oscar-reyes-of-tracetest.md index 8caa55c..d8dd7c6 100644 --- a/video-transcripts/transcripts/2024-06-17T19:00:22Z-otel-q-a-feat-daniel-dias-and-oscar-reyes-of-tracetest.md +++ b/video-transcripts/transcripts/2024-06-17T19:00:22Z-otel-q-a-feat-daniel-dias-and-oscar-reyes-of-tracetest.md @@ -10,142 +10,150 @@ URL: https://www.youtube.com/watch?v=KR8j11FECgc ## Summary -In this OTEL Q&A session, hosts Daniel Dias and Oscar Reyes from Tracetest discuss the integration of trace-based testing into the OpenTelemetry (OTEL) community demo, which simulates a telescope shop with several interacting microservices. Daniel, a software developer and CNCF ambassador from Brazil, and Oscar, a lead software engineer from Mexico, explain trace-based testing, which uses traces emitted by systems to make assertions about their state, particularly in complex, distributed services. They elaborate on how Tracetest implements this testing through a two-step process of triggering the system and pulling traces for validation. They also recount the challenges faced during integration, such as ensuring context propagation across service boundaries and adapting existing integration tests into trace-based tests. The integration has led to improved community practices, such as incorporating trace tests into CI/CD pipelines, enhancing testing efficiency and reliability. Overall, the discussion highlights the benefits of combining trace-based testing with OpenTelemetry to improve software quality in distributed systems. +In this OTEL Q&A session, Daniel Dias and Oscar Reyes from Tracetest discuss the integration of trace-based testing into the OpenTelemetry (OTEL) community demo. Daniel, a software developer from Brazil, and Oscar, a lead software engineer from Mexico, explain the concept of trace-based testing, which utilizes generated traces from systems to perform multi-layer testing on distributed services. They detail how Tracetest implements this through a three-step process of triggering the system, pulling traces, and making assertions. They also highlight their contributions to the OTEL demo, which emulates a telescope shop with various microservices, and how they transformed existing integration tests into trace-based tests to better validate interactions between services. Key challenges included ensuring context propagation across services, and the integration has led to improvements in the development process and increased reliance on Tracetest in the community. The session concludes with a discussion on the potential for further enhancements and the benefits of the integration, including community feedback that has helped improve Tracetest. ## Chapters -00:00:00 Welcome and intro -00:01:10 Guest introductions -00:02:20 Trace based testing explanation -00:04:00 Tracetest implementation overview -00:06:18 Automation and YAML usage -00:07:39 OTEL community demo introduction -00:09:04 Tracetest integration story -00:12:00 Traceability tests discussion -00:16:30 Challenges during integration -00:20:30 Benefits of trace based testing +00:00:00 Introductions +00:02:07 What is trace-based testing? +00:03:22 How Tracetest implements trace-based testing +00:05:50 Discussion on OTEL community demo +00:08:10 Overview of the OTEL demo +00:11:45 Integration of Tracetest into the OTEL demo +00:12:50 Challenges faced during integration +00:20:25 Benefits of trace-based testing integration +00:24:25 Community feedback and improvements +00:30:05 Addressing context propagation issues -**Host:** Welcome everyone who has been able to join us today. This is super exciting to have. We have both Daniel Dias and Oscar Reyes from Tracetest joining us today for OTEL Q&A, and we haven't had an OTEL Q&A for a while. So I'm so glad that both of you were able to join us today. For those of you who are here today, thank you for joining, and also tell your friends that we will be posting a recording of this on the OTEL YouTube channel. So that's OTEL official, for anyone who couldn't make it. And we'll also provide the links on socials once the recording is made available. +## Transcript -So with that, let's get started. Why don't I get you to introduce yourselves? +### [00:00:00] Introductions -[00:01:10] **Daniel:** I can go first. So hello everyone. I'm Daniel Dias, a Brazilian living and complaining in Argentina right now because of the weather. I am a software developer at Tracetest, and while I'm not helping the team evolve with new features, I'm bragging about the features somewhere. That's all. And Daniel, he's also an ambassador, so he's really proud to have one ambassador in CNCF ambassador too. CNCF ambassador, right? Yes. Newly minted, right? Just, yes. I became my ambassador last week, last month. Yay. +**Host:** Welcome everyone who has been able to join us today. This is super exciting, to have. We have both Daniel Dias and Oscar Reyes from Tracetest joining us today for OTEL Q&A, and we haven't had an OTEL Q&A for a while. So I'm so glad that both of you were able to join us today. For those of you who are here today, thank you for joining, and also tell your friends that we will be posting a recording of this on the OTEL YouTube channel. So that's OTEL official, for anyone who couldn't make it. We'll also provide the links on socials once the recording is made available. -**Oscar:** For my part, I'm Oscar Reyes. I'm Mexican. I live in Mexico. And I'm a lead software engineer here at Tracetest. And yeah, I'm really happy to be here and talk about Tracetest and OpenTelemetry. +With that, let's get started. Why don't I get you to introduce yourselves? -**Host:** Awesome. Okay. First things first, for folks on the call who might not be familiar, can you tell us what trace-based testing is? +**Daniel:** I can go first. So hello everyone. I'm Daniel Dias, a Brazilian living and complaining in Argentina right now because of the weather. I am a software developer at Tracetest, and while I'm not helping the team evolve with new features, I'm bragging about the features somewhere. That's all. And Daniel, he's also an ambassador, so he's really proud to have one ambassador in CNCF ambassador too. CNCF ambassador, right? Yes. Newly minted, right? Just, yes. I became my ambassador last week, last month. Yay. -[00:02:20] **Daniel:** When we call, say about trace-based tests, the main idea is that we want to, we use traces emitted by systems to assert how the screen state of the trace, the state is. So we call some component on the system, wait for a while until the trace is generated, and we start to make assertions to this trace to guarantee that everything is right. +**Oscar:** For my part, I'm Oscar Reyes. I'm Mexican. I live in Mexico. And I'm a lead software engineer here at Tracetest. I'm really happy to be here and talk about Tracetest and OpenTelemetry. -**Oscar:** Yeah, and this is really helpful for distributed services where you have a lot of moving pieces, and if the entire system is instrumented, you can get a lot of, you can have a lot of assertions around the generated OpenTelemetry data or the traces, right? You can validate that certain spans exist, but you can also go deeper and validate things like the attributes or the execution time of the spans, those sort of things. So it's a multi-layer level of testing and the entire spans or traces that your system generates. +### [00:02:07] What is trace-based testing? -**Daniel:** That's awesome. And the thing I love about trace-based testing is we're already emitting traces in our code. So why not take advantage of that? So I feel like it's a two-for-one deal. Which is very awesome. +**Host:** Awesome. Okay, first things first, for folks on the call who might not be familiar, can you tell us what trace based testing is? -**Host:** The other thing that I wanted to ask, so how does Tracetest implement trace-based testing? +**Oscar:** When we call, say about trace based tests, the main idea is that we want to use traces emitted by systems to assert how the screen state of the trace, the state is. So we call some component on the system, wait for a while until the trace is generated, and we start to make assertions to this trace to guarantee that everything is right. Yeah, and this is really helpful for distributed services where you have a lot of moving pieces, and if the entire system is instrumented, you can get a lot of assertions around the generated OpenTelemetry data or the traces, right? You can validate that certain spans exist, but you can also go deeper and validate things like the attributes or the execution time of the spans, those sort of things. So it's a multi-layer level of testing and the entire spans or traces that your system generates. -[00:04:00] **Oscar:** Maybe I can answer this one and you can complement what I say, Daniel, if you want to. What trace-based, the way that trace-based does it is that we have a two-step process. The first one is triggering your system, the system on their test. Currently, we support things like HTTP, gRPC requests. We also support Kafka, that was actually integrated by Daniel. And then we have other types of integrations for other types of systems like Cypress or Playwright. So that's the first half of what Tracetest does, triggers their system on the test. +### [00:03:22] How Tracetest implements trace-based testing -And then the system address will need to generate or will generate those instrument, the traces and the spans and put in somewhere, in a tracing backend. Tracing backend, it could be something like Jaeger or Tempo or any other providers that you will store the traces. And then the second half of what Tracetest will do is pull for those traces, based on the configuration that you have. So we will wait, depending on the way that system works, that system that generates all of the entire trace in one, a couple of seconds. You have other ones that generate that have long-lasting processes that could take five minutes. So we support all those variables. +**Daniel:** That's awesome. And the thing I love about trace based testing is, we're already emitting traces in our code. So why not take advantage of that? I feel like it's a two for one deal, which is very awesome. -And so we wait for the trace to be ready. We pull that into the Tracetest system. Based on the trace ID that we generate on the first step, that will work as a parent ID, and then the assertions on the spans, the sessions, and the test space will be triggered against that generated traces or telemetry data. So I guess, there will be a three-step process: trigger, trace, or getting the trace, and then test. That's how we're calling our three-step process. +**Host:** The other thing that I wanted to ask, so how does, so how does Tracetest implement trace based testing? + +**Oscar:** Maybe I can answer this one and you can complement what I say, Daniel, if you want to. What trace based, the way that trace based does it is that we have a two-step process. The first one is triggering your system on their test. Currently, we support things like HTTP, gRPC requests. We also support Kafka, that was actually integrated by Daniel. And then we have other types of integrations for other types of systems like Cypress or Playwright. So that's the first half of what Tracetest does, triggers their system on the test. + +And then the system address will need to generate or will generate those instrumented traces and the spans and put in somewhere, in a tracing backend. The tracing backend could be something like Jaeger or Tempo or any other providers that you will store the traces. And then the second half or what Tracetest will do is pull for those traces based on the configuration that you have. So we will wait, depending on the way that system works, that system that generates the entire trace in one, a couple of seconds. You have other ones that generate that have long-lasting processes that could take five minutes. So we support all those variables. And so we wait for the trace to be ready. We pull that into the Tracetest system. Based on the trace ID that we generate on the first step, that will work as the parent ID, and then the assertions on the spans, the sessions, and the test space will be triggered against that generated traces or telemetry data. + +So I guess there will be a three-step process: trigger, trace, or getting the trace, and then test. That's how we're calling our three-step process. **Daniel:** Cool. Do you have anything to add on to that or you're good? -**Oscar:** All right, cool. And if I remember correctly, because it's been a while since I played with Tracetest, but you can create your trace-based tests using YAML, right? Everyone's favorite thing that they love to hate. +**Oscar:** All right, cool. If I remember correctly, because it's been a while since I played with Tracetest, but you can create your trace based tests using YAML, right? Everyone's favorite thing that they love to hate. -**Daniel:** Yeah, I think one of the things that we have pushing in the past few months is not only allowing things to have these three steps, but also how they can automate and simplify the way to introduce trace-based testing in their systems. Either if you use it through a system of Git actions or CI/CD processes, sorry, or you have them in a, I don't know, in a cron job. +### [00:05:50] Discussion on OTEL community demo -[00:06:18] With the automate way that we provide, you can use our CLI, you can use the different tools that we provide, like our TypeScript library, so you can pretty much graph the specifications and the YAML files, as you said, even JSON files from the system. If you go to the UI or either use the CLI to export them, you can put it in your report and execute everything from there. So it's like you have multiple ways of interacting with the system and with the test definitions, either through the UI, through the CLI, or the libraries that we also support. +**Daniel:** Yeah, I think one of the things that we have pushing in the past few months is not only allowing things to have these three steps, but also how they can automate and simplify the way to introduce trace based testing in their systems. Either if you use it through a system of Git actions or CI/CD processes, or you have them in a, I don't know, in a cron job. With the automate way that we provide, you can use our CLI, you can use the different tools that we provide, like our TypeScript library, so you can pretty much graph the specifications and the YAML files, as you said, even JSON files from the system. If you go to the UI or either use the CLI to export them, you can put it in your report and execute everything from there. So it's like you have multiple ways of interacting with the system and with the test definitions, either through the UI, through the CLI, or the libraries that we also support. -**Host:** Cool. Awesome. Before we go on, I do want to mention to folks who are watching, if anyone has any questions, do feel free to pop them into the chat and we will also have a Q&A portion at the end. I just wanted to mention that quickly. +**Host:** Cool. Awesome. Before we go on, I do want to mention to folks who are watching, if anyone has any questions, do feel free to pop them into the chat, and we will also have a Q&A portion at the end. I just wanted to mention that quickly. -[00:07:39] Okay. I guess the main reason why we called Daniel and Oscar to do Q&A is that they did something super cool last fall, which was they integrated trace-based testing, a la Tracetest, into the OTEL community demo. So first of all, can one of you explain what the OTEL community demo is for folks who aren't familiar with it? +I guess the main reason why we called Daniel and Oscar to do Q&A is that they did something super cool last fall, which was they integrated trace based testing, a la Tracetest, into the OTEL community demo. So first of all, can one of you explain what the OTEL community demo is for folks who aren't familiar with it? -**Oscar:** I can answer that. The main idea of the OpenTelemetry demo is that it emulates a telescope shop, but with one major thing. Under this shop, we have a bunch of microservices that interact with each other, each one with one responsibility on the system. So we have one API to make payments, another one to do shipping and everything else. And the main idea of this demo is just to show, with a different technology, a different language, how you can integrate OpenTelemetry to your system and how you can see this data integrated across a bunch of mobile services. +### [00:08:10] Overview of the OTEL demo -**Daniel:** Awesome. Yeah. And it's, it's a great way for getting started with OTEL, especially if you're like, Oh, this is so overwhelming seeing it in a relatively complex example, just kind of goes to show the power of OTEL, which I always think is so cool. And it's not terribly awful to get started with the OTEL demo. I definitely recommend to folks who haven't tried it out to give it a go. +**Oscar:** I can answer that. The main idea of the OpenTelemetry demo is that it emulates a telescope shop, but with one major thing. Under this shop, we have a bunch of microservices that interact with each other, each one with one responsibility on the system. So we have one API to make payments, another one to do shipping, and everything else. The main idea of this demo is just to show how you can integrate OpenTelemetry to your system and how you can see this data integrated across a bunch of mobile services. -Okay. So now that we understand what the OTEL demo is, let's talk about how you got Tracetest integrated into the demo. +**Daniel:** Awesome. Yeah. And it's a great way for getting started with OTEL, especially if you're like, oh, this is so overwhelming seeing it in a relatively complex example, just kind of goes to show the power of OTEL, which I always think is so cool. And it's not terribly awful to get started with the OTEL demo. I definitely recommend to folks who haven't tried it out to give it a go. -**Oscar:** Maybe I can start with the first half of the story and then Daniel can continue after that. So almost a year and a half ago, we were looking for different partnerships or different ways to contribute with the community and just push OpenTelemetry further. We discovered the OpenTelemetry demo. And I was on a task on let's try to just contribute and see how we can help and grow the community. +**Host:** Okay, so now that we understand what the OTEL demo is, let's talk about how you got Tracetest integrated into the demo. -And the first thing that I was, that I worked on, it was to switch or migrate the existing front-end application that was built entirely in Go, Golang with a front, with a server-side rendered front end. Just templates, HTML templates, things like that to an actual front-end application. So we, I switched that to, from Go to Next.js. So Next.js on the, having React involved and also introducing the actual front-end instrumentation of the application by using the tools and the web, the browser site, OpenTelemetry libraries. So in that case, it is continuously connected across, starting from the browser all the way to the backend services. And that was pretty much my contribution, my contribution on the OpenTelemetry demo. +**Oscar:** Maybe I can start with the first half of the story and then Daniel can continue after that. So almost a year and a half ago, we were looking for different partnerships or different ways to contribute with the community and just push OpenTelemetry further. We discovered this OpenTelemetry demo, and I was on a task on let's try to contribute and see how we can help and grow the community. The first thing that I worked on was to switch or migrate the existing front end application that was built entirely in Go, Golang, with a server-side rendered front end, just templates, HTML templates, things like that to an actual front end application. So I switched that from Go to Next.js. So Next.js on the having React involved and also introducing the actual front end instrumentation of the application by using the tools and the browser site OpenTelemetry libraries. So in that case, it is continuously connected across, starting from the browser all the way to the backend services. That was pretty much my contribution, my contribution on the OpenTelemetry demo. **Daniel:** And then, after Daniel joined and did other stuff. So you want to go next? Continue, Daniel. -**Oscar:** After a while, we started to see the OpenTelemetry demo because it is great. Every time that you need to see, oh, I will start programming in Ruby and I need an API. If you go to OpenTelemetry demo, you will see an API there and you can see, oh, this is how you integrate things there or Python or everything else. But looking to the tests that we have in OpenTelemetry demo at the time, they had integration tests, but just on the API level. +### [00:11:45] Integration of Tracetest into the OTEL demo -[00:12:00] So we could test some features of each API, but we could not see how they act together. And sometimes this is painful, mainly when you have background processes. For instance, if you have an API that emits messages to a Kafka queue, and we have another API that reads data from this Kafka queue, how do you guarantee that this second service got the message correctly and is doing something correct? We did not have this at that time. Looking to this test site, I thought, what if we do a trace-based test? So we started discussing a little bit. One of the members of the OpenTelemetry team, also demo team, also asked us, because they say, oh guys, we developed some new things on the OpenTelemetry demo, but we did not notice that we changed the slide a little bit, some traces, and something is wrong. It's weird to understand, we need to manually inspect everything. We started to do traceability tests on that. +**Oscar:** After a while, we started to see the OpenTelemetry demo because it is great. Every time that you need to see, oh, I will start programming in Ruby and I need an API. If you go to OpenTelemetry demo, you will see an API there and you can see, oh, this is how you integrate things there or Python or everything else. But looking to the test that we have in OpenTelemetry demo at the time, they had integration tests, but just on the API level. So we could test some features of each API, but we could not see how they act together. And sometimes this is painful, mainly when you have background processes. For instance, if you have an API that emits messages to a Kafka query, and we have another API that reads data from this Kafka query, how do you guarantee that this second service got the message correctly and is doing something correct? We did not have this at that time. Looking to this test site, I thought, what if we do a trace based test? So we started discussing a little bit. One of the members of the OpenTelemetry demo team also asked us, because they said, oh guys, we developed some new things on the OpenTelemetry demo, but we did not notice that we changed the slide a little bit, some traces, and something is wrong. It's weird to understand, we need to manually inspect everything. We started to do traceability tests on that. We started looking at each one of the integration tests, and we started writing a counterpart traceability test, but for a small difference. Instead of just seeing the API response, we also are seeing what are the APIs that we are calling behind the scenes, and we are testing these APIs too. -**Oscar:** We started looking at each one of the integration tests, and we started writing a counterpart traceability test, but for a small difference. Instead of just seeing the API response, we also are seeing what are the APIs that we are calling behind the scenes, and we are testing these APIs too. And so my thing as well was that for us, part of that section, I don't know if it was a month or a couple of weeks, there was a specific service that was, the telemetry was failing or was broken, right? And the team didn't find out until that time passed and someone, I don't know, saw that problem. And then based on that also pushed the idea on using Tracetest to catch those problems as well, right? +### [00:12:50] Challenges faced during integration -And recently, we had another situation where the Kafka link, the Kafka SDK for Go changed it, changed the ownership from Shopify to IBM. And some things changed on this API. They use Tracetest to change the telemetry, update this library, and assert that everything was fine. This was another thing that was good too. +**Daniel:** And so my thing as well was that for us, part of that section, I don't know if it was a month or a couple of weeks, there was a specific service that was, the telemetry was failing or was broken, right? And the team didn't find out until that time passed and someone, I don't know, saw that problem. And then based on that also pushed the idea on using Tracetest to catch those problems as well, right? And recently, we had another situation where the Kafka link, the Kafka SDK for Go changed the ownership from Shopify to IBM. Some things changed on this API. They used Tracetest to change the telemetry, update this library, and assert that everything was fine. This was another thing that was good, too. -**Host:** Cool, that's awesome. Just to make sure I understand correctly, when you started writing the trace-based tests, was it, your initial process was basically converting the initial integration tests into the trace-based tests? So you already had, you didn't have to worry about coming up with the tests. It was essentially a translation and then you did not just checking for the response, but additional stuff basically. +**Host:** Cool, that's awesome. Just to make sure I understand correctly, when you started writing the trace based tests, was it, your initial process was basically converting the initial integration tests into the trace based test. So you already had, you didn't have to worry about coming up with the tests. It was essentially a translation, and then you did not just check for the response, but additional stuff basically. -**Oscar:** Exactly. One difference on this approach is that first I started with the language that we all love that is YAML. So I did a copy and paste on the test. But when you see traces, you need to think a little bit of what types of traces this API calls does. What I did was first write a MO test, run a little bit, and then I swapped it to Tracetest UI because there we can see how is the shape of the traces that we are generating? And I could play a little bit, see, oh, I saw that the checkout service emits this, the trace that I can assert. The shipping service emits another trace, another span, sorry. And I could tweak a little bit and start building more complex tasks with that same. +**Daniel:** Exactly. One difference on this approach is that first I started with the language that we all love that is YAML. So I did a copy and paste on the test. But when you see traces, you need to think a little bit of what types of traces this API calls does. What I did was first write a MO test, run a little bit, and then I swapped it to Tracetest UI because there we can see how is the shape of the traces that we are generating. I could play a little bit, see, oh, I saw that the checkout service emitted this, the trace that I can assert. The shipping service emitted another trace, another span, sorry. I could tweak a little bit and start building more complex tasks with that same. Oh, I want to test service A and see if services B, D, and E are working as well. I think that's a really interesting part of telemetry in general, right? Producing traces and having all of the services instrumented, because for someone that is not familiar with the system, it's easier to look at a trace and understand the process and the steps that are taken for a specific use case, instead of going line by line, looking at the code and trying to understand what is going on. -Oh, I want to test the service A and see if the service B, D, and E are working as well. And I think that's a really interesting part of telemetry in general, right? And producing traces and having all of the services instrumented, because for someone that is not familiar with the system, they can, it's easier to look at a trace and understand the process and the steps that are taken for a specific use case, instead of going line by line, looking at the code and trying to understand what is going on. +**Host:** Yeah. And how long did it take to create all the trace based tests, the initial set for the OTEL demo? Because I think it was, the announcement was made, I want to say, last KubeCon North America in Chicago, last fall kind of thing? Is that correct? Correct me if I'm wrong, please. -**Host:** And how long did it take to create all the trace-based tests, the initial set for the OTEL demo? Because I think it was, the announcement was made, I want to say, last KubeCon North America in Chicago, last fall kind of thing? Is that correct? Correct me if I'm wrong, please. +**Oscar:** Yes, it was fast. I believe that, also to that we discussed it with our team about what was happening and checking, inspecting the integration codes. We were able to, in two or three days, to build all the tests for each one of the services and start discussing with the team, opening a PR and start discussing with the team, changes and other things that we could do. -**Daniel:** Yes, it was fast. I believe that also to that we discussed it with our team about what was happening and checking, inspecting the integration codes. We were able to, in two or three days, to build all the tests for each one of the services and start discussing with the team, opening a PR and start discussing with the team, changes and other things that we could do. +**Host:** Awesome. That's so cool. And so once you implemented trace based testing into the OTEL demo, what were some of the big challenges, the gotchas, the unexpected things that you encountered as part of this? -[00:16:30] **Host:** Awesome. That's so cool. And, so once you implemented, in implementing the trace-based testing into the OTEL demo, what were some of the big challenges, the gotchas, the unexpected things that you encountered as part of this? +**Daniel:** The first thing that we noticed that we thought is how we could do this test without impacting the entire ecosystem. The first decision that we took is, we know that the OpenTelemetry demo uses Jaeger. So instead of, we will see the system outside of it. We will just check Jaeger and do every other operation on Jaeger. With that, it was pretty easy to do everything. The only thing that besides that is just to understand the services, understand how they are working to make our tests, it's generated some interesting things like, one service that I noticed that, first I thought that the JSON that we sent for the service was in camel case and later we discovered during the execution of this case that this YAML, this JSON was in a snake case. So we started to tweak a little bit and start seeing how the things are and do proper tests for it. -**Oscar:** The first thing that we noticed that we thought is how we could do this test without impacting the entire ecosystem. And the first decision that we took is, we know that OpenTelemetry demo uses Jaeger. So instead of, we will see the system outside of it. We will just check Jaeger and do every other operation on Jaeger. With that, it was pretty easy to do everything. The only thing that besides that is just to understand the services, understand how they are working to make our tests, it's generated some interesting things like, one service that I noticed that, first I thought that, the JSON that we sent for the service was in camel case and later we discovered during the execution of this case that this YAML, this JSON was in a snake case. So we started to tweak a little bit and start seeing how the things are and do proper tests for it. +**Host:** When you were writing the trace based tests, you said you were looking, you're looking at your, you were using Jaeger, as part of your basis. Because I know, Tracetest, I guess there's a couple of approaches to pulling the traces, right? One is either you pull it like in the collector. So you add a config to the collector, or you can pull it directly from any of the supported observability backends. So what was the approach? So I guess my question is, was it the approach that you took then was pulling it directly from Jaeger, like the traces? -**Host:** When you were writing the trace-based tests, you said you were looking, you're looking at your, you were using Jaeger, as part of your basis. Because I know, Tracetest, I guess there's a couple of approaches to pulling the traces, right? One is either you pull it like in the collector. So you add a config to the collector, or you can pull it directly from any of the supported observability backends. So what was the approach? So I guess my question is, was the approach that you took then was pulling it directly from Jaeger, like the traces? +**Oscar:** Yes, and to that, one thing that we noticed is that since the example sent all the telemetry data to Jaeger, Prometheus, and everything else, we were secure to use Jaeger as it is. Because we know that sometimes if you have a huge system, you might use sampling, for instance, to just send a small portion of the tests of the traces for a tracing back. It was not the case. So this was important because we know that, oh, if you have sampling or you have some specific configuration on the collector, maybe you want to do a specific configuration on the OTEL collector to send some of the just traces for Tracetest to do the things, but it was easier. The job was pretty easy that I thought. Since we had everything we ever needed. -**Oscar:** Yes, and to that, one thing that we noticed is that, since the example sent all the telemetry data to Jaeger, Prometheus, and everything else, we were secure to use Jaeger as it is. Because we know that sometimes if you have a huge system, you might use sampling, for instance, to just send a small portion of the tests of the traces for a tracing back. It was not the case. So this was important because we know that, oh, if you have sampling or you have some specific configuration on the collector, maybe you want to do a specific configuration on OTEL collector to send some of the just traces for Tracetest to do the things, but it was easier. The job was pretty easy that I thought. Since we had everything we ever needed. +**Host:** Yeah, that's an important distinction, too. Yeah, because I hadn't even thought of the fact that if you're pulling directly from the backend, you're basically pulling the traces that were emitted there versus if you're using the collector, if I recall correctly, you create a different, you create basically a second tracing pipeline, which then intercepts the traces, and then you use those to create your trace based tests, and then you can put whatever restrictions you want on that for your trace based test, which I guess gives you a little more freedom as well, like to play around and configure things. -**Host:** Yeah, that's an important distinction, too. Yeah, because I hadn't even thought of the fact that if you're pulling directly from the backend, you're basically pulling the traces that were emitted there versus if you're using the collector, if I recall correctly, you create a different, you create basically a second tracing pipeline, which then intercepts the traces, and then you use those to create your trace-based tests, and then you can put whatever restrictions you want on that for your trace-based test, which I guess gives you a little more freedom as well, like to play around and configure things. +**Daniel:** Exactly. -[00:20:30] **Oscar:** Exactly. Cool. That's awesome. And then on a similar vein, what are some of the benefits that you started seeing after the integration was like with trace-based testing? Did they just start receiving high praise from the maintainers in the community demo where they're like, Oh my God, this has changed my life? +### [00:20:25] Benefits of trace-based testing integration -**Daniel:** One of the game changer things that I think it was, when I, when we saw the OpenTelemetry demo, maintainers starting to integrate trace tests on their CI/CD pipeline, because this, this was, this is something difficult. And I remember at the time that discussing with them, every time that they needed to change something, they did a bunch of final testing. And since we are humans. Sometimes we might forget testing some service or thinking about a user case. But, what they did was, oh, we will create a pipeline. We will build the entire system to see if everything is right. And after building the system, we will run trace tests and check if everything was right. +**Host:** Cool. That's awesome. And then, on a similar vein, what are some of the benefits that you started seeing after the integration was, like with trace based testing? Did they just start receiving high praise from the maintainers in the community demo where they're like, oh my God, this has changed my life? -And by doing that, they started to validate some PRs and discover, for instance, that, oh, this PR updated a component that is breaking some traces. And with that, they could start to evaluate and say, oh, okay, let us fix this PR and guarantee that everything is right. And this was painful before because sometimes you could approve a PR, merge it on the database, on the code base, and just two weeks later discover that something was missing. +**Oscar:** One of the game changer things that I think it was, when I, when we saw the OpenTelemetry demo maintainers starting to integrate trace tests on their CI/CD pipeline, because this, this was, this is something difficult. I remember at the time that, discussing with them, every time that they needed to change something, they did a bunch of final testing. Since we are humans, sometimes we might forget testing some service or thinking about a user case. But what they did was, oh, we will create a pipeline. We will build the entire system to see if everything is right. After building the system, we will run trace tests and check if everything was right. By doing that, they started to validate some PRs and discover, for instance, that, oh, this PR updated a component that is breaking some traces. With that, they could start to evaluate and say, oh, okay, let us fix this PR and guarantee that everything is right. This was painful before because sometimes you could approve a PR, merge it on the database, on the code base, and just two weeks later discover that something was missing. -**Oscar:** Also want to call out the shout out that we got from Josh Lee. So I remember that when I first joined Tracetest, I saw one of his talks about trace-based tests, actually, something similar to quite what we're doing. And just having someone like him sharing out Tracetest or what it is and what it means for the OpenTelemetry community. It's just great. So I think that's part of what we, the work that we. +**Daniel:** I also want to call out the shout out that we got from Josh Lee. I remember that when I first joined Tracetest, I saw one of his talks about trace based tests, actually, something similar or quite what we're doing. Just having someone like him sharing out Tracetest or what it is and what it means for the OpenTelemetry community. It's just great. I think that's part of what we, the work that we, Daniel has done on the OpenTelemetry demo. -**Daniel:** That's so great. That's very cool to hear. And now that the trace-based testing has been integrated in the demo for several months, at this point, what changes have you seen since that initial integration, even to how you approach trade? Have there been any massive changes as to how you approach trace-based testing? Or has it mostly stayed the same? Have there been tweaks that you've had to make along the way? +**Host:** That's so great. That's very cool to hear. And now that the trace based testing has been integrated in the demo for several months, at this point, what changes have you seen since that initial integration, even to how you approach trace? Have there been any massive changes as to how you approach trace based testing? Or has it mostly stayed the same? Have there been tweaks that you've had to make along the way? -**Oscar:** The one thing that was great is that they started to build some traceability tests, as they, I remember that I talked with Juliano, that is one of the maintainers, and when this Kafka thing happened, he told me, Daniel, we noticed that the Kafka SDK changed and we wanted to create a basic, to do some changes on the system and create a basic traceability test in it. And it was amazing. - -**Daniel:** And another thing that we loved was that before we had three types of tests running on OpenTelemetry demo, a front-end test, using, I forgot the name of the library. +**Daniel:** The one thing that was great is that they started to build some traceability tests. I remember that I talked with Juliano, that is one of the maintainers, and when this Kafka thing happened, he told me, Daniel, we noticed that the Kafka SDK changed and we wanted to create a basic, to do some changes on the system and create a basic, a traceability test in it. It was amazing. Another thing that we loved was that, before we had three types of tests running on OpenTelemetry demo, a front end test, using, I forgot the name of the library. **Oscar:** Cypress. -**Daniel:** Cypress, they had an API call, API test with AVA, and they had trace test, and they noted, oh, we don't need AVA and Cypress anymore. Since we are focusing on seeing traces and guaranteeing that the telemetry is right and the system is working. Three steps is enough for that. And it was amazing and a huge responsibility for us because now we know that they rely on us. We need to have a good API to everything. +### [00:24:25] Community feedback and improvements -**Host:** That's so cool. That's such a success. And I love the fact, the great thing about the OTEL community demo too, is that it showcases OpenTelemetry all with open source tooling, including Tracetest, which I think is so cool and really speaks to the power of open source and the open source community. And you end up with this like really nice symbiotic relationship too, right? Because then, I assume you would get some more use cases for improving Tracetest as a whole based on how you see trace-based testing acting in the wild through the community demo. +**Daniel:** Cypress. They had an API call, API test with AVA, and they had trace test. They noted, oh, we don't need AVA and Cypress anymore since we are focusing on seeing traces and guaranteeing that the telemetry is right and the system is working. Three steps is enough for that. It was amazing and a huge responsibility for us because now we know that they rely on us. We need to have a good API to everything. -**Daniel:** During the time that we started to write tests for OpenTelemetry demo, we detected that some of the tests were huge because we needed to embed a protobuffer file inside of each test. And when we started to see that, we noticed that, oh, the developer experience is bad for that because you cannot see what is happening and this moved us to change the CLI and think, let us simplify the test to guarantee that you can see the test and see what matters. And it was good. It was a two-way road. We were able to help them, the OpenTelemetry team, but the feedback that they gave us helped us to improve Tracetest too. +**Host:** That's so cool. That's such a success. And I love the fact the great thing about the OTEL community demo too is that it showcases OpenTelemetry all with open source tooling, including Tracetest, which I think is so cool and really speaks to the power of open source and the open source community. You end up with this like really nice symbiotic relationship too, right? Because then, I assume you would get some more use cases for improving Tracetest as a whole based on how you see trace based testing acting in the wild through the community demo. -**Oscar:** That's awesome. And as you mentioned, you've got a few folks now writing their own trace-based tests. Have you felt then that, like just getting into that mindset of people writing, like other developers writing trace-based tests, has that, have you noticed, was that a major shift or was it organic once they saw the initial examples that were, that you both added to the repo? +**Oscar:** During the time that we started to write tests for OpenTelemetry demo, we detected that some of the tests were huge because we needed to embed a protobuffer file inside of each test. When we started to see that, we noticed that, oh, the developer experience is bad for that because you cannot see what is happening, and this moved us to change the CLI and think, let us simplify the test to guarantee that you can see the test and see what matters. It was a two-way road. We were able to help them, the OpenTelemetry team, but the feedback that they gave us helped us to improve Tracetest too. -**Daniel:** Believe that, that both things happened. For some developers that were used to OpenTelemetry, they noted that they could use this telemetry to test it, and it was. Sometimes it's way easier to test. For instance, I cannot think in an easy way to test a Kafka consumer without doing a bunch of code magic behind the scenes, and traces are good for that. +**Daniel:** That's awesome. And as you mentioned, you've got a few folks now writing their own trace based tests. Have you felt then that, like just getting into that mindset of people writing, like other developers writing trace based tests, has that, have you noticed, was that a major shift or was it organic once they saw the initial examples that were that you both added to the repo? -So we had some examples of people using serverless knowing that it is difficult to communicate with several components and using Tracetest to help them. And we saw another developer saying, oh, we are starting to implement OpenTelemetry. We noticed that, with OpenTelemetry, we can test our system quickly. Tracetest helped them to drive and implement more things in OpenTelemetry and start doing things there. I believe that these two cases happened. +**Oscar:** I believe that both things happened. For some developers that were used to OpenTelemetry, they noted that they could use this telemetry to test it, and sometimes it's way easier to test. For instance, I cannot think in an easy way to test a Kafka consumer without doing a bunch of code magic behind the scenes, and traces are good for that. We had some examples of people using serverless, knowing that it's difficult to communicate with several components and using Tracetest to help them. We saw another developer saying, oh, we are starting to implement OpenTelemetry. We noticed that with OpenTelemetry, we can test our system quickly. Tracetest helped them to drive and implement more things in OpenTelemetry and start doing things there. I believe that these two cases happened. -**Oscar:** Oh, that's so great. Final question. If you had a redo, would you change anything about how you integrated trace-based testing with the OTEL demo? +**Host:** Oh, that's so great. Final question. If you had a redo, would you change anything about how you integrated trace based testing with the OTEL demo? -**Daniel:** First glance, I believe that nothing because I believe it was a perfect match. So we could do everything that we needed there. The only thing, the only thing that I think that I could do, and I think that I might do, if you are watching me, Pierre, Juliano, and company, I will haunt you in the future for that, is to add more tasks and start thinking in more interesting use cases that we can do and thinking more telemetry that we can show that. +**Daniel:** First glance, I believe that nothing because I believe it was a perfect match. We could do everything that we needed there. The only thing, the only thing that I think that I could do, and I think that I might do, if you are watching me, Pierre, Juliano, and company, I will haunt you in the future for that, is to add more tasks and start thinking of more interesting use cases that we can do, and thinking more telemetry that we can show. -**Host:** Amazing. This is great. We do have a question from the audience. Daniel asked, one of the biggest pains for distributed systems is ensuring correct context propagation across service boundaries, especially when async operations require in-service context propagation across threads and context may be instrumented properly. Can trace tests help identify where context is being broken when it shouldn't? +**Host:** Amazing. This is great. We do have a question from the audience. Daniel asked, one of the biggest pains for distributed systems is ensuring correct context propagation across service boundaries, especially when async operations require in-service context propagation across threads and context may instrumented properly. Can trace tests help identify where context is being broken when it shouldn't? -**Oscar:** I think I can answer that question. First of all, I think we have all been there. We have all felt the pain of why is my trace not being propagated to this backend system or this part of the app. And yes, actually, a great thing that, or a good thing that we are, one, one kind of standard or thing that we're pushing is TDD, but in this case, Trace-Based Driven Development. +### [00:30:05] Addressing context propagation issues -Where users can create assertions and test specs based on what they would expect the trace to look like. So if you already know that after a queuing system, there should be a worker that would process that message, you can pretty much, from the beginning, from the get-go, create a definition that would match. I would expect that span to exist. So if that span doesn't exist, it's because the context propagation is pretty much not, didn't work or the span, that's why the span doesn't exist, right? It's not there. So with this kind of technique that you can use, you can validate that the expected spans should be there and help you with the problem of your context propagation problem as well. +**Oscar:** I think I can answer that question. First of all, I think we have all been there. We have all felt the pain of why is my trace not being propagated to this backend system or this part of the app? Yes, actually, a great thing that, or a good thing that we are, one kind of standard or thing that we're pushing is TDD, but in this case, Trace Based Driven Development. Where users can create assertions and test specs based on what they would expect the trace to look like. So if you already know that after a queuing system, there should be a worker that would process that message, you can pretty much, from the beginning, from the get-go, create a definition that would match. I would expect that span to exist. So if that span doesn't exist, it's because the context propagation is pretty much not, didn't work or the span, that's why the span doesn't exist, right? It's not there. So with this kind of technique that you can use, you can validate that the expected spans should be there and help you with the problem of your context propagation problem as well. **Host:** Awesome. So you've answered Dan's question. Thank you. And then there was a follow-up question, can Tracetests make assertions on baggage? -**Oscar:** It depends. If you are saying baggage, so for some specific metadata of OpenTelemetry, perhaps not. But what we can do today is that if you write custom attributes in your span, you define a bunch of attributes, you can test them. You can test them. Also, we are every time looking to the OpenTelemetry specification, seeing if there is more metadata that we should integrate to the API and doing it like error codes, span statuses, and everything else. I think we have something about span links, right? I want to do as well. I remember. Exactly. +**Oscar:** It depends. If you are saying baggage, so for some specific metadata of the OpenTelemetry, perhaps not. But what we can do today is that if you write custom attributes in your span, you define a bunch of attributes, you can test them. You can test them. Also, we are every time looking to the OpenTelemetry specification, seeing if there is more metadata that we should integrate into the API and doing it like error codes, span statuses, and everything else. I think we have something about span links, right? I want to do as well. I remember. + +**Daniel:** Exactly. + +**Host:** Nice. That's awesome. Does anyone else, who's listening have any additional questions? This has been really great. Thank you both Daniel and Oscar for joining today. This has been really awesome. I think it's really cool. I'm a big fan of the OTEL demo and I think having trace based testing integrated really, it's like a very tracing native approach to integration tests. I'm a huge fan, so it's super cool to see that integration in place and to see other folks, outside of both of you actually writing trace based tests. -**Host:** Nice. That's awesome. Does anyone else, who's listening have any additional questions? This has been really great. Thank you both, Daniel and Oscar for joining today. This has been really awesome. I think it's really cool. I'm a big fan of the OTEL demo and I think having trace-based testing integrated really, it's like a very tracing native approach to integration tests. I'm a huge fan, so it's super cool to see that integration in place and to see other folks, outside of both of you actually writing trace-based tests. Thank you. +**Daniel:** Thank you. -**Daniel:** Thank you, everyone. And thank you, Adriana, for putting this together and for having us. It was great. +**Oscar:** Thank you, everyone. And thank you, Adriana, for putting this together and for having us. It was great. ## Raw YouTube Transcript diff --git a/video-transcripts/transcripts/2024-08-01T05:59:23Z-otel-in-practice-otel-for-mobile-apps-with-hanson-ho-and-eliab-sisay.md b/video-transcripts/transcripts/2024-08-01T05:59:23Z-otel-in-practice-otel-for-mobile-apps-with-hanson-ho-and-eliab-sisay.md index 296cf76..7056bf8 100644 --- a/video-transcripts/transcripts/2024-08-01T05:59:23Z-otel-in-practice-otel-for-mobile-apps-with-hanson-ho-and-eliab-sisay.md +++ b/video-transcripts/transcripts/2024-08-01T05:59:23Z-otel-in-practice-otel-for-mobile-apps-with-hanson-ho-and-eliab-sisay.md @@ -10,193 +10,260 @@ URL: https://www.youtube.com/watch?v=mTcdwdWIFgI ## Summary -In this edition of "Otel and Practice," hosts Hansen and Eliah from Embrace discuss the challenges and opportunities of implementing data modeling and observability in mobile apps using OpenTelemetry (Otel). Eliah introduces Embrace's focus on mobile observability and the transition to Otel, emphasizing the unique complexities mobile apps face compared to backend systems, such as diverse hardware environments and the need for real-time performance monitoring. Hansen elaborates on these challenges, including the limitations of current telemetry systems, the importance of user experience, and the adaptation of Otel's protocols to better fit mobile applications. They also encourage audience involvement in the mobile observability community to help shape standards and best practices. The session concludes with a Q&A, addressing various topics such as instrumentation for different types of apps, golden signals for performance, and the readiness of Otel for production use in mobile environments. +In this edition of "Otel and Practice," hosts Hansen and Eliah from Embrace discuss the challenges and opportunities of implementing data modeling and observability using OpenTelemetry (OTel) in mobile apps. Eliah, a product manager at Embrace, introduces the significance of mobile observability, highlighting the unique issues faced by mobile applications compared to back-end systems, such as session tracking and performance monitoring. Hansen, an Android architect, delves deeper into the specific challenges of mobile environments, including dynamic runtime conditions, data transmission fragility, and the need for user-centric data capture. The presentation emphasizes the ongoing development of mobile observability standards and calls for greater community involvement in refining OTel for mobile use cases. The session concludes with a Q&A segment addressing various technical questions related to mobile observability and instrumentation. ## Chapters -00:00:00 Welcome and intro -00:01:20 Introduction of Eliah -00:02:53 Overview of mobile observability -00:05:00 Challenges in mobile observability -00:06:30 Call to action for involvement -00:07:00 Hansen's presentation on mobile differences -00:09:56 Unique challenges of mobile environments -00:12:30 OpenTelemetry challenges on mobile -00:19:00 Ongoing challenges and future considerations -00:22:00 Q&A session begins -00:44:00 Closing thoughts and wrap-up +00:00:00 Introductions +00:01:47 What is mobile observability? +00:03:32 Challenges of mobile observability +00:06:00 Guest introduction: Hansen +00:07:10 Unique challenges of mobile environments +00:10:00 Data capture and transmission issues +00:12:30 OpenTelemetry overview for mobile +00:17:30 Assumptions of telemetry on mobile +00:22:00 Discussion on mobile metrics +00:35:50 Q&A session on mobile observability -**Speaker 1:** Well hello everyone, welcome to another edition of Otel and Practice. Today we've got Hansen and Eliah from Embrace. They're going to take us through some of the challenges and some of the opportunities as well of doing a data modeling API design in mobile apps. In OpenTelemetry, we talk a lot about the backend, but we'll see how we can apply OpenTelemetry to the client side as well. +## Transcript -We will do a presentation first, and then there will be time for Q&A. I'll drop a link in the chat, and then I've created an Agile Coffee board. You can go there, you can add your questions, and then at the end of the talk, we'll go through the Q&A. You can vote for your questions as well, so feel free to add your questions as you see in the presentation, and then we'll go through them at the end. +### [00:00:00] Introductions + +**Speaker:** Well hello everyone, welcome to another edition of OTel in Practice. Today we've got Hansen and Eliah from Embrace. They're going to take us through some of the challenges and some of the opportunities as well of doing a data modeling AP design in mobile apps. In OpenTelemetry, we talk a lot about back end, but we'll see how we can apply OpenTelemetry to the client side as well. + +We will do a presentation first and then there will be time for Q&A. I'll drop a link in the chat, and then I've created an Agile Coffee board. You can go there, you can add your questions, and then at the end of the talk, we'll go through the Q&A. You can vote for your questions as well, so feel free to add your questions as you see in the presentation, and then we'll go through them at the end. So without further ado, I'll introduce Hansen. I think you're going to be kicking off, so take it away. **Hansen:** Great, I'm actually going to pass it off to Eliah to kick it off. -[00:01:20] **Eliah:** Nice. Hi everyone, yeah like it was mentioned, my name is Eliah Cisi. I am a product manager at Embrace. We are a mobile observability company. We provide developers with SDKs that they can integrate into their mobile apps to monitor performance in real time. Our main goal really is to help developers keep their apps running smoothly by proactively identifying and resolving issues, really with the objective of improving the overall user experience. +### [00:01:47] What is mobile observability? + +**Eliah:** Nice, uh hi everyone! Yeah, like it was mentioned, my name is Eliah Cisi. I am a product manager at Embrace. You can go to the next slide, Hansen. We are a mobile observability company. We provide developers with SDKs that they can integrate into their mobile apps to monitor performance in real time. Our main goal really is to help developers keep their apps running smoothly by proactively identifying and resolving issues, really with the objective of improving the overall user experience. -We're also heavily invested in OpenTelemetry. We began that transition about nine months ago, and it's really helped us provide our customers with a more unified and comprehensive view of app performance from what's happening client-side on the user's mobile device all the way to the backend services that power those experiences. +We're also heavily invested in OpenTelemetry. We began that transition about nine months ago and it's really helped us provide our customers with a more unified and comprehensive view of app performance from what's happening client side on the user's mobile device all the way to the backend services that power those experiences. -In doing that migration, we discovered that while the benefits of OpenTelemetry are numerous, there are some specific challenges that occur when trying to implement it in a mobile environment, and that's kind of what we're here to talk about today. I'll be honest and say that the term "we" is very generous. Also on the call is Hansen, who we talked about; he's an Android architect at Embrace, formerly a mobile performance engineer at Twitter. So I'm just here to do kind of a brief intro, and then I'll hand it off to Hansen, who will be doing a majority of the presentation. +In doing that migration, we discovered that while the benefits of OpenTelemetry are numerous, there are some specific challenges that occur when trying to implement it in a mobile environment, and that's kind of what we're here to talk about today. I'll be honest and say that the term "we" is very generous. Also on the call is Hansen, who we talked about. He's an Android architect at Embrace, formerly a mobile performance engineer at Twitter. I'm just here to do kind of a brief intro and then I'll hand it off to Hansen, who will be doing a majority of the presentation. -[00:02:53] **Hansen:** Next slide, thank you. One more. I want to start by just setting a little bit of context. In 2022, the average mobile user spent a little over four hours a day on their phone, and of that, over 90% of it, or about 90% of it, was within native mobile apps. By the end of this year, mobile apps are expected to generate over 900 billion dollars in revenue. If you think about your own mobile use, I'm willing to bet that the brand or companies that you interact with most have a native app that you use consistently. +Next slide, thank you. One more. I want to start by just setting a little bit of context. So in 2022, the average mobile user spent a little over four hours a day on their phone, and of that, over 90% of it or about 90% of it was within native mobile apps. By the end of this year, mobile apps are expected to generate over 900 billion dollars in revenue. If you yourself think about your own mobile use, I'm willing to bet that the brand or companies that you interact with most have a native app that you use consistently. -Now, as we think about the observability ecosystem, we've spent the last 5 to 10 years really trying to figure out how we do observability better for backend systems. There are a variety of standards that were created, most notably OpenCensus and OpenTracing, which form the foundation for where we are today with OpenTelemetry to provide the visibility into the health of distributed systems. +### [00:03:32] Challenges of mobile observability -Mobile apps are not a distributed system; they are installed software running on distributed compute resources that interact with distributed systems. There are a lot of unique challenges with that environment. Take session tracking for instance. In the mobile world, a user session can be anything from several seconds to several hours, and it can be interrupted by calls or network changes or the app running in the background, which is really quite different from what you see server-side. +Now as we think about the observability ecosystem, we've spent the last 5 to 10 years really trying to figure out how we do observability better for backend systems. There are a variety of standards that were created, most notably OpenCensus and OpenTracing, which form the foundation for where we are today with OpenTelemetry to provide the visibility into the health of distributed systems. + +Mobile apps are not a distributed system; they are installed software running on distributed compute resources that interact with distributed systems, and there's a lot of unique challenges with that environment. Take session tracking for instance. In the mobile world, a user session can be anything from several seconds to several hours, and it can be interrupted by calls or network changes or the app running in the background, which is really quite different from what you see server side. Mobile apps themselves are also extremely different. A gaming app, for example, needs to track things like frame rate and rendering times, while a financial app might focus more on transaction speeds and security events. Add to that the fragmentation in the mobile ecosystem, where you've got multiple operating systems and countless device manufacturers, and it's really easy to see how complex that becomes. -[00:05:00] We were looking at the number of unique devices for just one customer, and we saw that there were like 42,000 plus different combinations of device models and chipsets that could potentially exist, which is crazy. For most of its life cycle, observability and monitoring in the mobile ecosystem has been mostly proprietary systems designed by vendors. The typical starting point is Firebase Crashlytics, which is free but highly sampled and extremely limited in its feature set. So if you're a serious app developer, you're not going to use that, and that's led to a bunch of vendors building their own proprietary standards and trying to convince people that they should be serious about observability with their solution, but mostly it's just crash reporting and error tracking. +We were looking at the number of unique devices for just one customer and we saw that there were like 42,000 plus different combinations of device models and chipsets that could potentially exist, which is, you know, crazy. For most of its life cycle, observability and monitoring in the mobile ecosystem has been mostly proprietary systems designed by vendors. The typical starting point is Firebase Crashlytics, which is free but highly sampled and extremely limited in its feature set. So if you're a serious app developer, you're not going to use that. + +That's led to a bunch of vendors building their own proprietary standards and trying to convince people that they should be serious about observability with their solution, but mostly it's just crash reporting and error tracking. The result is that the paradigm looks something like this, and the unfortunate thing is that a lot of mobile engineers actually consider that to be kind of acceptable. -The result is that the paradigm looks something like this, and the unfortunate thing is that a lot of mobile engineers actually consider that to be kind of acceptable. When we talk to customers who truly care about the user experience that their customers are having on their mobile devices, they tell us that all of their customer-impacting SLOs are directly tied to the mobile device, but they don't have a good source of information that correlates that data to the work they're doing to build reliability and resiliency in their backend systems. +When we talk to customers who truly care about the user experience that their customers are having on their mobile devices, they tell us that all of their customer impacting SLOs are directly tied to the mobile device, but they don't have a good source of information that correlates that data to the work they're doing to build reliability and resiliency in their backend systems. -[00:06:30] That's what Hansen's going to be talking about, which is some of the challenges in this paradigm that we're facing today, and some of the opportunities and how we're working to address that. More than anything, I think today is really a call to action for the people in the Zoom, for the people watching. If you yourself, or if you have coworkers or people that are interested in the mobile ecosystem, we would love for you guys to get involved. Hansen's involved in the Android SIG and the client-side SIG. We have people working in the Swift SIG. We're working on OTEL contributions for React Native and Flutter, and we're really committed to making mobile observability as robust and standardized as it is for backend systems. We'd love to involve as many of you in that effort as possible. +### [00:06:00] Guest introduction: Hansen -With that, I'll hand it off to Hansen. +That's what Hansen's going to be talking about, which is some of the challenges in this paradigm that we're facing today, and some of the opportunities and how we're working to address that. More than anything, I think today is really a call to action for the people in the Zoom, for the people watching. If you yourself, or if you have co-workers or people that are interested in the mobile ecosystem, we would love for you guys to get involved. Hansen's involved in the Android SIG and the client side SIG. We have people working in the Swift SIG. We're working on OpenTelemetry contributions for React Native and Flutter, and we're really committed to making mobile observability as robust and standardized as it is for backend systems. We'd love to involve as many of you in that effort as possible. -[00:07:00] **Hansen:** Thanks, Eliah. So I am here to talk about how mobile is different in terms of observability. I think the first thing to approach this is that the crux of the problem is actually about simply moving tooling onto these mobile platforms and making them run because there are fundamental differences that are embedded in the assumptions that we make when we have durability and the questions that we ask of our tooling. Without first identifying and acknowledging these differences, we can't start improving the specs or tooling because we don't know what it should look like. +With that, I'll hand it off to Hansen. -To make the front of the horse look like the back of the horse, we've got to first figure out what the face should be. Here is what I'm trying to do: I'm just going to go over a little bit what the differences may be. I can spend a couple hours here talking about this, but hey, we don't have that much time, so I'm just going to concentrate this in three general areas where mobile is different. +### [00:07:10] Unique challenges of mobile environments -First is the runtime environment. We're talking about dynamic heterogeneous runtime environments running on devices that are unpredictable—42,000 unique chipsets and device combinations. Not only that, the environment that they run in is extremely dynamic as well. You walk into an elevator and you lose your mobile connection; you lose your network connection. You background the app to answer a notification, and that affects how the operating system runs the app. Not only that, the actual hardware that is running your app is quite severely limited. Phones that existed 10 years ago still even today, PHs are released last year that cost $99. The hardware mirrors that cost, and the OS not only limits what you can do with that already limited hardware, it also does it in a very unpredictable way. +**Hansen:** Thanks, Eliah. So I am here to talk about how mobile is different in terms of observability. I think the first thing to approach this is that the crux of the problem is actually about simply moving tooling onto these mobile platforms and making them run because there are fundamental differences that are embedded in the assumptions that we make when we have durability and the questions that we ask of our tooling. + +Without first identifying and acknowledging these differences, we can't start improving the specs or tooling because we don't know what it should look like. To make the front of the horse look like the back of the horse, we got to first figure out what the face should be. Here, I'm trying to go over a little bit of what the differences may be. I can spend a couple of hours here talking about this, but we don't have that much time, so I'm just going to concentrate this in three general areas where mobile is different. + +First is the runtime environment. We're talking about dynamic heterogeneous runtime environments running on devices that are unpredictable—42,000 unique chipsets and device combinations. Not only that, the environment that they run in is extremely dynamic as well. You walk into an elevator and you lose your mobile connection; you lose your network connection; you background the app to answer a notification—that affects how the operating system runs the app. + +Not only that, the actual hardware that is running your app is quite severely limited. Phones exist 10 years ago; still even today, PH are released last year that cost $99, and the hardware mirrors that cost. The OS not only limits what you could do with that already limited hardware, it also does it in a very unpredictable way. What you have is a dynamic environment where you don't know what is actually executing your app. Objective performance, which is typically what we measure, is only part of the equation because what makes an app slow differs in terms of where I'm using it or whether somebody else is using it. Perceived performance is also part of the equation, and SLOs have to somehow take that into account. -[00:09:56] Another aspect of mobile that makes it challenging and different than backend is the fragility of the capture and transmission pipeline. How to get data from these devices onto servers, I could do something with it. Data could be lost in multiple stages before it gets captured because it was a crash, before you persist the data or after we persist the data, because the network connection is unstable and we can't get the data over to the server. Even when the server gets the data, it could be delayed or out of order, so what you get may not be the full picture until several hours later when more data comes in. +Another aspect of mobile that makes it challenging and different than backend is the fragility of the capture and transmission pipeline—how to get data from these devices onto servers so we can do something with it. Data could be lost in multiple stages before it gets captured because it was a crash, before you persist the data or after we persist the data because the network connection is unstable. Even when the server gets the data, it could be delayed or out of order, so what you get may not be the full picture until several hours later when more data comes in. + +### [00:10:00] Data capture and transmission issues + +Lastly, the data we capture has to center around the user experience because operational runtime in device state is useful only in how they provide insight into individual user experiences end to end. Otherwise, it's just trivia. On mobile, we're observing millions of independent app instances, but each one of the measurements we get maps back to a user. It is not merely a state of health of the system. If something is slow, somebody is looking at a very slow phone. + +What's P99? P99 is 1% of all measurements are that slow or slower. So if you think that's an outlier, well, 1% of interactions being that slow means it is more than an outlier; it is something that we have to capture. + +Now let's talk about OpenTelemetry. In general, OpenTelemetry works really well as a lingua franca of observability. Its backend roots mean that it was designed to solve a problem with a very specific context. When we change that context to mobile, some parts start to not fit very well, and these are the ones I'm going to talk about. + +First, let's talk about spans. Spans are great in OpenTelemetry if you want to measure the duration of operations of applications that have a very fixed runtime environment, predictable runtime code path, so you can actually measure and calculate deviations based on some baselines. They are not so great if, say, duration is not an indicator of performance. If you want to measure a period of time, well, OpenTelemetry signals look like spans, but is it? Because duration is not an indicator of performance, and when you start aggregating and say, "Hey, what's P95?" that starts to not make sense. + +### [00:12:30] OpenTelemetry overview for mobile + +Also, operations run for a long time in OpenTelemetry. Unfortunately, you don't get to know what happened until it ends, either in a failure or success. But what if it gets interrupted in the middle through a crash that we don't know about? Mobile apps could be killed without actually alerting the app, so operations like that are gone or lost, and with OpenTelemetry itself, we wouldn't know about it because it's not done. That's kind of challenging for mobile. + +Also, operations that need to be contextualized with a lot of mutable state that are expensive to obtain—well, that's a challenge because in OpenTelemetry, you write the data into the spans of span events or attributes, but sometimes the act of getting these types of state takes a while for the mobile app to actually get from the OS, and for us to basically block the ending of a trace—that's not fantastic. + +The second point I want to talk about is that the protocols and APIs of OpenTelemetry make certain assumptions that are just not true on mobile. For example, assumption one is that telemetry is recorded and transmitted reliably, and you could trust that data that gets recorded will make it to the server collector in a reasonable amount of time. But as we mentioned before, that is not true in mobile and it could also not be true in a number of fascinating ways. + +So tooling that is usable on mobile has to take that into account and build resilience, like persistence for instance before export. Thanks to Cesar for doing that on Open Java or the OpenCensus Android extension; it's extremely helpful in production. In the future, perhaps there could be a place for ways for us to automatically transmit a group of related telemetry so that we don't get into this weird state in the backend wondering if more data is coming—that'd be nice. + +Another assumption is that recording and transmitting telemetry doesn't put a strain on system resources. The SDKs are well written and they perform well, so on the backend, it's no problem. However, on low-end devices and meter networks, it means that every signal captured reduces and potentially reduces performance or costs users money. We have to be a lot more careful in how we record data and how we transmit data, and even simply the act of getting the data to record can be expensive, so the consideration that we have to go into what to record is a lot greater and depends on the use case. + +Another assumption is that engineers that use the API understand low-level concepts like threads or context propagation. Even the idea of tracing and what spans are may not be universally understood if you change the audience to mobile engineers simply because the background, the variety of background of folks building mobile apps changes quite a bit. It could be somebody who's just done a six-month boot camp who is building a mobile app for the local grocery store, and they want observability too. They want to know why their stuff is slow, but they're used to higher-level constructs like coroutines when you're using threads, but not really—they don't know about it. + +How do you explain propagation when they don't understand the existence of a thread? Adapting the APIs to be thematic is one thing, but making those concepts understandable by those without CS backgrounds is another challenge right there. + +Lastly, another assumption is that traced operations have a clear execution boundary and code ownership. You have a service that creates a span and the runtime—generally a team owns that front to back, and you transmit the context via context propagation, etc., as you would in distributed tracing. Unfortunately, things are not as clean on mobile because a single span for an operation can go through several modules owned by several teams, and not all of them may be aware of all these problems. + +Instrumentation can be extremely brittle if you don't have the right infrastructure in place to catch regressions where implementation changes but the instrumentation doesn't. It's not as modular or nicely connected as you would for distributed tracing just because of how mobile apps are architected. + +### [00:17:30] Assumptions of telemetry on mobile + +The last OpenTelemetry thing I'm going to talk about is that mobile devices are millions of independent instances. OpenTelemetry generally assumes that the system being monitored and observed is one connected system. It could be made of dozens of microservices deployed in various ways, but effectively they all roll up. Metrics in that context make a lot of sense. OpenTelemetry metrics, but for mobile when we have disconnected app instances running, that starts to break down because metrics need to be grounded in the context of the system that means them in order for the baselines to be created and compared. + +Merging together metrics from phones of various models into one limits how useful those generated metrics are. What does P95 of heap size of an app at one minute mean when you look at the entire fleet of devices? I don't know. When it changes, when it increases or decreases, is that good or bad? Well, I don't know because we don't know the reasons; these are different systems. + +Not that metrics aren't useful on mobile; they're extremely useful, but you tend to need to have like-to-like comparison—apples to apples—and it tends to involve dimensions that are high in cardinality, and unfortunately, that just doesn't work fantastically well with OpenTelemetry metrics. Simply, the strict time-aligned aggregation is not really suitable for apps that have operations that have variable duration and have data coming in at any time. + +It just wasn't meant to do what mobile wants it to do, but that's okay because these are not insurmountable challenges. These are ongoing things that we could work with. The protocol and the SIGs and the folks to kind of improve. We are also, at Embrace, having workarounds to go around OpenTelemetry or perhaps use OpenTelemetry in, I would say, non-standard ways in order to get the data to do what we want to do. + +That's not the end state. The end state we want to see is a diverse ecosystem and tooling that builds on not only what folks in OpenTelemetry have done before but also extends support to the plethora of use cases that we're bringing up. Frankly, this is just the beginning because mobile apps that I'm familiar with run on a very small restricted set of circumstances. Phones and tablets are what I'm used to using. I haven't worked on cars or TVs or IoT, but those apps deserve observability as well. I'm sure as we take the tooling and the spec forward, more use cases will come on the board and say, "Hey, I want to connect my data from my TV to my backend data." + +What additional challenges are there when I don't even know how a TV works in terms of how it uses Android? But I'm sure it's different, and I'm sure there are new things. At this point, we're just exploring ourselves. We're trying to ask questions; we're trying to figure out if our assumptions are correct. Are there things we could change about how we used to do things in order to fit more into OpenTelemetry? But at the end of the day, we want everyone to come up with their use cases and help ask these questions of mobile and client use cases for OpenTelemetry. + +Great work has been done in the Android and Swift and client SIG already, but I think there could be more going forward. For folks listening, maybe the converted to OpenTelemetry, but hopefully other people also watch this and say, "Hey, you know what, I looked at OpenTelemetry; it didn't really fit, but after this presentation, maybe I can make it fit. Maybe I can use it in a way that is productive and actually truly have this become the lingua franca of observability for both the backend and the front end." + +That's enough of us talking. Any questions? I'm going to end the presentation if I can. There we go. + +### [00:22:00] Discussion on mobile metrics + +**Eliah:** Thank you very much, Hansen. For those that have joined a bit later and maybe haven't seen this message, we've got an Agile Coffee board you can put your questions in there. I think we've got a couple of questions. We'll start with one related to... -Lastly, the data we capture has to center the user experience because operational runtime and device state is useful only in how they provide insight into individual user experiences end to end. Otherwise, it's just trivia. On mobile, we're observing millions of independent app instances, but each one of the measurements we get maps back to a user. It is not merely a state of health of the system. If something is slow, somebody is looking at a very slow phone. It's like a P99—what does that mean? Well, P99 is 1% of all measurements are that slow or slower, so if you think that's an outlier, well, 1% of interactions being that slow means it is more than an outlier. It is something that we have to capture. +So we've got, in web CWV (Core Web Vitals), have become a standard to measure user experience for better awards. Do you see an emerging standard to measure equivalent concepts in mobile apps for sluggishness or speed? -Now let's talk about OTEL. In general, OTEL works really well as a lingua franca of observability. Its backend roots, though, mean that it was designed to solve a problem with a very specific context, and when we change that context to mobile, some parts start to not fit very well. +**Hansen:** Yes, so you know, with OpenTelemetry, everything is defined effectively as semantic conventions built on the existing signals. At Embrace, we've kind of modeled some of the TREC capture for things like ANRs on Android and various other kinds of mobile slowness/sluggishness. We did it in a certain way that works, but is it the best? We don't know. Once we figure it out, we'll want to work with the SIGs to submit conventions to model that. I believe especially with the introduction of these emerging specs of profiling and even entities, a lot of these problems that we have are going to be addressed. -The first point I want to talk about is spans. Spans are great in OTEL if you want to measure the duration of operations of applications that have a very fixed runtime environment, predictable runtime code path, so you can actually measure and calculate deviations based on some baselines. They are not so great if, say, duration is not an indicator of performance. If you want to measure a period of time, well, OTEL signals look like spans, but is it because duration is not an indicator of performance? +It's a matter of figuring out what we have and then defining them. If you're interested in defining certain slowness metrics or telemetry for mobile, let's talk. There's one that I'm in the middle of putting together with the help of a lot of folks from the client SIG—a crash convention that spans platforms that's different from exceptions and we have many more down the pipe. Anything that can be standardized as semantic inventions should be. So it's a very long way of saying yes, and please help. -[00:12:30] When you start aggregating and saying, "Hey, what's P95?" that starts to not make sense. Also, operations run for a long time in OTEL, but unfortunately you don't get to know what happened until it ends, either in a failure or success. What if it gets interrupted in the middle through a crash that we don't know about? Mobile apps could be killed without actually alerting the app, so operations like that are gone or lost, and with OTEL itself, we wouldn't know about it because it's not done, and that's kind of challenging for mobile. +**Eliah:** Good stuff. Let's go on to another one. There are some options available for OpenTelemetry instrumenting iOS or Android code. What about applications using Kotlin Multiplatform? What instrumentation options are available? Any additional considerations when instrumenting KMP? -Also, operations that need to be contextualized with a lot of mutable state that are expensive to obtain, well, that's a challenge because in OTEL you write the data into the spans of span events or attributes, but sometimes the act of getting these types of state takes a while for the mobile app to actually get from the OS, and for us to basically block the ending of a trace, just doesn't work well. +**Hansen:** So Kotlin Multiplatform is interesting. For those unfamiliar, it's similar to React Native or things like that, where you write it once and it kind of generates native apps for the various platforms. I'm sure I'm getting something wrong in the technicality there, but the idea is that you have one codebase for multiple platforms. -The second point I want to talk about is that the protocols and APIs of OTEL make certain assumptions that are just not true on mobile. For example, assumption one is that telemetry is recorded and transmitted reliably, and you could trust that data that gets recorded will make it to the server, the collector, in a reasonable amount of time. But as we mentioned before, that is not true in mobile, and it could also not be true in a number of fascinating ways. +There isn't an SDK that I'm aware of that works in such a way that is built natively into Kotlin Multiplatform that will emit OpenTelemetry telemetry. Whenever we have iOS or Android, we use a Java SDK at the core, and iOS uses the Swift SDK. For Kotlin Multiplatform, either there has to be a native SDK for Kotlin Multiplatform that will transpile into the native platforms, so not only iOS and Android, but there's also web and a whole bunch of different platforms. -So tooling that is usable on mobile has to take that into account and build resilience, like persistence, for instance, before export. Thanks to Cesar for doing that on OpenJava or the OpenCry Android extension; it's extremely helpful in production. In the future, perhaps there will be a place for ways for us to automatically transmit a group of related telemetry so that we don't get into this weird state in the backend wondering if more data is coming. That would be nice. +We are evaluating, at Embrace, how best to approach this. We think the native way is the best way, but we don't know. We're working through some of these problems. Not specifically Multiplatform, but with React Native, which is I think why it's more well-known and more well-adopted. -Another assumption is that recording and transmitting telemetry doesn't put a strain on system resources. The SDKs are well-written, and they perform well, so on the backend, use them, it's no problem. However, on low-end devices and metered networks, it means that every signal captured potentially reduces performance or costs users money. We have to be a lot more careful in how we record data and how we transmit data. Even simply the act of getting the data to record can be expensive, so the consideration that we have to go into what to record is a lot greater and depends on the use case. +There's definitely opportunity in Kotlin Multiplatform, but first we got to have an SDK before we have a way of getting OpenTelemetry telemetry working on that platform—like recording first before anything else can happen. If you're working on it, you should start talking to people and think about that because that would be really interesting as well. -Another assumption is that engineers that use the API understand low-level concepts like threads or context propagation. Even the idea of tracing and what spans are may not be universally understood if you change the audience to mobile engineers, simply because the background, the variety of background of folks building mobile apps changes quite a bit. It could be somebody who's just done a six-month boot camp who is building a mobile app for the local grocery store, and they want observability too. They want to know why their stuff is slow, but they're used to higher-level constructs like coroutines when you're using threads but not really. They don't know about it, and how do you explain propagation when they don't understand the existence of a thread? +**Eliah:** I think the next question is about one of your challenges. It sounds like one of the core challenges for mobile observability is the variability of runtime environments—software and hardware. Is this entirely an open problem, or are there suggestions on how to start tackling this? -Adapting the APIs to be ematic is one thing, but making those concepts understandable by those without CS backgrounds is another challenge right there. Lastly, another assumption is that traced operations have a clear execution boundary and code ownership. You have a service that creates a span, and the runtime, generally a team owns that front to back, and you transmit the context via context propagation, etc. Unfortunately, things are not as clean on mobile because a single span for an operation can go through several modules owned by several teams, and not all of them may be aware of all these problems. +**Hansen:** I think to do it natively in OpenTelemetry, the entities will have to. For those who may not be familiar with that particular working group, the entities working group is a way of capturing external mutable transitions of states that are independent from but related to apps. I think for us, we could see it capturing a lot of this variable state without having to directly. -Instrumentation can be extremely brittle if you don't have the right infrastructure in place to catch regressions where implementation changes but the instrumentation doesn't. It's not as modular, nicely connected, cleanly connected as you would for distributed tracing, just because of how mobile apps are architected. The last OTEL thing I'm going to talk about is that mobile devices are millions of independent instances. OTEL generally assumes that the system being monitored and observed is one connected system. +Well, actually, the implementation is not yet—I don’t know, maybe it could work. We've done certain things that we're not super... I mean, we've done certain things to work around this that may not be accepted or standard. We're using spans to log durations of interesting things happening, and then on the backend, kind of merging the stuff all together. That's not great, but it also allows us to capture the stuff independently and not have telemetry recording be blocked and not have to encode every change of network condition onto every piece of telemetry sent and have race conditions that will, especially on mobile apps, make the edges blurry. -It could be made of dozens of microservices deployed in various ways, but effectively they all roll up, and metrics in that context make a lot of sense—OTEL metrics. But for mobile, when we have disconnected app instances running, that starts to break down because metrics need to be grounded in the context of the system that emits them in order for the baselines to be created and compared. Munting together metrics from phones of various models into one limits how useful those generated metrics are. What does P95 of heap size of an app at one minute mean when you look at the entire fleet of devices? I don't know. Did it change when it increases or decreases? Is that good or bad? Well, I don't know because we don't know the reasons because these are different systems. +We have something working, but we're not sure if it's the best quite yet, and we're really looking for entities to be able to do that or help us. Does that answer your question? I think there were two parts; I might have only answered one part, I'm assuming. -[00:19:00] Not that metrics aren't useful on mobile; they're extremely useful, but you tend to need to have like-to-like comparisons, apples to apples, and it tends to involve dimensions that are high in cardinality, and unfortunately that just doesn't work fantastically well with OTEL metrics. Simply the strict time-aligned aggregation is not really suitable for apps that have operations that have variable duration and have data come in at any time. It just wasn't meant to do what mobile wants it to do. But that's okay because these are not insurmountable challenges. These are ongoing things that we could work with the protocol and the SIGs and the folks to kind of improve. +**Eliah:** So, moving on. I know that you're part of the client-side instrumentation SIG. This was quite an interesting one: how closely or not is mobile instrumentation to browser instrumentation? As far as tracking user journeys, for example, does the client-side instrumentation work gear towards one or the other, or are the data models for both browser and mobile considered the same or similar enough? -We're working with various SIGs to try to get some of these problems surfaced and addressed. We are also, at Embrace, having workarounds to go around OTEL or perhaps use OTEL in non-standard ways in order to get the data to do what we want it to do. But that's not the end state. The end state we want to see is a diverse ecosystem and tooling that builds on not only what folks in OTEL have done before but also extends support to the plethora of use cases that we're bringing up. +**Hansen:** I think there are differences, but certainly there are a lot more similarities between web apps and mobile apps than I would say between mobile apps and backend distributed traces. I think the differences are more nuanced, and I think there's enough similarity that anything that we should consider for web, we should consider for mobile and vice versa. -Frankly, this is just the beginning because mobile apps that I'm familiar with run on very small, restricted sets of circumstances and phones and tablets are what I'm used to using. I haven't worked on cars or TVs, IoT, but those apps deserve observability as well. I'm sure as we take the tooling and the specs forward, more use cases will come on board and say, "Hey, I want to connect my data from my TV to my backend data." What additional challenges are there when I don't even know how a TV works in terms of how it uses Android? I'm sure it's different, and I'm sure there are new things. +There may be cases where it is immediately un... this doesn't fit, but honestly, you could run a mobile application on a mobile browser on a mobile device, so all the environmental changes are effectively similar that they have to deal with. In that respect, they are very similar. -At this point, we're just exploring ourselves. We're trying to ask questions; we're trying to figure out if our assumptions are correct. Are there things we could change about how we used to do things in order to fit more into OTEL? But at the end of the day, we want everyone to come up with their use cases and help ask these questions of mobile and client use cases for OTEL. Great work has been done in the Android and Swift and client SIG already, but I think there could be more going forward. +We work very closely with the web folks. -Folks listening, maybe they're converted to OTEL, but hopefully other people also watch this and say, "Hey, I looked at OTEL, it didn't really fit, but after this presentation, maybe I can make it fit. Maybe I can use it in a way that is productive and actually truly have this become the lingua franca of observability for both the backend and the frontend." +**Eliah:** Moving on. What different types of mobile apps require different kinds of data models? For example, an online gaming app like Magic: The Gathering versus a social media app like Instagram or a messaging app. -That's enough of us talking. Any questions? I'm going to end the presentation if I can. There we go. +**Hansen:** So the question is, do they require different kinds of data models, those different types of apps? Certainly different semantic conventions, I would say. There are certain characteristics about the runtime that are more interesting to high frame rate apps like games, for instance. If you're using, I don't know, Salesforce online, frame rate or scroll jank is bad, but it doesn't deter from the experience that much, and also it's not as sensitive to say mobile device capabilities. -[00:22:00] **Eliah:** Thank you very much, Hansen. For those that have joined a bit later, maybe haven't seen this message, we've got an Agile Coffee board. You can add your questions in there. I think we've got a couple of questions. We'll start with one related to web CWV. Vitals have become a standard to measure user experience for better awards. Do you see an emerging standard to measure equivalent concepts in mobile apps for sluggishness or speed? +But if you're running a game and your frame rate drops by half in critical instances, you want to know about it. Having more detailed data for that kind of stuff will be applicable to certain domains and not others. -**Hansen:** Yes, so you know, with OTEL, everything is defined effectively as semantic conventions built on the existing signals. At Embrace, we've kind of modeled some of the TRec capture for things like ANRs on Android and various other kinds of mobile slowness or sluggishness. We did it in a certain way that works, but is it the best? We don't know. Once we figure it out, we'll want to work with the SIGs to submit conventions to model that. I believe especially with the introduction of these emerging specs of profiling and even entities, a lot of these problems that we have are going to be addressed. So it's a matter of figuring out what we have and then defining them. If you're interested in defining certain slowness metrics or telemetry for mobile, let's talk. +I think that becomes more of a challenge of tooling and instrumentation rather than the spec. The spec as it is, you know, with logs and spans—well, events actually specifically—I think give us enough of the building blocks to model these different use cases. -There's one that I'm in the middle of putting together with the help of a lot of folks from the client SIG, a crash convention that spans platforms that's different from exceptions. We have many more down the pipe and it's limited by the abandonments that we have, but anything that can be should be standardized as semantic conventions. So it's a very long way of saying yes, and please help. +The same challenges that we have in terms of transmission and things like that, as long as those are dealt with, I think a good portion of that stuff is going to be taken care of. Now there may be additional things that I'm not aware of that will certainly need to have different types of consideration, but I think before we run, let's just crawl. -**Eliah:** Good stuff. Let's go on to another one. There are some options available for OTEL instrumenting iOS or Android code. What about applications using Kotlin Multiplatform? What instrumentation options are available? Any additional considerations when instrumenting KMP? +I think getting Unity and others, those different apps that we may not typically think about, console apps for instance. You have McDonald's and you have that thing that opens for 24 hours—that's a different use case than a mobile app that you background all the time after seconds. -**Hansen:** Kotlin Multiplatform is interesting. For those unfamiliar, it's similar to, you know, React Native or things like that, where you write it once and it kind of generates native apps for the various platforms. I'm sure I'm getting something wrong in the technicality there, but the idea is that you have one codebase for multiple platforms. There isn't an SDK that I'm aware of that works in such a way that is built natively into Kotlin Multiplatform that will emit OTEL telemetry. Whenever we have iOS or Android, we use a Java SDK at the core, and iOS uses the Swift SDK. +I think with the work that we're doing now, hopefully we'll move things forward enough so that we can start looking at some of these more challenging issues, like frame rates on Unity. -For Kotlin Multiplatform, either there has to be a native SDK for Kotlin Multiplatform that will transpile into the native platforms, not only iOS and Android, but web and a whole bunch of different platforms, or there has to be a way of getting a bridge built to the other SDKs. We are evaluating, at Embrace, how best to approach this. We think the native way is the best way, but we don't know. We're working through some of these problems, not specifically Multiplatform, but with React Native, which is I think why it's more well-known and more well-adopted. +**Eliah:** I think this next one relates back to some of the earlier questions. You mention that duration is not an indicator of performance. Are there golden signals for mobile irrespective of device type and OS version, or do we always need to take these factors into consideration? -There's definitely opportunity in Kotlin Multiplatform, but first, we got to have an SDK before we got to have a way of getting OTEL telemetry working on that platform, like recording first before anything else can happen. If you're working on it, you know, you should start talking to people and think about that because that would be really interesting as well. +**Hansen:** I think the golden signal is whether the user is happy or not, and on mobile, there are indicators of whether the user is happy in terms of whether they actually come back and use the app again. -**Eliah:** Thank you. I think the next question is about one of your challenges. It sounds like one of the core challenges for mobile observability is the variability of runtime environments. Software and hardware— is this entirely an open problem or are there suggestions on how to start tackling this? +Things like, you know, simply the DA—whether the user comes back, a usage rate. Just because one operation is slow, the effects could be multiplied if you have many, many, many instances of these slow operations. You basically get fed up with the app. -**Hansen:** I think to do it natively in OTEL, entities will have to, for those who may not be familiar with that particular working group, entities is a way of capturing external mutable transition of states that are independent from but related to apps. I think for us, we could see it capturing a lot of this variable state without having to directly—well actually the implementation is not yet, I don't know, maybe it could work. +At the very end, the very highest level, whether the app is being used is a good indicator. But going lower level, because that level is almost too late sometimes—whether an operation succeeded. Users tend to give up on operations if things take too long to load. They background the app, or they hop to another page or whatever it is. -We have done certain things to work around this that may not be accepted or enatic. We're using spans to log durations of interesting things happening, and then on the backend kind of merging the stuff all together. That's not great, but it also allows us to capture the stuff independently and not have telemetry recording be blocked and also not have to encode every change of network condition onto every piece of telemetry sent and have race conditions that will, especially on mobile apps, make the edges blurry. +Looking at the abandoned rate of operations is useful if you're tracking whether a particular operation is taking too long. So looking at duration in those cases, even if duration is important, abandoned rate is actually super important as well because you can actually have a case where your P50, P5 goes down because more people give up. -So we have something working, but we're not sure if it's the best quite yet, and we're really looking for entities to be able to do that or help us. Does that answer your question? I think there were two parts; I might have only answered one part, I'm assuming so. +So your success rate reduces, and your performance increases. You're like, "What's that?" Well, it's because people are...it's so slow people are giving up, so the population is different underneath. You're losing people already, so all the people remaining are the fast people. -**Eliah:** I think we've got a few questions, so I'm going to move to the next one. I know that you're part of the client-side instrumentation SIG, so this was quite an interesting one. How closely or not is mobile instrumentation to browser instrumentation as far as tracking user journeys? For example, if the client-side instrumentation work gears towards one or the other, or are the data models for both browser and mobile considered the same or similar enough? +This is actually a key problem in mobile performance: people look at the current state and they're like, "Oh yeah, P99, that looks great." Well, you haven't considered people for whom your app is way too slow, and you can't even use it. -**Hansen:** I think there are differences, but certainly there are a lot more similarities between web apps and mobile apps than I would say between mobile apps and backend distributed traces. I think the differences are more nuanced, and I think there's enough similarity that anything that we should consider for web we should consider for mobile and vice versa. There may be cases where it is immediately, you know, unfit, but honestly, you can run a mobile application on a mobile browser, on a mobile device. +Or they use it, they install it, it's too slow, they uninstall it. The inability to track those you've lost is a huge miss in mobile observability. I think OpenTelemetry has facilities to track it. We can end a span with an unsuccessful ending, so data can be collected to take care of this use case. -All the environmental changes are effectively similar that they have to deal with, so I think in that respect, they are very, very similar. So yeah, we work very closely with the web folks. +To me, the most important thing to track is not just duration; it's whether or not it actually succeeded. -**Eliah:** Moving on, what different types of mobile apps require different kinds of data models? For example, an online game like Magic the Gathering versus a social media app like Instagram or a messaging app. +**Eliah:** Can I add just one other thing there from my product hat here? I think the golden signals that are asked in that question really depend on the type of app that you have. -**Hansen:** The question is, do they require different kinds of data models, those different types of apps? Certainly different semantic conventions, I would say. There are certain characteristics about the runtime that are more interesting to high frame rate apps like games, for instance. If you're using Salesforce Online, scroll jank is bad, but it doesn't deter from the experience that much, and also it's not as sensitive to mobile device capabilities. +### [00:35:50] Q&A session on mobile observability -But if you're running a game and your frame rate drops by half in critical instances, you want to know about it. Having more detailed data for that kind of stuff will be applicable to certain domains and not others. But I think that becomes more of a challenge of tooling and instrumentation rather than the spec. I think the spec as it is, you know, with logs and spans, well, events actually specifically, and spans—hopefully a way of classifying spans in the future—give us enough of the building blocks to model these different use cases. +Hansen talked about startup and abandon rate and slow frame rates—all of those can have different impacts on your users depending on the intent of your app. Whether it's, like we talked about this earlier, a gaming app or a financial services app or whatever that may be. -The same challenges that we have in terms of transmission and things like that, as long as those are dealt with, I think a good portion of that stuff is going to be taken care of. Now there may be additional things that I'm not aware of that will certainly need to have different types of consideration. But I think before we run, let's just crawl, and I think getting unity and others, those different apps that we may not typically think about, console apps for instance. You have McDonald's, and you have that thing that opens for 24 hours—that's a different use case than a mobile app that you background all the time after seconds. +If I were a PM or working on a mobile app, I would really go back to what are the primary key workflows that we need our users to do to make this a successful transaction? I use that term very broadly for them, and then you go instrument those based on whatever that is. -I think with the work that we're doing now, hopefully, we'll move things forward enough so that we can start looking at some of these more challenging issues like frame rates on Unity and things like that. +For a gaming app, your golden signal may be frame rate because you know if your frame rate is as high as possible and your users are having a really frictionless experience, you see gameplay time increase, and you see all these other second-order effects really turn into positive impacts on those second-order effects. -**Eliah:** I think this next one relates back to some of the earlier questions, but you mentioned that duration is not an indicator of performance. Are there golden signals for mobile, irrespective of device type and OS version, or do we always need to take these factors into consideration? +Whereas for a retail app, it's really about checkout, right? Like how fast is that checkout flow for me? Are the operations that play are checkout flow, are they performant? Are they reliable? Etc. -**Hansen:** I think the golden signal is whether the user is happy or not. On mobile, there are indicators of whether the user is happy in terms of whether they actually come back and use the app again. Things like whether the user comes back, a usage rate, because just because one operation is slow, the effects could be multiplied if you have many slow operations. You basically get fed up with the app. +It really, really depends on what your app is trying to do, and also the audience is important too. Are they using your app because they have to, or are they using your app because they choose to? If you're a game and you're slow, I can just play another game. -At the very end, the very highest level, whether the app is being used is a good indicator. But going lower level, because that level is almost too late sometimes, whether an operation succeeded—users tend to give up on operations if things take too long to load. They background the app or hop to another page or whatever it is, so looking at the abandoned rate of operations is useful if you're tracking whether a particular operation is taking too long. +If your workplace uses Microsoft—oh, I shouldn't say—uses certain apps that you have no choice but to use, you know, yeah, it's slow, but you don't have a choice. So the golden signal there may be a little bit less than if folks have more options to go. -Looking at duration in those cases, even if duration is important, the abandon rate is actually super important as well because you can have a case where your P50 or P95 goes down because more people give up. So your success rate reduces, and your performance increases. You're like, "What's that?" Well, it's because people are giving up. The population is different underneath; you're losing people already, so all the people remaining are the fast people. +**Eliah:** Next question. Do you know of some use cases of OpenTelemetry working on video streaming apps like Netflix? -In fact, this is actually a key problem in mobile performance: people look at the current state and say, "Oh yeah, P99, that looks great." Well, you haven't considered people for whom your app is way too slow and you can't even use it, or they use it, they install it, it's too slow, they uninstall it. This kind of self-fulfilling prophecy of, "Oh, I don't have to invest in apps because people experience high-quality fast," well, it's because you've lost all the people, and the inability to track those you've lost is a huge miss in mobile observability. +**Hansen:** Not that I'm aware of. If they exist, I think the instrumentation would be fairly bespoke because—oh no, not that I'm aware of. But it would be a very interesting use case. -I think OTEL has facilities to track it. We can end a span with an unsuccessful ending, so data can be collected to take care of this use case. I think that to me is the most important thing to track: it's not just duration; it's whether or not it actually succeeded. +**Eliah:** I think we've got two more. Resource utilization like CPU or memory is normally represented as a gauge in a time metric on a backend system. Do time series make sense in mobile apps at all? -**Eliah:** Can I add just one other thing there? Just from my product hat here, I think the golden signals that are asked in that question really depend on the type of app that you have. Hansen talked about startup and abandon rates and slow frame rates—all of those can have different impacts on your users depending on the intent of your app, whether it's, like we talked about this earlier, whether it's a gaming app or a financial services app or whatever that may be. +**Hansen:** Certain time series, I would say utilization less so because the app is not the only utilizers of resources. You could be running very, you know, expectedly and something higher priority—the OS decides to schedule somebody, you know, starts a video in the corner of their tablet; they have multi-screen enabled, and suddenly the fast cores are now going to the mobile tablet video. -If I were a PM or working on a mobile app, I would go back to what are the primary key workflows that we need our users to do to make this a successful transaction, and I use that term very broadly for them. Then you go instrument those based on whatever that is. For a gaming app, your golden signal may be frame rate because you know if your frame rate is as high as possible and your users are having a really frictionless experience, you see gameplay time increase. You see all these other second-order effects really turn into positive impacts on those second-order effects. Whereas for a retail app, it's really about checkout—how fast is that checkout flow for me? Are the operations that are part of that checkout flow performant? Are they reliable, etc.? +Well, your app is running and suddenly things are slow. Suddenly you are getting issues or you didn't get before. It's good to track the utilization of the CPU, but there are probably other things you would want to know about, like that the utilization is supposed to tell you information that you can't get otherwise. -It really depends on what your app is trying to do, and also the audience is important too. Are they using your app because they have to, or are they using your app because they choose to? If you're a game and you're slow, I can just play another game. If your workplace uses Microsoft—oh, no, I shouldn't say—uses certain apps that you have no choice but to use, you know, it's slow, but you don't have a choice, so the golden signal there may be a little bit less than if folks have more options to go. +On mobile, there are just so many different things that could affect utilization that you almost need to capture so much other contextual information for that to be useful that if you're going to do that, you may as well go direct and say, "Hey, are we seeing lag? Are we seeing failures? Are we seeing unexpected occurrences in the app in certain instances?" -**Hansen:** Next question? +Knowing that your app is not being prioritized is important, but how big of the heap is, for instance, you know, the OS changes that on you so much. It'll GC you out in the most inappropriate places simply because it wants more resources for other apps. It'll kill your app in the background because another app is running and needs it, and it's no fault of yours that your thing got killed faster. -**Eliah:** Do you know of some use cases of OTEL working on video streaming apps like Netflix? +Resource utilization—there may be use cases where it's useful, but I think for me, it's harder to use the data that makes sense. -**Hansen:** Not that I'm aware of, and if they exist, I think the instrumentation would be fairly bespoke because—no, not that I'm aware of, but it would be a very interesting use case. +**Eliah:** I think this is the last question we've got. Do you have some idea of when OpenTelemetry mobile instrumentation might be ready for use in production? Is it months, is it years? -**Eliah:** I think we've got two more. Resource utilization like CPU or memory is normally represented as a gauge in a time series format on a backend system. Do time series make sense in mobile apps at all? +**Hansen:** It's ready now! It depends on what you want to do. There are SDKs out there. The Swift SDK and the various Java SDKs—if you want to kind of roll your own, it works. The Android OpenTelemetry project is something you can just drop in your app and it'll kind of do some monitoring for you. -**Hansen:** Certain time series, I would say utilization less so because the app is not the only utilizer of resources. You could be running very expectedly, and something higher priority, the OS decides to schedule. Somebody starts a video in the corner of their tablet; they have multi-screen enabled, and suddenly the fast cores are now going to the mobile tablet video. Well, your app is running, and suddenly things are slow, so suddenly you are getting issues that you didn't get before. +The Embrace SDKs—you can actually use it without using Embrace; you can just drop it in, configure your exporters to go to your own collectors, and use our implementation of the tracing API to have instrumentation libraries like data sessions and have it all sent to your own servers without Embrace being involved. -It's good to track the utilization of the CPU, but there are probably other things you would want to know about that the utilization is supposed to tell you. On mobile, there are just so many different things that could affect utilization that you almost need to capture so much other contextual information for that to be useful. If you're going to do that, you may as well go direct and say, "Hey, are we seeing lag? Are we seeing failures? Are we seeing unexpected occurrences in the app in certain instances?" +I'm sure there are other implementations out there as well, but it depends on what you want and what you need. There are things off the shelf you can use for free, and you can also roll your own with the SDKs. -Knowing that your app is not being prioritized is important, but how big of a heap, for instance, you know, those changes that on you so much. It'll GC you out of, in the most inappropriate places, simply because it wants more resources for other apps. It'll kill your app in the background because another app is running and needs it, and it's no fault of yours that your thing got killed faster. +All the challenges I was talking about really are to build a platform that is encompassing of all the corner cases that we want to support for all our customers. If you have a specific app with a specific use case with a specific thing you want to measure, you may not need any of this other stuff. -Resource utilization—there may be use cases where it's useful, but I think for me, it's harder to use the data and make sense of it. +If you're not sharing that code, if you're just kind of using it on your own, well, who cares about semantic conventions if no one else is going to look at that data? Now you're quite locked in to your own instrumentation, which is not a good thing, but if it works for you, it works for you. -**Eliah:** This is the last question we've got. Do you have some idea of when OTEL mobile instrumentation might be ready for use in production? Is it months? Is it years? +So I would go ahead and fork some of these repos and just try it out. They should work; they do work. There are apps that are in production with all of these SDKs. -**Hansen:** It's ready now; it depends on what you want to do. There are SDKs out there, directly the Swift SDK and the various JavaScript SDKs. If you want to kind of roll your own, it works. The Android Open project is something you can drag and drop into your app, and it'll kind of do some monitoring for you. The Embrace SDKs, you can actually use it without using Embrace. You can just drop it in, configure your exporters to go to your own collectors, use our implementation of the tracing API to have instrumentation libraries, like data sessions, and have it all sent to your own servers without Embrace being involved. +**Eliah:** That's all we had, so thank you very much, Hansen and Eliah! Is there any closing thoughts or anything you would like to add to finish off? -There are, I'm sure, other implementations out there as well, but it depends on what you want and what you need. There are things off the shelf you can use for free, and you can also roll your own with the SDKs. All the challenges I was talking about really is to build a platform that is encompassing of all the corner cases that we want to support for all our customers. +**Eliah:** I mean, I just go back to kind of where we started this. As Hansen's been talking about throughout this all, we really, really, really want to get more people involved who care about mobile in the working groups in the SIGs. -If you have a specific app with a specific use case, with a specific thing you want to measure, you may not need any of this other stuff. If you're not sharing that code, if you're just using it on your own, well, who cares about semantic conventions if no one else is going to look at that data? You're quite locked into your own instrumentation, which is not a good thing, but if it works for you, it works for you. So I would go ahead and fork some of these repos and just try it out. They should work; they do work. There are apps that are in production with all of these SDKs. +We are opinionated, and we come to it with our own experiences and the work that we've done, but we're by no means the final arbiter of what is right and wrong in mobile. The only way we can work through those is by having a variety of use cases. -**Eliah:** That's all we had, so thank you very much, Hansen and Eliah. Is there any closing thoughts, anything you would like to add to finish off? +I think that Netflix question is a good example of one that we don't deal with a lot, and so we may not be close to all the intricacies that come with running a video service at that scale or other services at that scale. -**Eliah:** I would just go back to kind of where we started this. I think as Hansen's been talking about through this at all is that we really, really, really want to get more people involved who care about mobile in the working groups, in the SIGs. We are opinionated and we come to it with our own experiences and the work that we've done, but we're by no means the final arbiter of what is right and wrong in mobile. +If you care about mobile—even if you may not be working directly on it—or if you have teams that work directly on it, we would love for you guys to get involved in the SIGs and provide your perspective and input. -The only way we can work through those is by having a variety of use cases. I think that Netflix question is a good example of one that we don't deal with a lot, and so we may not be close to all the intricacies that come with running a video service at that scale or other services at that scale. If you care about mobile, even if you're not working directly on it, or if you have teams that work directly on it, we would love for you guys to get involved in the SIGs and provide your perspective and input. +**Hansen:** The ecosystem is just emerging, I think, for mobile. There isn't enough diversity there—enough kind of folks leveraging the SDKs and building instrumentations for mobile use cases and mobile libraries that are popular. -[00:44:00] **Hansen:** The ecosystem is just emerging, I think, for mobile. There isn't enough diversity there; there aren't enough folks leveraging the SDKs and building instrumentation for mobile use cases and mobile libraries that are popular. The closing thought is that if you're using OTEL or if your backend is using OTEL and you're not, you could actually get a lot of mileage just by talking the same language and using the same signals. +The closing thought is that if you're using OpenTelemetry or if your backend is using OpenTelemetry and you're not, you could actually get a lot of mileage just by talking the same language and using the same signals. -I used to not think there was a ton of overlap in terms of, "Well, just mobile folks can just have that data, and then backend folks, all you have to do is encode all your conditionals and conditions and your requests in the back." There’s all that data, right? Well, it's not so easy to ingest all that in a performant way at every request. I've come around to understanding the utility of having two separate sets of data that can be connected. +I used to not think there was a ton of overlap in terms of like, well, just mobile folks can just have that data, and then backend folks—all you have to do is encode all your conditionals and your requests in the back, and has all that data right? Well, it's not so easy to ingest all that crap in a performant way at every request. -If you're a backend person and your company has mobile apps that you know use unnamed observability companies that may be free or may be very not free, but that don't really talk to your backend signals, well, have a look at OTEL and see what you can get in terms of things that are frankly even better, especially if it's provided by folks who only do mobile. There are a lot of things that are not captured if you just use folks that—well, I forget it, I'm not going to say it. +I've come around to understanding the utility of having two separate sets of data that can be connected. This is what it does. If you're a backend person and your company has mobile apps that use unnamed observability companies that may be free or maybe very not free but that don't really talk to your backend signals, have a look at OpenTelemetry and see what you can get in terms of things that are frankly even better, especially if it's provided by folks who only do mobile. -Yes, join us. I guess that's the thing: join us on the CNCF LA. I'll post the link in the channel for OTEL client-side telemetry. We also have the OTEL SIG and user channel if you're an end user and you want to discuss more things about how you're approaching OpenTelemetry. +There are a lot of things that are not captured if you just use folks that—well, I forget it, I'm not going to...never mind, blah blah blah. Yes, join us! I guess that's the thing—join us on the CNCF. I'll post the link in the channel. OpenTelemetry client-side telemetry—we also have the OpenTelemetry SIG and user channel. If you're an end user and you want to discuss more things about how you're approaching OpenTelemetry, so yeah, thanks again, both Hansen and Eliah, and we'll hopefully see you in the next edition of OTel in Practice. -Thank you again, both Hansen and Eliah, and we'll hopefully see you in the next edition of Otel and Practice. Thank you. Bye-bye. +Thank you, bye-bye. ## Raw YouTube Transcript diff --git a/video-transcripts/transcripts/2024-08-08T05:55:11Z-otel-q-a-feat-eliab-sisay-and-austin-emmons-of-embrace.md b/video-transcripts/transcripts/2024-08-08T05:55:11Z-otel-q-a-feat-eliab-sisay-and-austin-emmons-of-embrace.md index 331e035..63c41d5 100644 --- a/video-transcripts/transcripts/2024-08-08T05:55:11Z-otel-q-a-feat-eliab-sisay-and-austin-emmons-of-embrace.md +++ b/video-transcripts/transcripts/2024-08-08T05:55:11Z-otel-q-a-feat-eliab-sisay-and-austin-emmons-of-embrace.md @@ -10,157 +10,196 @@ URL: https://www.youtube.com/watch?v=A-t1hZdh7JY ## Summary -In this YouTube video titled "Otel Q&A," Elli and Austin from Embrace discuss their company's mobile observability solutions, focusing on the integration of OpenTelemetry into their SDKs for iOS and Android. Embrace aims to help developers monitor app performance in real-time, identifying and resolving issues to enhance user experience. They emphasize the importance of OpenTelemetry in creating a unified observability framework, enabling developers to understand both front-end and back-end performance. The conversation touches on technical challenges in mobile environments, the significance of community contributions, and educational resources for understanding observability in mobile applications. They also highlight the need for better integration between mobile and backend observability practices, stressing the importance of user experience and performance in mobile development. +In the YouTube video titled "Otel Q&A," hosts Elli and Austin from Embrace discuss the integration of OpenTelemetry into their mobile observability SDKs. Embrace is a mobile observability company that provides developers with tools to monitor app performance in real time, aiming to enhance user experience by identifying and resolving issues proactively. The conversation covers the architecture and technology stack used at Embrace, including their shift to OpenTelemetry, which helps unify observability across mobile and backend services. They also share insights into the challenges faced during this transition, such as adapting to varying mobile environments and educating teams about observability concepts. The duo emphasizes the importance of community contributions and collaboration within the OpenTelemetry ecosystem, highlighting their ongoing engagement with the community to improve standards and practices. They encourage developers to participate in discussions and contribute to OpenTelemetry, addressing common feelings of impostor syndrome when joining open-source projects. ## Chapters -00:00:00 Welcome and introductions -00:01:10 Overview of Embrace -00:03:40 Embrace architecture and tech stack -00:05:30 OpenTelemetry integration discussion -00:08:02 SDKs as core product -00:10:10 Internal telemetry dashboards -00:11:30 Challenges implementing OpenTelemetry -00:14:50 Rewriting SDKs for OpenTelemetry -00:16:30 Advantages of Embrace SDKs -00:18:00 Community contributions and collaboration -00:21:30 Audience Q&A session +00:00:00 Introductions +00:01:50 What does Embrace do? +00:04:35 Discussion on architecture and tech stack +00:11:00 Challenges in implementing OpenTelemetry +00:18:20 Benefits of using OpenTelemetry +00:22:55 Community contributions and collaboration +00:30:15 Discussion on mobile observability challenges +00:36:35 Audience Q&A session +00:44:00 Proposal for modeling client-side crashes +00:49:50 Wrap-up and final thoughts -**Elli:** Welcome to Otel Q&A! We've got folks from Embrace here. I guess let's why don't you introduce yourselves. I can start. My name is Elli. I lead product for our data collection and ingestion team here at Embrace. My team is responsible for a suite of mobile SDKs across iOS, Android, Unity, Flutter, and React Native. +## Transcript -**Austin:** And I'm Austin Emmans. I'm a developer on the iOS team, part of the iOS SDK development. I talk to the other teams about OpenTelemetry in general, but my main focus is Apple platforms. +### [00:00:00] Introductions + +**Elli:** Welcome to Otel Q&A! We've got folks from Embrace here. I guess let's start with introductions. I can start. My name is Elli. I lead product for our data collection and ingestion team here at Embrace. My team is responsible for a suite of mobile SDKs across iOS, Android, Unity, Flutter, and React Native. + +**Austin:** And I'm Austin Emmans. I'm a developer on the iOS team, and I lead the iOS SDK development. I also talk to the other teams about OpenTelemetry in general, but my main focus is Apple platforms. **Elli:** Awesome! Can one of you tell us what Embrace does? -**Austin:** Yeah, totally! We are a mobile observability company. We provide developers with SDKs that integrate into their mobile apps to monitor performance in real time. Our main goal is to help developers keep their apps running smoothly by proactively identifying and resolving issues, really with the objective of improving the overall user experience. We're heavily invested in OpenTelemetry, which I know we'll dig into a little bit later. We began that transition about nine months ago, and it's really helped us provide our customers with a more unified, comprehensive view of app performance—from what's happening client-side on the user's mobile device all the way to the backend services that power those experiences. By doing that, we enable developers to understand the full impact of any issues and optimize their apps more effectively. We've open-sourced all of our SDKs that I talked about earlier, and we encourage everyone to check those out in our GitHub repo, play around with them, and we'd love feedback. +### [00:01:50] What does Embrace do? + +**Austin:** Yeah, totally! We are a mobile observability company. We provide developers with SDKs that integrate into their mobile apps to monitor performance in real time. Our main goal is to help developers keep their apps running smoothly by proactively identifying and resolving issues, really with the objective of improving the overall user experience. We're heavily invested in OpenTelemetry, which I know we'll dig into a little bit later. We began that transition about nine months ago, and it's really helped us provide our customers with a more unified, comprehensive view of app performance, from what's happening client-side on the user's mobile device all the way to the backend services that power those experiences. By doing that, we really enable developers to understand the full impact of any issues and optimize their apps more effectively. We've open-sourced all of our SDKs that I talked about earlier, and we would encourage everyone to go check those out in our GitHub repo, play around with them, and we'd love feedback. + +**Elli:** It's really cool that as a mobile observability company, you're basically dogfooding by using OpenTelemetry. It sounds like you didn't start in that direction, but you moved in that direction. I can't wait to dig into that a little bit more. Before we get into that, can you describe the architecture that Embrace uses and any of the programming languages that are being used, like deployment environment? What's your tech landscape look like? + +**Austin:** I could take that one. Being an SDK developer, I focus mostly on the iOS side of things. As we took on this journey, we kind of rewrote the iOS SDK with a Swift-first approach, which takes on a Swift Package Manager project structure. The Android side is similar; it takes on a Kotlin-first approach. There's a lot of existing Java in that SDK, and they do a really good job at making sure that the interface on the Android side feels familiar if you're coming from Java. Our backend, though, is standard microservice architecture. We deploy using Argo CD and primarily run in a Kubernetes cluster. We have a couple of different data stores, including ClickHouse and Cassandra. The code for the backend is mostly in Go, with some Python sprinkled in here and there. -[00:01:10] **Elli:** Awesome! It's really cool that, as a mobile observability company, you are basically dogfooding, I guess, by using OpenTelemetry. It sounds like you didn't start in that direction but moved in that direction, and I can't wait to dig into that a little bit more. Before we get into that, can you describe the architecture that Embrace uses? Any of the programming languages that are being used? What's your tech landscape look like? +**Elli:** Cool! Awesome! Now that we've got some of the background, I'm really excited to hear about your OpenTelemetry integration. So first off, how did you learn about OpenTelemetry, and then why did you decide to integrate OpenTelemetry into the product? -[00:03:40] **Austin:** I could take that one. Being an SDK developer, I focus mostly on the iOS side of things. As we took on this journey, we kind of rewrote the iOS SDK with a Swift-first approach. That takes on a Swift Package Manager project structure. The Android side is similar; it takes on a Kotlin-first approach. There's a lot of existing Java in that SDK, and they do a really good job at making sure that the interface on the Android side feels familiar if you're a Java developer. Our backend, though, is standard microservice architecture. We deploy using Argo CD and primarily run in a Kubernetes cluster. We have a couple of different data stores depending on how we're accessing the data, which include ClickHouse and Cassandra. The code for the backend is mostly Go, with some Python sprinkled in here and there. +### [00:04:35] Discussion on architecture and tech stack -**Elli:** Cool! Awesome. Now that we've got some of the background, I'm really excited to hear about your OpenTelemetry integration. First off, how did you learn about OpenTelemetry, and why did you decide to integrate OpenTelemetry into the product? +**Austin:** So, I think as recently as five or six years ago, there were a variety of open standards that led to OpenTelemetry, most notably OpenCensus and OpenTracing. Once those combined into OpenTelemetry, we just saw the community really coalesce around that as the standard for modern observability practices. We decided to use OpenTelemetry because that fit perfectly with our vision of modernizing observability through open standards. OpenTelemetry provides a transparent, portable, flexible way to collect data, and we felt that was really essential for creating a unified observability framework that ties both the front end and the back end of applications together. As we were talking to customers, we kept hearing that one of the biggest challenges that SREs and developer teams consistently faced was marrying insights from their user-facing web and mobile applications into their observability practice. In an ideal world, your front-end teams are collecting data about what's happening on the end-user—the health of your end-user experiences—and your backend teams are collecting data about the health of infrastructure and services. Today, it's common for those to be entirely separate tools. They don't share a common set of telemetry, which prevents engineering teams from speaking the same language. Companies really want to work on what matters most, and it requires understanding where to invest their engineering resources to deliver the best user experiences. We saw that as an opportunity and wanted to help solve that challenge by collaborating with the OpenTelemetry community to drive the future of open standards for observability, specifically within our expertise of mobile. Our goal is to provide developers with a comprehensive view of their app's performance. -[00:05:30] **Austin:** I think as recently as five to six years ago, there were a variety of open standards that led to OpenTelemetry, most notably OpenCensus and OpenTracing. Once those combined into OpenTelemetry, we saw the community really coalesce around that as the standard for modern observability practices. We decided to use OpenTelemetry because it fit perfectly with our vision of modernizing observability through open standards. OpenTelemetry provides a transparent, portable, flexible way to collect data that we felt was really essential for creating a unified observability framework that ties both the front end and the back end of applications. As we were talking to customers, we kept hearing that one of the biggest challenges that SREs and developer teams consistently faced was marrying insights from their user-facing web and mobile applications into their observability practice. In an ideal world, your front end teams are collecting data about what's happening to the end user experiences, and your backend teams are collecting data about the health of infrastructure and services. Today, it's common for those to be entirely separate tools. They don't share a common set of telemetry, aren't interoperable, and really prevent engineering teams from speaking the same language. Companies want to work on what matters most, and it requires understanding where to invest their engineering resources to deliver the best user experiences. We saw that as an opportunity. We wanted to help solve that challenge by collaborating with the OpenTelemetry community to drive the future of open standards for observability, specifically within our expertise of mobile. Our goal is to provide developers with a comprehensive view of their apps' performance. Today, we collect the full technical and behavioral details of every user session with our OpenTelemetry-compatible SDKs. Users can even extend that instrumentation to any custom library in their app using OpenTelemetry signals and leverage our platform to contextualize the added instrumentation. Because we're using OpenTelemetry at the core of our SDKs, they can easily integrate with any backend of their choice or other observability tools, really giving users of our SDKs more flexibility to help them avoid being tied to one vendor. Of course, we think our product has some pretty cool stuff for mobile development teams in terms of workflow and helping them understand data and resolve things quickly, but we really think contributing to the community and helping mobile engineering teams modernize—and helping SRE and DevOps teams take that data and make sense of it as part of their entire system—will ultimately empower them to build better, more resilient mobile apps. +**Elli:** That's awesome! -**Elli:** As a follow-up question, you mentioned that you have OpenTelemetry built into your SDKs. Do you have OpenTelemetry built into your core product as well? +**Austin:** Today, we collect the full technical and behavioral details of every user session with our OpenTelemetry-compatible SDKs. Users can even extend that instrumentation to any custom library in their app using OpenTelemetry signals and then leverage our platform to contextualize the added instrumentation. Because we're using OpenTelemetry at the core of our SDKs, they can easily integrate with any backend of their choice or other observability tools, really giving users of our SDKs more flexibility to help them avoid being locked into one vendor. Now of course, we think our product has some pretty cool stuff for mobile development teams in terms of workflow and helping them understand data and resolve things quickly, but really we think contributing to the community and helping mobile engineering teams modernize, and helping SRE and DevOps teams be able to take that data and make sense of it as part of their entire system will ultimately empower them to build better, more resilient mobile apps. + +**Elli:** As a follow-up question, do you have OpenTelemetry built into your core product as well? **Austin:** I would say the SDKs are our core product that our customers use. We also have a frontend dashboard. -**Elli:** Yeah, the front end is what I'm thinking of. Austin, can you speak to that? +**Elli:** Right, the frontend is what I'm thinking of. Austin, can you speak to that? + +**Austin:** Sure! There are some features that we have, like Data Destinations, where we would collect metrics or some of the metrics that we aggregate in our backend, and then export as OpenTelemetry signals. So if you're not using the SDK and exporting the OpenTelemetry signals directly from a user's device, you might be sending data to us and then getting signals sent from our backend to whatever other provider you might be using to hopefully get the mobile data and your backend data in the same place. + +**Elli:** Gotcha! And on a similar vein, how do you interact with the telemetry that's coming from the applications and services in your organization? + +**Austin:** We have a bunch of internal dashboards. We host our own Grafana instance and can run reports to monitor the standard things that you would on a backend: uptime, response times, throughput. Our DevOps team and backend teams have those monitors, and me, as a front-end developer and SDK developer, looking at how our customers use the product, I can pull reports and generate my own visualizations just to see how features are being used, how they are being used, and how clean the data is. If they're adding custom properties to their telemetry, then I can assess whether they're using it in the correct way or the way we'd expect. That is really insightful just in terms of what documentation we need to provide and how we should help people because sometimes, if you're not looking at this data, then you're kind of at a loss or you're just guessing. For internal visibility, it's mostly internal Grafana dashboards and running our own custom queries against those. + +**Elli:** Awesome! You touched upon something that's so interesting and that we don't hear too much about in the context of observability. Because we always think observability helps us with troubleshooting, but observability also has that added benefit of understanding how users interact with the system. It’s really cool that this is one of the things you’re doing with the telemetry data that you're gathering. I wanted to call that out because I think that's very interesting and not often talked about. -[00:08:02] **Austin:** I'm more involved on the SDK side. There are some features that we have, like Data Destinations, where we would collect metrics. Some of the metrics that we aggregate in our backend, we then export as OpenTelemetry signals. If you're not using the SDK and exporting the OpenTelemetry signals directly from a user's device, you might be sending data to us and then getting signals sent from our backend to wherever you've configured—whatever other provider you might be using—to hopefully get the mobile data and your backend data in the same place. +**Austin:** Yeah, definitely! -**Elli:** Gotcha! On a similar vein, how do you interact with the telemetry that's coming from the applications and services in your organization? +### [00:11:00] Challenges in implementing OpenTelemetry -[00:10:10] **Austin:** We have a bunch of internal dashboards. We host our own Grafana instance, and we can run reports and monitor the standard things that you would on a backend: uptime, response times, throughput. Our DevOps team and backend teams have those monitors. As a front-end developer and SDK developer, looking at how our customers use the product, I can pull reports and generate my own visualizations just to see if this feature is being used, how it's being used, and how clean the data is. If they're adding custom properties to their telemetry, we can see if they're using it in the correct way or the way we'd expect. That is really insightful in terms of what documentation we need to provide and how we should help people because sometimes, if you're not looking at this data, then you're kind of at a loss or just guessing. For internal visibility, it's mostly internal Grafana dashboards and running our own custom queries against those. +**Elli:** Now, when you first decided to implement OpenTelemetry in your SDKs, what were some of the challenges that you encountered? -**Elli:** Awesome! You touched upon something that's so interesting and that we don't hear too much about in the context of observability, which is that observability helps us with troubleshooting, but it also has that added benefit of understanding how users interact with the system. It's really cool that that is one of the things you're doing with the telemetry data that you're gathering. I wanted to call that out because I think that's very interesting and not often talked about. +**Austin:** Yeah, I mean it was a process. Not that we really had to convince ourselves; I think there was a big upwell and a big push. But just from a software point of view, it's tough to bring in a dependency to a project because it's an unknown. At that point in time, we were pretty unfamiliar with the OpenTelemetry Swift SDK itself. We had to bring it in, and there was some pushback on the team about whether this is a dependency that we want to manage. We had to do our due diligence and some very tedious work to make sure that it fit how we expected it to, and verify that it would work how we expected it to. We also had to ensure that the performance matched the scale that we were expecting. That's just very tedious work, especially when you're compressing it into some deadlines. But that's not really OpenTelemetry specific; that's just software and dependencies in general. -Now, on to some meatier stuff. When you first decided to implement OpenTelemetry in your SDK, what were some of the challenges that you encountered? +A challenge that was probably more specific to OpenTelemetry and observability is that our first instinct is when we instrument something, we want to just get it done. You have to catch yourself and say, "Oh, there's actually some prior art here, so let's go search for these." OpenTelemetry has specifications and the shape of the data model that it should be. On top of that, there are all these existing semantic conventions on how to use something and how to fit into a system. When you record a span as a network request, what attributes should be used for that network request? Where does the URL go? What is the name of that span? Because we are all in, we want to make sure we adhere to those semantic conventions. It's tough when there's something very similar to what you're instrumenting, and you have to have discussions about whether this is something new or if we should leverage some of that and massage it a little bit to fit our needs. That's also just a tedious process to ensure you're not stepping on any toes but you're innovating and ultimately getting what you need done. -[00:11:30] **Austin:** Yeah, I mean, it was a process. Not that we really had to convince ourselves; I think there was a big upswell and a big push. Just from a software point of view, it's tough to bring in a dependency to a project because it's an unknown. At that point in time, we were pretty unfamiliar with the OpenTelemetry Swift SDK itself. We had to do our due diligence and some very tedious work to make sure that it fit how we expected it to, to ensure that any edge cases that we could think of we could test and verify that it would work how we expected it to, and then also make sure that the performance matched the scale we were expecting. That's just very tedious work, especially when you're compressing it into deadlines. That's not really OpenTelemetry specific; that's just software and dependencies in general. +**Elli:** When you did this, were you updating your existing SDK or was this a complete rewrite when you made the decision to use OpenTelemetry? -A challenge that was probably more specific to OpenTelemetry and observability is that our first instinct is when we instrument something, we want to just go instrument and get it done. You kind of have to catch yourself and say, "Oh, there's actually some prior art here," and search for these. OpenTelemetry has the specification and the shape of the data model and what it should be, and then on top of that, there are all these existing semantic conventions on how to use something and how to fit into a system—how to record a span for a network request, what attributes you should use for that network request, where the URL goes, what the name of that span is, and so on. Because we are all in, we want to make sure we adhere to those semantic conventions. It's tough when there's something very similar to what you're instrumenting, and you have to have discussions about whether this is something new, whether it's the same, or whether we should leverage some of that and massage it a little bit to fit our needs. That's just a tedious process to ensure that you're not stepping on any toes, but you're innovating and ultimately getting what you need done. +**Austin:** For the iOS team, we took the opportunity to rewrite into Swift. We had an existing observability SDK that was mostly Objective-C. Being a larger shift, we were able to take this opportunity to modernize it and present it to Apple developers in the year 2024. Our Android team’s SDK was a little younger, if that's a term you can use for SDKs, but they were able to maintain that same core code base and then start shifting just the data model piece. I think they also took the opportunity to modernize a lot of the architecture because it changes a bunch when you're pulling in something like the OpenTelemetry SDK. There are a lot of concepts in there that are very useful that you either massage to fit or you just completely replace. It was different for each platform, but yeah. -**Elli:** When you did this, were you just updating your existing SDK, or was this a complete rewrite when you made this decision to use OpenTelemetry? +**Elli:** Some folks who are listening in or who will listen to this in the future might be wondering, OpenTelemetry does have some SDKs tailored for mobile, so what's the advantage of using the Embrace SDKs in that case? -[00:14:50] **Austin:** For the iOS team, we took the opportunity to rewrite into Swift. We had an existing observability SDK that was mostly Objective-C. Being a larger shift, we were able to take this opportunity to really modernize it and present it to Apple developers in the year 2024. Our Android team— their SDK was a little younger, if that's a term you can use for SDK—but they were able to maintain that core codebase and then start shifting just the data model piece. I think they also took the opportunity to modernize a lot of the architecture because it changes a bunch when you're pulling in something like the OpenTelemetry SDK. There are a lot of concepts in there that are very useful that you either massage to fit or completely replace. It was different for each platform, but yeah. +**Austin:** It's really just to simplify. We use the SDKs as dependencies; they underlie what we do, and we're trying to add a layer on top of that to make it a little more accessible. We manage a lot of the setup process for the SDK and try to streamline that. We also want to minimize the mental overhead of, "Okay, when I'm creating a span, what do I need to do?" Hopefully, that's one simple call into our SDK instead of getting your tracer provider, building a tracer from that, taking that, and creating a span builder. Once you have that span, there are a couple of steps that the SDKs themselves have, and that's part of the spec. There's good separation and reason for that separation, but we want to streamline that. -**Elli:** Some folks who are listening in or who will listen to this in the future might be wondering, you know, OpenTelemetry does have some SDKs tailored for mobile, so what's the advantage of using the Embrace SDKs in that case? +If you're manually instrumenting things, we also want to add our own instrumentation that is mobile-specific. That is the space that we're really in and we are experts in. When you're on an iOS device, there are things that occur that are very specific to the iOS system or just users of that application that don't apply to a backend system. We want to make sure that we can automate that instrumentation so that users don't need to reimplement that or worry about it. There’s definitely a push for us to get that instrumentation upstream because the goal is it benefits the community and then it's kind of off our maintenance burden. I mentioned there were two layers of the spec and those semantic conventions. The third, on top of that, is the Embrace semantic conventions that we're hopefully keeping as thin as possible and we're pushing down into those OpenTelemetry semantic conventions when we do create or think of new things. -[00:16:30] **Austin:** It's really just to simplify. We use the SDKs as dependencies; they underlie what we do, and we're trying to add a layer on top of that to make it a little more accessible. We manage a lot of the setup process for the SDK and try to simplify and streamline that. We also want to minimize the mental overhead of, "Okay, when I'm creating a span, what do I need to do?" Hopefully that's one simple call into our SDK instead of getting your tracer provider, building a tracer from that, taking that, and creating a span builder. Once you have that span, there are a couple of steps that the SDKs themselves have; that's part of the spec and there's good separation for a reason. But we want to streamline that. If you're manually instrumenting things, we also want to add our own instrumentation that is mobile specific, which is the space that we're really in and are experts in. When you're on an iOS device, there are things that occur that are very specific to the iOS system or just users of that application that don't really apply to a backend system. We want to make sure that we can automate that instrumentation so that the users don't need to reimplement that or worry about it. There's definitely a push for us to get that instrumentation upstream because the goal is to benefit the community, and then it's off our maintenance burden. I mentioned there were two layers of the spec and those semantic conventions. The third on top of that is the Embrace semantic conventions that we're hopefully keeping as thin as possible and pushing down into those OpenTelemetry semantic conventions when we do create or think of new things. +**Elli:** That's awesome! So basically your SDK serves as a wrapper, but then also any sort of things that you come up with that would benefit the community, you contribute them back upstream, which is very cool. I think that really speaks to the power of community because I think OpenTelemetry’s success is due to the fact that we have so much support from various vendors who have all decided not to work on an island. I think this is extremely beneficial to the community. -[00:18:00] **Elli:** That's awesome! Basically, your SDKs serve as a wrapper, but then also any sort of things that you come up with that would benefit the community, you contribute them back upstream, which is very cool. That really speaks to the power of community because I think OpenTelemetry's success is due to the fact that we have so much support from various vendors who have all decided not to work on an island. I really want to emphasize that stuff like this is extremely beneficial to the community. +### [00:18:20] Benefits of using OpenTelemetry Now, when you switched your SDKs to using OpenTelemetry, what kind of benefits did you start seeing? -**Austin:** For me, it was the common language. It became very easy to discuss with people what OpenTelemetry is, using terms like trace, span, and log, and what those glossary items are. One thing that really bugged me before is that pretty much every vendor models these concepts but calls them different things, and we were doing that too, so we're at fault as well. Being able to use just a common envelope and a common language was a massive benefit when we started switching to OpenTelemetry, even at a non-technical level. +**Austin:** For me, it was the common language, and it became very easy to discuss with people what OpenTelemetry is using: a trace, a span, a log, and what those glossary items are. One thing that really bugged me before is that pretty much every vendor models these concepts but calls them different things. We were doing that before too, so we're at fault as well. Being able to use just the common envelope and a common language was a massive benefit when we started switching to OpenTelemetry, just at the non-technical level. -**Elli:** Can I add one thing to that? I think from an organizational perspective, it also helped to break down some silos for us internally. Even across our SDKs, across the different platforms, prior to OpenTelemetry, our Swift SDK, our Android SDK, React Native, Unity, etc., were all built with specific implementation details for those platforms in mind and kind of lived a little bit in a silo. When you think about our customers, a customer of ours that has an iOS or an Android app builds it specific to those platforms, but in terms of performance, their mobile teams want to understand that a consumer is having a great experience regardless of whether they're on an iOS device or an Android device. Building the underlying SDKs in silos internally or I should say the other way around, when we moved to OpenTelemetry, being able to use those common signals helped us break down those silos such that we could really focus on the insights that we're giving to our customers, which is really the ultimate value that we aim to provide. +**Elli:** Can I add one thing to that? From an organizational perspective, it also helped to break down some silos for us internally. Even across our SDKs, across the different platforms, prior to OpenTelemetry, our Swift SDK, our Android SDK, React Native, Unity, etc., were all kind of built with specific implementation details for those platforms in mind. They lived a little bit in a silo. But when you think about our customers, a customer of ours that has an iOS or an Android app doesn't build it specific to those platforms, but in terms of performance, their mobile teams want to understand that a consumer is having a great experience regardless of whether they're on an iOS device or an Android device. Building the underlying SDKs in silos internally meant we couldn't focus on the insights we were providing to our customers, which is really the ultimate value that we aim to provide. -**Elli:** That's awesome! That's a really great point. Now I want to switch gears a little bit because, Austin, you mentioned that you make some contributions upstream to the client-side telemetry. Can you tell us about your involvement with that SIG and how you started getting involved? +**Elli:** That's awesome! That's a really great point. -[00:21:30] **Austin:** I started attending the client-side SIG probably five or six months ago, right at the beginning of 2024. My coworker Hansen had been joining those before me, and I had been joining the OpenTelemetry Swift SIG meetings before that. Somebody just popped their head into a Swift SIG meeting and said, "Hey, we have this client-side SIG; there's not really any iOS representation. Would somebody please volunteer to come hang out with us?" I volunteered, and I've been part of that SIG since, joining every week. It's been great! Community ideas are shared; we talk about what's going on, what proposals are happening, or what problems people are trying to solve. When the time comes, it could just be unmuting yourself on this Zoom call and sharing an idea or even just sharing how it works for you. That could be going into the GitHub repos for some of the semantic conventions, adding a comment, and starting the discussion there or responding and continuing the discussion. Hansen has done a better job than I have, but he's led some semantic convention proposals for things that are mobile-specific or client-side specific. Some things like a user session are a little more formalized and standardized than they are now, and a big thing for mobile apps is when a mobile app crashes—how do we model that as telemetry? Now that the events data model has taken shape in the spec, there's a semantic convention proposal around modeling client-side crashes as events. That will be very useful because we deal with crash reporting as well at Embrace, and to formalize that and say, "Okay, here's how other people interpret crash reports," gives that flexibility to send that data anywhere, ridding ourselves of vendor lock-in because it's burdensome when you have it. +Now, I want to switch gears a little bit because Austin, you mentioned that you make some contributions upstream for the client-side telemetry. Tell us about your involvement with that SIG and how you started getting involved with that SIG as well. -**Elli:** That's so awesome! When you started attending those OpenTelemetry meetings, did you do that when the decision was made to start using OpenTelemetry internally, or when did that come about? +**Austin:** I started attending the client-side SIG probably five or six months ago, right at the beginning of 2024. My coworker Hansen had been joining those before me, and I had been joining the OpenTelemetry Swift SIG meetings before that as well. Somebody just popped their head into the Swift SIG meeting and said, "Hey, we have this client-side SIG. There's not really any iOS representation. Would somebody please volunteer to come hang out with us?" I volunteered and have been part of that SIG since, joining every week. It's been great! Community ideas are shared, we talk about what's going on, what proposals are happening, or what problems people are trying to solve. When the time comes, it could just be unmuting yourself on this Zoom call and sharing an idea or sharing how it works for you. That could also be going into the GitHub repos for some of the semantic conventions and just adding a comment to start the discussion or responding and continuing the discussion there, or it could be proposing some new things. Hansen has done a better job than I have, but he's led some semantic convention proposals for things that are mobile-specific or client-side specific. -**Austin:** I would say the decision was made when we started to switch, like, "Okay, we want to take this on; we want to be part of this community." I procrastinated a little bit just because I'm a person. I didn't actually start joining until the beginning of the year, but that's just the realistic approach, I guess. It's tough because you don't think that you have anything to contribute, and so there's an impostor syndrome that happens. It takes a bit to rid yourself of that, and I'm here to tell you to rid yourself of that, please join! We love hearing voices from all types of people using the tools provided. You can just come and hang out and chat, and if something comes up that you're comfortable with, then unmute yourself and join the chat verbally. +**Elli:** That's great! -**Elli:** That's great! Thank you for calling that out. I think it's so common. The impostor syndrome is very real; it's very scary. You are being so vulnerable there, especially when you join a group that seems to be very well-established and appears to be full of very intelligent people. You start thinking, "Oh my God, am I worthy of this?" and the answer, as you said, is yes, you are worthy of it. Everyone has a contribution to make, and it can be any kind of contribution. +### [00:22:55] Community contributions and collaboration -So often, I've used something from an open-source project and followed whatever in the documentation, and the docs are wrong. My first reaction is to be mad, like, "Oh my God, these idiots don't know what they're doing." But you should turn that frown upside down because if you notice an issue in the documentation, there's nothing preventing you from filing a pull request to fix that documentation and prevent other people from getting frustrated. That is the simplest thing you can do, and it makes such a huge impact. It doesn't have to necessarily be code; it can be documentation, and it can even be a minor correction. That's my little PSA there. +**Austin:** Some things like a user session are now formalized and standardized. A big thing for mobile apps is when a mobile app crashes—how do we model that as telemetry? Now that the standalone event data model has taken shape in the spec, there's a semantic convention proposal around modeling client-side crashes as an event. That will be very useful because we deal with crash reporting as well at Embrace. To formalize that and say, "Okay, here's how other people interpret crash reports," gives that flexibility to send that data anywhere, ridding ourselves of that vendor lock-in, which is burdensome. -**Austin:** That was exactly our approach too! We're like, "Awesome! We want some first issues; let's go read the documentation and see if there's anything we can adjust or make more clear." +**Elli:** That's so awesome! When you started attending those OpenTelemetry meetings, did you start doing that when the decision was made to start using OpenTelemetry internally? When did that come about? -**Elli:** That's so great! I love hearing these stories, and I feel like these are the types of things we just need to keep repeating in our community—telling people, "Yes, this is a great way to contribute." We talked about the benefits that you saw with integrating OpenTelemetry into your SDKs. What are some challenges you've seen in OpenTelemetry, especially in the mobile space, now that you've dipped your toes a little bit more into that community? +**Austin:** I would say the decision was made when we started switching to OpenTelemetry, saying, "Okay, we want to take this on. We want to be part of this community." I procrastinated a little bit because I’m human, and I didn't actually start joining until the beginning of the year, but that's just the realistic approach, I guess. It's tough because you don't think that you have anything to contribute, and there’s an impostor syndrome that happens. It takes a bit to rid yourself of that. I'm here to tell you to rid yourself of that, please join! We love hearing voices from all types of people using the tools that are provided. You can even just come and watch; you can hang out and chat. If something comes up that you're comfortable with, then unmute yourself and join the chat verbally. -**Austin:** I would say the biggest challenge that we faced is really—we have this layer above the OpenTelemetry SDK, and we're trying to streamline the process of getting into observability. It's mostly because mobile developers just aren't familiar with this territory. What’s interesting is that a lot of developers are, and there are a lot of existing tools out there that log events and things, but we're still trying to flatten the learning curve for observability. A lot of the challenge is just educating and figuring out what clicks for people, why this is valuable, and why it's useful. Then going from there, educating them on how to use our product specifically is probably one of the biggest non-technical challenges. +**Elli:** That’s great! Thank you for calling that out as well specifically, because I think it’s so common. When people join open source projects, the impostor syndrome is very real. It’s very scary. You are being so vulnerable there, and especially when you join a group that seems to be well-established and appears to be full of very intelligent people, you start thinking, "Oh my God, am I worthy of this?" The answer, as you said, is yes, you are worthy of it. Everyone has a contribution to make, and it can be any kind of contribution. -The biggest technical challenge is that in the mobile space, the environment is just chaotic—it's something an app developer does not control. This is not a server in a box on some air-conditioned shelf; it is in somebody's pocket. It could be on a nightstand—you have no idea on the network status, how much battery is left on that device, or how angry the operating system is and whether or not it's just going to kill your app for who knows what reason. The variability is ever-present, and we need to be able to recover if anything goes wrong. We have to handle things like, "Okay, we're trying to write to disk locally to recover this data, but there's no disk space. What do we do?" or "The battery is at 2%. What do we do?" It's just a little more hectic than you would expect or hope for if you're an app developer. The worst thing is if you're an app developer that's worried about performance; these things are out of your control, but the user is going to blame you if the app is not performing well. If they are on 2% battery and they're rushing to call a car to get to wherever they need to be, you need to ensure that they can complete that operation before the system decides to sleep. +**Austin:** Exactly! -**Elli:** Yeah, it's definitely a whole different ball game in the mobile space compared to the non-mobile space, where it feels like there are fewer unknowns. +**Elli:** I can't emphasize enough that so often, I have seen people using something from an open-source project, and they follow whatever is in the documentation, and then they notice that the docs are wrong. Their first reaction is to be mad, thinking, "Oh my God, these idiots don't know what they're doing." You know what? Turn that frown upside down because if you notice an issue in the documentation, if there's a boo-boo, there's nothing that prevents you from filing a pull request to fix that documentation to prevent other people from getting frustrated by it being wrong. That is the simplest thing you can do, and it makes such a huge impact. It doesn’t have to necessarily be code; it can be documentation, and it can even be a minor correction. That's my little PSA there! -**Austin:** Right, and we're monitoring more than just the application's performance. We're also looking at how the user is interacting with this application and what their behavior is. You don't really have to deal with that if you're in a microservice architecture—there are very consistent entry points and exit points. In any client-side application, not just mobile-specific but even in a web browser, the user might be doing something on the page that you just don't know how they got there. They came in with state that you didn't expect, and you need to understand how to reproduce the issue. It works on my machine, but that's not an excuse. +**Austin:** That was exactly our approach too. We want some first issues; let's go read the documentation. Is there anything that we can adjust or make more clear? -**Elli:** Absolutely! Now, switching back to contributing to OpenTelemetry, you mentioned getting started in contributing. Now that you've been doing it for several months, what has the contribution experience been like for you? +**Elli:** That's so great! I love hearing these stories, and I feel like these are the types of things that we just need to keep repeating over and over in our community and tell people, "Yes, you should be doing observability for your mobile apps." Now, we've talked about the benefits that you saw with integrating OpenTelemetry into your SDKs. What are some of the challenges that you've seen in OpenTelemetry, especially in the mobile space, now that you've dipped your toes a little bit more into that community? -**Austin:** It's good! I don't really have any bad or ugly experiences, which maybe is just me. When I joined the Swift SIG, it was like a week or two in—these are weekly meetings—so my second or third meeting, we were discussing an issue. Nacho just said, "Oh hey, Austin, can you take a stab at this?" I was like, "Okay, sure! You trust me to do that? That's awesome!" That was kind of a good sign of faith and confidence. I thought, "What can I do to help out?" Some of the things we're trying to do to help out are to push proposals upstream into the semantic conventions and make updates to the documentation to flatten that learning curve. You know, specific examples on how to instrument, like what this view controller is doing or making a network request with URL session—these are very common patterns for iOS developers. It just makes it very easy if there's a snippet out there that you can grab and tweak for your application. Those are the contributions we're trying to make. +**Austin:** I would say the biggest challenge we faced is that we’re trying to streamline the process of getting into observability, and it's mostly because mobile developers just aren't familiar with this territory. What’s interesting is a lot of developers are familiar, and there are existing tools out there that will log events. But we're still trying to flatten the learning curve for observability for people. A lot of the challenges are just educating and figuring out what clicks for people, why this is valuable, and why it’s useful, and then going from there. Then, educating them on how to use our product specifically. That’s probably one of the biggest non-technical challenges. -In terms of feedback for the SIGs, I don't know, be less friendly? I don't have much feedback; it's been great! +### [00:30:15] Discussion on mobile observability challenges -**Elli:** That's awesome to hear! In terms of improvements, do you have anything you'd want to see tweaked? +Technically, the biggest challenge is that in the mobile space, the environment is just chaotic. It's something an app developer does not control. This is not a server in a box on some air-conditioned shelf that you have on key and lock; it is in somebody's pocket, hopefully. It could be on a nightstand. You have no idea about the network status, how much battery is left on that device, or how angry that operating system is and whether or not it's just going to kill your app for who knows what reason. The variability is ever-present, and we need to be able to recover if anything goes wrong. We have to handle things like, "Okay, we're trying to write to disk locally to recover this data, but there's no disk space. What do we do?" The battery is at 2%. What do we do? It’s just a little more hectic than you would expect or hope for if you're an app developer. The worst thing is if you're an app developer worried about performance, these things are out of your control, but the user is going to blame you if the app is not performing well. If they are on 2% battery and they're rushing to call a car to get wherever they need to be, you need to make sure that they can complete that operation before the system just decides to put the app to sleep. -**Austin:** I don’t know if it’s an improvement, but finding that prior art is tough sometimes. A lot of these proposals are ongoing, and many things are just marked experimental, so you don't know how experimental they are. There's a process for every proposal or specification change, and making sure that those signals are up to date would be very useful and helpful as proposals go through the process. But that's just bookkeeping. If you're joining and talking about things, that's something that if you just ask a question, someone would help you out very quickly. +**Elli:** Yeah, it's definitely a whole different ball game in the mobile space compared to the non-mobile space where it feels like there are more unknowns that you're having to deal with. -**Elli:** That's actually a really great callout. +**Austin:** Exactly! We're monitoring more than just the application's performance as well. There’s the user interaction with this application and what their behavior is. That's something you don’t really deal with if you're in a microservice architecture. There are very consistent entry points and exit points. In any client-side application—not just mobile-specific but even in a web browser—the user might be doing something on the page that you just don’t know how they got there. They came in with state that you didn't expect, so now you need to understand how you actually got to this issue. You can't just say, "It works on my machine," because that's not an excuse. -**Austin:** I think it takes a bit longer than you would normally expect to get contributions accepted and move through that process, like Austin is talking about. But one thing to be aware of is that this is by design. When you're used to working at a startup, we try to make really quick decisions—implement them and see what happens. Working in a community like this takes more consensus. The goal is to create a unified solution that anyone can use, and that means decisions take a little longer than you might be used to when you're new. +**Elli:** Absolutely! Now, switching back to contributing to OpenTelemetry, you mentioned getting started in contributing. Now that you've been doing it for several months, how has it been? One of the things our SIG does is we want to understand what the contribution experience is for OpenTelemetry so we can provide feedback back to the SIGs to improve that. If they're not aware of the feedback, what can they do to improve? What are some of the good, the bad, and the ugly that you've experienced with becoming a contributor? -**Elli:** That's a really great point! Now we are finished with the main questions portion. Does anyone from our audience have questions they would like to ask? +**Austin:** It’s good! I don’t really have any bad or ugly experiences, which maybe means it’s just me! When I joined the Swift SIG, I think it was like a week or two in and these are weekly meetings, so my second or third meeting, we were discussing an issue, and Nacho just said, "Oh hey, Austin, can you take a stab at this?" I was like, "Oh, okay, sure! You trust me to do that? That's awesome!" That was a good sign of good faith and confidence. Then you go from there and think, "What can I do to help out?" -**Paige:** Hello! I am really curious what learning resources you would recommend—if there was a particular blog or metaphor or tutorial that helped you have that aha moment for you both. The follow-up is any helpful metaphors you use for teaching your customers and end users. +Some of the things we’re trying to do to help out are to push proposals upstream into the semantic conventions, make updates to the documentation to flatten that learning curve, and provide specific examples on how to instrument. For example, on iOS, maybe what this view controller is doing or how to make a network request with URLSession. These are very common patterns for iOS developers, and it just makes it easy if there's a snippet out there that you can grab and tweak in your application. Those are the types of contributions we’re trying to make. In terms of feedback for the SIGs, I don’t know, maybe be less friendly? I don’t have much feedback; it’s been great! + +**Elli:** That’s awesome to hear! In terms of improvement, do you see anything that could use a little tweak? + +**Austin:** I don’t know if it’s an improvement, but finding that prior art is tough sometimes, mostly because some of these proposals are ongoing. A lot of things are just marked experimental, and you don’t know how experimental that is. It’s supposed to go through stages—experimental, beta, stable. There are two or three stages for every proposal or specification change. Making sure those signals are up to date would be very useful and helpful as a proposal goes through the process. But that’s just bookkeeping, so if you're joining and you're talking about things, that's something that if you just ask a question, someone would help you out very quickly as well. + +### [00:36:35] Audience Q&A session + +**Elli:** That’s actually a really great call-out. Thanks for pointing that out! So, we’re finished now with the main questions portion. Does anyone from our audience have questions that they would like to ask? + +**Paige:** Hello! I am really curious what learning resources you would recommend. If there was a particular blog or metaphor or tutorial that helped you have that aha moment, and then the kind of follow-up is any helpful metaphors that you use for teaching your customers and kind of end users? **Elli:** Paige, real quick before we answer that, can you tell us who you are and what your goal is? -**Paige:** Yes! I'm Paige Cruz. I work as a principal developer advocate over at Kronos. I used to be in SRE, mostly focused on those beautiful backend systems—the microservices running in beautifully air-conditioned data centers. I'm looking to learn more about mobile. I'm very interested in making sure observability is useful for everybody up and down the stack, and mobile is an area that I need to brush up on, so I was here to learn. +**Paige:** Yes! I'm Paige Cruz. I work as a principal developer advocate over at Kronos. I used to be in SRE, mostly focused on those beautiful backend systems, the microservices running in beautiful air-conditioned data centers. I'm looking to learn more about mobile. I'm very interested in making sure observability is useful for everybody up and down the stack, and mobile is an area that I need to brush up on. You all are the experts, so I was here to learn. + +**Elli:** Thank you! Just wanted to make sure I knew who I was talking to. I appreciate that! Actually, it's funny that you mention blogs and books and resources. It is required reading at Embrace: the Learning OpenTelemetry book by Ted Young and Austin Parker. We've started a book club around that at Embrace for everyone in the company. I think it addresses one of the other challenges we face that's not technical. When you think about observability concepts in OpenTelemetry, it's been primarily focused on the backend to date. Taking those concepts and introducing them to front-end and mobile developers, and mobile teams, and why they’re important in observability as a whole is a challenge. Before we can even get them to start implementing our solutions or helping them understand why they're important, they need to understand the fundamental concepts first. In our journey over the last nine months, it's been various levels of education through our entire org—not just myself and Austin who are involved deeply in the SDKs but our CSM teams, our go-to-market teams, our SDRs, all of them—so we can all kind of speak the same language and understand the benefits of OpenTelemetry not just for backend observability but for client-side and mobile as well. + +Austin, I don't know if you have any other resources? + +**Austin:** Oh, and we attend a bunch of conferences too! Just being in the same space as people and learning what they're going through has been super helpful. + +### [00:49:50] Wrap-up and final thoughts + +**Paige:** For me, it was really eye-opening. In an org, we sent all of our application data to the same observability platform. I was mostly focused on the website. For fun, I looked at the mobile side just to see what was going on there, just poking around. I saw, "Oh my God, 90% of our requests came in through our iOS and Android apps!" I thought, "I should honestly ignore the website because to the business, the impact would be so much greater if we were to make these optimizations." Business observability is not always talked about inside the org, so really even just asking folks what the split is between entry points from mobile versus web can be a thread to start pulling on. Then you also get your managers and other parts of the business, like PMs, interested and invited to the party. Last thing is at SREcon this year, I posted a link in the chat. Isabelle gave a great talk titled "The Invisible Front Door: Reliability Gaps in the Front End," and touched a bit on mobile monitoring and how SREs really need to turn their attention toward this space. -**Elli:** Thank you! Just wanted to make sure I knew who I was talking to. Actually, it's funny that you mention blogs and books and resources. It is required reading at Embrace—the learning OpenTelemetry book by Ted Young and Austin Parker. We've started a book club around that at Embrace for everyone in the company. I think it addresses one of the other challenges we face that’s not technical. When you think about observability concepts in OpenTelemetry, it’s been primarily focused on backend to date. Taking those concepts and introducing them to front-end and mobile developers and mobile teams and why they're important in observability as a whole is a challenge. Before we can help them implement our solutions or understand why they're important, they need to understand the fundamental concepts. In our journey over the last nine months, it’s been various levels of education through our entire org, not just myself and Austin who are involved deeply in the SDKs but also our CSM teams, go-to-market teams, and SDRs, so we can all speak the same language and understand the benefits of OpenTelemetry—not just for backend observability but for client-side and mobile as well. Austin, I don't know if you have others. +**Austin:** That’s awesome! Thank you for sharing that! -**Austin:** Oh, and we attend a bunch of conferences too. Just being in the same space as people and learning what they're going through has been super helpful. +**Paige:** I was curious, Austin, you were talking about the challenges with the mobile environment just being chaotic. I think related to that, you mentioned the data model for crash events. I was just kind of curious what's going on with all that. Are you... yeah, what's in development, and what are the current topics in that area? -**Elli:** I can't remember when it clicked for me, but I'm a very visual person and a very hands-on learner. I think it was probably the first time I saw a trace as a trace tree visualized out in a timeline. I think that's when it was just like, "Oh, this is my code running right now, and I can see it and the differences." If I'm comparing two that should be identical but aren't because of stuff, that's when it was like, "Okay, this is interesting and useful." +**Austin:** I was at OpenTelemetry Day this week, so I didn't attend the client SIG this week because most people were in person. The latest I heard, which would be as of last week—nine days ago—the proposal is up in the GitHub repo. I believe the PR has been made, but maybe it's just in a Google doc. There's a document shared that describes a crash report as an event. The big thing is that events are built on top of logs and log records. Hansen says in the chat that the proposal is not up and the PR is not up yet, but soon! So, heads up everyone! We’re discussing and there will be a proposal very soon to use an event. Events are built on logs, but the key difference is there's an event name attribute that you can key off of. The value of that attribute is meant to represent the schema of the log body. -**Paige:** Does anyone have any other questions? +What we're kind of working through is that not all crash reports are the same in a mobile environment. On Android, you have the JVM, which is the Java Virtual Machine, and that runtime. Then there are the native crashes below that, mostly C and C++ crash reports. On iOS, you have Swift crash reports. There are some C++ crash reports that are kind of weird. We have these hybrid apps, React Native and others, that are interesting because you might actually get two crash reports and two stack traces for a single crash. There’s some weirdness there because if a React Native app crashes, it might crash at the bridge layer or in the JavaScript, then throw an exception down to the native code. You want to capture all of this stuff, and it’s very different. Part of what’s going into the proposal is figuring out how to represent this in a format that's consumable but also understand that it can be very different when an OpenTelemetry collector receives this data. It’s something we really struggled with when we came into OpenTelemetry—how to represent crashes. We found a way to shove everything in as a log record right now, but we said, "This isn't good enough. This could be better. Let's try to make this better." -**Audience Member:** I was curious, Austin. You mentioned the challenges with the mobile environment just being chaotic, and you mentioned the data model for crash events. I was just kind of curious what's going on with that. Are you developing anything, and what are the current topics in that area? +**Paige:** Interesting! Okay, yeah. I guess I didn't think about low battery being something that would affect my apps, of course. Like you said, "Stupid app!" -**Austin:** I missed the client SIG this week because it was OpenTelemetry Day, but the latest I heard, as of last week, is that the proposal is up in the GitHub repo. The PR has been made—maybe it's just in a Google Doc, but there’s a document shared that describes a crash report as an event. The big thing is that events are built on top of logs, and the key difference is there's an event name attribute that you can key off of. The value of that attribute represents the schema of the log body. The other attributes in that log record are important. If you start calling an event with a name, like "client crash," then the body can take a shape that's a little more formal. The interesting part is that not all crash reports are the same in a mobile environment. On Android, you have the JVM, which is the Java virtual machine and that runtime. Then there are the native crashes below that—mostly C and C++ crash reports. On iOS, you have Swift crash reports; there are some C++ crash reports that are kind of weird. Now we have these hybrid apps, React Native and others, which are interesting because you might get two crash reports and two stack traces for a single crash. There's some weirdness there because if a React Native app crashes, it might crash at the bridge layer or in the JavaScript and throw an exception down to the native code. You want to capture all this. What's going into the proposal is figuring out how to represent this in a format that's consumable while understanding that it can be very different when an OpenTelemetry collector receives this data. I definitely recommend checking out the proposal as soon as it's up. Sorry, Hansen, for putting the pressure on you, but we're looking forward to it because it's one of the biggest things that when we came into OpenTelemetry, we asked ourselves, "How are we going to represent crashes?" We struggled with that, but we found a way—we just kind of shove everything in as a log record right now and said, "This isn't good enough; this could be better. Let's try to make this better." +**Austin:** Yeah! Low battery is one where the systems will start throttling the processor. A lot of mobile devices now have high-efficiency processor cores and the crazy-fast processor cores. When the system sees that the battery is getting low, it starts throttling things and pushing them to those high-efficiency CPUs. That has a downstream effect on your logic; it's now less performant because it's either bottlenecked and waiting behind something or just the processor is slower. It’s just naturally not as quick as it would be if it were charging or had a full battery. -**Audience Member:** Interesting! Okay, I guess I didn't think about low battery as being something that would affect my apps, of course. Like you said, "Stupid app!" +**Paige:** Interesting! Thank you for sharing that. I'm sure I'll have more questions once I've processed this more, but this is good! -**Austin:** Low battery is one where the systems will start throttling the processor. Mobile devices have these high-efficiency processor cores and the AI crazy-fast processor cores. When the system sees that the battery is getting low, it starts throttling things and pushing things to those high-efficiency CPUs. That has a downstream effect on your logic, making it less performant because it's either bottlenecked and waiting behind something or just the processor is slower. It's just naturally not as quick as it would be if it was charging or had a full battery. +**Austin:** No problem! -**Audience Member:** Interesting! Thank you for sharing. I'm sure I'll have more questions once I've processed this more, but this is good. +**Elli:** Well, as we prepare to wrap up, we will turn the tables and allow you to ask us questions. -**Elli:** No problem! Well, as we prepare to wrap up, we will turn the tables and allow you to ask us questions. +**Austin:** I'm curious. I think both of you, Paige and Reese, are in developer relations, and Paige, you work as an SRE. To the other attendees as well, I'm not sure exactly what your roles are, but I’d be curious to know to what extent mobile observability is part of your current observability practice and how that fits in or how you marry that with your backend observability that you have in place. -**Austin:** I'm curious. I think both of you are—I think Reese, you're a developer relations engineer, and Paige, you work as an SRE. To the other attendees as well, I'm not sure exactly your roles, but I'd be curious to what extent mobile observability is part of your current observability practice and how that fits in or how you marry that with your backend observability that you have in place. +**Reese:** I don’t personally monitor any mobile applications. I work with Adriana in the APIs SIG. Anecdotally, I’ve heard a lot of end users being interested in this space. Like Paige, I want to learn more about it because I just don’t know much about mobile observability, and I’m really interested. -**Audience Member:** I don't personally monitor any mobile applications. I work with Adriana in the OpenTelemetry SIG. Anecdotally, I've heard a lot of end users being interested in this space. Like Paige, I want to learn more about it because I just don't really know much about mobile observability, and I'm really interested. +**Paige:** Honestly, I think it was learning that Embrace existed. I'm like, "Oh, of course, you'd want to do observability for your mobile applications as well!" That was when I made the mental connection of, "Duh, we should be doing that! But why aren't we talking about it?" I see this as an opportunity to bring awareness to folks that, of course, you should be doing observability for your mobile apps. -**Audience Member:** Honestly, I think it was learning that Embrace existed. I'm like, "Oh, of course you'd want to do observability for your mobile applications as well!" That was when I made the mental connection of, "Duh! Why aren't we talking about it?" I see this as an opportunity to bring awareness to folks that, of course, you should be doing observability for your mobile apps. +**Austin:** That triggers something I noted down for this question section. What tactics have you found work best in that regard? For us, we have deep expertise in mobile. We've just started working in OpenTelemetry over the last six to nine months, and we want to bring that mobile expertise to the OpenTelemetry community. There’s a thing around education from the mobile perspective that we also have to do. What have you found that’s particularly impactful in that regard, whether it’s something you’ve done or you’ve seen done? -**Austin:** That triggers something you said earlier that I meant to note down for this question section. What tactics have you found work best in that regard? For instance, we have deep expertise in mobile. We've just started working in OpenTelemetry over the last six to nine months, and we want to bring that mobile expertise to the OpenTelemetry community. I think there's a thing around education from the mobile perspective that we also have to do that relates to a lot of the questions we talked about today. What tactics have you found to be particularly impactful in that regard, whether it's something you've done or you've seen done in terms of educating others on the nuances, whether for us it'll be mobile or it could be whatever platform? +**Reese:** I would say talks, blogs... -**Audience Member:** I would say talks, blogs—apply to talk at conferences to bring awareness for sure! Blog posts, YouTube videos, get on people's podcasts is another way to bring that awareness. +**Paige:** Yeah, applying to talk at conferences will bring awareness for sure. Blog posts, YouTube videos, getting on people's podcasts is another way to bring awareness. -**Audience Member:** For me, it was really eye-opening. At an org, we sent all of our application data to the same observability platform. I was mostly focused on the website and just for fun looked at the mobile side to see what was going on there. I saw, "Oh my God, like 90% of our requests came in through our iOS and Android apps!" I was like, "I should just honestly ignore the website because the impact would be so much greater if we made these optimizations." Business observability is not always talked about inside orgs, so really even just asking folks what the split is between entry points from mobile versus web can really be a thread to start pulling on. You also get your managers and other parts of the business, like PMs, interested and invited to the party. +**Austin:** For me, it was really eye-opening. We sent all of our application data to the same observability platform, and I was mostly focused on the website. For fun, I looked at the mobile side just to see what was going on there, poking around. I saw, "Oh my God, 90% of our requests came in through our iOS and Android apps!" I thought, "I should ignore the website because to the business, the impact would be so much greater if we made these optimizations." Business observability is not always talked about inside the org, so just asking folks what the split is between entry points from mobile versus web can really be a thread to start pulling on. Then you also get your managers and other parts of the business, like PMs, interested in invited to the party. -**Audience Member:** Last thing is at SREcon this year, I posted a link in the chat. Isabelle gave a great talk called "The Invisible Front Door: Reliability Gaps in the Front End," and touched a bit on mobile monitoring and how SREs really need to turn their eyes towards this space. +**Paige:** Last thing is at SREcon this year, Isabelle gave a great talk titled "The Invisible Front Door: Reliability Gaps in the Front End," and touched a bit on mobile monitoring and how SREs really need to turn their attention toward this space. -**Elli:** That's awesome! Thank you for sharing that. +**Austin:** That's awesome! Thank you for sharing that! -**Austin:** Well, we are almost out of time! I thank you, Austin and Elli, for joining us today. This has been really great and educational. Thank you to everyone who joined us as well. See you for Otel in practice! +**Elli:** Well, we are almost out of time, so I thank you, Austin and Elli, for joining us today. This has been really great, really educational. Thank you to everyone who joined us as well. See you for Otel in practice! ## Raw YouTube Transcript diff --git a/video-transcripts/transcripts/2024-10-04T15:54:54Z-otel-q-a-with-steven-swartz.md b/video-transcripts/transcripts/2024-10-04T15:54:54Z-otel-q-a-with-steven-swartz.md index 990764c..84eed22 100644 --- a/video-transcripts/transcripts/2024-10-04T15:54:54Z-otel-q-a-with-steven-swartz.md +++ b/video-transcripts/transcripts/2024-10-04T15:54:54Z-otel-q-a-with-steven-swartz.md @@ -10,293 +10,274 @@ URL: https://www.youtube.com/watch?v=3c9Bnldt128 ## Summary -In this YouTube Q&A session, Steven Schwarz, a Form Engineering expert from a payment processing company, discusses his experiences with observability, particularly focusing on OpenTelemetry (OTel). He shares insights into his team's migration from a proprietary telemetry solution to OTel, highlighting the challenges and advantages of using OTel's architecture, including Kubernetes and the OpenTelemetry operator. Steven explains their use of sidecar collectors, auto-instrumentation, and the management of resources, as well as the importance of sampling strategies and the integration of metrics monitoring. The discussion touches on the learning curve for teams transitioning to OTel, the significance of proper documentation, and the need for community support in navigating complex setups. The session concludes with audience questions addressing various technical aspects of using OTel, showcasing the engagement and interest in observability practices. +In this YouTube video, a Q&A session features Stephen Schwarz, a Form Engineer at a payment processing company, discussing his experiences with observability infrastructure and the use of OpenTelemetry. He elaborates on his team's migration from a proprietary observability vendor to OpenTelemetry, highlighting the benefits of cost control and improved support. Stephen shares insights into their architecture, which includes Kubernetes clusters and the OpenTelemetry operator for auto-instrumentation, and discusses challenges related to scaling, resource allocation, and tail sampling. The conversation also touches on the use of Prometheus for metrics, the importance of community contributions, and the educational aspect of manual versus auto-instrumentation within his organization. The session concludes with Stephen expressing interest in further community engagement and the ongoing need for better documentation and example configurations. ## Chapters -00:00:00 Welcome and introduction -00:01:00 Guest introduction: Stephen Schwarz -00:02:40 Architecture and deployment overview -00:03:50 OpenTelemetry operator usage -00:06:01 Migration from proprietary solution -00:08:10 Challenges in scaling collectors -00:09:36 Load balancing collectors -00:12:40 Experience with OpenTelemetry operator -00:14:50 Building custom collector distribution -00:17:00 Manual vs. auto instrumentation -00:20:00 Audience Q&A session +00:00:00 Introductions +00:01:36 Stephen's background and role +00:02:24 Discussion on architecture +00:05:28 OpenTelemetry operator usage +00:07:20 Migration from proprietary vendor +00:10:24 Challenges in scaling collectors +00:12:58 Tail sampling discussion +00:20:00 Manual vs auto instrumentation +00:32:00 Feedback for OpenTelemetry maintainers +00:42:52 Q&A session with audience questions -**Speaker 1:** Well, thank you everyone for joining. We have quite a good audience for the OTEL Q&A. We have Steven Schwarz today with us. +## Transcript -Just a quick note on the format, which Dan actually posted in the chat. I'll be asking Steph some questions, and then afterwards, time permitting, you'll have the opportunity to post some questions for Steven as well in the link that Dan provided. +### [00:00:00] Introductions -All right, so first things first, Steven, do you want to introduce yourself? +**Speaker 1:** Well, thank you everyone for joining. We have quite a good audience for the OTEL Q&A. We have Steven Schwarz today with us. Now, just a quick note on the format, which Dan actually posted in the chat. Awesome! So, I'll be asking Steph some questions, and then afterwards, time permitting, you'll have the opportunity to post some questions for Steven as well in the link that Dan provided. -[00:01:00] **Steven:** Yeah, sure. My title at work is Form Engineering, and I work for a company that does Payment Processing. So you know, when you're working with payments, you're working with your money; you need to be kind of exact with your observability. +All right, so first things first, Stephen, do you want to introduce yourself? -What my team does is we manage all of the observability infrastructure. We've got about a hundred technical folks that we're managing it for, and we try to be sort of like the experts in observability and share some of the best practices and help them solve their problems. +**Stephen:** Yeah, sure. So, my title at work is Form Engineering, and I work for a company that does Payment Processing. So, you know, when you're working with payments, you're working with your money, you need to be kind of exact with your observability. What my team does is we manage all of the observability infrastructure. We've got about a hundred technical folks that we're managing it for, and we try to be sort of like the experts in observability and share some of the best practices and help them solve their problems. -I recently joined this team about almost a year ago, and the project that I started on was actually migrating from another observability vendor, where we were using proprietary instrumentation, and we adopted OpenTelemetry and moved to a new observability vendor. +### [00:01:36] Stephen's background and role -So hopefully, some of those learnings are useful to other folks going through that process. Also worth calling out is that I really enjoyed contributing to the OpenTelemetry Collector contrib repo. I had to do that to unblock myself with a few bugs that we faced, but it was also just a really good way to understand what the collectors are doing under the hood. I was completely new to Go, but the code is really approachable. So yeah, I'd recommend taking a look if you haven't. It made my job a lot easier for sure. +I recently joined this team about almost a year ago, and the project that I started on was actually migrating from another observability vendor where we were using proprietary instrumentation, and we adopted OpenTelemetry and moved to a new observability vendor. So, hopefully, some of those learnings are useful to other folks going through that process. Also worth calling out is that I really enjoyed contributing to the OpenTelemetry collector contrib repo. I had to do that to unblock myself with a few bugs that we faced, but it was also just a really good way to understand what the collectors are doing under the hood. I was completely new to Go, but the code is really approachable. So yeah, I would recommend taking a look if you haven't. It made my job a lot easier for sure. -[00:02:40] **Speaker 1:** That's great! Thanks for the intro. Now, can you tell us a little bit about the architectures that you use in your organization? Like what programming languages are used and the deployment landscape, that kind of thing? +### [00:02:24] Discussion on architecture -**Steven:** Yeah, so I have that architecture diagram I shared with you. Would it be a good time to share that? +**Speaker 1:** That's great! Thanks for the intro. Now, can you tell us a little bit about the architectures that you use in your organization? Like what programming languages are used and the deployment landscape, that kind of thing? + +**Stephen:** Yeah, so I have that architecture diagram I shared with you. Would it be a good time to share that? **Speaker 1:** Yeah, go for it. -**Steven:** Okay, all right. So I believe my screen is visible. Everyone can see a diagram? +**Stephen:** Okay, all right. So, I believe my screen is visible. Everyone can see a diagram? **Speaker 1:** Yep, I can see it. Hopefully, everyone else can as well. -**Steven:** Well, okay. So going kind of left to right, we can dive into specific parts that seem interesting. - -So we are running on Kubernetes. We have multiple production clusters, and within each of those clusters, we are using the OpenTelemetry operator. That gives us a way to inject common configuration of the SDK into our teams' applications, into their containers. It also lets us automatically turn on auto instrumentation, so we get consistent spans collected from all the apps. +**Stephen:** Well, okay. So, going kind of left to right, we can dive into specific parts that seem interesting. So we are running on Kubernetes. We have multiple production clusters, and within each of those clusters, we are using the OpenTelemetry operator. That gives us a way to inject common configuration of the SDK into our teams' applications and into their containers. It also lets us automatically turn on auto instrumentation, so we get consistent spans collected from all of the apps. -[00:03:50] Another unique part is we are using OpenTelemetry. We're running OpenTelemetry as a sidecar, meaning that alongside each instance of the application, we have a mini collector running, and the app can send all their telemetry over localhost to that sidecar. +Another sort of unique part is we are using the OpenTelemetry collector as a sidecar, meaning that alongside each instance of the application, we have a mini collector running, and the app can send all their telemetry over localhost to that sidecar. We are currently sending metrics and spans. Logs, it's a bit of a long story, but we aren't currently using logs through OpenTelemetry. -So we are currently sending metrics and spans. Logs, it's a bit of a long story, but we aren't currently using logs through OpenTelemetry. For metrics, we use OpenTelemetry, but as sort of like a hop. We also use Prometheus to scrape the sidecar before it pushes to Grafana. +For metrics, we use OpenTelemetry as sort of like a hop. We also use Prometheus to scrape the sidecar before it pushes to Grafana. Another interesting challenge was getting tail sampling to work because we have spans coming from multiple Kubernetes clusters, and we only want to keep some of them. But we don't want to make the decision about which traces we keep until we have all the spans. Maybe we wait to see if there's an error. So, we have a separate cluster that all the spans go to in order to make that decision. We load balance just to help scale that; we have these collectors in that common cluster. They just load balance by Trace ID, so we don't have a ton of spans piling up on one collector. That's sort of like a high-level overview of our setup. -Another interesting challenge was getting tail sampling to work because we have spans coming from multiple Kubernetes clusters, and we only want to keep some of them. But we don't want to make the decision about which traces we keep until we have all the spans—maybe we wait to see if there's an error. +### [00:05:28] OpenTelemetry operator usage -So we have a separate cluster that all the spans go to, to make that decision. We load balance just to help scale that. We have these collectors in that common cluster; they just load balance by Trace ID, so we don't have a ton of spans piling up on one collector. Yeah, that's sort of like a high-level overview of our setup. +**Speaker 1:** That's awesome! It's really cool to see this kind of setup out in real life because I do feel like that's kind of a burning question with a lot of our practitioners: how do you set up your collectors? It's very nice to see. And cool that you're using the OpenTelemetry operator. I think you're one of the few people that I've spoken with recently who has been using the operator. What aspects of the operator are you currently using? Do you leverage the auto instrumentation? -**Speaker 1:** That's awesome! It's really cool to see this kind of setup out in real life because I think that's, I do feel like that's kind of a burning question with a lot of our practitioners: how do you set up your collectors? So it's very nice to see. +**Stephen:** Yeah, so the main ones we're using are just injecting the common SDK configuration. That saves us a ton of time. We don't have to get teams to make code changes; we can just manage that centrally and kind of push it out to all the teams. Same with the auto instrumentation; they just add a line to their configuration of their pod, and then they get the instrumentation. The other ones you mentioned we aren't using, but we are using the sidecar, which is the OpenTelemetry operator inject. -And cool that you're using the OpenTelemetry operator. I think you're one of the few people that I've spoken with. Not to say people don't use the operator, but you're definitely one of the few people I've spoken with recently who has been using the operator. +**Speaker 1:** Cool, cool. I have to put a plug for the operator. Auto instrumentation is actually so cool. You literally don't have to write code; you just put an annotation in the manifest. -What aspects of the operator are you currently using? Do you leverage the auto instrumentation? Do you leverage the OpAmp capabilities? What's the other one? Do you use the Collector deployment capabilities? Do you use all three or a combination? +**Stephen:** Yeah, you know, the least effort you can make for teams, the better. Even getting them to add the annotation can sometimes take time. So, I think it made the migration go much faster, being able to minimize the amount of work they have to do. -**Steven:** So the main ones we're using are just injecting the common SDK configuration. That saves us a ton of time. We don't have to get teams to make code changes; we can just manage that centrally and push it out to all the teams. +### [00:07:20] Migration from proprietary vendor -Same with the auto instrumentation. They just add a line to their configuration of their pod, and then they get the instrumentation. The other ones you mentioned we aren't using, but we are using the sidecar, which is the OpenTelemetry operator inject. +**Speaker 1:** Absolutely. Now, I wanted to ask because you mentioned that you had migrated from a previous vendor using their own proprietary telemetry solution. What made you decide to do that migration? -**Speaker 1:** Cool, cool. I have to put a plug for the operator. Auto instrumentation is actually so cool. You literally don't have to write code; you just put an annotation in the manifest. And the instrumentation CR, of course, that you have to figure, but it's pretty cool, I have to say. +**Stephen:** Yeah, so cost was a big one. It was hard to control with the previous vendor, and I guess support wasn't good. So those are, I think, the main drivers. A lot of stakeholders were unhappy with it, and we wanted a bit more. The collector gives us a lot of control over cost that this other vendor doesn't. -**Steven:** Yeah, you know, the least effort you can make for teams, the better. Even getting them to add the annotation can sometimes take time, so I think it made the migration go much faster by being able to minimize the amount of work they have to do. +**Speaker 1:** Right, that makes a lot of sense. Now, another thing that I wanted to ask in terms of scaling your collectors: have you experienced any challenges in scaling your collectors? Because you mentioned that you have a sidecar deployment pattern. How many different sidecars are you running at a particular point in time? Is it a lot that you're managing? -[00:06:01] **Speaker 1:** Absolutely. Now, I wanted to ask because you mentioned that you had migrated from a previous vendor using their own proprietary telemetry solution. What made you decide to do that migration? +**Stephen:** Yeah, so for the sidecars, we run one for each pod or instance of the application, so that kind of almost scales on its own, which is quite nice. So that works. The tricky part with the sidecar is configuring how many resources to allocate, like how much memory to ask from Kubernetes. Just because every application is slightly different in their volume, and there's not really a way currently for a specific application to say that they want a specific amount of memory. We need to configure that in our centralized configuration, so it creates a lot of coordination overhead if we want to tune that right. But we've only had to tune it for one application so far, one of our monoliths. So, that has actually been quite easy to scale, the sidecar. -**Steven:** Yeah, so cost was a big one. It was hard to control with the previous vendor. I guess support wasn't good, so those are the main drivers. A lot of stakeholders were unhappy with it, and we wanted a bit more. The Collector gives us a lot of control over cost that this other vendor doesn't. +**Speaker 1:** Okay, that's great! Now, what about your load-balanced collectors? Did you encounter any challenges in setting that up? -[00:08:10] **Speaker 1:** Right, right. Yeah, that makes a lot of sense. Now, another thing that I wanted to ask in terms of scaling your collectors: have you experienced any challenges in scaling your collectors? Because you mentioned that you have a sidecar deployment pattern. How many different sidecars are you running at a particular point in time? Is it a lot that you're managing? +**Stephen:** Yeah, well, I mean, at the beginning, there was a regression that caused a memory leak in one of the components we’re using, the span metrics one. That was a problem; that was actually one of the things that I fixed as an open source contribution. I guess our setup is quite memory intensive because we have to cache our metrics in memory for Prometheus to scrape them. So yeah, it's really been a matter of keeping memory at a reasonable level. -**Steven:** Yeah, so for the sidecars, we run one for each pod or instance of the application, so that kind of almost scales on its own, which is quite nice. That works. +### [00:10:24] Challenges in scaling collectors -The tricky part with the sidecar is configuring how many resources to allocate, like how much memory to ask from Kubernetes, just because every application is slightly different in their volume. There currently isn't a way for a specific application to say that they want a specific amount of memory. We need to configure that in our centralized configuration, so it creates a lot of coordination overhead if we want to tune that. +Right now, we use the Kubernetes horizontal pod autoscaler that watches the memory of each of the instances of the collector and makes scaling decisions based on that. It works pretty well. It took some tinkering. One thing I've noticed—and maybe other folks have a good solution for this—is when our vendor Grafana has an outage, we find the telemetry starts to pile up while waiting for the service to be restored, and we see this big spike in CPU and memory. It's quite hard to manage that. I think if the outage were to go for quite a while, probably the memory usage would just start to kill the collectors or be very high. -But we've only had to tune it for one application so far—one of our monoliths. So that has actually been quite easy to scale, the sidecar. +**Speaker 1:** Right, right. Now, another question that I had for you, just going back to the OpenTelemetry operator: what made you decide to use the operator in the first place? Is it something that you were made aware of, or did you hear about it from folks in the community? -[00:09:36] **Speaker 1:** Okay, that's great! Now, what about your load-balanced collectors? Did you encounter any challenges in setting that up? +**Stephen:** I heard about it actually from a colleague. It was their recommendation, but it fit with the kind of platform ethos of trying to get out of the way of the developers and have a way to do things without the developers having to get involved. -**Steven:** Yeah, well, I mean at the beginning, there was a regression that caused a memory leak in one of the components we're using—the span metrics one. So that was a problem. That was actually one of the things that I fixed as an open-source contribution. +**Speaker 1:** Right, fair enough. That is one thing I like about the operator. Now, what was your experience in running it? Was it something that you felt was straightforward? -I guess our setup is quite memory-intensive because we have to cache our metrics in memory for Prometheus to scrape them. So yeah, it's really been a matter of keeping memory at a reasonable level. Right now, we use the Kubernetes horizontal pod autoscaler that watches the memory of each of the instances of the collector and makes scaling decisions based on that. +**Stephen:** I'd say generally, compared to the collector itself, it was very smooth. I'm not sure why that is, but yeah, it worked quite well. The maintainers were very responsive in the Slack channels when we did have minor issues. -It works pretty well; it took some tinkering. One thing I've noticed, and maybe other people have a good solution for this, is when our vendor has an outage, I find the telemetry starts to pile up, waiting for service to be restored, and we see this big spike in CPU and memory. So it's quite hard to... I guess if the outage were to go for quite a while, probably the memory usage would just start to kill the collectors or be very high. So that's a bit of a problem. +### [00:12:58] Tail sampling discussion -**Speaker 1:** Right, right. Now, another question that I had for you, just going back to the OTEL operator: what made you decide to use the operator in the first place? Is it something that you had been made aware of, or did you hear about it from folks in the community? +**Speaker 1:** Awesome! That's good to hear. I've always had a really good experience with the OpenTelemetry operator maintainers, so that's nice to hear as well. Great! -**Steven:** I heard about it actually from a colleague. It was their recommendation, but it fit with the kind of platform ethos of trying to get out of the way of the developers and have a way to do things without the developers having to get involved. +Now, the other thing that I wanted to ask is, are you building your own collector distribution? -**Speaker 1:** Right, right. Yeah, fair enough. That is one thing I like about the operator. Now, what was your experience in running it? Was it something that you felt was straightforward? +**Stephen:** Yeah, we are. That was because when we faced some of those bugs, we couldn't wait until it was pushed to the main distribution, so we forked it just to fix it earlier. Once that happened, we started to make other changes, so it's a bit hard to get off the fork now, but it's something we do want to do. -Because one of the things that we really value about these sessions is not only learning how you're using these things in real life but also learning your user experience with using components of OpenTelemetry. Was it something you felt was documented well enough? Did you have to do a little bit of digging, poke around, ask some questions in the operator Slack? +**Speaker 1:** Now, does that involve using the OTEL collector builder tool when you're deploying your collectors? -[00:12:40] **Steven:** I'd say generally, compared to the collector itself, it was very smooth. I'm not sure why that is, but yeah, it worked quite well. The maintainers were very responsive in the Slack channels when we did have minor issues. +**Stephen:** Yeah, it does. I looked at basically the Dockerfile, how they build the container, and just kind of copied the commands from there. -**Speaker 1:** Awesome! That's good to hear. I've always had a really good experience with the OTEL operator maintainers, so that's nice to hear as well. +**Speaker 1:** Oh, okay, okay. Have you used the actual tool where you specify the components, like which receivers, processors, etc., that you want to include? Have you tried using that tool? -Great! Now, the other thing that I wanted to ask is, when you're using your collectors, are you building your own collector distribution? +**Stephen:** Yeah, yes, that's what we're using to build it. We copied the list of components. -**Steven:** Yeah, we are. That was because when we faced some of those bugs, we couldn't wait until it was pushed to the main distribution, so we forked it just to fix it earlier. Once that happened, we started to make other changes, so it's a bit hard to get off the fork now, but it's something we do want to do. +**Speaker 1:** Oh, okay, okay, got it. Sorry, I misunderstood you. Awesome! You mentioned you did your own customizations, and that kind of forced you to contribute back to the collector. How was the experience of contributing back to the collector overall? -**Speaker 1:** Now, does that involve using the OTEL Collector Builder tool when you're deploying your collectors? +**Stephen:** It was a good experience. People were helpful during the code review. It took a little while to get the code merged; you’d have a separate person for approving it, and then it would sit unmerged for a few weeks, and then you get all these merge conflicts because the versions would be upgraded. I had to go back and update the Go mod files a bunch of times, so that was I think maybe there’s room to merge it a bit faster once someone approves it, but I don’t know the rationale behind that. -**Steven:** Yeah, it does. I looked at the Dockerfile, how they build the container, and just kind of copied the commands from there. +**Speaker 1:** Fair enough, fair enough. And the other thing I wanted to ask: you mentioned that you use auto instrumentation. Have you done any manual instrumentation as well? -**Speaker 1:** Oh, okay, okay. Have you used the actual tool where you specify the components, like which receivers, which processors, etc., that you want to include? Have you tried using that tool? +**Stephen:** Yeah, we had to show teams because we had manual instrumentation using that proprietary format, so we had to show teams how to convert it over to the OTEL format and also educate them on how to use OTEL manual instrumentation. -**Steven:** Yeah, that's what we're using to build it. We copied the list of components. +**Speaker 1:** How did that go? Did you find that teams were receptive to that? -[00:14:50] **Speaker 1:** Oh, okay, okay. Got it, got it. Sorry, I misunderstood you. Awesome! You mentioned you did your own customizations, and that kind of forced you to contribute back to the collector. How was the experience of contributing back to the collector overall? +**Stephen:** It's still, I think they're slowly doing it. I think tracing isn't maybe as obvious, so I think probably I just need more examples of why it's useful to give them the motivation to add more metadata to their spans, for example. -**Steven:** It was a good experience. People were helpful during the code review. It took a little while to get the code merged. I noticed you'd have a separate person for approving it, and then it would sit unmerged for a few weeks, and then you get all these merge conflicts because the versions would be upgraded. So I had to go back and update the Go mod files a bunch of times. +**Speaker 1:** Right, right. Now, does your team ever get asked to instrument stuff for other teams? -That was a bit of a challenge. I think there's room to merge it a bit faster once someone approves it, but I don't know the rationale behind that. +**Stephen:** Not specifically for their applications. -[00:17:00] **Speaker 1:** Fair enough, fair enough. The other thing I wanted to ask, you mentioned that you use auto instrumentation. Have you done any manual instrumentation as well? +**Speaker 1:** I'm very glad to hear that. One of my observability pet peeves! -**Steven:** Yeah, we had to show teams because we had manual instrumentation using that proprietary format. We had to show teams how to convert it over to the OTEL format and also educate them on how to use OTEL manual instrumentation. +**Stephen:** Well, that's good to hear! -**Speaker 1:** And how did that go? Did you find that teams were receptive to that? +**Speaker 1:** And it looks like, in terms of instrumentation, from your diagram, it looks primarily like a Java app, is that correct? -**Steven:** I think they're slowly doing it. I think tracing isn't maybe as obvious, so I probably just need more examples of why it's useful to give them the motivation to add more metadata to their spans, for example. +**Stephen:** Yeah, mostly Java. We do have some Go as well. -**Speaker 1:** Right, right. Now, does your team ever get asked to instrument stuff for other teams? +**Speaker 1:** Okay. Now, how did you find it in terms of the learning curve for instrumenting stuff in OpenTelemetry? How did you find it since you are in a position to teach folks how to instrument their code with OpenTelemetry? -**Steven:** Not specifically for their applications. +**Stephen:** I think it was a big learning curve for our team as far as understanding how the manual instrumentation works for both Java and Go. The Java SDK has done a really good job with it; I think the language just makes it easier. -**Speaker 1:** Yeah, I'm very glad to hear that. One of my observability pet peeves! Well, that's good to hear. +**Speaker 1:** Right. -Now, in terms of the instrumentation, it looks from your diagram that it's primarily a Java app, is that correct? +**Stephen:** We use that Micrometer, which is a Java library for emitting metrics, so they didn't actually have to change anything to get the metrics to work. -**Steven:** Yeah, mostly Java. We do have some Go as well. +**Speaker 1:** Oh, okay, got it. -**Speaker 1:** Okay. Now, how did you find it in terms of the learning curve for instrumenting stuff in OpenTelemetry? Since you're in a position to teach folks how to instrument their code with OpenTelemetry, was it a big learning curve for your team as far as understanding how the manual instrumentation works for both Java and Go? Did you notice anything that was maybe easier in one language compared to the other, or where you had to go back to the SIGs to ask for clarification? +**Stephen:** The span API is pretty intuitive as well. For Go, I mean, it's not my personal strength, but I found it's a bit more manual. It was definitely harder to use than the Java one. -**Steven:** So Java, they've done a really good job with it. I think the language just makes it easier. Because we use Micrometer, which is a Java library for emitting metrics, they didn't actually have to change anything to get the metrics to work. - -The span API is pretty intuitive as well. For Go, I mean, it's not my personal strength, but I found it's a bit more manual. It was definitely harder to use than the Java one. I'm curious if anyone's used the auto instrumentation; I haven't had a chance to play around with that, but it would be nice if we could get that working. +**Speaker 1:** Right. I'm curious if anyone's used the auto instrumentation. I haven't had a chance to play around with that, but that would be nice if we could get that working. **Speaker 1:** For folks who are listening, does anyone have experience with that that would like to share? Feel free to raise your hand if you do have any experience. -No takers? Alright. +**Speaker 1:** No takers? All right. -[00:20:00] Okay, I guess I'm just about done asking the questions. The only other thing I was going to ask is do you have any general feedback that you wanted to provide to the OpenTelemetry maintainers around your OpenTelemetry experience, as far as using components? You mentioned the contribution experience; you said it was a little bit challenging with approving the PRs. Anything else that you wanted to add to that? +### [00:20:00] Manual vs auto instrumentation -**Steven:** I think it felt like we were kind of figuring out a lot of stuff for ourselves. It took us quite a few iterations to get to this point. It might be hard to have a recipe for every combination of vendor and backend, and all that, so that could be part of the problem. +**Speaker 1:** Okay, I guess I'm just about done asking the questions. The only other thing is that I was going to ask is if you have any general feedback that you wanted to provide to the OpenTelemetry maintainers around your OpenTelemetry experience, as far as using components like ease of use. You mentioned the contribution experience; you said it was a little challenging with approving the PRs. Anything else that you wanted to add to that? -I guess more example configuration would be helpful. For example, the load balancing—originally, we actually had another set of collectors that all they did was load balancing, but we realized we can actually load balance this deployment of collectors to itself, and that just is way easier to maintain. +**Stephen:** I think it felt like we were figuring a lot of stuff out for ourselves. It took us quite a few iterations to get to this point. It might be hard to have a test for every combination of vendor and backend and all that, so that could be part of the problem. I guess like more example configurations would be helpful, for example, the load balancing. Originally, we had another set of collectors that all they did was load balancing, but we realized we can actually load balance this deployment of collectors. -Yeah, and I guess more example configuration for common problems, like the tail sampling that we're doing here. +**Speaker 1:** Right, right. -**Speaker 1:** That's really good feedback. I have heard from other folks as well that we have the OpenTelemetry demo, which is a great showcase. I think it's really focused on application instrumentation, which is awesome, but I think there is definitely a desire from the community for more collector-specific use cases, and specifically stuff that's a little bit more complex than what you would necessarily see in the demo. +**Stephen:** That just is way easier to maintain. I guess more example configurations for common problems like tail sampling that we're doing here would be helpful. -**Steven:** For sure. Another one is monitoring your collectors. I found I had to dig through the codebase to find what metrics are exposed and what they are doing. I feel like there are probably some common alerts or things that for most setups would be useful. +**Speaker 1:** That's a really good piece of feedback. I have heard from other folks as well that we have the OpenTelemetry demo, which is a great showcase focused on application instrumentation, but there is definitely a desire from the community for more collector-specific use cases, especially for things that are a little more complex than what you would necessarily see in the demo. -Maybe someone already has this, but like, you know, some documentation like, "Hey, these are useful things to monitor." The dashboard for Grafana they have was pretty useful, but I think on the alert side, I didn’t find anything there. +**Stephen:** For sure. Another one is monitoring your collectors. I found I had to kind of dig through the codebase to find what metrics are exposed, like what are they doing. I feel like there are probably some common alerts or things that would be useful for most setups. Maybe someone already has this, but hey, these are useful things to monitor. The dashboard for Grafana they have was pretty useful, but I think on the alert side, I didn't find anything there. -**Speaker 1:** Right, right. Cool! Well, thank you for the feedback on that. Now, I guess we can do one of two things. We've got the Lean Coffee board where folks can post questions for Steven. I'm actually going to check the board. +**Speaker 1:** Right, cool. Thank you for the feedback on that. Now, I guess we could do one of two things. We've got the lean coffee board where folks can post questions for Steven, so I'm actually going to check the board. -Oh, we have some questions. Yeah, let's go through that, and then if we have some time afterwards, then Steven, you can ask some questions to our viewers as well. +**Speaker 1:** Oh, we have some questions! Yeah, let's go through that, and then if we have some time afterwards, then Steven, you can ask some questions to our viewers as well. -Dan, do you want to take it on from here on the questions from the board? +**Speaker 1:** Dan, do you want to take it on from here on the questions from the board? **Dan:** Sure thing! We've got many questions from the audience, which is great. Also, feel free to go and vote for the ones that you think are the most interesting. -I have marked one of them as answered already because we talked about the collector builder and how you build your own dist for the collector on your sidecar. - -Starting from the one that we talked about tail sampling, a little bit about how you use the tail sampling, but I wanted to ask this question here: do you only use tail-based sampling, or do you also use head-based, and any tips or thoughts on mixing these approaches? - -**Steven:** Yeah, so we only use tail sampling. I guess it's more work to get it set up, but when you have the full trace to make your sampling decision, I find that you can make the best decision. That's why we lean on tail sampling completely. I guess with head-based sampling, there's always a risk you're going to miss something important, right? Because it's completely probabilistic. +I have marked one of them as answered already because we talked about the collector builder and how you build your own distribution for the collector on your sidecar. Starting from the one that we talked about tail-based sampling: do you only use tail-based sampling, or do you also use head-based? Any tips or thoughts on mixing these approaches? -**Dan:** Good point! Moving on to the next one: how do you handle container sizing on the collector sidecars? You mentioned this already, that you had to customize or tune one of the collector sizes. Do you have one size that fits all, or anything you would like to improve in that setup, or resource allocation? +**Stephen:** Yeah, so we only use tail sampling. I guess it's more work to get it set up, but when you have the full trace to make your sampling decision, I find that you can make the best decision. That's why we just lean on tail sampling completely. I guess with head-based sampling, there's always a risk you're going to miss something important because it's completely probabilistic. -**Steven:** I think what would be nice is if pods could add an annotation saying this is the resources I need. Instead, we have to do something kind of hacky. For each sidecar, it's a separate Kubernetes resource or configuration file. +**Dan:** Right, good point! Moving on to the next one: how do you handle container sizing on the collector sidecars? You mentioned this already, that you had to customize or tune one of the collector sizes. Do you have one size that fits all, or anything you would like to improve in that setup or resource allocation? -So what we do is we have a for loop that goes through and creates those configurations for different resource combinations. It's something we have to coordinate, like, "Hey, you have to update the sidecar configuration that your pod wants to use to this OTEL sidecar, 500 megabytes, one CPU." It's a bit of extra coordination work there. +**Stephen:** I think what would be nice is if pods could add an annotation saying this is the resources I need. Instead, what we have to do is a bit hacky. For each sidecar, it's like a separate Kubernetes resource or configuration file. We have a for loop that goes through and creates those configurations for different resource combinations. So, it's something we have to coordinate like, "Hey, you have to update the sidecar configuration that your pod wants to use for this OTEL sidecar, 500 megabytes, one CPU." It's a bit of extra coordination work there. -**Dan:** Cool! The next one is related to serverless. Do you use serverless, and any tips or go-to's to look out for in instrumenting serverless workloads with OpenTelemetry? +**Dan:** Cool! The next one is related to serverless. Do you use serverless, and do you have any tips or go-tos for instrumenting serverless workloads with OpenTelemetry? -**Steven:** We don't use serverless, so I don't have any advice there. +**Stephen:** We don't use serverless, so I don't have any advice there. -**Dan:** That's fine! I wanted to ask that in case you had some experience with that. Also, you mentioned you use FluentD for logs reading from standard out. Are you considering moving to the OTEL log file receiver? +**Dan:** That's fine! I wanted to ask that in case you had some experience with that. Also, you mentioned that you use FluentD for logs reading from standard out. Are you considering moving to the OTEL log file receiver? -**Steven:** Yeah, it's something I'd like to explore. We sort of had a bit of an issue when we first tried using logs. I'm curious if anyone has found a good solution for this. What we wanted to do is store our logs on disk so if there's an outage, we don't lose them. We ran into a bug with the durable storage extension in the collector, and that kind of spooked us. We already had FluentD running, so that's why we kind of just pivoted to that. But I don't know; that was a while ago. Maybe people have had good success with getting that durability for their logs. +**Stephen:** Yeah, it's something I'd like to explore. We sort of had a bit of an issue when we first tried using logs. I'm curious if anyone has found a good solution for this. What we wanted to do is store our logs on disk so if there's an outage, we don't lose them. We ran into a bug with the durable storage extension in the collector, and that kind of spooked us. We already had FluentD running, so that's why we kind of just pivoted to that. But I don't know, that was a while ago, and maybe people have had good success with getting that durability for their logs. -**Dan:** Just to add on this, with FluentD, are you decorating your logs with some of the semantic conventions or trace ID, span ID, so you can then correlate with your traces? +**Dan:** Just to add on this, with FluentD, are you decorating your logs with some of the semantic conventions or trace ID, span ID, that you can then correlate with your traces? -**Steven:** Yeah, so we definitely have the trace IDs and the span IDs, and we're actually adopting a specification for the event—they call it log events. It's sort of a structured format for logs in the OpenTelemetry specification, so we're trying that out as well. +**Stephen:** Yeah, we definitely have the trace IDs and the span IDs, and we're actually adopting a specification for the event—they call it log events—so it's sort of a structured format for logs in the OpenTelemetry specification. So, we're trying that out as well. **Dan:** Cool! Do you use OTEL to monitor OTEL collectors? -**Steven:** Yeah, it's like a big—it's always called monitoring your monitoring. We use Prometheus to get the metrics from the collectors and FluentD to collect the logs from the collectors, and that’s worked pretty well. - -**Dan:** Okay, moving on to a bit more of your team. Sounds like you folks have a dedicated platform team that manages the OTEL collectors in your organization. Do you have any thoughts on the opposite of this, where you have no collector gateway or no collector endpoint, basically 100% of your telemetry data is then sent to your observability provider? - -**Steven:** I guess in our case, we decided to use the collectors so that we could do sampling to control costs of our traces. We also try to have some durability; if the vendor goes down, we can move that retry logic off to the collector. - -I'm curious if anyone has done that in production. It would definitely simplify things, but I guess there are some tradeoffs as well. - -**Dan:** Right, there are tradeoffs as well of authentication to a provider. It does sell in one place as well, some of the benefits, pros and cons I guess. - -The next one is related to auto instrumentation and the OTEL operator. You inject those instrumentation packages, and we also talked about the fact that you are also using manual instrumentation. I guess the question is, how do you find if your teams found that auto instrumentation is sufficient, or is there a lot of custom instrumentation that's needed? - -**Steven:** One thing I haven't talked about that has been great for us is the span metrics component. We generate metrics—frequency, latency, error counts—for every single span. +**Stephen:** Yeah, it's like a big monitoring your monitoring, but we use Prometheus to get the metrics from the collectors and FluentD to collect the logs from the collectors, and that's worked pretty well. -Because we're using the auto instrumentation, every application emits the spans in the same format. Out of the box, without adding any instrumentation, we get some what folks call APM or RED metrics—like all your API calls, all your database queries. We have metrics on that automatically. +**Dan:** Okay, moving on to a bit more about your teams. It sounds like you folks have a dedicated platform team that manages the OTEL collectors in your organization. Do you have any thoughts on the opposite of this, where you have no collector gateway or no collector endpoint, basically 100% of your telemetry data is then sent to your observability provider? -That goes quite a long way, and we can build common dashboards and common alerts. All these apps just get out of the box because of that standardization, so it's a really good starting point. If teams want to add metadata, they can. I would definitely recommend span metrics if that's an option for people. +**Stephen:** Yeah, I guess in our case, we decided to use the collectors so that we could do sampling to control costs of our traces and also to try and have some durability. If the vendor goes down, we can kind of move that retry logic off to the collector. I'm curious if anyone is doing that in production. It would definitely simplify things, but I guess there's some trade-offs as well. -**Dan:** I'm going to add a bit to this one as well. Do you find that if you're moving from a world where you had to instrument everything, and now there is a lot of auto instrumentation out there, do you find that engineers normally tend to not know about that auto instrumentation, and they do it themselves? How do you manage that sort of education piece, saying that you may not need to instrument things? Has that been a challenge within your organization, or has everyone just...? +**Dan:** Right, there's a trade-off as well of authentication to a provider and having everything centralized. -**Steven:** I'd say it is. We have some redundant, like for example our API metrics—Spring Boot and Java automatically emit some metrics, so a lot of applications were getting some redundant data. So yeah, that has been a problem, but I guess it just comes down to education. +**Stephen:** Right. -Then having a way of tracking the cardinality, when we've seen, "Oh, you're sending a ton of API metrics, and we already captured the span metrics," we can reach out to that team and say, "Oh, you can probably disable this. You don't really need this. We don't even pay for it." +**Dan:** The next one is related to auto instrumentation and how you inject those instrumentation packages. We also talked about the fact that you are using custom manual instrumentation. I guess the question is: how do you find if your teams found that auto instrumentation is sufficient, or is there a lot of custom instrumentation that's needed? -**Dan:** Cool! Regarding sampling, I'm assuming this is related to sampling policies. Can you advise on whether that’s the collector where you manage the policies for all traces, or can teams dial that config up and down themselves? +**Stephen:** One thing I haven't talked about that has been great for us is the span metrics component. We generate metrics for frequency, latency, error counts for every single span. Because we're using the auto instrumentation, every application emits the spans in the same format, so basically out of the box, without adding any instrumentation, we get some metrics, like APM or RED metrics for all your API calls and database queries. We have metrics on that automatically, so that goes quite a long way, and we can build common dashboards and common alerts that all these apps just get out of the box because of that standardization. -**Steven:** Ours is pretty simple. We sample errors; we keep traces that are longer than something like five seconds. We do have an escape hatch if they add an attribute to their span to force sampling. So far, no one has caused any problems with that, but it might in the future. Teams may overuse it, but that works so far. +**Dan:** That's really cool! So, if teams want to add metadata, they can, but you would definitely recommend span metrics if that's an option for people. -**Dan:** Related to some of that SDK config that you currently inject through environment variables, do you currently use any type of other config that may not be supported through environment variables, like metric views, for example? Do you have a way to apply defaults across the board for that type of config? +**Stephen:** Yeah, I would definitely recommend span metrics. -**Steven:** We haven't had to do that too much, and when we have, we do have—our company is lucky in that we have a tool that can use to generate applications from scratch. There’s a way to sort of regenerate it. +**Dan:** I'm going to add a bit to this one as well: do you find that if you're moving from a world where you had to instrument everything, and now there is a lot of auto instrumentation out there, do you find that engineers normally tend to not know about that auto instrumentation and they do it themselves? How do you manage that learning or education piece, saying that you may not need to instrument things? Has it been a challenge within your organization? -The regeneration doesn't work too well, but we know that most teams have a pretty similar format to their application, so when we do need to make a change that we can't do with the operator, it's usually just boilerplate copy-paste. We can show them a merge request, and they can copy it in. But it does take a couple of months for every team to adopt that. +### [00:32:00] Feedback for OpenTelemetry maintainers -**Dan:** Cool! Okay, I guess one related to supply chain security with the OTEL SDKs. How do you handle supply chain security so that the OTEL SDKs are safe to include them in application code, or do you build them from source yourself and then run your own security tests? +**Stephen:** I would say it is. We have some redundant metrics; for example, our API metrics like Spring Boot and Java automatically emit some metrics, so a lot of applications were getting some redundant data. So, yeah, that has been a problem, but I guess it just comes down to education and then having a way of tracking the cardinality. When we've seen that, like, "Oh, you're sending a ton of API metrics," and we already have captured the span metrics, we can reach out to that team and say, "Oh, you can probably disable this; you don't really need this." -**Steven:** Are you thinking this would be, you know, if there's a CVE in one of the SDK versions, how would we sort of manage patching that for everyone or downstream or upstream dependencies from OTEL packages? +**Dan:** Cool! Regarding sampling, I'm assuming this is related to sampling policies. Can you advise on whether the collector is where you manage the policies for all traces, or can teams dial that config up and down themselves? -Yeah, for different reasons, I kind of created a wrapper dependency that wraps all the OTEL library versions we use and we know they work together just because we found some versions were kind of incompatible. +**Stephen:** Ours is pretty simple. We sample errors; we keep traces if they are longer than something like five seconds. We do have an escape hatch if they add an attribute to their span to force sampling. So far, no one has caused any problems. It might in the future, you know, teams overuse it, but that works so far. -From the app team's perspective, they depend on one dependency, and then they get all the OTEL dependencies as transitive dependencies. If we need to upgrade it very quickly, I guess it would just be one thing they need to update. +**Dan:** Related to some of that SDK config that you currently inject through environment variables, do you currently use any type of config that may not be supported through environment variables, like metric views, for example? Do you have a way to apply defaults across the board for that type of config? -Although that works in theory, it has also caused dependency conflicts just because of how Maven works in Java, so I don't know if that's actually the best thing to do—it might be more of a headache than it's worth. +**Stephen:** We haven't had to do that too much, and when we have, we do have—our company's lucky in that we have a tool that we can use to generate applications from scratch. There's a way to sort of regenerate, but it doesn't work too well. Most teams have a pretty similar format to their application, so when we do need to make a change that we can't do with the operator, it's usually just boilerplate copy-paste. We can show them a merge request, and they can copy it in, but it does take a couple of months for every team to adopt that. -**Dan:** Thank you! Okay, moving on to the next one. Related to the OTEL operator and automatic instrumentation: is that your preferred solution, or can the OTEL operator be used in parallel with SDKs, for example, in the case of .NET? Do you provide some balance of scalability and configurability? Is there a recommendation here? +**Dan:** Cool! I guess one related to supply chain security with the OTEL SDKs: how do you handle supply chain security so that OTEL SDKs are safe to include them in application code? Do you build them from source yourself and then run your own security tests? -**Steven:** My immediate concern is that the manual instrumentation inside the application could make it difficult to configure and manage at scale. So far, we do—you can't override the environment variables that are injected into the application automatically by the platform team. +**Stephen:** For different reasons, I kind of created a wrapper dependency that wraps all the OTEL library versions that we use, and we know they work together. Just because we found some versions were kind of incompatible. From the app teams' perspective, they depend on one dependency, and then they get all the OTEL dependencies as transitive dependencies. I suppose if we need to upgrade it very quickly, I guess it would just be one thing that they need to update. Although that creates dependency conflicts just because at least how Maven works in Java, so I don't know if that's actually the best thing to do; it might be more of a headache than it's worth. -So far, I guess it hasn't been too much of a problem. Teams haven't really wanted to deviate too much from the configuration we've given them, but I’d be curious long term if that does cause problems. +**Dan:** Thank you! Okay, moving on to the next one: related to the OTEL operator and automatic instrumentation, is that your preferred solution, or can the OTEL operator be used in parallel with SDKs, for example, the .NET case? Do you provide some balance of scalability and configurability? Is there a recommendation here? -**Dan:** Thank you! This one is interesting: are your devs using the LGTM Docker container from Grafana to develop OTEL implementation and then do local troubleshooting? +**Stephen:** I mean, so far, you can't override the environment variables that are injected into the application automatically by the platform team. So far, it hasn't been too much of a problem; teams haven't really wanted to deviate too much from the configuration that we've given them. But I'd be curious longer term if that does cause problems. -**Steven:** Not yet. Right now, they can only really test it by deploying it to our clusters, so we do need a better solution there so they can test locally. +**Dan:** Thank you! This is an interesting one: are your devs using the LGTM Docker container from Grafana to develop OTEL implementation and then do local troubleshooting? -**Dan:** I personally love that there are more and more solutions coming up that are open-source that allow you to see that telemetry in your own computer for testing. So yeah, a lot of really cool solutions out there at the moment. +**Stephen:** Not yet. Right now, they can only really test by deploying it to our clusters, so we do need a better solution there. -If you had a magic wand, what's the thing that you would love to add, remove, or change in OTEL and why? +**Dan:** I personally love there are more and more solutions coming up that are open source that allow you to see that telemetry on your own computer. -**Steven:** That is a logic question! I think this is maybe short-sighted, but if we could get rid of Prometheus in our architecture, the reason we can't right now is because we rely on Prometheus to aggregate our data together for cost savings. +**Stephen:** Yeah, a lot of really cool solutions out there at the moment. -For span metrics, we generate like 50 histogram buckets for every span, so it gets really expensive. If each pod has its own set of 50 buckets, our cost scales with the number of pods. But we use Prometheus to aggregate all the counts from all the pods, and that cuts our cost significantly. +**Dan:** If you had a magic wand, what's the thing that you would love to add, remove, or change in OTEL and why? -I don't think the collector can do that today, so yeah, that would make our architecture simpler. +**Stephen:** That's a logic question! I think this is maybe short-sighted, but if we could get rid of Prometheus in our architecture—the reasons we can't right now is because we rely on Prometheus to aggregate our data together for cost savings. Source span metrics—we generate like 50 buckets, histogram buckets for every span, so it gets really expensive. If each pod has its own set of 50 buckets, our costs scale with the number of pods. We use Prometheus to aggregate all the counts from all the pods, and that cuts our cost significantly. I don't think the collector can do that today, so that would just make our architecture simpler. **Dan:** Makes sense! Are you using multiple OTEL backends to process some of the things internally compared to your vendor, or is everything going to the vendor? -**Steven:** We have one vendor if that was the question that we're pushing to. +**Stephen:** We have one vendor, if that was the question, that we're pushing to. -**Dan:** So I guess you're not pushing anything internally to anywhere else; just everything goes to... +**Dan:** I guess you're not pushing anything internally to anywhere else; everything goes to the vendor. -**Steven:** Oh right, yeah, we don't. +**Stephen:** Right, yeah. -**Dan:** Okay! I think we've got time for one last one, and that is: given that the collector can quickly become a bottleneck due to back pressure, do you see any disadvantages to having the SDK export that data directly to the backend, except for the cost? +**Dan:** I think we've got time for one last one, and that is: given that the collector can quickly become a bottleneck due to back pressure, do you see any disadvantages to having the SDK export that data directly to the backend except for the cost? -**Steven:** Yeah, I think that was similar to the other question about why not just push directly to your vendor from your pod. It does make it easier to kind of scale; you don't have to worry. Well, yeah, that's a good question. I guess you're shifting the back pressure then to your app. +**Stephen:** Yeah, I think that was similar to the other question about why not just push directly to your vendor from your pod. It does make it easier to scale. You don't have to worry—well, yeah, that's a good question. I guess you're shifting the back pressure then to your app. -It might be easier to handle if it's a smaller amount of data, but yeah, that one I'm not too sure about. +**Dan:** Right! -**Dan:** That would be interesting to discuss. It might be easier to handle but also more impactful for the application. I guess there are pros and cons there as well, right? +**Stephen:** It might be easier to handle if it's a smaller amount of data, but I don't know. I'm not too sure about that. -So if you have all your telemetry shipped from your pod as soon as possible, there are benefits for that as well in terms of impacts. +**Dan:** Yeah, that would be interesting to discuss! -**Steven:** Yeah, exactly! +**Stephen:** Yeah, maybe easier to handle but also more impactful for the application, so I guess there are pros and cons there as well. -**Dan:** Okay, I think that's all we have time for. Thank you very much, Steven, for joining us. We had quite a lot of questions we went through from the audience. Thanks for everyone that joined and asked questions as well. It's super nice to see engagement in a session Q&A. +**Dan:** Right, if you have all your telemetry shipped from your pod as soon as possible, there are benefits for that as well, in terms of impacts. -Do you want to say a few words to say goodbye? +**Stephen:** Right. -**Steven:** Yeah, thanks, Dan, and thanks for doing the questions. It's really nice to see the engagement from folks in the Q&A. I think this is a topic that really interests people, so we love it when you engage. Also, if anyone out there would like to participate in OTEL Q&A or OTEL in practice, we are always looking for folks to share their use cases with us. +**Dan:** I think that's everything. Thank you very much, Stephen, for joining us. That was quite a lot of questions we went through from the audience. Thanks to everyone that joined and asked questions as well. Super nice to see the engagement in this Q&A session. Do you want to say a few words to say goodbye? -You can DM either Dan Reyes or me, or if you want, you can also message us in the user channel in the CNCF Slack. We would be happy to hear your use cases and have conversations. Also, as I mentioned, the recording for this will be up on YouTube, so anyone that you know that wishes they had attended but could not, let them know. +### [00:42:52] Q&A session with audience questions -We usually post on the OTEL socials and also in the channel once the video is up. Thank you, Steven, for joining us and answering all the various questions. We really appreciate your time. +**Stephen:** Yeah, thanks, Dan, and thanks for doing the questions. It's really nice to see the engagement from folks in the Q&A. I think this is a topic that really interests people, so we love it when you engage. Also, if anyone out there would like to participate in OTEL Q&A or OTEL in practice, we are always looking for folks to share their use cases with us. You can either DM either Dan or me, or you can also just message us in the user channel in the CNCF Slack. We would be happy to hear your use cases and have conversations. Also, as I mentioned, the recording for this will be up on YouTube, so anyone that you know that wishes they had attended but could not attend, let them know. We usually post on the OTEL socials and also in the channel once the video is up. Thank you, Steven, for joining us and answering all the various questions. We really appreciate your time! -**Steven:** Yeah, that was a lot of fun! Thank you for hosting this. +**Steven:** Yeah, that was a lot of fun. Thank you for hosting this! ## Raw YouTube Transcript diff --git a/video-transcripts/transcripts/2024-11-04T21:03:07Z-otel-q-a-with-dan-ravenstone.md b/video-transcripts/transcripts/2024-11-04T21:03:07Z-otel-q-a-with-dan-ravenstone.md index 4eb3a83..5a3e940 100644 --- a/video-transcripts/transcripts/2024-11-04T21:03:07Z-otel-q-a-with-dan-ravenstone.md +++ b/video-transcripts/transcripts/2024-11-04T21:03:07Z-otel-q-a-with-dan-ravenstone.md @@ -10,358 +10,204 @@ URL: https://www.youtube.com/watch?v=BMkckCQN8eg ## Summary -In this YouTube video, Dan Ravenstone, a staff engineer at Top Hat, discusses his extensive experience in the fields of monitoring and observability, particularly focusing on the transition to open telemetry. The conversation is led by Adrian, who shares insights on the upcoming Cucon event in Salt Lake City. Dan introduces his background, highlighting his journey from traditional monitoring tools to modern observability practices, and emphasizes the importance of understanding user experience as a key driver for effective observability. The discussion also tackles challenges organizations face in adopting observability, including cultural resistance, the importance of communication, and the need for developers to take ownership of instrumentation. Dan shares anecdotes from his experience at Top Hat, where they are migrating to open telemetry and highlights the necessity of aligning observability practices with user needs to drive meaningful outcomes. Throughout the Q&A, they address various audience questions related to telemetry quality, instrumentation strategies, and the impact of user experience on organizational priorities. +In this YouTube Q&A session, Dan Ravenstone, a staff engineer at Top Hat, discusses the transition from monitoring to observability, sharing his extensive experience in the field. The conversation, led by Adrian, touches on the significance of open telemetry and the challenges organizations face in adopting it, particularly concerning cultural shifts and communication within teams. Dan highlights the importance of understanding user experiences when determining what to instrument and how to implement observability effectively. He emphasizes that more data does not necessarily equate to better observability and discusses the need for teams to move away from outdated practices based on logs to leverage the full benefits of distributed tracing. Overall, the discussion underscores the necessity of aligning observability practices with user-centric approaches to ensure service reliability and business success, especially in a field where user experience directly impacts financial outcomes. ## Chapters -00:00:00 Welcome and intro -00:01:20 Dan Ravenstone introduction -00:03:40 Observability evolution -00:04:00 Challenges in alerting -00:07:00 User experience and observability -00:09:30 Observability adoption challenges -00:10:40 Vendor-specific libraries issues -00:12:50 Cultural shifts in instrumentation -00:20:00 Communication challenges -00:30:10 Instrumentation culture at Top Hat -00:34:22 Collector landscape discussion -00:40:11 Questions from the audience +00:00:00 Introductions +00:06:30 Guest introduction: Dan Ravenstone +00:10:15 Discussion on monitoring vs observability +00:15:47 Importance of user experience in observability +00:19:50 Challenges in adopting open telemetry +00:25:30 Discussion on vendor libraries vs open telemetry +00:30:05 Communication challenges in observability +00:34:30 Cultural shifts in instrumentation +00:40:00 Instrumentation culture at Top Hat +00:45:30 Q&A session with audience questions -**Host:** Welcome everyone to the Otel Q&A, and thanks Dan for joining us for Otel Q&A, especially on such short notice. I'm glad we're able to squeeze you in for October, especially with KubeCon coming up in November. So for folks who are going, a few of us will be there. Dan and I will be there from the Otel team and user Sig, so it'll be busy. +## Transcript -**Dan:** Yeah, it's going to be busy. When is that happening, by the way? +### [00:00:00] Introductions -**Host:** It's in Salt Lake City. +**Speaker 1:** welcome everyone to otel Q&A and thanks Dan for joining us for otel Q&A especially on we got you on such short notice. I'm glad we're able to squeeze you in for October, especially me too. Cucon is coming up in November, so for folks who are going, a few of us will be there. Dan and I will be there from theel and user Sig, so it's going to be busy. -**Dan:** Salt Lake City. Nice! I've never been there myself personally. +**Dan:** yeah, it's going to be busy. Where is that happening by the way? -**Host:** Yeah, me neither. +**Speaker 1:** it's in Salt Lake City. -**Dan:** Oh, so this will be an experience for you too. +**Dan:** Salt Lake City, nice. I've never been there myself personally. -**Host:** Yeah, looking exciting. +**Speaker 1:** yeah, me neither. -**Dan:** Um, okay, let's start asking you questions. I realize I'm the one here who is supposed to be with the breaks. Folks, it's going to be a fun one. +**Dan:** oh, so this be an experience for you too? -**Host:** Oh yeah, not sure about informative, but it will be fun. +**Speaker 1:** yeah. -**Dan:** Well, here we go. Okay, let's start. Why don't you introduce yourself and tell us what you do? +**Dan:** exciting. -[00:01:20] **Dan:** Hello, I am Dan Ravenstone. I am a staff engineer at Top Hat. I've been there for, I guess, two and a half years maybe? No, a little bit longer than that. Come May, I think it'll be three. A little bit of my background: I have been doing monitoring and observability for the bulk of my technical career. I started back in, um, where I actually refer to as like where the bug bit, the monitoring bug bit me, was when I was at a company called Affiliat, which was at Young and 401 area in Toronto. Affiliat managed the info.org domain registry in the back end of it, for back in the day. I started there in tech support, but then I one day opened up my big mouth and said, "Hey listen, I'm tired of our customers calling us up and telling us our stuff is down. Can we fix this?" And so they said, "Well then you ask for it, you get it." So I went out and that's how I started learning about monitoring tools. +**Speaker 1:** um, start asking you questions. I realize I'm the one here supposed to be with the breaks. Folks, it's gonna be a fun one. -**Dan:** And that's back in the day of Big Brother, Nagios, Cacti, all those fun things. And I just loved it. This kind of like, I think one of the things I liked about monitoring in those days was that I like looking for patterns. I like detective novels; I like all the fun things. And what this does is it kind of allows for somebody to be on the front lines in operations and dive in and find problems and report them back. +**Dan:** oh yeah, not sure about informative but it will be fun. -**Dan:** The better your monitoring tools were, the more likely you were able to follow and find the problems a lot sooner before they actually become customer impact. And that has always been my kind of push when I did this. And of course, as things progressed and modernized and changed, we got into the wonderful world of observability and a whole different way of looking at it. I fell in love with that obviously as well and tried to grow with the times and see how being proactive is a lot better than being reactive, which is kind of like, you would think is an obvious thing, but it takes a lot of people time to kind of get their heads wrapped around. +**Speaker 1:** well here we go. Okay, let's start. Uh, let's start with the question. So Dan, why don't you introduce yourself and tell us what you do? -**Dan:** So yeah, that is kind of how I got to this part of the world and a little bit about me. There's tons of other things, but we don't want to go into those because then we'll get lost in the weeds and we'll never get out and we'll be lost forever. +**Dan:** hello, I am Dan Ravenstone. I am a staff engineer at Top Hat. I've been there for, I guess, two and a half years maybe? No, a little bit longer than that. Come May, I think it'll be three. A little of my background, I have been doing monitoring observability for the bulk of my technical career. I started back in um, where I actually this and I usually refer to as like where the bug bit, the monitoring bug bit me was when I was at a company called affiliat, which was at Young in 401 area in Toronto. Affiliat managed the info.org regist domain registry in the back end of it, uh for back in the day. I started there in texport, but then I one day opened up my big mouth and said, "Hey listen, I'm tired of our customers calling us up and telling us our stuff is down. Can we fix this?" And so they said, "Well then you ask for it, you get it." So I went out and that's how I started learning about monitoring tools. And that's back in the day of Big Brother, Nagios, Cacti, all those fun things. And I just loved it. This kind of like, I think one of the things I liked about monitoring in those days was that like I like looking for patterns. I like detective novels. I like all the fun things. And what this do is like kind of allow for somebody to kind of like be in the front lines in the operations and dive in and find problems and report them back. And then like the better, more, the better your monitoring tools were, the more likely you were able to follow and find the problems a lot sooner before they become actually customer impact. And I was always been my kind of like push when I did this. And of course, as things progressed and modern and changed, we got into the wonderful world of observability and a whole different way of looking at it. And I fell in love with that obviously as well and tried to grow with the times and see how I, you know, how being proactive is a lot better than being reactive, which is kind of like, you would think is an obvious thing, but it takes a lot of people time to kind of get their heads wrapped around. Um, so yeah, that is kind of uh how I got to the, how I got to this part of the world and a little bit about me. There's tons of other things, but we don't want to go into those because then we'll get lost in the weeds and we'll never get out and we'll be lost forever. -[00:03:40] **Host:** Sounds good! Well, thanks. And, you know, you've been in this for a while. Like, when it started out with monitoring, evolved into observability. What kinds of big changes have you seen in the course of that time? +**Speaker 1:** sounds good. Well, thanks. Um, and you know, like you've been in this for a while, like when it started out with monitoring evolved into observability, what kinds of like big changes have you seen um in the course of that time? -[00:04:00] **Dan:** Let me start off with the things I haven't seen change, which is still a problem: alerting. Okay, alerting still has not changed, surprisingly. But what has changed, and I think this is one of the things that we're trying to get to—I focus on alerting because I was giving myself time to think—what has changed though in observability and in OpenTelemetry especially, too, is it allowed for us to actually look at our services a lot differently where monitoring was kind of a very largely reactive thing and we only had the symptoms of what was going on. +### [00:06:30] Guest introduction: Dan Ravenstone -**Dan:** So if everybody has ever used Nagios, they're well aware of the sort of out-of-the-box HTTP monitoring sort of plugin you usually include, which is like your CPU, your memory, your disk space, and these things have been carried throughout time to be indicators of a problem. But that's usually not quite the case anymore, is it? Especially when we have Kubernetes and containers and other tools now that these things shouldn't really play a role for us. So it's more like, what else is impacting the service? We never really had the ability to look at it until we started aggregating our logs. And even when we aggregated our logs, it was still strangely problematic because, you know, we had guidelines on how to do structured logging and that kind of thing, but still people would do their own thing. +**Dan:** uh, let me start off the things I haven't, I haven't seen change, which is still a problem, alert. Okay, alerting still has not changed surprisingly. Um, but what has changed, and I think this is one of the things that um we're trying to get to, I, I so I focus on alerting because I was by myself time to think. What has changed? So now that you know my secret, I just thought myself a couple more minutes. So, but what has changed though in observability and in the open Telemetry especially too is it allowed for us to actually look at our services a lot differently where monitoring was kind of a very largely reactive kind of thing, and we only had the symptoms of what was going on. So if everybody has ever used Nagios, is well aware of the sort of out of the box HST monitoring sort of plugin you usually include, which is like your CPU, your memory, your disc SP, and these things have been carried throughout time in, you know, to con be indicators of a problem, but that's usually not quite the case anymore, is it? Um, especially way when we have Kubernetes and containers and other tools now that these things shouldn't really play a role in us. So it's like more like, so what else is impacting the service? And we never really had those, that ability to look at it until we started sort of aggregating our logs. And even when we aggregated our logs, it was still strangely problematic, um, because you know, we had guidelines on how to do structured logging and that kind of thing, but still people would do their own thing and things were never, and so even with all the good documentation in sort of best practice around it, we still logs were still problematic. And again, you still have to still collect a bunch of data before you can actually understand, "Oh wait a minute, this is a problem," before actually kind of knowing beforehand. What observability has done has taught us is how to sort of like set things up in a more of a way that we're looking at it as an overall user experience and we don't get sort of sucked into minutia sometimes. And there's because there's a lot of red herrings in operations when you're especially when you're in the middle of incident, you're looking at all this data and you're like, "What is the actual problem?" We have high CPU over here and we have slow load times over here, we have latency here, we have some errors here, but which one of these is the smoking, I would not, I don't want to use a gun, but for lack of a better word, smoking gun, right? And I see that with observability has changed how we look at that. However, that's for this probably this group a lot of people are familiar with it. There's a whole world of engineers out there who are still having, are still trying to get their heads wrapped around this concept. And this is why I'm glad we have these conversations and I try to get those people, this is like, this is targeted towards them of why this is so important, why you need to start looking at this because there's a number of good things. One, you know what the user is experiencing and that's all that matters. If you know what the user is experiencing, that means you can translate that to how much you're making, like very, very, like very, it's a cost thing and it's a money thing. If you know if your users are happy, that means you're making money because they're spending money because they're going to keep using your service. If they're not happy, they're not going to be using your service and they're going to walk away and use something else or just not use it that often and try to avoid as much as possible. That means you're not gonna be making the money anyway. Um, and I'm gonna get in the weed, so I'll, you're gonna hear me say that a lot because I just do. So you Adrian, it's all up to you to rope me back in. -**Dan:** And things were never, and so even with all the good documentation and sort of best practices around it, logs were still problematic. And again, you still have to collect a bunch of data before you can actually understand, oh wait a minute, this is a problem before actually kind of knowing beforehand what observability has taught us is how to sort of like set things up in more of a way that we're looking at it as an overall user experience. We don't get sort of sucked into minutia sometimes, and there's because there's a lot of red herrings in operations, especially when you're in the middle of an incident. You're looking at all this data and you're like, what is the actual problem? We have high CPU over here, and we have slow load times over here, we have latency here, we have some errors here, but which one of these is the smoking— I would not want to use a gun, but for lack of a better word, smoking gun, right? +**Speaker 1:** don’t worry, don’t worry. But I do wanna, um, you said something really interesting, um, in terms of translating it, um, in terms of money because you know, like observability adoption and maybe I'm getting ahead of myself here, but I like the way you said, um, in terms of observability adoption, like it's as much a top-down thing as a bottom-up thing. And one way to speak to the benefits of observability is in dollars and cents and executives speak in dollars and cents. So, um, proving the worth of having an observable system I think becomes really, really important and being able to use that language, um, with executives to be able to convince them I think is very valuable. -**Dan:** And I see that with observability has changed how we look at that. However, that's for this group; a lot of people are familiar with it. There's a whole world of engineers out there who are still trying to get their heads wrapped around this concept. And this is why I'm glad we have these conversations. I try to get those people, this is targeted towards them, of why this is so important, why you need to start looking at this. +**Dan:** so, and, and that's what I've been trying to focus more on. And like there are like there's quantifiable, quantifiable reasons why to look at this. We can do that why observability, excuse me, uh, provides value toward us at the end of the day. But I think we're still getting our, we're still trying to get that figured out, I feel like that. Um, anyway, you know what before we jump in, let's keep moving forward because I, I, I could go into a whole other area and I don't know, I don't want to sort of suck a lot of time in there, but it is one of those things I think that does have, is worth having conversations not with just those in the community but also talk, like I mean if you're, you know out there find out what your what the observability sort of platform or the concepts are your where you work, what are the drivers, why aren't you doing it? Why aren't your, why aren't you using these things? And ask those questions and then like, you know, that maybe we could, that could help and feed that back to community because that could help us build out the reasons why you should be doing this because it does make better sense eventually. Um, but yeah, that's a sort of a general call to anyone who's out there who needs, who wants to sort of get going on this and need help. -[00:07:00] **Dan:** Because there's a number of good things. One, you know what the user is experiencing, and that's all that matters. If you know what the user is experiencing, that means you can translate that to how much you're making. It's a cost thing and it's a money thing. If you know if your users are happy, that means you're making money because they're spending money because they're going to keep using your service. If they're not happy, they're not going to be using your service, and they're going to walk away and use something else or just not use it that often and try to avoid it as much as possible. That means you're not going to be making the money anyway. +**Speaker 1:** yeah, I completely agree. And I think this, uh, this takes us to our next question, um, which is, uh, you know, like you've, you've done this for a while now, like first the monitoring space now moving into the observability space. What do you think are some of the main challenges that most organizations are facing these days when it comes to observability? -**Dan:** And I'm going to get in the weeds, so you're going to hear me say that a lot because I just do. So, Adrian, it's all up to you to rope me back in. +### [00:10:15] Discussion on monitoring vs observability -**Host:** Don't worry, don't worry. But I do want to— you said something really interesting in terms of translating it in terms of money because, you know, like observability adoption— and maybe I'm getting ahead of myself here— but I like the way you said in terms of observability adoption, like it's as much a top-down thing as a bottom-up thing. And one way to speak to the benefits of observability is in dollars and cents, and executives speak in dollars and cents. +**Dan:** uh, there's a lot of challenges and I've been trying to get my head wrapped around some of this stuff. One is communication, I think so some of it, I think it's mostly cultural really. If you think about like, I mean there's not a technical reason why people shouldn't go to this other than resourcing in time. Uh, so and, and I mean this might be jumping ahead of a little bit, but where I am right now, we, we are using a current vendor, uh, which will remain nameless, but we use a current vendor right now for our monitoring capabilities, but we're getting away from them because of the library we're using is, is vendor specific and that means that we're basically, when it comes to doing any kind of telemetry, it has to use that particular library and theirs is only getting, and that's, that's one challenge because if you think about like, okay, so you're using this one vendor, it's cost you x amount per month or per year, it's exuberant. However to get translate or get off of that one and go into another one where you have more options like with open Telemetry takes time. It takes, it's not like an, like anybody who's ever gone on any kind of migration from one tool to another knows how long it takes. And if you're, and you have to have a few things in place to actually say why you should do this. So one is like, of course, obviously the cost, right? Well, but you know, it's hard for some executives to say, well, we're spending this but it'll cost us this much to get over here and then we'll see the cost value. So there's this huge challenge of just trying to shift things over, spend the time to do it and then do it properly. And that's a whole other challenge is to say, okay, yeah, doing open Telemetry is one thing, but doing it properly, like every other software or any other tool that you have out there, you have to still use it for the right reasons and do it properly. If you don't, you'll still run into issues and whatever you're trying to sort of get away from over here will still might come up in a different way over here. So there's those, that's, that's one of the challenges is just getting that, just doing the whole shift over. Uh, I think that's a where a lot of people and they don't know how to do it. And so, and what happens too is then they do a shift, they try it out and then they get burned for because, you know, they don't know how to do sampling or something like that and then they feed it off to some tool like let's say a tool that you get charged by event volume, you're going to, even though they're using otel, they still became like, oh, I saw one company who did that. They start shifting using otel, but they started setting all this volume, all this data to it. They didn't do it right. They were having like a hundred thousand spans per trace, which is a huge amount and they weren't really getting a lot of value and their costs are soaring. And so it's like, well, you know, then they kind of just drop and they, well, we'll figure it out another way. We'll just use some other tools to figure out when there's problems with our service, which is sort of like a defe kind of attitude, but it does happen because they don't, you know, because there's a lot of work involved doing this is more than just a simple lift and shift. It's a lot more involved to it. Um, trying to think of other challenges, like those, that to me is like some of the big, some of the bigger challenges and I think a lot of people could probably emphasize with that one as well. Um, there are others obviously, but I mean, I don't, we don't have a whole lot of time and we could probably go on just you and I alone, I bet you we both have experiences on some of those other challenges we have to experience. But I mean, uh, I like to see like, like how do I get, how do we get past that is kind of like where I want to target, how do we get past these challenges and move forward, which I don't have a proper answer to, but I will want to talk about it. -**Host:** So proving the worth of having an observable system, I think, becomes really really important. And being able to use that language with executives to be able to convince them, I think, is very valuable. +**Speaker 1:** you touched upon something that um, I think is worth mentioning as well, which is, um, moving away from like vendor libraries to open Telemetry. Um, because you know, even, even though I, I think one of the, one of the wonderful things about open Telemetry is that it's vendor neutral, um, and so it makes it once you've moved all of your instrumentation to that, super easy to switch vendors, I would say relatively easy, right? Because they're still like, yeah, like at least on from the instrumentation standpoint, it makes it super easy. Um, but, but it, the challenge though is convincing folks to move away from those vendor libraries and I think that becomes, um, that becomes a bit of a sticking point because it's like they, you need to move away from them because maybe that vendor is too expensive and you don't want to go with them and you want to try something fresh, but there's so much time and effort and learning curve all that stuff associated with reinstrumentation. And that what are your thoughts around that? -**Dan:** And that's what I've been trying to focus more on. There are quantifiable reasons why to look at this. We can do that. Why observability, excuse me, provides value toward us at the end of the day. But I think we're still trying to get that figured out. I feel like that, um— anyway, you know what? Before we jump in, let's keep moving forward because I could go into a whole other area and I don't want to sort of suck a lot of time in there. +### [00:15:47] Importance of user experience in observability -**Dan:** But it is one of those things I think that is worth having conversations, not just with those in the community, but also talk. Like, I mean, if you're out there, find out what the observability sort of platform or the concepts are where you work. What are the drivers? Why aren't you doing it? Why aren't you using these things? And ask those questions. And then maybe we could help feed that back to the community because that could help us build out the reasons why you should be doing this because it does make better sense eventually. +**Dan:** well, and that's a very, very powerful point you're making because first off with vendor libraries you get in the habit of doing things a certain way, right? And so again, I do not want to, there's no like, but the thing is, so some vendors may do things one way, but it's not part of the industry standard or the best practice we see across the industry. So they may like name things a certain way or do things a certain way and people get used to that. So not only is it just shifting a library over, if it was just a matter of shifting a library to one or the other, that's in itself is a challenge, but there's also all the behavior that gets attributed to using that older library that some, for some things you got to go, well listen, you don't need to look at it like that anymore. You need to look at it like this now. And so it's not just a technical shift, it's also a cultural shift and how they even like envision how their service is working with the new library with using open Telemetry. Um, because open Telemetry is current, it's trying to, it's keep, it's adding new things as we speak as every day. Uh, it's, you know, it's, you know, we're seeing a lot more love now for mobile, which is wonderful. I'm so excited to see some of that because I feel that's largely been ignored for a long time. But we're seeing these things getting pulled together and we're having this, and it's kind of like keep following these sort of patterns and following these, this, these standards. And so make forces the developers to think a certain way. A lot of them are not used to that yet. A lot of them still want to develop. And I realize one of the things I noticed too was for developers perspective, I don't, I Adrian, you, you do, you're a coder by nature, right? Like that's kind of, yeah, yeah, that's when you, yeah, that's your back. So it's not mine, but I know that when I did do some coding though, I would always print a line at certain parts in my code to say, "Okay, where am I at?" So you're even like logging was, is kind of part of your local development experience, totally. Right? Now we're telling them, don't worry so much about logging, use instrumentation. Well, you still have to have a mechanism for them to actually print that to screen. And so it's just changing how they even develop in the beginning. Um, and that, and that's, I think where we're facing some challenges there is that it's not just a, it's, it's technical, it's cultural, it's, it's about even how, what you've been taught in the universities and how you even taught how to develop is changing a little bit and not everybody's quick to embrace change, not, and you know, especially when you've got your CEO or your, or your, or your, um, your features team or product team saying we need this out now. Well, you know, you're not going to worry about trying to learn a new way of doing things and it's, you know, oh well now how my service is working from open Telemetry perspective. It's like, well I just need to get this out so I just slap these log lines in, boom, gone. It passed, you know, and we're going now it's in production and we move on. Right? So there's that, that it's that kind of like also encouraging and supporting our developers to have the time to learn this and work with it so they can actually use it to their benefit and perhaps be able to do better code before it even hits staging or development and then, and then once it's a production, you know how it should be based on all your stuff, but that's, it's getting that linking those people together, getting them to work together in that thinking, that's, that's a huge challenge. -[00:09:30] **Host:** I completely agree, and I think this takes us to our next question, which is, you know, like you've done this for a while now— first the monitoring space, now moving into the observability space. What do you think are some of the main challenges that most organizations are facing these days when it comes to observability? +**Speaker 1:** yeah, absolutely. And, and getting, getting them used to, as you said, like the nuances of, you know, instrumenting with open Telemetry, which open Telemetry wants to standardize how you're doing like your main signals in a certain way, right? Which as you said, maybe different from how you've been used to doing it, whether it's like through your print statements or vendor libraries or whatever. It's kind of like, so back in the day, um, this might be pre a lot of folks, I don't know, but I used to work a lot with SNMP and what I liked about SNMP was it was basically a protocol that was agreed upon by the community and it was very basic, but even then I saw this with logging too, but even with SNMP, I would still come across MIBs, I would still come across libraries that were not properly done. So it required me even like because I knew it, I would have to go in and kind of redo maybe a MIB to so I can translate the OID properly so when it goes into our, or create a MIB because it wasn't available, it just, it just had these voids available from an interface or something and you just have to guess what they were. Um, even with standards in place like that, something has been in our sort of part of our, our industry for so long, no longer really being used anymore. Um, well actually still is by network devices, but that's not hopefully we don't have to worry about that right now. Um, is, is that, you know, even with those things in place, it still takes time for people to even then there's still mistakes and things happen. So with a new thing like open Telemetry and, and being setting standards, it's taking a long time for people to understand how to use it to do it, to set it up properly and we get their most value out of that instrumentation, the most value out of using that from the beginning of the development cycle. -**Dan:** There's a lot of challenges and I've been trying to get my head wrapped around some of this stuff. One is communication, I think. Some of it I think is mostly cultural, really, if you think about it. I mean, there's not a technical reason why people shouldn't go to this other than resourcing and time. +### [00:19:50] Challenges in adopting open telemetry -[00:10:40] **Dan:** And I mean, this might be jumping ahead a little bit, but where I am right now, we are using a current vendor, which will remain nameless, but we use a current vendor right now for our monitoring and observability purposes. But we're getting away from them because the library we're using is vendor specific, and that means that we're basically—when it comes to doing any kind of telemetry, it has to use that particular library. And theirs is only getting— and that's one challenge because if you think about it, like, okay, so you're using this one vendor, it's costing you X amount per month or per year. It's exorbitant. +**Speaker 1:** yeah, absolutely. Um, I do want to switch gears a little bit and talk um, a little bit about specifically um, the challenges that you're facing currently in, in your role um, when it comes to observability. -**Dan:** However, to get off of that one and go into another one where you have more options, like with OpenTelemetry, takes time. It's not like, anybody who's ever gone on any kind of migration from one tool to another knows how long it takes. And you have to have a few things in place to actually say why you should do this. So one is, of course, obviously the cost, right? But you know, it's hard for some executives to say, well, we're spending this, but it'll cost us this much to get over here and then we'll see the cost value. +### [00:30:05] Communication challenges in observability -**Dan:** So there's this huge challenge of just trying to shift things over, spend the time to do it, and then do it properly. And that's a whole other challenge is to say, okay, yeah, doing OpenTelemetry is one thing, but doing it properly, like every other software or any other tool that you have out there, you have to still use it for the right reasons and do it properly. If you don't, you'll still run into issues, and whatever you're trying to sort of get away from over here will still might come up in a different way over here. +**Dan:** oh, I'm really not my own mental health problems. Okay, that's different. Um, yeah, so my, my challenges have pretty much always been, I feel like it's always been the same. It's communication and not because I'm my challenge for me. So I've been in this business for a while. So what happens sometimes is as I make the mistake or the or make the assumption that folks know what I'm talking about and I have to be careful sometimes. Like for, like I can have these conversations with many people in the same community is like within the monitoring observability community, and we could go on for hours about little nuances of something, whatever, maybe people logging or tracing anything like that. And we can go on and talk about that as ad nauseum. However, when you, when you're talking to other people and you, you talk about basic things like um, like the red method, so rate of errors, um, rate of requests, durational requests, which is a very basic kind of concept with the monitoring world, um, and which has helped kind of like we see a lot in even with observability too to a certain degree, uh, people don't understand that. And you make the assumption that they actually kind of look at things in that light, but they don't. They still look at it from the old school. And then like honestly, like at some places I've been, it's like they still see CPU as an indicator of a performance problem, but it shouldn't be in my opinion. It should because that's not CPU can, is cheap. You can get more. So that should be a scaling problem. Make sure you just, you're scaling properly, you should be good. However, I make the assumption people know what I'm talking about and that's one thing is I got to con, so I have to constantly be patient and communicate the same message over and over again to make sure they understand because people, even if I tell them two or three times, they still don't clue in. I had one conversation with one engineer for about a month. Um, and it took a month for them to actually understand what I was driving at. And this is like because they were in different time zones, so that's why it took a little bit longer because and so, but it took a while before they actually, "Oh, that's what you meant." And as soon as they were able to make that change, everything was working better and like, "Oh, now I see where you're driving at. I now see the point you're trying to make." And it's just you have to be diligent, you have to be patient, you have to communicate, and you have to change how you communicate sometimes too. Like you just can't say the same words over. You have to rethink, "Okay, you're not getting what I'm saying. What will help you understand?" And then try to reward it so people understand. Um, and you met my friend Damian who, who I work with, he's actually the same place as I am, but Damian who also works with me on a lot of stuff and we, he's more of a, he has a different background, but he sometimes steps in and sort of translate what I'm saying because sometimes it helps to have a different, because he puts a different spin on it and all, "Oh, now I get it." So that's those are some of the biggest things I find, um, to today are still the biggest sort of people get it. It's just a matter of once, but just getting them just lead you leave the horse to water type thing, right? But I am, it's just leading them there. Once they're there, they eventually do start to drink because they kind of get, "Oh, I see this is good for me. I will do it now." -**Dan:** So that's one of the challenges. Just doing the whole shift over, I think that's where a lot of people— and they don't know how to do it. And so what happens too is then they do a shift, they try it out, and then they get burned because, you know, they don't know how to do sampling or something like that. And then they feed it off to some tool like, let's say, a tool that you get charged by event volume. They're going to— even though they're using OpenTelemetry, they still became like, oh, I saw one company who did that. They started shifting to using OpenTelemetry, but they started sending all this volume, all this data to it. They didn't do it right; they were having like a hundred thousand spans per trace, which is a huge amount, and they weren't really getting a lot of value. +**Speaker 1:** for sure. But getting there, there is sometimes you gotta kind of be patient. You gotta pause for a while and say, "Okay, yeah, you want to look at the pretty flowers for a moment, that's cool. You watch TR some grass, great." Okay, finally I'm equating developers and engineers to horses, so hopefully nobody will take offense to that, but in the nicest possible way. -**Dan:** And their costs were soaring. And so it's like, well, you know, then they kind of just dropped and said, well, we'll figure it out another way. We'll just use some other tools to figure out when there's problems with our service, which is sort of like a defeatist kind of attitude. But it does happen because they don't, you know, because there's a lot of work involved. Doing this is more than just a simple lift and shift; it's a lot more involved. +**Dan:** honestly, I mean, you know, it's not like I'm trying to hurt. I think you like you such a good point though with, with like experimenting with different ways of doing the messaging because as you said, like you can't just keep repeating the same thing and then hoping that people are going to get it. Um, you have to, you have to come up with, with different ways of saying it and, and another thing that you and I have discussed before also is you can't tell them that they have an ugly baby. You can't say, "Your thing sucks. My way is the better deal," because people will get very offended, rightfully so. So you have to be very tactful in your messaging. -[00:12:50] **Dan:** I'm trying to think of other challenges— those to me are some of the bigger challenges. And I think a lot of people could probably empathize with that one as well. There are others, obviously, but I mean, we don't have a whole lot of time. We could probably go on just you and I alone. I bet you we both have experiences on some of those other challenges we have to experience. But I mean, I like to see like, how do I get past that is kind of where I want to target. How do we get past these challenges and move forward, which I don't have a proper answer to, but I do want to talk about it. +**Dan:** exactly. No, you're right because they'll just take offense and then they'll walk and then everything you say afterwards it falls on deaf ears. So you don't want to attack them. You've got to draw them out. And we, we wear a lot of different hats in this role when it comes to observability, I feel. It's not just about knowing how to do it technically, it's about how to assisting folks to see how they can learn from it and grow with it too. And that is a huge challenge. Uh, that's why I've done like a lot of my videos I've done too, like, uh, not too many of them been exposed publicly. I've done them internally with at work, but I will put on different characters. Like I would honestly like do different characters to help people understand just so they have something so it's not just another boring, "Oh here we go, get told off about monitoring observability. Yes, we know Dan logs are not monitoring." Okay, like I, but I try to build it up so they, it resonates a little bit because they're more engaged because, "Oh Dan's put on a funny wig and he's making a funny voice," but that point made actually does make sense. What, so then you know, it starts to click eventually. I hope, maybe I'm wrong. Maybe they just look at me and laugh and just walk away. -**Host:** You touched upon something that I think is worth mentioning as well, which is moving away from vendor libraries to OpenTelemetry. Because, you know, even though I think one of the wonderful things about OpenTelemetry is that it's vendor-neutral, and so it makes it, once you've moved all of your instrumentation to that, super easy to switch vendors. I would say relatively easy, right? Because they're still— like from the instrumentation standpoint, it makes it super easy. +**Speaker 1:** nothing sticks. -**Host:** But the challenge though is convincing folks to move away from those vendor libraries and I think that becomes a bit of a sticking point because you need to move away from them because maybe that vendor is too expensive and you don't want to go with them, and you want to try something fresh, but there's so much time and effort and learning curve and all that stuff associated with reinstrumentation. What are your thoughts around that? +**Dan:** nothing sticks, yeah. -**Dan:** Well, and that's a very powerful point you’re making because first off, with vendor libraries, you get in the habit of doing things a certain way, right? So again, I do not want to—there's no—like, but the thing is, so some vendors may do things one way, but it's not part of the industry standard or the best practice we see across the industry. So they may like name things a certain way or do things a certain way, and people get used to that. +**Speaker 1:** I like that though because you're, you're also like putting on different personas, right? And I think people also res, like different personas resonate with, with different people. They have to, at the end of the day, it's got to be how, how does this benefit me, right? Because otherwise if they don't see where they fit into all this, then it's falling on deaf ears. -**Dan:** So not only is it just a matter of shifting a library over—if it was just a matter of shifting a library to one or the other, that's in itself a challenge. But there's also all the behavior that gets attributed to using that older library. For some things, you got to go, well listen, you don't need to look at it like that anymore. You need to look at it like this now. And so it's not just a technical shift; it's also a cultural shift and how they even envision how their service is working with the new library, with using OpenTelemetry. +**Dan:** exactly, how does it benefit me? That's a good valid point and, and it's getting them to see that there is a benefit to them. It's just like, "Alright, well you just have to sort of showcase this several different times." Um, I know somewhere along the way I heard somebody say that how when you're me creating a message for your organization, you got to do it like seven times in like different formats to get it so people all kind of get it and think about it and understand it. And I think that's got some truth to it to be honest because even to this day, I'm still telling people, "By the way, I've, you know, this is the reason why we're doing this," and you kind of go through that process. So, um, yeah, no, it's totally, totally true. -**Dan:** Because OpenTelemetry is current; it's adding new things as we speak every day. We're seeing a lot more love now for mobile, which is wonderful. I'm so excited to see some of that because I feel that's largely been ignored for a long time. But we're seeing these things getting pulled together, and we're having this— and it's kind of like keep following these sort of patterns and following these standards. +**Dan:** yeah, and I, I think there's also the seeing is believing sort of thing to it too, which is like, yeah, okay, understand how like open Telemetry works in theory, right? And but it's not until you put it in practice and instrument your code for the first time and even if it's like some, you know, I added a trace, I added a log, whatever, then you see it go through the collector and enter your observability backend and you're like, "What? My life is transformed!" Just even, and you go like, "Now what I do?" -**Dan:** And so it forces the developers to think a certain way. A lot of them are not used to that yet. A lot of them still want to develop—and I realize one of the things I noticed too was from a developer's perspective— I don't know if Adrian, you do—you're a coder by nature, right? Like that's kind of—yeah, that's your back, so it's not mine. But I know that when I did do some coding though, I would always print a line at certain parts in my code to say, okay, where am I at? +**Dan:** yeah, right. It's like you crave more, right? Like give them taste it and it proofers in the pudding, as the saying goes, is a very, very true too because I have seen a lot of success once you get the ball rolling, they start to implement something and I, and I, I've seen it over the years in different tooling. Um, I think only once have I seen it not work, but that's only because I, that was more my arrogance than anything else. So I've dropped arrogance at the door after that and never did it again because I want to use a specific tool for the job and realize that that's not the way to approach it. Um, and that's the other thing too as engineers, as leaders, technical leaders, we should never push a tool in my opinion. I think we should always look at what the organization needs, what's most important thing to most value to the organization and then making sure we have the right practices in place, the right standards in place and that will event then we can figure out what tooling we'll use after that. Tooling should be always the last thing you should decide on how you want to visualize or work with your data, that's my just my two cents. I don't know how the people that might cause problems, but who knows. -**Dan:** So even logging was part of your local development experience, right? Now we're telling them, don't worry so much about logging; use instrumentation. Well, you still have to have a mechanism for them to actually print that to the screen. And so it's just changing how they even develop in the beginning. And that, I think, is where we're facing some challenges. It's not just a technical shift; it's cultural. It's about even how what you've been taught in the universities and how you've been taught how to develop is changing a little bit. +**Speaker 1:** now here's a question for you along the lines of tooling because we see this a lot in tech where you've got like a team, they start using a tool, but then the corporate standard for tooling comes out. How do you reconcile, you know, the teams that are off doing their, their own thing with the corporate standard? Because sometimes it's, it's really hard to get folks unstuck from that. -**Dan:** And not everybody's quick to embrace change. And you know, especially when you've got your CEO or your features team or product team saying, we need this out now. Well, you know, you're not going to worry about trying to learn a new way of doing things. And you’re saying, oh, well now how is my service working from an OpenTelemetry perspective? It's like, well I just need to get this out, so I just slap these log lines in, boom, gone. It passed, you know, and we're going now; it's in production, and we move on, right? +**Dan:** you, you can, you know what, you got to, I honestly personally that when it comes to that thing is like, okay, try to get them to, as long as they're following best practices, following industry standards, then you can only push so far. You know, like you could say, "Hey listen, you know, stop using this particular tool because it's archaic, it doesn't give you the same value for what you have." But you know, and then you're going to have push back. I mean, it doesn't matter where you are, even when everybody's on board, you're going to get push back because people want to do things their own way, right? There's always that kind of like, "I know better, I know what I'm doing," and you have to be delicate and you have to be patient and you have to be tactful. Um, so it's better to just, I would say just try to encourage them to use best practice and standards, but don't try to push them off the tool if that's, I mean, if somebody, if somebody higher up wants to make that decision and call them out on it, let them do it. But you want to, I think it's more important to have the standards in place because if you can kind of tie everything together and you have a proper process flow and everybody can still can't get to the root of the cause of the issue, then that's better than just saying, "You need to use x for your stuff moving forward, tough nuts," because that, that kind of thing doesn't work either in my opinion. -**Dan:** So there's that kind of like also encouraging and supporting our developers to have the time to learn this and work with it so they can actually use it to their benefit and perhaps be able to do better code before it even hits staging or development. And then once it's in production, you know how it should be based on all your stuff. But that's—it's getting that linking those people together, getting them to work together in that thinking. That's a huge challenge. +**Speaker 1:** yeah, that makes a lot of sense, especially like, you know, I, I'd almost prefer to like, "Okay, you use whatever observability backend you want as long as we're all instrumenting with open Telemetry." I feel like that, that is, that is the main thing. -**Host:** Yeah, absolutely. And getting them used to, as you said, the nuances of, you know, instrumenting with OpenTelemetry, which OpenTelemetry wants to standardize how you're doing your main signals in a certain way, right? Which, as you said, may be different from how you've been used to doing it, whether it's like through your print statements or vendor libraries or whatever. +**Dan:** yeah, that is the bar. Exactly. Now when you, um, when you joined your organization, um, were, like was Top Hat doing observability at the time? What was, what was the landscape like at the time? -**Host:** It's kind of like—so back in the day, this might be pre a lot of folks, I don't know, but I used to work a lot with SNMP. And what I liked about SNMP was it was basically a protocol that was agreed upon by the community, and it was very basic. But even then, I saw this with logging too, but even with SNMP, I would still come across MIBs. I would still come across libraries that were not properly done, so it required me even—because I knew it, I would have to go in and kind of redo maybe a MIB to translate the OID properly, so when it goes into our— or create a MIB because it wasn't available. It just had these OIDs that were available from an interface or something, and you just had to guess what they were. +**Dan:** so at that time, so we were at, so we were not, but we were, um, so this is one of those things where I kind of like I had to be very patient, but they were like, so when I first joined, we were using a vendor. Um, everything was, there was instrumentation in place, there was logging in place, there were, um, well I should say there was traces in place and there was logging in place. They were linked, but they weren't, um, it was done, the initially was rolled out about five years ago years ago, but no other real work was done, just kind of building on what they already knew moving forward. I was asked to come on and actually bring to take the company off the vendor and migrate to open Telemetry. So that was two and a half years ago and we are almost there. Like this is, uh, I think, uh, we have one more service to migrate over to open Telemetry this month and then we're done. So it's taken to an almost about two and a half years to get to kind of get to this point. -**Host:** Even with standards in place like that, something that has been in our sort of part of our industry for so long, no longer really being used anymore—well, actually, it still is by network devices, but that's not—hopefully we don't have to worry about that right now. +**Speaker 1:** amazing. -**Host:** It’s that, you know, even with those things in place, it still takes time for people to even—there are still mistakes and things that happen. So with a new thing like OpenTelemetry and being setting standards, it's taking a long time for people to understand how to use it to get the most value out of that instrumentation, the most value out of using that from the beginning of the development cycle. +**Dan:** yeah. It was hard obviously, but it was also, it was challenging. Um, had to be a lot of patience involved. Um, and what I did though in the beginning was actually kind of go back to remember the, uh, when we were at Monorama together, I did a talk on alerting and I did that talk because I kind of was, that was a precursor to where I really wanted to go, which is actually getting into SLOs, right? But that was sort of like, "Here, here's how you can first sort of understand your landscape when it comes to alerts and why." So you've got, and if you can categorize them in a certain way and realize a bulk of them were a lot of noise and doesn't have any value, help people start seeing those things in that kind of light and then start showing the value of using service level objectives and how that can actually help with end user journeys, help actually bring the user experience home to the developer, so they understand, "Okay, I don't need to alert on, you know, when the load balancer is hitting 5xx errors when I should be alerting on whether or not the service is being impacted by that and how it's being impacted." And then, yeah, I mean those 5xx are important, but where are they important? How is that impacting the user experience? If it's for some feature that we don't really care about yet, we don't care about the 5xx because we don't need, or we don't need to be alerted about that in the middle of the night. But there's, so that this was how we were trying to get that process going and then they start doing the migration. "Okay, now that we know how our service should look, now we can do our instrumentation on that," moving forward. So we've, we've done started off a lot of instrumentation, but now we have those user journeys to reflect upon and go, "Okay, we're missing this, this function in our auto instrumentation. Let's make sure that's part of that because that impacts the user's experience." So yeah, so it's been a while, uh, but we are getting closer and closer to getting over open Telemetry. We've set a lot of service level objectives. Two of them have already helped out, like alerted us like two days before an incident actually happened. So we're already seeing some value in this. So it does, again, proof this is in the pudding, right? And we're already seeing some of that and we're, you know, it's, it's great. A lot of churn, but in a lot of like turmoil, a lot of folks were like, "Oh no, we're losing this," but they're seeing that they're getting more value out of how we're doing things moving forward and they're able to query things better because they're able to have more richer data available at their fingertips. -**Dan:** Yeah, absolutely. I do want to switch gears a little bit and talk a little bit about specifically the challenges that you're facing currently in your role when it comes to observability. +**Speaker 1:** so now in terms of vendor, did you end up switching vendors from the one that you were using when you first came in or? -[00:20:00] **Dan:** Oh, I'm really not my own mental health problems. Okay, that's different. Yeah, so my challenges have pretty much always been—I feel like it's always been the same—it's communication. And not because I'm—my challenge for me, so I've been in this business for a while, so what happens sometimes is I make the mistake or the assumption that folks know what I'm talking about. +**Dan:** so we had the one vendor we were using, we're still on that same vendor, uh, but as of the end of this month, we're no longer going to be using that vendor. We'll be on another vendor moving forward because we're pulling all their vendor specific SDKs out and we're just putting in open Telemetry in and we're doing that replacement. So it's kind of like now we're getting to the point where the last in that, in those open Telemetry SDKs are pointing to a different vendor for our visualization of data. -**Dan:** I have to be careful sometimes. I can have these conversations with many people in the same community, like within the monitoring and observability community, and we could go on for hours about little nuances of something, whatever it may be—logging, or tracing, anything like that. And we can go on and talk about that ad nauseum. However, when you're talking to other people and you talk about basic things like the rate of errors, rate of requests, duration of requests, which is a very basic kind of concept in the monitoring world, which is helped—kind of like we see a lot in observability too to a certain degree—people don't understand that. +### [00:34:30] Cultural shifts in instrumentation -**Dan:** You make the assumption that they actually kind of look at things in that light, but they don't. They still look at it from the old school and then—like honestly, at some places I've been, it's like they still see CPU as an indicator of a performance problem. But it shouldn't be, in my opinion. It should be because CPU can be cheap; you can get more. So that should be a scaling problem. Make sure you're scaling properly; you should be good. +**Speaker 1:** cool. Awesome. Um, alright, moving on. Um, yeah, so the next question I had is, um, you know, now that you've gone through the exercise of, um, you know, in of having of doing the instrumentation, so actually around instrumentation, is there, um, what what's the instrumentation culture like at Top Hat? Um, is it because you know, one, one thing that I've spoken about in, in the past is like it really instrumentation has to be the responsibility of the developers. Do you think that there is that attitude of developers instrumenting their own code where you're at? -**Dan:** However, I make the assumption people know what I'm talking about, and that's one thing is I got to—so I have to constantly be patient and communicate the same message over and over again to make sure they understand. Because even if I tell them two or three times, they still don't clue in. I had one conversation with one engineer for about a month, and it took a month for them to actually understand what I was driving at. +**Dan:** not as of yet. Um, and there's a, the reason why, and this is nothing against anybody, so I hope anybody who's at Top Hat listening don't take offense. I love you all. Um, no, but it, it's because we were kind of like once we started getting going on this whole transition, we had a very short period of time. So there's been a few instances where staff engineers and principal engineers have stepped in and done some of the work, laid the groundwork for the other team and they kind of just went, flipped over things and did things. The most effort they put in was actually in the user journeys themselves. So the auto instrumentation has been more largely being kind of like we've, we moved this, put this in and they've just kind of looked, done it. So they haven't really, it hasn't part of been ingrained in part of their cultural thinking yet. It's still largely kind of like a checkbox for them to sort of, "Okay, we've gotten, we've, we've moved, migrated to open Telemetry, job done, move on to something else." And little do they know that there's, there's a lot more work involved, but don't, don't scare them off yet because it will help them. But it is something that has to be, I think, and what I we've tried to do is try to make it so as easy as possible, less not as impacting as far as their day-to-day life, but try to slowly get them to start looking at things and so going over things and saying, and, and using the tooling that we have available saying, "Here we know you did it in this particular tool, this is how you should do it in this tool," and providing that data and then doing training on that too. So trying, so they're slowly, it's slowly becoming part, but I would not say it's 100, like it's far from 100% yet. It's still, there's a lot of work to still to be done there just so that they start to, and they all understand why it's, it's part of the value, right? Um, it's only like I think right now it's still a handful and I could be wrong. Maybe I might get in from snack later. I'll get, you know, told, "No, we all love it." Who knows? -**Dan:** And this is like because they were in different time zones, so that's why it took a little bit longer. But it took a while before they actually, oh, that's what you meant. And as soon as they were able to make that change, everything was working better. And like, oh, now I see where you're driving at. I now see the point you're trying to make. And it's just you have to be diligent; you have to be patient; you have to communicate. +**Speaker 1:** in terms of instrumentation, like what, uh, what languages, um, are part of like your, your application landscape? -**Dan:** And you have to change how you communicate sometimes too. Like you just can't say the same words over—you have to rethink, okay, you're not getting what I'm saying. What will help you understand? And then try to reword it so people understand. +**Dan:** um, mostly Python. -**Dan:** Adrian, you met my friend Damian, who I work with. He's actually at the same place as I am. But Damian, who also works with me on a lot of stuff, he has a different background, but he sometimes steps in and sort of translates what I'm saying because sometimes it helps to have a different—because he puts a different spin on it. And oh, now I get it. +**Speaker 1:** oh, okay, okay. -**Dan:** So those are some of the biggest things I find today are still the biggest sort of people get it. It's just a matter of once—just getting them—just lead the horse to water type thing, right? But I am—it's just leading them there. Once they're there, they eventually do start to drink because they kind of get, oh, I see this is good for me; I will do it now. +**Dan:** so mostly, so we're, what's that? -**Host:** For sure. But getting there, sometimes you gotta kind of be patient. You gotta pause for a while and say, okay, yeah, you want to look at the pretty flowers for a moment. That's cool. You watch the grass grow. Great. +**Speaker 1:** oh, I was gonna say, and you alluded to some auto instrumentation there. -**Dan:** Okay, finally, I'm equating developers and engineers to horses, so hopefully nobody will take offense to that, but in the nicest possible way. Honestly, I mean, you make such a good point though with experimenting with different ways of doing the messaging because, as you said, you can't just keep repeating the same thing and then hoping that people are going to get it. +### [00:40:00] Instrumentation culture at Top Hat -**Dan:** You have to come up with different ways of saying it. And another thing that you and I have discussed before also is you can't tell them that they have an ugly baby. You can't say, your thing sucks; my way is the better deal because people will get very offended, rightfully so. So you have to be very tactful in your messaging. +**Dan:** yeah, so we did auto instrumentation in our Python code. Um, and so we, and we use a couple different frameworks, but I think we're just trying to work towards getting to one framework. Um, and then we have, and then yes, it's fortunately there's not, so when it comes to, so we can, you can actually almost kind of get away with like doing a wrapper sort of library for people and have them just use that and then yeah, to get. And that's kind of what we did. We have this little wrapper library set up that uses otel with the, you know, all the different things that they need to pull in there and people just kind of just dumped it in and did auto instrumentation the way they went. So they kind of like, so it's kind of like all done for them and they just kind of include this in a way they go. Um, but I think as we progress, it's going to be like, "Okay, let's start target this is my next stage will be talking to first off the PMS talking to them, okay, this is what we have so far, these are the gaps we have in our telemetry data, these are the SLOs we have set so far. Let's decide on how that should look because your team needs to own this. Your team needs to act on this. So when your error budget gets exhausted, your team needs to take that on in the next sprint." And that's one of, and like so that, and again, the technical side is not that difficult. It's the culture change process change requires a lot of effort, right? Because now you have to get people to say, "Okay, well instead of when we burn through our error budget, we need to take deal with this the next sprint," well that's usually, "Well that are this future that we need to push out," which, which one do we do? And so, but they have to, that's what I said when I first started this is that you have to agree upon this process. You have to think about you are making a contract with the company to say, "Yeah, we will act on these SLOs because they will impact the user journey. They will impact our users and that's what's most important to us today." -**Dan:** Exactly. No, you're— and that's right because they'll just take offense, and then everything you say afterward falls on deaf ears. So you don't want to attack them; you've got to draw them out. We wear a lot of different hats in this role when it comes to observability, I feel. It's not just about knowing how to do it technically; it's about assisting folks to see how they can learn from it and grow with it too. And that is a huge challenge. +**Speaker 1:** so, for those who are not aware, Top Hat is an educational platform. So what our users are teachers or professors and students. So imagine, you know, if they're taking an exam and then everything goes sideways, that's going to be really, I mean, anybody who has kids or has been to university knows that that could be a huge frustrating factor when you're not, you know, if that's not working properly. -**Dan:** That's why I've done a lot of my videos, not too many of them have been exposed publicly; I've done them internally at work. But I will put on different characters. Like I would honestly do different characters to help people understand just so they have something. So it’s not just another boring—oh, here we go, getting told off about monitoring observability. Yes, we know Dan, logs are not monitoring. Okay, but I try to build it up so they resonate a little bit because they're more engaged. +**Dan:** so yeah, it's important that, um, oh I did get a message. Somebody did tell me we all love it. Thanks Mark. Yeah, so um, yeah, uh, now I just completely thrown off there. Uh, what was I talking about? ADHD. -**Dan:** Because, oh, Dan's put on a funny wig and he's making a funny voice, but that point actually does make sense. So then, you know, it starts to click eventually, I hope. Maybe I'm wrong; maybe they just look at me and laugh and just walk away. +**Speaker 1:** you were talking about the SLOs and how frustrating it is, uh, if, if the consumers of your platform, um, if it's not working as they expected. -**Host:** Nothing sticks. +**Dan:** yeah, yeah. So, and that's this kind of fundamental shift is going to SLOs and helping them see that and get through that. Um, yeah, because I think now I just went off the rails. I so, but that's all good. -**Dan:** Nothing sticks! I like that though because you're also putting on different personas, right? And I think people also resonate with different personas. They have to, at the end of the day, it's got to be how does this benefit me, right? Because otherwise, if they don't see where they fit into all this, then it's falling on deaf ears. +**Speaker 1:** cool. I have one more question that I wanted to ask and then we actually do have some questions from, uh, from folks. See that our on our board, which I'm very excited about. Um, final, final question. Um, I guess there, the two questions I guess, um, are you, um, if, if are you using Kubernetes and if so, are you using the otel operator, um, to manage your collectors and to manage your auto instrumentation? -**Dan:** Exactly, how does it benefit me? That's a good valid point. And it's getting them to see that there is a benefit to them. It's just like, alright, well, you just have to sort of showcase this several different times. +**Dan:** no, no. -**Dan:** I know somewhere along the way I heard somebody say that when you're creating a message for your organization, you've got to do it like seven times in different formats to get it so people all kind of get it and think about it and understand it. And I think that's got some truth to it, to be honest, because even to this day, I'm still telling people, by the way, this is the reason why we're doing this. And you kind of go through that process. +**Speaker 1:** yeah, so we, um, I'm glad we're not, um, just because, and this is not because I think that Kubernetes has its place in, like all tools has its place and I think for what we're doing today, it would not make sense. It would add a layer of complexity that would completely destroy our service. So glad we're not using Kubernetes. I don't think it, not saying that there's anything wrong with Kubernetes, I just think that you, if you're going to run a Kubernetes environment, you need to know it. -**Host:** Yeah, no, it's totally true. +**Speaker 1:** yeah, yeah. -**Dan:** Yeah, and I think there's also the seeing is believing sort of thing to it too, which is like, yeah, okay, understand how OpenTelemetry works in theory, right? And, but it's not until you put it in practice and instrument your code for the first time, and even if it's like, you know, I added a trace, I added a log, whatever—then you see it go through the collector and into your observability backend. And you're like, what? My life is transformed just even—and you go like, now what I do, yeah, right? +**Dan:** right? And so if you don't know what you're doing with Kubernetes, you can really, really mess up your service. And I've been at places where we only had one or two people who knew Kubernetes and we had a number of problems always with our clusters. So I'm, when we're ready though, I think we'll probably make, probably make that push, but right now we're just in containers. So it's all just through, yeah, just containers, but not a Kubernetes. -**Dan:** It's like you crave more, right? Give them a taste; the proof is in the pudding, as the saying goes, is very, very true too because I have seen a lot of success once you get the ball rolling. They start to implement something, and I’ve seen it over the years in different tooling. I think only once have I seen it not work, but that's only because that was more my arrogance than anything else. So I've dropped arrogance at the door after that and never did it again because I wanted to use a specific tool for the job and realized that that's not the way to approach it. +**Speaker 1:** gotcha, gotcha. So you're using like another container management service, like a cloud provided one kind of thing? -**Host:** And that's the other thing too; as engineers, as leaders, technical leaders, we should never push a tool, in my opinion. I think we should always look at what the organization needs, what's the most important thing, what's the most value to the organization, and then make sure we have the right practices in place, the right standards in place, and that will eventually then we can figure out what tooling we'll use after that. Tooling should be always the last thing you should decide on how you want to visualize or work with your data. That's just my two cents. I don't know how the people that might cause problems, but who knows? +**Dan:** a cloud provider container service, yes. -**Dan:** Now here's a question for you along the lines of tooling because we see this a lot in tech where you've got a team, they start using a tool, but then the corporate standard for tooling comes out. How do you reconcile the teams that are off doing their own thing with the corporate standard? Because sometimes it's really hard to get folks unstuck from that. +**Speaker 1:** gotcha, gotcha. You're trying to avoid all names of vendors right now. -**Dan:** You can, you know, try to force them, but some folks are very adamant and passionate about the tooling that they use, right? +**Dan:** yeah, yeah, let's be agnostic. -**Host:** They are, and you know what? You've got to—I honestly personally, when it comes to that thing, is like, okay, try to get them to—as long as they're following best practices, following industry standards, then you can only push so far. You know, like you could say, hey listen, you know, stop using this particular tool because it's archaic; it doesn't give you the same value for what you have. But, you know, and then you—you’re going to have pushback. I mean, it doesn't matter where you are. Even when everybody's on board, you're going to get pushback because people want to do things their own way, right? +**Speaker 1:** and, and final question before we get to the questions. Um, collectors, what's your collector landscape like? -**Host:** There's always that kind of like I know better; I know what I'm doing. +**Dan:** so we're using a combination of otel collector, um, as a SAR and then to a primary one that aggregates it. And then for our traces, we use a with one company that we're work where we're moving towards. Uh, they have their own collector that we send our traces to that we can do additional sampling rules on and send off. So, um, yeah, yeah. So we, um, so right now, because, and I really, really love the fact that you could choose, you could do send from code itself, but I love the otel collector because simply because you have these processors, you have these exporters and you can send to different places as you see fit. I've had conversations with our data team saying, "Hey listen, if you want stuff, otel is collecting a lot of this right and we don't have to send all of it to our for production. Like as far as what we're looking at as far as like would give us an idea of the user experience, but there may be things you are collecting that from our otel that we can send to an S3 bucket that you can pull in later for whatever you need. However, um, you know, we have, we're still haven't gotten to that point, but that's the beauty of the otel collectors that you can kind of like just point to different things and I love that because then it there's no code change. So if we decide we no longer want to use this or we want to add logging or we want to add set get, we start using, we don't use Prometheus now, but let's say we start using Prometheus as an open source tool, um, as an example though, uh, you would, we can easily just start getting that information right off the mark. So it, I just, if anything try not to use the sampling and the old push from this code itself. Use a collector because I love the fact that because it makes life so much easier. Then you can just, if you have to do little tiny changes of tweaks to your sampling or whatever, you don't have to go push it out to like the production environment again. It's just a small change on your collectors themselves, which makes life so much easier. That's just how I think of it. -**Dan:** Yeah, and you have to be delicate, and you have to be patient, and you have to be tactful. So it's better to just—I would say just try to encourage them to use best practices and standards, but don't try to push them off the tool if that's—I mean, if somebody higher up wants to make that decision and call them out on it, let them do it. +**Speaker 1:** absolutely. And, and you're using all three like major signals, traces, logs, metrics right now like for otel or mostly traces right now? -**Dan:** But you want to—I think it's more important to have the standards in place because if you can tie everything together and you have a proper process flow and everybody can still get to the root cause of the issue, then that's better than just saying you need to use X for your stuff moving forward. Tough luck, because that kind of thing doesn't work either, in my opinion. +**Dan:** um, so there was a decision to not do so metrics. We still are capturing the infrastructure metrics from the cloud and the logs are primarily staying in our cloud provider. So all of our services are, are just logging by default from container to the logging, um, in our cloud provider and they are staying there for the time being. Um, because that was another thing to, to start tackling and I didn't, I just, I felt like that'd be too much for folks as they're transitioning from the one vendor to open Telemetry for them to like get their heads wrapped around would be like, "Oh, by the way you also need to have structured logs too and this is how you should do it." No, this is just gonna, it'll make more confusion. And so I figured just, just focusing on one and then adding things as they go along as they, as they prove useful. So yeah, that's how we kind of approach that whole process. -**Host:** Yeah, that makes a lot of sense, especially like, you know, I'd almost prefer to like, okay, you use whatever observability backend you want as long as we're all instrumenting with OpenTelemetry. I feel like that is the main thing. +**Speaker 1:** makes sense, baby steps. -**Dan:** Yeah, that is the bar. +**Dan:** exactly. Cool. Um, okay, so we're going to turn to the board and if you want, you can take a peek at the questions. They're on the, on the chat window, there's a link to the board. Um, take a look. So, um, let's have a look at the questions. So the first one is what is your opinion on Telemetry quality as a way to reduce spend? Is more data always better observability? Great question. -**Host:** Now when you joined your organization, was Top Hat doing observability at the time? What was the landscape like at the time? +### [00:45:30] Q&A session with audience questions -[00:30:10] **Dan:** So at that time, we were not, but we were. So this is one of those things where I kind of had to be very patient. But they were like—so when I first joined, we were using a vendor. Everything was there; there was instrumentation in place. There was logging in place. Well, I should say there were traces in place and there was logging in place. They were linked, but they weren't—it was done—initially it was rolled out about five years ago, but no real work was done; it was just kind of building on what they already knew moving forward. +**Dan:** awesome question and that's a tough one. Um, I don't think I can, I hope I can answer that in my opinion. Um, it's about quality. You need to actually, because really with the end of, because you can have a bunch of things going through, but if you're sort of, so let's just take a, a standard trace, right? If you have, um, a bunch of spans in there that are doing sort of some things that you really know it's just back and forth and you really like there's, I've noticed some traces where there, it there's, um, I forget what it's called and I, I wish I had a sample the code before. It's been like two weeks ago one of our engineers cleaned this up and what he did though is he looked and realized we're making these calls in the code that we're, we're adding span events to and we don't need to because so you're looking at the quality there and then you're not, you're seeing what's invaluable and what's not. So more data doesn't always mean better telemetry. Quality data means better telemetry. So in my opinion, you've got to look at what you're getting, making sure does that reflect the user experience? If it doesn't reflect the user experience or is not exposing things where you would say, "Oh you know, like that's kind of like the whole the concept of the unknowns," right? Like you're exposing those unknowns. You're not able to see those things and then you're not, the quality, it doesn't matter how much you have if you're not seeing that stuff, then it doesn't mean you're, it's the quality of the data that's most important and that will help you reduce spend because then you know what's good. You can sample that partially out and you're going to make sure that those, that the error prone things bubble up and become more apparent when you're looking at your data. Hopefully that answered the question. -**Dan:** I was asked to come on and actually bring—take the company off the vendor and migrate to OpenTelemetry, so that was two and a half years ago. We are almost there. I think we have one more service to migrate over to OpenTelemetry this month, and then we're done. So it's taken us almost about two and a half years to get to this point. +**Speaker 1:** yeah, that was a really great answer. Um, yeah and I, I agree with you. It's, it's really like what's the point of capturing in telemetry for your system that's useless to you? Like that's just, and I think that's, um, that's a pitfall that a lot of people run into also with auto instrumentation initially, right? Because there's a lot of stuff coming in and, and, and fortunately now I think there's a way to actually switch off auto instrumentation for certain libraries so that you can, you can really zero in on the stuff that's, uh, super meaningful to you. -**Host:** Amazing! +**Dan:** so yay, yay, cool. Okay, next question. Um, how do you approach the what to instrument and how much to instrument questions with your teams? -**Dan:** Yeah, it was hard, obviously, but it was also—it was challenging. It had to be a lot of patience involved. And what I did though in the beginning was actually kind of go back to—remember when we were at Monorama together? I did a talk on alerting, and I did that talk because I kind of was—that was a precursor to where I really wanted to go, which is actually getting into SLOs, right? +**Dan:** so that kind of is it linked to the previous question too in my opinion? Um, because what to instrument is again goes back to, this is so I went through the whole process of working with the teams, the development teams and the product teams to understand what the user journey looks like. So I went through and said, "Okay, what is the, so we did the user journey, we looked at their servers and I asked this like some did more than others, uh, but more or less it was like I asked him what are the typical interactions a user would have your service and write it down and then write in in so in plain language and then in plain language describe the tech, what's happening in the background and then we also talk about the prerequisites of what people expect when they interact with that system. So they have to be logged in, they have to have this tech, maybe this cookie or this cache or whatever. Um, and then what are the dependencies? And then we just kind of work through and go, "Okay, what are the things that where a user will potentially see problems based on this information?" And we went through that. So that's how I kind of approach what to instrument because then once you kind of know, "Okay, so we know that user logs in, so they know they have to do that. This, they have to interact with maybe, uh, let's hypothetically say they interact with a database, they interact with an authorization service, maybe they interact with something in the backend for something else, uh, another database." And so all these queries are checked in there. Now that all those things, all those functions are like talking to these different things, if they don't work means they can't log in. Therefore we have key spots what they should be mostly looking at to make sure there are always going to be times where people miss things. This is why we have the SLOs. This is why we have to have this constant part of our cycle to go back and make sure that those things are identified. So, you know, we're having problems with our servers, but we don't know why. That means maybe you're not instrumenting. So what that means you have to go revisit some of the things you did because maybe you did some changes that no longer uses this kind of database, uses maybe a cache service or uses a Redis cluster instead of these things. And so does your code or your user journey reflect that change? And does your instrumentation reflect that change? And if not, let's make sure they do. So, but it always boils down to the user journey and what the user is experiencing. If as long as we're capturing that, you're like 80% of the way, in my opinion, 80% of the way there. Uh, because you at least have a general idea, but you have to know, you have to have that conversation. -**Dan:** But that was sort of like, here’s how you can first sort of understand your landscape when it comes to alerts and why you’ve got to categorize them in a certain way and realize a bulk of them were a lot of noise and didn’t have any value. Help people start seeing those things in that kind of light and then start showing the value of using service level objectives and how that can actually help with end-user journeys, help actually bring the user experience home to the developer. +**Speaker 1:** awesome, thank you. Okay, next question, the Q. Um, in an org that has adopted distributed tracing, how do you make individual teams leave behind their outdated practices based on logs? How do you show value? -**Dan:** So they understand, okay, I don’t need to alert on, you know, when the load balancer is hitting 5xx errors when I should be alerting on whether or not the service is being impacted by that and how it’s being impacted. And then, yeah, I mean those 5xx errors are important, but where are they important? How is that impacting the user experience? If it's for some feature that we don’t really care about, we don’t care about the 5xx because we don’t need or we don’t need to be alerted about that in the middle of the night. +**Dan:** that was a spicy question. That's a spicy one. Y slap them around and tell them what? No, you do not. That's not the way to do things. You have to be very, exactly. That's number one, do not call her baby ugly. Um, that's a tough one because you really, really have to be patient and there sometimes you do have to say, "Okay, listen, this has been dictated by the AL ops that we're doing this," and you try to offer them as much help as possible, but you have to, you got to find out too why they're reticent about switching over. Uh, having these conversations is important because then you understand what, how they're looking at things and then you can restructure your discussions around that. Uh, so, oh I like using logging because you know, I get to see immediately when I'm doing my development, I can see it point out on the screen, I know where things are broken so I can go back and fix and blum blum blum and that's so you know that that's why they love logging as an example. So how you have to then help them bridge that gap and say, "Okay, well this is how you would do it with tracing and this is why you would probably want to do it because then with distributed tracing you're not just getting the value out of your service but you're seeing how service A interacted with your service, service B and how service B is going to interact with service C and how all that kind of works together. So this is how, why you want to have these pieces in the puzzle so you can get that full picture." But it takes time, it takes patience, it takes, and again, going back to which one of the things we talked about before, restructuring how you approach the, their, their, the, uh, the challenge of switching over. It's not just repeating yourself the same words over and over again because eventually people are just going to tune you out. Like at my work, I think they tune me out half the time. -**Dan:** But that was how we were trying to get that process going. And then they start doing the migration. Okay, now that we know how our service should look, now we can do our instrumentation on that moving forward. So we've started off a lot of instrumentation, but now we have those user journeys to reflect upon and go, okay, we’re missing this function in our auto instrumentation. Let’s make sure that’s part of that because that impacts the user’s experience. +**Speaker 1:** okay, uh final question we can squeeze it in hopefully in two minutes. Um, really love what you said about the tooling, uh, is the last thing you should think about and that you should lead with the question what is most important to your organization? What are some of the questions you ask to understand what is most important to someone's organization? -**Dan:** So yeah, it’s been a while, but we are getting closer and closer to getting over to OpenTelemetry. We've set a lot of service level objectives—two of them have already helped out, like alerted us like two days before an incident actually happened. So we're already seeing some value in this. So it does, again, proof is in the pudding, right? And we're already seeing some of that, and we're, you know, it's great. A lot of churn, but a lot of like turmoil. A lot of folks were like, oh no, we’re losing this, but they’re seeing that they’re getting more value out of how we’re doing things moving forward, and they’re able to query things better because they’re able to have more richer data available at their fingertips. +**Dan:** it's always about the user in my opinion. It's always about the user. Um, because first because you're looking at if the user is not happy and you can't measure that or you don't know that, then you're in trouble, right? Uh, and I always use streaming services like video streaming services as an example because, and we, everybody has tuned into one of many of these streaming services we have now today. So if you pull up a movie, you push play, it gets all pixelated for the first three seconds and then it clears up after that. Do you care? No. So if a person comes by, but does the streaming service know that? They probably are measuring that because they want to make sure that that doesn't become a bigger issue because they know that runway of where you're okay to where you're not okay and you're going to get frustrated and go, "I'm gonna go switch to this now because they have better quality." And so it's not, it's always about what the user, how we are we measuring the user experience? Do we know what the user experience is like? Do we know what frustrates them? And if we don't, how do we get there? And that's how we get, so that is usually with the tooling that we have available today, today with the SDKs and other things and the philosophies we have. And eventually then you can go because you know what, the tools comes out all the time. There's so many tools out there that the landscape has gotten bigger and bigger as people become more and more involved. But doesn't message mean they're all good? And you know, but you have to find the one that kind of works for you at the time. And this is why we like open Telemetry is because you can switch from one to the other to the other without with very minimal change. And that's why I like about getting people on open Telemetry. But when it comes to the tooling, that should be your last thing to think about. Don't build your solution around this tool in your spot. Build your solution about what your users are thinking and that's how I think a lot of people that resonates when you start talking in that light, especially with the higher ups anyway. -**Host:** So in terms of vendor, did you end up switching vendors from the one that you were using when you first came in? +**Speaker 1:** so I think we got them all in. -**Dan:** So we had the one vendor we were using; we're still on that same vendor. But as of the end of this month, we're no longer going to be using that vendor. We'll be on another vendor moving forward because we're pulling all their vendor-specific SDKs out, and we're just putting in OpenTelemetry, and we're doing that replacement. So it's kind of like now we're getting to the point where the last of those OpenTelemetry SDKs are pointing to a different vendor for our visualization of data. +**Dan:** yeah, absolutely. -[00:34:22] **Host:** Cool, awesome. Alright, moving on. The next question I had is, you know, now that you've gone through the exercise of doing the instrumentation, what's the instrumentation culture like at Top Hat? +**Speaker 1:** well, we got through all the questions. Yay! We filled up this hour quite nicely. So thank you so much, Dan, for joining us for hotel Q&A. Stay tuned for our next hotel Q&A and or hotel in practice. And again, thank you, Dan, for joining today. I really appreciate it. -**Dan:** Um, not as of yet. And there's a reason why, and this is nothing against anybody, so I hope anybody who's at Top Hat listening doesn't take offense. I love you all! +**Dan:** oh yeah, that you know, it's my pleasure. Thank you for having me. Um, you know how much I love talking about these things. So, you know, I, I think I actually did not go down a rabbit hole today. I think I was pretty good. -**Dan:** No, but it's because we were kind of like, once we started getting going on this whole transition, we had a very short period of time. So there have been a few instances where staff engineers and principal engineers have stepped in and done some of the work, laid the groundwork for the other team, and they kind of just flipped over things and did things. The most effort they put in was actually in the user journeys themselves. +**Speaker 1:** it's perfect. It was great. Thank you again. -**Dan:** So the auto instrumentation has been more largely kind of like, we moved this, put this in, and they’ve just looked at it and done it, so they haven't really— it hasn't been ingrained in part of their cultural thinking yet. It's still largely kind of like a checkbox for them to sort of, okay, we've gotten, we've moved, migrated to OpenTelemetry, job done, move on to something else. - -**Dan:** And little do they know that there's a lot more work involved. But don't scare them off yet because it will help them. But it is something that has to be—I think we've tried to do it so as easy as possible, less not as impacting as far as their day-to-day life, but try to slowly get them to start looking at things and so going over things and saying— and using the tooling that we have available saying, here, we know you did it in this particular tool; this is how you should do it in this tool, and providing that data and then doing training on that too. - -**Dan:** So trying—so it's slowly becoming part, but I would not say it’s 100%—it’s far from 100% yet. There’s still a lot of work to be done there just so that they start to understand why it’s part of the value, right? - -**Host:** In terms of instrumentation, like what languages are part of your application landscape? - -**Dan:** Mostly Python. - -**Host:** Okay, okay. So mostly—so you alluded to some auto instrumentation there. - -**Dan:** Yeah, so we did auto instrumentation in our Python code, and we use a couple different frameworks, but I think we're just trying to work towards getting to one framework. And then we have—unfortunately, there’s not—so we can almost kind of get away with doing a wrapper sort of library for people and have them just use that. - -**Dan:** And that’s kind of what we did. We have this little wrapper library set up that uses OpenTelemetry with all the different things that they need to pull in there, and people just kind of just dump it in and do auto instrumentation and away they go. So it’s kind of like all done for them, and they just kind of include this and away they go. - -**Dan:** But I think as we progress, it’s going to be like, okay, let’s start targeting—this is my next stage—talking to first off the PMs, talking to them: okay, this is what we have so far, these are the gaps we have in our telemetry data, these are the SLOs we have set so far; let’s decide on how that should look because your team needs to own this. - -**Dan:** Your team needs to act on this. So when your error budget gets exhausted, your team needs to take that on in the next sprint. And that’s one of—again, the technical side is not that difficult; it's the cultural change process change requires a lot of effort, right? Because now you have to get people to say, okay, well, instead of when we burn through our error budget, we need to deal with this next sprint—well, that's usually, well, that feature that we need to push out. - -**Dan:** Which one do we do? And so—but that’s what I said when I first started this is that you have to agree upon this process. You have to think about you are making a contract with the company to say, yeah, we will act on these SLOs because they will impact the user journey. They will impact our users, and that's what's most important to us today. - -**Host:** For those who are not aware, Top Hat is an educational platform, so our users are teachers or professors and students. So imagine, you know, if they're taking an exam and then everything goes sideways, that's going to be really— I mean, anybody who has kids or has been to university knows that that could be a huge frustrating factor when you're not—you know, if that's not working properly. - -**Dan:** Yeah, so it's important that—oh, I did get a message. Somebody did tell me, we all love it. Thanks, Mark! - -**Dan:** Yeah, so now I just completely thrown off there. What was I talking about? - -**Host:** ADHD! - -**Dan:** You were talking about the SLOs and how frustrating it is if the consumers of your platform if it's not working as they expected. - -**Dan:** Yeah. So, and that’s this kind of fundamental shift is going to SLOs and helping them see that and get through that. - -**Host:** Yeah, because I think—now I just went off the rails. - -**Dan:** I can relate! So, but that’s all good. - -[00:40:11] **Host:** Cool, I have one more question that I wanted to ask, and then we actually do have some questions from folks. See that our on our board, which I’m very excited about. Final, final question. I guess there are two questions, I guess. Are you using Kubernetes, and if so, are you using the OpenTelemetry operator to manage your collectors and to manage your auto instrumentation? - -**Dan:** No, no. - -**Host:** Yeah, so we—I'm glad we're not just because—and this is not because I think that Kubernetes has its place and all tools have their place. And I think for what we're doing today, it would not make sense. It would add a layer of complexity that would completely destroy our service. So glad we're not using Kubernetes; I don't think it—not saying that there’s anything wrong with Kubernetes; I just think that if you’re going to run a Kubernetes environment, you need to know it. - -**Dan:** Yeah, absolutely. - -**Host:** And if you don't know what you're doing with Kubernetes, you can really mess up your service. And I’ve been at places where we only had one or two people who knew Kubernetes and we had a number of problems always with our clusters. So I’m—when we’re ready though, I think we’ll probably make that push. But right now, we’re just in containers, so it’s all just through—yeah, just containers, but not Kubernetes. - -**Host:** Gotcha, gotcha. So you're using like another container management service, like a cloud-provided one kind of thing? - -**Dan:** A cloud provider container service, yes. - -**Host:** Gotcha, gotcha. You're trying to avoid all names of vendors right now. - -**Dan:** Yeah, yeah. Let's be agnostic. - -**Host:** And final question before we get to the questions. Collectors, what's your collector landscape like? - -**Dan:** So we're using a combination of OpenTelemetry Collector as a SAR, and then a primary one that aggregates it. And then for our traces, we use one company that we're working with. They have their own collector that we send our traces to that we can do additional sampling rules on and send off. - -**Dan:** Yeah, so we—right now because, and I really, really love the fact that you could choose. You could send from code itself, but I love the OpenTelemetry Collector simply because you have these processors, you have these exporters, and you can send to different places as you see fit. I've had conversations with our data team saying, hey listen, if you want stuff, OpenTelemetry is collecting a lot of this, right? - -**Dan:** And we don’t have to send all of it to our— for production, as far as what we're looking at, as far as like would give us an idea of the user experience. But there may be things you are collecting from OpenTelemetry that we can send to an S3 bucket that you can pull in later for whatever you need. - -**Dan:** However, you know, we still haven't gotten to that point, but that's the beauty of the OpenTelemetry collectors, that you can kind of like just point to different things. And I love that because then there's no code change. So if we decide we no longer want to use this or we want to add logging or we want to add—let's say we start using Prometheus as an open-source tool, as an example, though, you would—we can easily just start getting that information right off the mark. - -**Dan:** So I just—if anything, try not to use the sampling and the old push from this code itself. Use a collector because I love the fact that it makes life so much easier. Then you can just—if you have to do little tiny changes or tweaks to your sampling or whatever, you don’t have to go push it out to like the production environment again. It’s just a small change on your collectors themselves, which makes life so much easier. That's just how I think of it. - -**Host:** Absolutely. And you're using all three major signals: traces, logs, metrics right now for OpenTelemetry, or mostly traces right now? - -**Dan:** So there was a decision to not do—so metrics we still are capturing the infrastructure metrics from the cloud, and the logs are primarily staying in our cloud provider. So all of our services are just logging by default from container to the logging in our cloud provider, and they are staying there for the time being because that was another thing to start tackling. - -**Dan:** And I didn't—I just—I felt like that’d be too much for folks as they're transitioning from the one vendor to OpenTelemetry for them to like get their heads wrapped around would be like, oh, by the way, you also need to have structured logs too, and this is how you should do it. No, this is just going to—it’ll make more confusion. - -**Dan:** And so I figured just focusing on one and then adding things as they go along, as they prove useful, so yeah, that's how we kind of approach that whole process. - -**Host:** Makes sense, baby steps. - -**Dan:** Exactly, cool. Okay, so we're going to turn to the board. And if you want, you can take a peek at the questions; they're on the chat window. There's a link to the board. - -**Host:** Take a look. So let's have a look at the questions. - -**Host:** So the first one is, what is your opinion on telemetry quality as a way to reduce spend? Is more data always better observability? - -**Dan:** Great question! Awesome question, and that's a tough one. I don't think I can—I hope I can answer that. In my opinion, it's—quality is—you need to actually—because really with the end of the—because you can have a bunch of things going through, but if you're sort of—so let's just take a standard trace, right? If you have a bunch of spans in there that are doing some things that you really know it's just back and forth and you really—like there's—I’ve noticed some traces where there— I forget what it's called, and I wish I had a sample of the code before. It's been like two weeks ago; one of our engineers cleaned this up, and what he did though is he looked and realized we’re making these calls in the code that we’re adding span events to, and we don’t need to because—so you're looking at the quality there, and then you're not—you're seeing what's invaluable and what's not. - -**Dan:** So more data doesn't always mean better telemetry; quality data means better telemetry. So in my opinion, you've got to look at what you're getting, making sure—does that reflect the user experience? If it doesn't reflect the user experience or is not exposing things where you would say, oh, you know, like that's kind of like the whole concept of the unknowns, right? Like you're exposing those unknowns. You're not able to see those things, and then you're not—the quality doesn't matter how much you have if you're not seeing that stuff, then it doesn't mean you're—it's the quality of the data that's most important, and that will help you reduce spend because then you know what's good, you can sample that partially out, and you're going to make sure that those error-prone things bubble up and become more apparent when you're looking at your data. Hopefully, that answered the question. - -**Host:** Yeah, that was a really great answer. - -**Dan:** Yeah, and I agree with you. It's really like what's the point of capturing telemetry for your system that's useless to you? Like that's just—and I think that's a pitfall that a lot of people run into also with auto instrumentation initially, right? Because there's a lot of stuff coming in. And fortunately now, I think there's a way to actually switch off auto instrumentation for certain libraries so that you can really zero in on the stuff that's super meaningful to you. - -**Host:** So, yay! - -**Dan:** Yay! - -**Host:** Cool. Okay, next question. How do you approach the what to instrument and how much to instrument questions with your teams? - -**Dan:** So that kind of is linked to the previous question too, in my opinion, because what to instrument is again—it goes back to this. So I went through the whole process of working with the development teams and the product teams to understand what the user journey looks like. So I went through and said, okay, what is the—so we did the user journey. We looked at their servers, and I asked—some did more than others—but more or less it was like I asked them what are the typical interactions a user would have with your service? And write it down. - -**Dan:** And then write in plain language, and then in plain language describe the tech, what's happening in the background, and then we also talk about the prerequisites of what people expect when they interact with that system. So they have to be logged in, they have to have this tech—maybe this cookie or this cache or whatever—and then what are the dependencies? And then we just kind of work through and go, okay, what are the things that where a user will potentially see problems based on this information? - -**Dan:** And we went through that. So that's how I kind of approach what to instrument because then once you kind of know, okay, so we know that user logs in, so they know they have to do that. They have to interact with maybe—let's hypothetically say they interact with a database, they interact with an authorization service, maybe they interact with something in the back end for something else—another database—and so all these queries are checked in there. - -**Dan:** Now that all those things—all those functions are like talking to these different things. If they don't work, it means they can't log in. Therefore, we have key spots what they should be mostly looking at to make sure. There are always going to be times where people miss things. This is why we have the SLOs. This is why we have to have this constant part of our cycle to go back and make sure that those things are identified. So you know, we’re having problems with our servers, but we don’t know why that means maybe you’re not instrumenting. - -**Dan:** So what that means you have to go revisit some of the things you did because maybe you did some changes that no longer use this kind of database. Maybe it uses a cache service or uses a Redis cluster instead of these things. And so does your code or your user journey reflect that change? And does your instrumentation reflect that change? And if not, let’s make sure they do. - -**Dan:** So, but it always boils down to the user journey and what the user is experiencing. If as long as we're capturing that, you're like 80% of the way—in my opinion, 80% of the way there, because you at least have a general idea. But you have to know—you have to have that conversation. - -**Host:** Awesome, thank you. - -**Host:** Okay, next question. In an organization that has adopted distributed tracing, how do you make individual teams leave behind their outdated practices based on logs? How do you show value? - -**Dan:** That was a spicy question! That's a spicy one. You slap them around and tell them what—no, you do not. That's not the way to do things. You have to be very—exactly, that’s number one: do not call their baby ugly. - -**Dan:** Um, that's a tough one because you really, really have to be patient. And sometimes you do have to say, okay, listen, this has been dictated by the upper management that we're doing this. And you try to offer them as much help as possible, but you have to find out too why they’re reticent about switching over. - -**Dan:** Having these conversations is important because then you understand how they're looking at things, and then you can restructure your discussions around that. So, oh, I like using logging because, you know, I get to see immediately when I'm doing my development. I can see it point out on the screen; I know where things are broken, so I can go back and fix it and blum blum blum. And that's why they love logging, as an example. - -**Dan:** So how you have to then help them bridge that gap and say, okay, well this is how you would do it with tracing, and this is why you would probably want to do it because then with distributed tracing, you're not just getting the value out of your service, but you're seeing how service A interacted with your service, service B, and how service B is going to interact with service C and how all that kind of works together. - -**Dan:** So this is how—why you want to have these pieces in the puzzle so you can get that full picture. But it takes time; it takes patience. It takes—and again, going back to which one of the things we talked about before, restructuring how you approach their challenge of switching over. It's not just repeating yourself with the same words over and over again because eventually, people are just going to tune you out. - -**Dan:** Like at my work, I think they tune me out half the time, okay? - -**Host:** Final question; we can squeeze it in hopefully in two minutes. - -**Host:** I really love what you said about the tooling. Is the last thing you should think about, and that you should lead with the question, what is most important to your organization? What are some of the questions you ask to understand what is most important to someone's organization? - -**Dan:** It's always about the user, in my opinion. It's always about the user because first—because you're looking at—if the user is not happy and you can't measure that or you don't know that, then you're in trouble, right? - -**Dan:** And I always use streaming services, like video streaming services, as an example because everybody has tuned into one of many of these streaming services we have now today. So if you pull up a movie, you push play, and it gets all pixelated for the first three seconds, and then it clears up after that, do you care? No. - -**Dan:** So if a person comes by—do the streaming service know that? They probably are measuring that because they want to make sure that that doesn't become a bigger issue because they know that runway of where you're okay to where you're not okay and you're going to get frustrated and go, I'm going to go switch to this now because they have better quality. - -**Dan:** And so it's not—it's always about what the user—how we are—we measuring the user experience. Do we know what the user experience is like? Do we know what frustrates them? And if we don't, how do we get there? And that's how we get—so that is usually with the tooling that we have available today, with the SDKs and other things and the philosophies we have. - -**Dan:** Eventually, then you can go—because you know what? The tools come out all the time. There are so many tools out there that the landscape has gotten bigger and bigger as people become more and more involved, but it doesn't mean they're all good. And you know, but you have to find the one that kind of works for you at the time. - -**Dan:** And this is why we like OpenTelemetry is because you can switch from one to the other to the other without with very minimal change. And that’s why I like about getting people on OpenTelemetry. But when it comes to the tooling, that should be your last thing to think about. Don't build your solution around this tool in your spot. Build your solution around what your users are thinking, and that's how I think a lot of people—that resonates when you start talking in that light, especially with the higher-ups. - -**Host:** Anyway, I think we got through them all. - -**Dan:** Yeah, absolutely. - -**Host:** Well, we got through all the questions! Yay! We filled up this hour quite nicely. So thank you so much, Dan, for joining us for Otel Q&A. Stay tuned for our next Otel Q&A and/or Otel in Practice. And again, thank you, Dan, for joining today. I really appreciate it. - -**Dan:** Oh yeah, you know, it’s my pleasure. Thank you for having me. You know how much I love talking about these things. So, you know, I think I actually did not go down a rabbit hole today. I think I was pretty good. - -**Host:** Perfect! It was great. Thank you again! - -**Dan:** Alright, thank you everyone! Thank you for showing up. See you! +**Dan:** all right. Thank you everyone. Thank you for showing up. See you. ## Raw YouTube Transcript diff --git a/video-transcripts/transcripts/2024-11-15T06:30:37Z-humans-of-otel-live-from-kubecon-na-2024.md b/video-transcripts/transcripts/2024-11-15T06:30:37Z-humans-of-otel-live-from-kubecon-na-2024.md index ea5ccf8..b0d4df8 100644 --- a/video-transcripts/transcripts/2024-11-15T06:30:37Z-humans-of-otel-live-from-kubecon-na-2024.md +++ b/video-transcripts/transcripts/2024-11-15T06:30:37Z-humans-of-otel-live-from-kubecon-na-2024.md @@ -10,335 +10,390 @@ URL: https://www.youtube.com/watch?v=LjD454EPMgQ ## Summary -In this live segment from KubeCon North America 2024, host Rees is joined by special guests Hazel and Ted, who are prominent figures in the OpenTelemetry community. They discuss Hazel's journey into observability, the significance of OpenTelemetry, and the challenges of integrating observability into platform engineering. Hazel emphasizes the importance of teaching others about observability tools and her commitment to empowering users, while Ted shares insights on the OpenTelemetry project's governance and community dynamics. They highlight the project's goals of unifying tracing, metrics, and logs to create a coherent view of systems, as well as the need for collaboration across various tech and business sectors to enhance observability practices. The conversation underscores the project's evolution, future aspirations, and the ongoing need for community engagement and feedback. +In this live session from KubeCon North America 2024, hosts Rees and Adriana Vela engage with Hazel, a prominent figure in the OpenTelemetry community, discussing her journey into observability and her insights on the evolution of OpenTelemetry. Hazel shares her frustrations with existing tools that drove her towards OpenTelemetry, emphasizing the importance of effective observability in platform engineering and SRE. She introduces the concept of "Observability 3.0," which aims to integrate non-technical perspectives into observability practices, fostering a holistic understanding across business functions. The conversation also touches on the challenges of community engagement and the complexities of maintaining a vendor-neutral environment in open-source projects like OpenTelemetry. Ted Young, a co-founder of OpenTelemetry, later joins to discuss the project's ongoing evolution, including the integration of various data signals and future developments aimed at enhancing user experience and data analysis capabilities. The dialogue highlights the collaborative spirit of the OpenTelemetry community and the importance of user feedback in shaping the project's direction. ## Chapters -00:00:00 Welcome and intro -00:01:20 Guest introductions -00:02:30 Hazel's observability journey -00:04:50 OpenTelemetry discussion -00:06:00 Platform engineering insights -00:08:20 Observability challenges -00:10:00 OpenTelemetry project overview -00:12:10 Observability 3.0 concept -00:22:00 NLY Foundation background -00:29:10 Transition to next guest -00:30:00 Ted Young introduction -00:31:16 OpenTelemetry governance and community -00:36:30 Community trust and collaboration -00:40:00 Project structure and incentives -00:44:35 Updates from project -00:55:00 Future of OpenTelemetry -00:58:00 Closing remarks and survey reminder +00:00:00 Introductions +00:02:06 Guest introduction: Hazel +00:05:30 Discussion about observability journey +00:10:12 Importance of observability in platform engineering +00:19:07 Introduction of Observability 3.0 +00:23:06 Discussion about the nly foundation +00:37:48 Guest introduction: Ted Young +00:40:57 Discussion about OpenTelemetry governance +00:45:09 Updates on OpenTelemetry project +00:57:45 Call to action for audience engagement -**Reys:** Well, he hasn't done the countdown, so YouTube... Oh, is this on the channel? Why should I just shut up, right? So, I launch the counter of 10 minutes. That's too much, probably drop it. No, we still have this. Oh, it's OpenTelemetry official. Oh, he's checking the OpenTelemetry channel to make sure that it’s queuing up on their live. Okay, cool, perfect. Are you ready, M? +## Transcript -**M:** Yeah, and I will in... +### [00:00:00] Introductions -[00:01:20] **Reys:** Hello, live from Salt Lake City, I'm Reys, and I'm here with some special guests. We are live at KubeCon North America 2024, and we're here to talk with a couple of really cool people in the OpenTelemetry community about how they got involved and kind of learn a little bit more about them. Let's go! +**Reys:** Well, he hasn't done the countdown so YouTube... Oh, is this on the channel? Why should I just shut up, right? So I launch the counter of 10 minutes. That's too much, probably drop it. No, we still have this. Oh, it's OpenTelemetry, official OpenTelemetry! -**Adriana:** All right, good morning, good morning, good morning! I'm really glad to be here. So, for those of you who don't know, I am Adriana Vela, and I work with Reys on the OpenTelemetry and User SIG. We're excited to be talking to our awesome OpenTelemetry guests today. +Oh, he's checking the OpenTelemetry channel to make sure that it's queuing up on their live. Okay, cool, perfect. Are you ready? -**Reys:** So, Hazel, how did you get involved in OpenTelemetry? What is your story? How did you get involved in observability? +**Adriana:** Yeah, and I will. Hello, live from Salt Lake City! I'm Reys and I'm here with some special guests. We are live at KubeCon North America 2024, and we're here to talk with a couple of really cool people in the OpenTelemetry community just about how they got involved and kind of learn a little bit more about them. Let's go! -[00:02:30] **Hazel:** So, I think I got involved with observability in a very similar way that a lot of people do, which is to say I didn't. What happened was I got really, really frustrated at things happening, and I needed to figure out how to actually ask better questions about my systems and figure out what was going on and dig into them. The tools that I had weren't very good, and so I dug around looking for better tools, better ways to do things, and more importantly, better ways to teach other people what we're doing and how it works, because one of my superpowers is being able to take the entire system and hold it into my head and be able to sort of intuitively think about it. Like, when I was in an operating systems class in college, the point of the class was to kind of almost force you to learn how to debug. But I never did, and I still don't know how to debug, because I just looked at the code and mentally traced the code in my head and debugged panic flaws in the sheer by just like staring at it. But I can't teach that to people, and I can't show them how to think about that. So I've always said I want to figure this out with just, you know, print F debugging or whatever I want to do, but how do I actually take the vibes and turn them into a tool and an explanation and a technique and a way to teach people that you have this? You already know the answers. You know how to find things. You're already like 80 or 90% in the way there. It's not this magical new toolkit; it's this asking questions that are meaningful to you, getting those answers that are useful, and then doing something useful with that, like learning. After effectively on that, and you don't do that, you already know how to do that, and just taking that and looking for really, really nice things ended up getting me into that observability space where I met a lot of people that were also trying to do that. +### [00:02:06] Guest introduction: Hazel -**Reys:** And how did you get into OpenTelemetry itself? +**Hazel:** All right, good morning, good morning, good morning! I'm really glad to be here. So for those of you who don't know, I am Hazel, and I have thoughts, lots of thoughts. They never stop thinking, and I am super glad to be here and talk with you all and hopefully make you laugh so hard you cry a little bit. Thank you! -[00:04:50] **Hazel:** So, OpenTelemetry... For me, it was actually I looked at the shape of all the different vendor-specific things or the different instrumentation of like logging or metrics, or we didn't really have traces when I first started. It was actually really frustrating because I really, really hate doing things that are undifferentiated in a different way. I would much, much rather do a bit of work to figure out how can I slice it down? How can I think about this concept of optionality and putting everything together? Because one of the best things about Linux is it turns out you don't need to write a kernel in order to get something working; you can use one. One of the best things about Kubernetes is sure, you write, you know, your own custom flavor on top, but it's the thin veneer over this core. So for observability, I started looking for that core, the essence of it that you could build that thin veneer, that layer on top, and I found that in OpenTelemetry. +**Adriana:** And I'm Adriana Vela, and I work with Reys on the OpenTelemetry and user SIG, so we're excited to be talking to our awesome OpenTelemetry guests today. So, Hazel, how did you get involved in OpenTelemetry? Like, what is your story? How did you get involved? We could start at the beginning. -**Reys:** That's amazing! So how long have you been using the project or contributing to the project? +**Hazel:** How did you get involved in observability? So I think I got involved with observability in a very similar way that a lot of people do, which is to say I didn't. And what happened was I got really, really frustrated at things happening, and I needed to figure out how to actually ask better questions about my systems and figure out what was going on and dig into them. The tools that I had weren't very good, and so I dug around looking for better tools, better ways to do things, and more importantly, better ways to teach other people what we're doing and how it works. -[00:06:00] **Hazel:** So I've been using the project actually primarily, usually not directly, because I'm low enough on that platform side. What I'm almost always looking to do is enable people, and so I have looked at the OpenTelemetry collector a lot, being able to say how can I empower people? How can I enable them? How can I take them from these different places where maybe they're using one vendor here and one vendor on another team and one vendor on another team or a third-party integration or the OpenTelemetry code? So as far as being able to enable people and think about that, I started, I want to say, really digging deep in building those platforms for other people around in 2019. +Because one of my superpowers is being able to take the entire system and hold it in my head and be able to sort of intuitively think about it. Like when I was in an operating systems class in college, the point of the class was to kind of almost force you to learn how to debug. But I never did, and I still don't know how to debug. I just looked at the code and mentally traced the code in my head and debugged panic gloss in the debugger by just staring at it. But I can't teach that to people, and I can't show them how to think about that. So I've always said then, "Okay, I can figure this out with just, you know, print F debugging or whatever I want to do." -**Reys:** Okay, wow! So basically from the beginning, really? +But how do I actually take the vibes and turn them into a tool and an explanation and a technique and a way to teach people that you have this? You already know the answers; you know how to find things. You're already like 80 or 90% in the way there. It's not this magical new toolkit; it's this asking questions that are meaningful to you, getting those answers that are useful, and then doing something useful with that—like learning. And then after you effectively do that, you already know how to do that. And just taking that and looking for really nice things ended up getting me into that observability space where I met a lot of people that were also trying to do that. -**Hazel:** Yeah, yeah, that's great! Awesome! Which is so weird because like 2019 was not that long ago; it's like two weeks ago. +**Adriana:** And how did you get into OpenTelemetry itself? -**Reys:** I know, right? It does feel like it, absolutely. Yeah, I was going to say, you're also heavily involved in the platform engineering space, and I feel like observability is one of those things. Like, we often talk about it as a separate thing, but really, like, you can't have effective platform engineering without observability. You can't have effective SRE without observability. And I know you have many thoughts. +### [00:05:30] Discussion about observability journey -[00:08:20] **Hazel:** Yeah, so it's actually something that irks me a little bit a lot of the time because I get why this happens. You have a massively wide platform, and you have a massively wide tool chain, and people keep adding more and more context into something, and the context goes deep, and it goes wide, and it goes up, and it goes in all the different places, and you can't possibly hold it all in your head unless, you know, you're weird like me and you can hold way too many things in your head. But you do end up in this situation where, yeah, OpenTelemetry, you can dive so deep into it; it can become your whole thing; it can become like the only thing you really think about. When you see a lot of companies with the platform teams, they start off with an IT team that's really just rebranded Ops. Then they take the rebranded Ops team, they split it into two or three teams, and then one of those teams ends up being in charge of, you know, the Kubernetes part, and that gets labeled the platform team for some reason. Then the platform team gets turned into two or three different teams, and then you keep going on, and you know someone gets stuck with the observability thing, and it ends up being really complex because there's a lot of moving parts. So I see how things get split out and sort of fragmented, you know, much in the same way as the CNCF landscape is so massive. Everything has its own kind of corner; nobody really talks to each other. But I wish people would do that because they often end up re-implementing the same thing over and over. +**Hazel:** So OpenTelemetry... For me, it was actually I looked at the shape of all the different vendor-specific things or the different instrumentation of, like, logging or metric. We didn't really have traces when I first started, and it was actually really frustrating because I really, really hate doing things that are undifferentiated in a different way. I would much, much rather do a bit of work to figure out, like, how can I streamline down? How can I think about this concept of optionality and putting everything together? Because one of the best things about Linux is it turns out you don't need to write a kernel in order to get something working; you can use one. -**Reys:** Right, exactly. +And one of the best things about Kubernetes is, sure, you write, you know, your own custom flavor on top, but it's the thin veneer over this core. And so for observability, I started looking for that core, the essence of it, that you could build that thin veneer, that layer on top, and I found that in OpenTelemetry. -[00:10:00] **Hazel:** So like in the profiling side of things, we have this Open Profiling aspect, and they're looking at a lot of the OpenTelemetry stuff, and they're reinventing a lot of the same discussions that they need to have. A lot of saying, do you need standards? Do you need naming? Do you need plugins? How does the collector work? And I'm like we solved that five years ago! Like, look at the pre-existing stuff, tweak it a bit, and then you think about it. Or you have the event streaming architecture people where you have like database streaming, you have lake house people, and you have business analytics and business intelligence and all these different data science types of things. It turns out that if you are trying to take a metric load of data and derive useful actionable insights from it and then share that with people who are doing data science, that's kind of what we decided to call it, you know, 20 years ago. Then we forgot about that, and we reinvented OpenTelemetry instead of making a data pipeline. And then we reinvented profiling instead of making a data pipeline, and now we're kind of looking at all these things, and then we're going, "Oh, we should do this thing," and then the, you know, the Power BI or the business analytics people in the corner are just crying a little bit because they've been doing this since the '80s. +**Adriana:** That's amazing! So how long have you been using the project or contributing to the project? -**Reys:** Right. +**Hazel:** So I've been using the project actually indirectly. I'm low enough on that platform side. What I'm almost always looking to do is enable people. And so I have looked at the OpenTelemetry collector a lot, being able to say, "How can I empower people? How can I enable them? How can I take them from these different places of maybe they're using one vendor here and one vendor on another team and one vendor on another team or a third-party integration or the OpenTelemetry code?" So as far as being able to enable people and think about that, I started, I want to say, really, really digging deep in building those platforms for other people around in 2019. -**Hazel:** And then way over here in the financial side, did you know this is one of my favorite things actually? So KDB is a financial analytic state database, and it is a column database. If you look at high-frequency trading or, you know, data analytics people that specialize in the financial world, they actually predate the usage of all of these, like, oh no, queries that rely on indices. You can just grab the data, and they've been doing that for 40 years. They had to, and nobody else ever really thought to look it up. It's fun to see all the communities reinvent things over and over, bring their own flavor and context into it, and then hopefully we can, with platform engineering, start stirring all the people together, talking to them, and getting them to actually look outside the little window and go, "Oh, we solved the same problem!" Well, that's cool. How can I learn from how you approached it, and how can you learn from how I approached it, and can we build kind of a more common thing that's really awesome? +**Adriana:** Okay, wow! So basically from the beginning, really? -**Reys:** Given that there's so many interesting points that you just brought up and like the way your brain works and how you can look at something and kind of intuitively understand what's happening, what pieces of the open projects are intriguing you? +**Hazel:** Yeah, yeah, that's great! Awesome. -**Hazel:** So one of the things that's really intriguing to me about the OpenTelemetry project is you have this dichotomy of OpenTelemetry as the end-user kind of part where you have this SDK and this API and there's how you use it, and you have this mental model. Then you have the operational side of how you collect the data, how you store it, how you enrich it, how you sample, and how you do all these things, and they're all deeply disconnected. They really don't need to be, but the reason they're often deeply disconnected is because they're all non-intentionally or maliciously horrifically lying to each other. So the mental model, for example, that you have with traces is, oh, there's like this little box and it's this start of it and the end of it, and in the box, I put all my information stuff, and I draw some information on the outside of it. Then in the information stuff, I have another box, then I have another box, then I have another box. I have so many boxes! When you actually use the SDK, when you actually send things, that's a complete lie. You're not doing a box; you are reinventing the concept of distributed transactions really badly and without transactional semantics because you're just firing stuff off like a stream of events. But you don't have like this read-ahead lock or any of the other stuff that the database people invented in the '60s. So there's that kind of lie of it. Then when you get into the operational side, increasingly, it turns out sending everything is expensive, you know, B, really, really time-consuming, C, actually a waste of effort and time and resources, and you shouldn't do that. So you need to take that, enrich it, correlate other data, look at better things, figure out associations, sample it, and sort of figure out and finesse how you understand the data in this very operational context, which we often bundle that into the OpenTelemetry collector. But we never ever talk about the OpenTelemetry collector as like a required thing or even, you know, a thing, and then so it hampers a lot of the potential of the SDK make, because for example, you can't update a span because a span is treated as an append-only immutable log. Except it's a box, but no, it's a dependently immutable log; it's a stream of events. But at the same time, it turns out if you break this at any point because not all are reliable, you completely destroy the entire concept of what you're doing and embreeze all your ingestion pipelines and embreeze your UI of everything building everything, and it's super annoying. +**Adriana:** Which is so weird because like 2019 was not that long ago. It's like two weeks ago! -**Reys:** Right. +**Hazel:** I know, right? It does feel like it! -**Hazel:** So I would love for people to think about what it's like if we made the concept of a collector or this concept of enriching, rewriting the tree, flattening the tree, doing all these weird sorts of correlation concepts or flattening or changing the shape of stuff to make it more malleable for you. What if that was a more integral part of how we designed the SDK? And if we designed the SDK with more use cases in mind, how do we do so in a way that gives people a simple mental model but doesn't lie to them about what they can expect out in the platform? So like there are client SDKs, for example, that don't send the data at all ever until they have received on the client the ending span, which means that if your span is 10 seconds long and on second 9 and 2, the client process gets shut down and does not have time to send that span, you lose all of that data. But if you send it all to the collector immediately, you have so much traffic you can't really afford to do that. But if you were able to send like a snapshot thing of a read-ahead log of this isn't done yet, but I'm going to window it like databases do in the '70s, then okay, you can work with it. You can work with it, and then you can patch it up. Some vendors have actually started to work cleverly around OpenTelemetry and work cleverly around the specification in the middle part to allow that capability in places where they need it, like mobile clients, and then patch it up and make it, you know, OpenTelemetry compliant by the time you send it to your backend. So if we open up the Pandora's box a little bit of, okay, maybe this is kind of necessary, maybe we need to think about this in a weird way of we have to have all the things talk to each other, and we need to make it more possible to do these things that make the mental model a bit more coherent, when can we do that? +**Adriana:** I was going to say you're also heavily involved in the platform engineering space, and I feel like observability is one of those things. We often talk about it as a separate thing, but really, you can't have effective platform engineering without observability. You can't have effective SRE without observability. And I know you have many thoughts. -**Reys:** That's a lot! You got thoughts! +**Hazel:** Yeah, so it's actually something that irks me a little bit a lot of the time because I get why this happens. You have a massively wide platform and you have a massively wide toolchain, and people keep adding more and more context into something, and the context goes deep and it goes wide and it goes up and it goes in all the different places. You can't possibly hold it all in your head unless you know you're weird like me and can hold way too many things in your head. -**Hazel:** Um, I wanted to just switch gears a little bit because before we started the stream, we were talking about, you know, you have many thoughts on many things, and one of them was Observability 3.0. Now we've heard Observability 2.0 has kind of come into our vernacular lately, and you were talking now about Observability 3.0. So can you tell us what you mean by that? +But you do end up in this situation where, yeah, OpenTelemetry, you can dive so deep into it. It can become your whole thing. It can become like the only thing you really think about. And when you see a lot of companies with the platform teams, they start off with an SRE team that's really just a rebranded Ops team. Then they take the rebranded Ops team, they split into, you know, two, three teams, and then one of those teams ends up being in charge of, you know, the Kubernetes part, and that, you know, gets labeled the platform team for some reason. -**Hazel:** So, what I mean by that is a really, really fun thing where it gets into the heart of what I like about observability, which is essentially the same thing that everybody ignores, and I would like them to not ignore that. And so if I brand it a little bit with like a cute little sticker, maybe people will care. So Observability 1.0 and 2.0 is sort of a reframing of the natural progression that has happened in observability and the vendors and the capabilities and the needs of the platform and what we need to do in order to ask the right questions. Before, it was sort of this is observability, and this is observability, and then we're like, well, no, it is all observability. It's all about asking questions. It's more what's the fidelity of the questions? What type of questions can you ask? How much information is there? How rich is it? What are the properties of asking those types of questions? So if we think of Observability 2.0 as sort of being the ultimate insight in the tech context of a company or a program or a platform, can we ask essentially a question? That's kind of the ultimate goal of observability 2.0 to me. +And then the platform team gets turned into two, three different teams. And then you keep going on, and you know someone gets stuck with the observability thing, and it ends up being really complex because there are a lot of moving parts. So I see how things get split out and sort of fragmented, you know, much in the same way as the CNCF landscape is so massive; everything has its own kind of corner, and nobody really talks to each other. -So for me, Observability 3.0 is defined by the idea that you need to take non-tech people and non-tech problems and the larger context of the business and embed it into your ability to reason about your system. You need to take the system and embed that into the rest of the company so the company can reason about that. So Observability 3.0 has a stark difference in that it's not about tooling; it's not about capabilities. It's not about all the sort of the technical side; it's very much now we need to take the tech and we need to take the people and the processes and this, you know, massive budget that the IT industry gets and stop spending it without accountability and stop spending it as this massive black hole where money goes in, magic stuff comes out, and nobody can explain why. So can your business analysts, can your marketing people, can your salespeople, can your product people, can your everybody else, your customer success team, can they all sit there and understand how to better serve the customer and how to better interoperate with the technology people without the technology people having screen feeding to them? Can you give them the capability to do a better job, and can you take what they know and take all this cool stuff that they do and put it into context that lets platform engineers and product engineers and backend and all these people do a really, really good thing of being able to present technological options and be able to actually interoperate with what the company needs, as opposed to what the company has asked for, you know, in a guessing manner? To me, Observability 3.0, to kind of sum it up, is the business context comes in tech, the tech context goes into the business, and they become one harmonious concept. +But I wish people would do that because they often end up re-implementing the same thing over and over. So like in the profiling side of things we have this Open Profiling aspect, and they're looking at a lot of the OpenTelemetry stuff, and they're reinventing a lot of the same discussions that they need to have. A lot of the saying, "Do you need standards? Do you need name events? Do you need plugins? How does the collector work?" And I'm like, "We solved that five years ago! Look at the pre-existing stuff, tweak it a bit, and then you think about it." -**Reys:** That's amazing, and I'm sure this is going to spark a lot of conversations! I'm really excited to see other people's thoughts and kind of hopefully, you know, see your vision. +### [00:10:12] Importance of observability in platform engineering -**Adriana:** I also wanted to chat with you; you know, OpenTelemetry is an open-source project. Obviously, we at KubeCon, and you work at the NLY Foundation, and I would love for the audience to learn more about kind of the story behind the foundation because, yeah, you shared it earlier, and I would love for you to share that. +Or you have the event streaming architecture people where you have, like, database streaming, you have lake house people, and you have business analytics and business intelligence and all these different data science types of things and data science-related stuff. And it turns out that if you are trying to take a metric buttload of data and derive useful actionable insights from it and then share that with people who are doing data science, that's kind of what we decided to call it, you know, 20 years ago. -[00:12:10] **Hazel:** The NLY Foundation was started by Chris Noda, and it was one of those last things that she did before she passed away. The NLY Foundation actually kind of started as an idea that germinated from Hacker, the Mastodon instance that we all built together, made into this massive thing, and then did a huge migration. We live-streamed it, and everybody learned from it; it was awesome. The reason in Hacker that we chose the technology that we did is because you had a bunch of brains; you had a bunch of people that are kind of the world’s support to doing a lot of things, but we didn't want to rely on that because we wanted to experiment with the idea of what does it mean for a community to deeply own their own community, their own concept, their own architecture, their everything? This was right around the time when the community, especially the tech community, was first, you know, grappling with the idea of we built this little thing, and now it's kind of being taken away, and we don't have control over our own, you know, engagement platform, our own sort of communication. How do we never lose it again? Then it turns out that as the massive migration initially happened, we almost closed registrations on Hacker, not due to operational concerns, but due to a massive legal liability. The course of how federation works on Mastodon ends up being a very, very abusable vector for putting illegal content somewhere because that content has to get syndicated onto your server. If you run a Mastodon instance, you are liable for anything that touches your computer, but you don't really necessarily control what that is, and so that is a very difficult concept. Nova kind of ran into this with her legal background and her legal brain. She was like, "Oh, absolutely no, no, no, no, no, no, no." She ended up immediately finding lawyers, sitting down, and building like an LLC. We wrapped Hacker around NLY and very quietly spread this kind of information to a lot of other larger instances of, "You need to now care about this; you need to immediately start caring about this because you are at risk of a huge vector." There's a huge surge of potential heat coming, but it made us realize as we sat down and we kind of solved this problem, when you go from a fun toy project and you take this toy project, and it becomes a community, there's this invisible cliff of I have like my vision, I have my dream, my people, and now there's everything. I can't half-ass my licensing anymore; I can't half-ass my contributor CLA agreement. How do I get people from large companies or from regulated industries to be able to contribute? It turns out you can't just make their repository open source; you have to actually make it so that their paperwork is okay with it. +And then we forgot about that, and we reinvented OpenTelemetry instead of making a data pipeline. And then we reinvented profiling instead of making a data pipeline. And now we're kind of looking at all these things and then we're going, "Oh, we should do this thing!" And then the, you know, Power BI or the business analytics people in the corner are just crying a little bit because they've been doing this since the '80s. -**Reys:** Right. +And then way over here in the financial side, did you know this is one of my favorite things? Actually, so KDB is a financial analytic state machine database, and it is a column database. So if you look at high-frequency trading or, you know, data analytics, people that specialize in the financial world, they actually predate the usage of all of these, like, oh no queries that rely on indices. You can just grab the data, and they've been doing that for 40 years. -**Hazel:** And then how about international people? How about people with disabilities? How about people with different needs? How about people with different backgrounds? Okay, you know, we haven't even begun to solve the how you fund open source. How do you find open source? No, nobody knows the answer; nobody's even gone close. People are trying, like Eva Black, as she says, doing a fantastic job trying to create this sort of top-down way and path and avenue for the ability for companies to pay open source and for governments to pay and for all these things to happen. It's going to take time, but as a community, we need to bottom-up sort of figure out how do we pass this cliff? It's a massive cliff, and nobody really knows it's coming, and you don't have time to prepare for it because you don't get to choose when you become adopted and beloved by the community. Nobody sits down and says, "We're a community now." You just look around one day, and you go, "These are all my friends, and I love them, and this is great!" and then, oh no! It turns out that's only step two because it turns out that you can't really call the CNCF community anymore; it's an ecosystem. That's kind of that third massive cliff that very few people talk about, where you have this one project and this community, and that's cool. The community maybe grows other things when it reaches this stage of becoming a generation and incubation hub of innovation and experimentation, and it starts generating a fractal of different communities that all come together in this ecosystem of this massive idea exchange of community knowledge sharing and community growth when it becomes this messy, inarticulate sort of ill-defined but beautifully growing organic thing that takes on its own life. That's an ecosystem, and we don't even really know how to build those. We definitely don't know how to fund them; we definitely don't even know the differences between all the legal and the compliance. How do you facilitate that? How do you get there? So we looked at this problem, Nova and I, and a bunch of other people, and we thought someone needs to start thinking about this. It's going to take a while; it’s going to take several years, maybe even a decade or two, to really deeply help the world understand how to pass those cliffs and turn them into this growing ramp of taking ideas and sharing with the world in a way that generates further ideas in a way that becomes this ever-growing thing. The NLY Foundation, which was originally sort of kind of starting as like a legal cover and legal ability to do this, always was intended to be a vehicle that helps you understand that problem and helps figure out how do we make this beautiful growth and knowledge sharing possible. +So you had to, and nobody else ever really thought to look it up. And so it's fun to see all the communities reinvent things over and over, bring their own flavor and context into it, and then hopefully we can, with platform engineering, start stirring all the people together, talking to them, and getting them to actually look outside their little window and go, "Oh, we solved the same problem! Well, that's cool. How can I learn from how you approached it, and how can you learn from how I approached it, and can we build kind of a more common thing that's like really awesome?" -**Reys:** That's so great! I think are we coming up on time? +Given that there's so many interesting points that you just brought up and like the way your brain works and how you can look at something and intuitively understand what's happening, what pieces of the open projects are intriguing you? -[00:29:10] **Adriana:** So we are coming up on time. We are going to have our next guest on. Thank you so much, Hazel. It was lovely to talk with her, and we will share socials, like how you can get in touch with her at the end, and as well as like how you can get involved in the future Humans of OpenTelemetry segment as well. I'm really excited to see what kind of interactions come up from these conversations that we just had. And yeah, whatever questions, comments, thoughts you have, definitely please let us know. We would love to hear them and help connect you with Hazel so we can learn even more and move forward on kind of her vision. Also, secretly jealous of her mental superpowers. I'm going to have to see if I can pick up any tips, but now we have our next guest on, and I'm so excited! Hopefully, you can see his cat pants because they're amazing. +**Hazel:** So one of the things that's really intriguing to me about the OpenTelemetry project is that you have this dichotomy of OpenTelemetry as the end-user kind of part where you have this SDK and this API and there's how you use it, and you have this mental model. And then you have the operational side of how you collect the data, how you store it, how you enrich it, how you sample, and how you do all these things. And they're all deeply disconnected, and they really don't need to be. -**Ted:** Hello! +But the reason they're often deeply disconnected is because they're all, non-intentionally or maliciously, horrifically lying to each other. So the mental model, for example, that you have with traces is, oh, there's like this little box and it's the start of it and the end of it, and in the box, I put all my information stuff and I draw some information on the outside of it. And then in the information stuff, I had another box, then I have another box, and I have another box. I have so many boxes! -**Reys:** Hey! Hello, how's it going? +And when you actually use the SDK, when you actually send things, that's a complete lie. You're not doing a box; you are reinventing the concept of distributed transactions really badly and without transactional semantics because you're just firing stuff off like a stream of events. But you don't have, like, this read-ahead lock or any of the other stuff that the database people invented in the '60s. And so there's that kind of lie of it. -**Ted:** Going great! How are you all doing today? +And then when you get into the operational side, increasingly it turns out sending everything is expensive, really time-consuming, and actually a waste of effort and time and resources, and you shouldn't do that. So you need to take that, enrich it, correlate other data, look at better things, figure out associations, sample it, and sort of figure out and finesse how you understand the data in this very operational context, which we often bundle that into the OpenTelemetry collector. -**Reys:** Not bad! It's KubeCon! Awesome times! It's like a big friend reunion! Day two of the main KubeCon events, which, you know, everyone is on fumes at this point. +But we never ever talk about the OpenTelemetry collector as like a required thing or even, you know, a thing, and then it hampers a lot of the potential of the SDK because, for example, you can't update a span because a span is treated as an append-only immutable log, except it's a box. But no, it's a dependently immutable log; it's a stream of events. But at the same time, it turns out if you break this at any point because of how unreliable you are, you completely mess up the entire concept of what you're doing and you mess up all your ingestion pipelines, and it's super annoying. -**Ted:** Yeah, we had a busy start because we had Observability Day, which just started with Rejects, and then, yep, Observability Day and KubeCon Day one yesterday. +So I would love for people to think about what it's like if we made the concept of a collector or this concept of enriching, rewriting the tree, flattening the tree, doing all these weird sorts of correlation concepts, or flattening or changing the shape of stuff to make it more malleable for you. What if that was a more integral part of how we designed the SDK? And if we design the SDK with more use cases in mind, how do we do so in a way that gives people a simple mental model but doesn't lie to them about what they can expect out in the platform? -**Reys:** I know! It just feels like we've been here forever, so we're so excited to have you! Can you tell us a little bit about your role in OpenTelemetry? I'm sure a lot of our viewers who are more familiar with the project are already aware of who you are, but we'd love an introduction. +So like there are client SDKs, for example, that don't send the data at all ever until they have received on the client the ending span, which means that if your span is 10 seconds long and on second 9 and 2, the client process gets shut down and does not have time to send that span, you lose all of that data. But if you send it all to the collector immediately, you have so much traffic you can't really afford to do that. But if you were able to send like a snapshot thing of a write-ahead log of "this isn't done yet, but I'm going to window it" like databases do in the '70s, then okay, you can work with it! You can work with it, and then you can patch it up. -[00:31:16] **Ted:** Yeah, sure! My name’s Ted Young, and I'm one of the co-founders of the project, coming from the OpenTracing side of the family. For people who don't know, OpenTelemetry is actually a merging of two prior projects: OpenTracing and OpenCensus. OpenCensus was mostly Google people and some Microsoft people, and OpenTracing was everybody else. Then we merged to form OpenTelemetry, and I continue to work on the project as a member of the governance committee, mostly just focusing on the various spec SIGs, implementation SIGs that need an extra helping hand when it comes to finding consensus and figuring out what design is actually going to work. +And some vendors have actually started to work cleverly around OpenTelemetry and work cleverly around the specification in the middle part to allow that capability in places where they need it, like mobile clients, and then patch it up and make it, you know, OpenTelemetry compliant by the time you send it to your backend. So if we open up the Pandora's box a little bit of, okay, maybe this is kind of necessary, maybe we need to think about this in a weird way of we have to have all the things talk to each other, and we need to make it more possible to do these things that make the mental model a bit more coherent, when can we do that? That's a lot! -**Reys:** Right. +**Adriana:** You've got thoughts! I wanted to just switch gears a little bit because before we started the stream, we were talking about, you know, you have many thoughts on many things, and one of them was observability 3.0. Now we've heard observability 2.0 has kind of come into our vernacular lately, and you were talking now about observability 3.0. So can you tell us what you mean by that? -**Ted:** There are some areas in OpenTelemetry where it's technically challenging enough that ironically it's easy to come to a design, or at any rate, it's easy to determine which design is the correct one, right? Because it's so technically challenging that the requirements are very rigorous. But then there's other parts of OpenTelemetry where it's like a little more squishy, where there's like one way that would be a really, really good way, and then there's some other ways that would like sort of work, right? You can't say they definitely would not work, but they're not like great. But when you have that situation, it's so, so, so much harder to find consensus, right? Because people will lock in to a particular idea, and then if it's possible to find some way to make it work, because they're engineers, they will continue to promote, but "But there's this way it could work!" and trying to like find consensus there. +**Hazel:** So what I mean by that is a really, really fun thing where it gets into the heart of what I like about observability, which is essentially the same thing that everybody ignores, and I would like them to not ignore that. And so if I brand it a little bit with like a cute little sticker, maybe people will care. -**Reys:** Right. +So observability 1.0 and 2.0 is sort of a reframing of the natural progression that has happened in observability and the vendors and the capabilities and the needs of the platform and what we need to do in order to ask the right questions. Before it was sort of, "This is observability, and this is observability." And then we're like, "Well, no, it is all observability." It's all about asking questions. It's more, "What's the fidelity of the questions? What type of questions can you ask? How much information is there? How rich is it? What are the properties of asking those types of questions?" -**Ted:** Yeah, of being like, "Yes, okay, that would work, but this would work better," and getting everyone to be like, "Well, even though you prefer that one, would you agree that you're not going to convince everyone at this point to go that way?" Like, "Yes!" And like, "Would you be able to get everything done with this other one?" Like, "Yes!" And like, "Okay, well then would you be willing to come on board with this and help us move this forward because we all actually need it to get done?" So that's like a bit of just more like community organizing that requires maybe kind of like an engineering design background. +### [00:19:07] Introduction of Observability 3.0 -**Reys:** Yeah, and I feel like that's where I provide the most value these days to the project. +And so if we think of observability 2.0 as sort of being the ultimate insight in the tech context of a company or a program or a platform, can we ask essentially in question that's kind of the ultimate goal of observability 2.0 to me? So for me, observability 3.0 is defined by the idea that you need to take non-tech people and non-tech problems and the larger context of the business and embed it into your ability to reason about your system. And you need to take the system and embed that into the rest of the company so the company can reason about that. -**Ted:** That's so awesome! Cool! I feel like there's also, you have to like have some good diplomatic skills as well. +So observability 3.0 has a stark difference in that it's not about tooling. It's not about capabilities. It's not about all the sort of the technical side. It's very much now we need to take the tech and we need to take the people and the processes and this, you know, massive budget that the IT industry gets and stop spending it without accountability and stop spending it as this massive black hole where money goes in, magic stuff comes out, and nobody can explain why. -**Reys:** Right. +So can your business analysts, can your marketing people, can your sales people, can your product people, can your everybody else, your customer success team, can they all sit there and understand how to better serve the customer and how to better interoperate with the technology people without the technology people having to screen feed to them? Can you give them the capability to do a better job? And can you take what they know and take all this cool stuff that they do and put it into context that lets platform engineers and product engineers and backend engineers and all these people do a really, really good thing of being able to present technological options and be able to actually interoperate with what the company needs? -**Ted:** Building trust in the community is important, just absolutely having people know that you don't have an agenda. +It has a potential, not just what the company has asked for, you know, in a guessing manner. To me, observability 3.0, they kind of sum it up as the business context comes in tech, the tech context goes into the business, and they become one harmonious concept. -**Reys:** Yeah, yeah. Right. And that you really are just there to listen to everyone and help everyone really clarify the requirements and help everyone really determine what the best design solution would be for those requirements. +**Adriana:** That's amazing! And I'm sure this is going to spark a lot of conversations, and I'm really excited to see, you know, other people's thoughts and kind of hopefully see your vision. I also wanted to chat with you. You know, OpenTelemetry is an open-source project, obviously. We’re at KubeCon, and you work at the OpenTelemetry Foundation. I would love to, you know, have the audience learn more about kind of the story behind the foundation, because, yeah, you shared it earlier, and I would love for you to share that. -**Ted:** Yeah, yeah, yeah. And you know, on a similar vein, it always brings me back to like the whole thing with OpenTelemetry being such a lovely community and that it really takes the whole thing of being vendor-neutral very seriously. I mean, we're all at three different places, and we all get along. Like, we're all competitors, but like not really, because we're all moving towards the same goal. I was wondering if you could comment on like what has to go on to really continue to foster that sense of community and not in us versus them. +**Hazel:** The OpenTelemetry Foundation was started by Chris Nova, and it was one of those last things that she did before she passed away. The OpenTelemetry Foundation actually kind of started as an idea that germinated from Hacker, the Mastodon instance that we all built together, made into this massive thing, and then did a huge migration. It was live! We live-streamed it, and everybody learned from it; it was awesome. -**Ted:** Yeah, that is great! So I really think there's two parts. The simple part is that like humans, generally speaking, are good people. And also, even though we might all work at different companies now, the reality is for the most part, there are just like many tech domains. There's like an observability scene, right? There's like the engineering and product people or whatever who's just like, that's their bag, and they're good at it, and they just tend to kind of like circle through the different orgs and companies that happen to be paying people to work on that stuff in that time. So I think that's the thing that makes it easier, right? Like it's not like we're from different planets that are in some intergalactic war, and we have like nothing in common or something. We've got all this stuff to overcome. It's just like we all know each other. +### [00:23:06] Discussion about the nly foundation -**Reys:** Right. +And the reason in Hacker that we chose the technology that we did is because you had a bunch of brains, you had a bunch of people that are kind of the world’s support to doing a lot of things, but we didn't want to rely on that because we wanted to experiment with the idea of, "What does it mean for a community to deeply own their own community, their own concept, their own architecture, their everything?" And this was right around the time when the community, especially the tech community, was first, you know, grappling with the idea of, "We built this little thing, and now it's kind of being taken away, and we don't have control over our own engagement platform, our own sort of communication. How do we never lose it again?" -**Ted:** And when you're around long enough, it's like everyone cycles two or three jobs, so you stop really looking at people as like representatives of a particular company. +And then it turns out that as the massive migration initially happened, we almost closed registrations on Hacker not due to operational concerns, but due to a massive legal liability. And the course of how federation works on Mastodon ends up being a very, very abusable vector for putting illegal content somewhere because that content has to get syndicated onto your server. And so if you run a Mastodon instance, you are liable for anything that touches your computer, but you don't really necessarily control what that is. And so that is a very difficult concept. -**Reys:** It's so true! +And so Nova kind of ran into this, and with her legal background and her legal brain, she was like, "Oh, absolutely no, no, no, no, no, no." And then so she ended up immediately finding lawyers, sitting down, and building an LLC. And we wrapped Hacker around an LLC and very quietly spread this kind of information to a lot of other larger instances of, "You need to now care about it; you need to immediately start caring about this because you are at risk of a huge vector. There's a huge surge of potential heat coming." -**Ted:** Yeah. I think one of the things too, just from like my, you know, very short experience, is even when they are moving companies, people want to stay involved in the project. They're actively seeking out roles where they can continue contributing to the project in their role, so I think that speaks volumes. +But it made us realize, as we sat down and we kind of solved this problem, when you go from a fun toy project and you take this toy project and it becomes a community, there's this invisible cliff of, "I have my vision, I have my dream, my people, and now there's everything." Like, I can't half-ass my licensing anymore, I can't half-ass my contributor CLA agreement. How do I get people from large SE companies or from regulated industries to be able to contribute? It turns out you can't just make their repository open source; you have to actually make it so that their paperwork on there is okay with it. -**Ted:** The fact that you can look at our maintainers and community members and there's like quite a long lifespan at this point. I actually haven't run the math, but I would not be shocked if like the average age of a maintainer in OpenTelemetry was measured in years at this point. Like, on average, it's like two or three years. I wouldn't be shocked if it's that high. Like people really do stick around. +And then how about international people? How about people with disabilities? How about people with, you know, different needs? How about people with different... that happens, okay? You know, we haven't even begun to solve the how you for open source, how do you find open source? Nobody knows the answer! Nobody's even gone close! People are trying, like Eva Black, as she says, doing a fantastic job trying to create this sort of top-down way and path and avenue for the ability for companies to pay open source and for governments to pay and for all these things to happen. -**Reys:** It's so true! +It's going to take time, but as a community, we need to bottom-up sort of figure out how do we pass this cliff? And it's a massive cliff, and nobody really knows it's coming. You don't have time to prepare for it because you don't get to choose when you become adopted and beloved by community. Nobody sits down and says, "We're community now." You just look around one day and you go, "These are all my friends, and I love them, and this is great!" And then, oh no! -[00:36:30] **Ted:** And the second part of that is structuring the project because companies, there are very few like companies, but there are a lot of bad incentives. So when we started the project, coming from other open-source projects where I had borne witness and had to deal with all the fallout of these bad incentives or muddy incentives causing various companies to like test the waters and like get into stuff with each other. So when we started this project, we had a goal based off of our experiences of like how do we actually structure this project in a way that makes it obvious that these bad incentives are not present? Where people can look at that and be like, "Oh yeah, that won't work, so we're not going to bother to try to see if we can just buy the whole TC, right, and take over the project." +And then it turns out that's only step two because it turns out that you can't really call the CNCF community anymore—it’s an ecosystem. And that's kind of the third massive cliff that very, very few people talk about where you have this one project and this community, and that's cool, and the community maybe grows other things when it reaches this state of becoming a generation and incubation hub of innovation and experimentation, and it starts generating a fractal of different communities that all come together in this ecosystem of this massive idea exchange of community knowledge sharing and community growth when it becomes this messy, inarticulate sort of ill-defined but beautifully growing organic thing that takes on its own life—that's an ecosystem. -**Reys:** Right. +And we don't even really know how to build those. We definitely don't know how to fund them. We definitely don't even know the differences between all the legal and the compliance. How do you facilitate that? How do you get there? And so we looked at this problem, Nova and I, and a bunch of other people, and we said, "Someone needs to start thinking about this." It's going to take a while; it's going to take several years, maybe even a decade or two to really deeply help the world understand how to pass those cliffs and turn them into this growing ramp of taking ideas and sharing with the world in a way that generates further ideas in a way that becomes this ever-growing thing. -**Ted:** By like, "What if just everyone who works on who's in a leadership position works for Splunk or something?" Right? Like you can't do that because once you get more than two people on the TC who work at one company, and you want a third person, well one of those first two people has to drop out or remove companies or something. +The OpenTelemetry Foundation, which was originally sort of kind of starting as like a legal cover and legal ability to do this, always was intended to be a vehicle that helps you understand that problem and helps figure out how do we make this beautiful growth and knowledge sharing possible. -**Reys:** Right. +**Adriana:** That's so great! I think, are we coming up on time? -**Ted:** Yeah, and then when it comes to the actual shape of the project, prior projects, even these big industry things like Kubernetes and whatever, they tend to have this veneer of utopia like, "Oh, we're all just getting together for this altruistic reason of making container scheduling work!" And it is like as an individual, it's somewhat altruistic in that like I'm really interested in it. I would love to like push this domain forward. I want a real distributed operating system. I don't even want Kubernetes; I want the next thing, but sorry, Kubernetes! +**Reys:** We are coming up on time. We are going to have our next guest on. Thank you so much, Hazel! It was lovely to talk with her, and we will share socials—like how you can get in touch with her at the end, and as well as like how you can get involved in the future Humans of OpenTelemetry segment as well. And I'm really excited to see what kind of interactions come up from these conversations that we just had. -**Reys:** Right. +And yeah, whatever questions, comments, thoughts you have, definitely please let us know. We would love to hear them and help connect you with Hazel so we can learn even more and move forward to kind of her vision. And also, secretly jealous of her mental superpowers! I'm going to have to see if I can pick up any tips! -**Ted:** But I also want the next OpenTelemetry; I want to keep pushing this stuff. +But now we have our next guest on, and I'm so excited! Hopefully, you can see his cat pants because they're amazing. Ted, hello! -**Reys:** Exactly! +**Ted:** Hey, hello! How's it going? -**Ted:** So I'm motivated, but like these are like billion-dollar projects almost, right? The amount of engineering effort that goes into Kubernetes... If you calculate the salaries that I'm sure that's at this point, that's probably a billion dollars. Maybe that's crazy; maybe it's like a hundred million. +**Adriana:** Going great! How are you all doing today? -**Reys:** It's a chunk of change. +**Ted:** Not bad! It's KubeCon—awesome times! It's like a big friend reunion! -**Ted:** It's a chunk of change! A lot of money when you think about developer salaries and how many people work on these projects. OpenTelemetry, on average, like monthly average number of contributors is like a couple hundred engineers, I think. +**Reys:** Day two of the main KubeCon events, which, you know, everyone is on fumes at this point! + +**Adriana:** Yeah, we had a busy start because we had Observability Day, which just restarted with Rejected, and then Observability today and KubeCon Day one yesterday. + +**Ted:** I know! It just feels like we've been here forever! + +**Adriana:** So we're so excited to have you. Can you tell us a little bit about your role in OpenTelemetry? I'm sure a lot of our viewers who are more familiar with the project are already aware of who you are, but would love an introduction. + +**Ted:** Yeah, sure! My name's Ted Young, I'm one of the co-founders of the project, and I’m coming from the OpenTracing side of the family. People don't know OpenTelemetry is actually a merging of two prior projects: OpenTracing and OpenCensus. OpenCensus was mostly Google people and some Microsoft people, and OpenTracing was everybody else. + +Then we merged to form OpenTelemetry, and I continue to work on the project as a member of the governance committee, mostly just focusing on the various spec SIGs, implementation SIGs that need an extra helping hand when it comes to finding consensus, figuring out what design is actually going to work. + +There are some areas in OpenTelemetry where it's technically challenging enough that ironically it's easy to come to a design or, at any rate, it's easy to determine which design is the correct one, right? Because it's so technically challenging that the requirements are very rigorous. But then there are like other parts of OpenTelemetry where it's like a little more squishy, where there's like one way that would be a really, really good way and then there's some other ways that would sort of work, right? You can't say they definitely would not work. + +**Adriana:** Yeah, right! + +**Ted:** But they're not like great. But when you have that situation, it's so, so, so much harder to find consensus, right? Because people will lock into a particular idea, and then if it's possible to find some way to make it work—because they're engineers—they will continue to promote, "But there's this way it could work!" -**Reys:** Wow! +And trying to find consensus there is about being like, "Yes, okay, that would work, but this would work better." And getting everyone to be like, "Well, even though you prefer that one, would you agree that you're not going to convince everyone at this point to go that way?" Like, "Yes!" And like, "And would you be able to get everything done with this other one?" Like, "Yes!" And like, "Okay, well then would you be willing to come on board with this and help us move this forward?" Because we all actually like need it to get done. -**Ted:** On average, it's a couple hundred engineers pushing some kind of PR or something at any given moment on OpenTelemetry. It's a lot of people; there’s a lot of money. The only reason people are getting paid to do that is because there are some incentives. +So that's a bit of just more like community organizing that requires maybe kind of an engineering design background. -**Reys:** Yeah. +**Adriana:** Yeah, and I feel like that's where I provide the most value these days to the project. -[00:40:00] **Ted:** And the problem Kubernetes had when it first started and other projects that I worked on is there was all this interest and all this willingness to pay people to work on bootstrapping Kubernetes, but why? That question was actually nebulous for maybe some vendors or cloud providers. It was clear how they were going to make money and thus why they should be paying people to work on this, but for a lot of companies, a lot of groups getting started, especially the startups, it was not clear how they were going to make money off of Kubernetes. +**Ted:** That's so awesome! -**Reys:** Right. +**Adriana:** Cool! And I feel like there's also—you have to have some good diplomatic skills as well, right? -**Ted:** So it was like step one: spend all this money to bootstrap this thing; step two: figure out how to exploit it. And just leaving that door open, right? Just the fact that it wasn't really clearly defined how Kubernetes was to be organized, how everyone was expected to make money, what part of this is going to be some shared open stack versus what part are we going to like sell things or whatever? That was all just like a big question mark in the early days and should all the code live in a Kubernetes GitHub? Or should various startups and companies be allowed to have complete ownership of some component that lives within Kubernetes and thus it lives within that company's repo? +**Ted:** Yeah! Building trust in the community is important, just absolutely. Having people know that you don't have an agenda, right? And that you really are just there to, like, listen to everyone and help everyone really clarify the requirements and help everyone really determine what the best design solution would be for those requirements. -**Reys:** Right. +**Reys:** And on a similar vein, it always brings me back to like the whole thing with OpenTelemetry being such a lovely community, and that it really takes the whole thing of being vendor-neutral very seriously. I mean, we're all at three different places, and we all get along. Like, we're all competitors, but like not really because we're all moving towards the same goal. -**Ted:** All things like this led to just like a lot of problematic stuff, right? Like what if one company kind of owns a component of OpenTelemetry? They're going to build a lot of like startup brand around that component naturally, right? That would be the natural play, would be to talk that component up and how it's kind of yours. +And I was wondering if you could comment on like what has to go on to really continue to foster that sense of community and not an us versus them? -**Reys:** Right. +**Ted:** Yeah, that is great! So I really think there's two parts. The simple part is that like humans, generally speaking, are good people. And also, even though we might all work at different companies now, the reality is for the most part, there's just like many tech domains. There's like an observability scene, right? There's like the engineering and product people, whatever, that's just like that's their bag and they're good at it, and they just tend to kind of like circle through the different orgs and companies that happen to be paying people to work on that stuff in that time. -**Ted:** And then OpenTelemetry, a project, was like, "That was a cool component, but now we're going in a different direction, so we should deprecate that one and get a new thing." Well, that would be like the kiss of death to a startup. +So I think that's the thing that makes it easier, right? It's not like we're from different planets that are in some intergalactic war and we have like nothing in common or something. We've got all this stuff to overcome; it's just like we all know each other! -**Reys:** For sure. +And when you're around long enough, it's like everyone cycles two or three jobs, so you stop really looking at people as like representatives of a particular company. -**Ted:** So you can start to see how like just leaving the door open starts to like create a situation where things like people's incentives can start to get at loggerheads. +**Adriana:** It's so true! -**Reys:** Right. +**Ted:** Yeah! I think one of the things too, just from my, you know, very short experience, is even when they are moving companies, people want to stay involved in the project. Like, they're actively seeking out roles where they can continue contributing to the project in their role, so I think that speaks volumes, honestly. -**Ted:** And working on containerization stuff that led to these situations where you'd be in spec meetings, right? People are making some technical proposals, and everyone else is like, "I can't tell if this is a good idea or not. This might be a good idea, or this might be the beginning of some sneaky play to like insert some nasty thing so that this company can pull some crap on the community later." +**Ted:** The fact that you can look at our maintainers and community members, and there's quite a long lifespan at this point. I actually haven't run the math, but I would not be shocked if like the average age of a maintainer in OpenTelemetry was measured in years at this point. Like, on average, it's like two or three years. I wouldn't be shocked if it's that high. People really do stick around! -**Reys:** Right. +**Adriana:** It's so true! -**Ted:** And why are they doing that? They're evil? No, but because they need to make money off of all this money that they're spending. So when we started OpenTelemetry, we felt it needed to be very, very, very clear how people were going to make money and how this project was going to provide value for end users, for vendors, for cloud providers, for all the people involved that need to be crystal clear. Because otherwise, we were going to end up in that situation. +**Ted:** And the second part of that is structuring the project because companies—there are very few like companies, but there are a lot of bad incentives. So when we started the project, coming from other open-source projects where I had borne witness and had to deal with all the fallout of these bad incentives or muddy incentives causing various companies to like test the waters and like get into stuff with each other. -**Reys:** Right. +So when we started this project, we had a goal based off of our experiences of like how do we actually structure this project in a way that makes it obvious that these bad incentives are not present? Where people can look at that and be like, "Oh yeah, that won't work, so we're not going to bother to try to see if we can just buy the whole TC, right, and take over the project by like, what if just everyone who works on—who's in a leadership position works for Splunk or something, right? Like, you can't do that because once you get more than two people on the TC who work at one company, and you want a third person, well, one of those first two people has to drop out or move companies or something." -**Ted:** But because we learned our lessons, we did make it crystal clear. +### [00:37:48] Guest introduction: Ted Young -**Reys:** Good! +And then when it comes to the actual shape of the project, prior projects—even these big industry things like Kubernetes and whatever—they tend to have this like veneer of utopia, like, "Oh, we're all just getting together for this altruistic reason of making container scheduling work." And it is like, as an individual, it's somewhat altruistic in that like I'm really interested in it. I would love to like push this domain forwards. + +I want a real distributed operating system; I don't even want Kubernetes! I want the next thing! + +**Adriana:** Exactly! + +**Ted:** But sorry, Kubernetes! But I also want the next OpenTelemetry, you know? I want to keep pushing this stuff. + +**Adriana:** Exactly! + +**Ted:** So I'm motivated, but like these are like billion-dollar projects, almost, right? The amount of engineering effort that goes into Kubernetes, if you calculate the salaries, that I'm sure that's at this point probably a billion dollars. Maybe that's crazy; maybe it's like a hundred million. + +**Adriana:** Yeah, I mean, it's a chunk of change! + +**Ted:** It's a chunk of change! A lot of money when you think about developer salaries and how many people work on these projects. OpenTelemetry, on average, like monthly average number of contributors is like a couple hundred engineers, I think. -**Ted:** Yeah, right! All the code lives in OpenTelemetry. Do you want to donate something to OpenTelemetry? That's fabulous! You need to move it into OpenTelemetry; you need to change the copyright and everything to be the OpenTelemetry authors. You need to completely relinquish any individual ownership of this; it's now collectively owned by OpenTelemetry and is now part of the CNCF. It's no longer part of your startup. +**Adriana:** Wow! -**Reys:** Right. +**Ted:** On average, it's a couple hundred engineers pushing some kind of PR or something at any given moment on OpenTelemetry. It's a lot of people, a lot of money! And the only reason people are getting paid to do that is because there is some incentive. -**Ted:** And also, like, what's the boundary of the project? OpenTelemetry, we're going to standardize the data being emitted by all of these systems, but storing and analyzing that data, that's never going to be part of OpenTelemetry because you wouldn't want to standardize that part. That's like green field; that's the part where we're trying to figure out the futures. Like, what can we do with this information? And we want everyone to compete on that, and that's also where everyone's going to make money and making that very, very clear really did a lot in the early days as far as bringing in the second and third wave of contributing companies. +**Adriana:** Yeah, which is so nice to have that because otherwise, like, we're so busy already. The fact that you have like all these vendors who are backing OpenTelemetry, I think speaks volumes for it because I don't think we'd be where we are without that. -**Reys:** Right. +**Ted:** But a problem Kubernetes had when it first started and other projects that I worked on is there was all this interest and all this willingness to pay people to work on bootstrapping Kubernetes, but why? That question was actually nebulous for maybe some vendors or cloud providers. It was clear how they were going to make money and thus why they should be paying people to work on this. -**Ted:** Like you have this first wave of people who are like, "We're just making this major bet because it's super clear this would be good for us." Then this second wave of companies that are like, "Well, okay, if those guys are all in, that kind of changes the calculus for me; I guess we should get involved." +But for a lot of companies, a lot of groups getting started, especially the startups, it was not clear how they were going to make money off of Kubernetes. So it was like step one, spend all this money to bootstrap this thing; step two, figure out how to exploit it! -**Reys:** Right. +And just leaving that door open, right? Just the fact that it wasn't really clearly defined how Kubernetes was should be organized, how everyone's expected to make money, what part of this is going to be some shared open stack versus what part are we going to sell things or whatever? That was all just like a big question mark in the early days. -**Ted:** And then once those groups get involved, there's like a third group that's like, "Well, now that all of those people are involved, I guess that changes it for us." And the fact that it was super clear what you would get out of this if you put something into it made that happen faster. +And should all the code live in a Kubernetes GitHub, or should various startups and companies be allowed to have complete ownership of some component that lives within Kubernetes, and thus it lives within that company's repo? -**Reys:** So you mentioned, you know, OpenTelemetry is the merging of like OpenTracing and OpenCensus, thank you. I was like OpenTraces! Okay, day three is okay, sorry. +All things like this led to just like a lot of problematic stuff! -**Ted:** No worries! +**Adriana:** Right! -[00:44:35] **Reys:** And you know, obviously, you guys did the project update yesterday; we heard a lot of new cool stuff coming on the pipeline. Was this part of your vision like at the beginning of the foundation of OpenTelemetry? +### [00:40:57] Discussion about OpenTelemetry governance -**Ted:** Or is this kind of... you mean like where the project is today? +**Ted:** Right? Like what if one company kind of owns a component of OpenTelemetry? They're going to build a lot of like startup brand around that component, naturally, right? That would be the natural play, would be to talk that component up and how it's kind of yours! -**Reys:** Yeah! +And then OpenTelemetry, a project, was like, "That was a cool component, but now we're going in a different direction, so we should deprecate that one and get a new thing." -**Ted:** Yeah! I mean, we've really like our original mandate. We're at kind of this interesting spot in OpenTelemetry where we had this original mandate, which was to unify tracing, metrics, and logs. Those are the primary signals that we have available. But the problem in the past was these were siloed systems, and what you're trying to do is get a completely connected coherent view of the system. +Well, that would be like the kiss of death to a startup! -**Reys:** Right. +So you can start to see how just leaving the door open starts to create a situation where things like people's incentives can start to get at loggerheads. -[00:30:00] **Ted:** And those were the three major signals that we wanted to tackle first. So that was like our original pitch: we're going to merge these three signals into one graph of data that can then be walked by a computer system, and we can dump all of this analysis off onto the computers instead of doing it in our brains when it comes to finding correlations across all of this information. That was a great pitch; it's been part of actually transforming this whole industry, right? As the data completely changes, the products, of course, have to completely change. If you're going from a bunch of siloed systems to like a unified platform, you can see how that's just going to create a complete sea change within the industry because, well, if that's the future, then what's going to happen? Well, all these companies are going to consolidate. Right? One company that was doing logging or whatever is going to acquire some companies that did the other signals, and then they're going to try to merge all of that into a platform. Or if they're a new company, they're right out of the gate, they're going to be like, "Are we a platform? Or how do we fit into this new platform world where all the data is going into one spot?" +**Adriana:** Yeah, yeah! -**Reys:** Right. +**Ted:** And working on containerization stuff that led to these situations where you'd be in spec meetings, right? People are making some technical proposals, and everyone else is like, "I can't tell if this is a good idea or not. This might be a good idea, or this might be the beginning of some sneaky play to insert some nasty thing so that this company can pull some crap on the community later." -**Ted:** So that's like a complete industry shift that probably would have happened anyways without OpenTelemetry, but OpenTelemetry is like this massive accelerant. So you incentivize basically the unification of the signals—not just supporting the three signals but actually intertwining them so that you can really get the most out of observability, really. +And why are they doing that? They're evil? No, but because they need to make money off of all this money that they're spending. -**Reys:** Right! +So when we started OpenTelemetry, we felt it needed to be very, very, very clear how people were going to make money and how this project was going to provide value for end users, for vendors, for cloud providers, for all the people who are going to be involved that need to be crystal clear. -**Ted:** Yeah, and I call, you know, the old version of it, I call the three browser tabs of observability because calling it three pillars gives it too much credit. Like, there was some intentional structure there; there was that was like never an intentional design; it's like a terrible design. What should we do? We should silo all of the data and use our brains to try to figure out how what connects to what. Like that's obviously dumb. So obviously, we didn't design that; it just kind of accrued over time. +Because otherwise, we were going to end up in that situation. But because we learned our lessons, we did make it crystal clear! -**Reys:** Right. +**Adriana:** Yeah! -**Ted:** And going from that to what I call the braid of data. So it is all this data, but it is being braided and connected in various ways. +**Ted:** Right? All the code lives in OpenTelemetry. Do you want to donate something to OpenTelemetry? That's fabulous! You need to move it into OpenTelemetry. You need to change the copyright and everything to be the OpenTelemetry authors. You need to completely relinquish any individual ownership of this; it's now collectively owned by OpenTelemetry and is now part of the CNCF; it's no longer part of your startup. -**Reys:** Right. +And also, like what's the boundary of the project? OpenTelemetry, we're going to standardize the data being emitted by all of these systems. But storing and analyzing that data, that's never going to be part of OpenTelemetry because you wouldn't want to standardize that part. You want to—that's like green field, that's the part where we're trying to figure out the futures. Like, what can we do with this information? And we want everyone to compete on that, and that's also where everyone's going to make money. -**Ted:** And you can think of the value that comes out of a braid, right? You can think of like the individual strands in a rope, and then you can think about a rope and how much more you can do with a rope than just pieces of straw. +**Adriana:** Yeah, exactly! -**Reys:** Yeah. +**Ted:** And making that very, very clear really did a lot in the early days as far as bringing in the second and third wave of contributing companies, right? Like, you have this first wave of people who are like, "We're just making this major bet because it's super clear this would be good for us." And then this second wave of companies that are like, "Well, okay, if those guys are all in, that kind of changes the calculus for me. I guess we should get involved." -**Ted:** It's just so much stronger, right? +And then once those groups get involved, there's like a third group, it's like, "Well, now that all of those people are involved, I guess that changes it for us." And the fact that it was super clear what you would get out of this if you put something into it made that happen faster. -**Reys:** Exactly! +**Adriana:** So you mentioned OpenTelemetry is the merging of OpenCensus and OpenTracing, and you know, obviously you guys did the project update yesterday. We heard a lot of new cool stuff coming down the pipeline. Was this part of your vision at the beginning of the foundation of OpenTelemetry, or is this kind of— + +**Ted:** You mean like where the project is today? + +**Adriana:** Yeah! + +### [00:45:09] Updates on OpenTelemetry project + +**Ted:** Yeah, I mean, we've really, like, our original mandate— we're at kind of this interesting spot in OpenTelemetry where we had this original mandate, which was to unify tracing, metrics, and logs. Those are the primary signals that we have available. But the problem in the past was these were siloed systems, and what you're trying to do is get a completely connected, coherent view of the system, right? + +And those were the three major signals that we wanted to tackle first, so that was like our original pitch. We're going to merge these three signals into one graph of data that can then be walked by a computer system, and we can dump all of this analysis off onto the computers instead of doing it in our brains when it comes to finding correlations across all of this information. + +And that was a great pitch! It's been part of actually transforming this whole industry, right? As the data completely changes, the products, of course, have to completely change. And if you're going from a bunch of siloed systems to like a unified platform, you can see how that's just going to create a complete sea change within the industry. + +Because, well, if that's the future, then what's going to happen? Well, all these companies are going to consolidate, right? One company that was doing logging or whatever is going to acquire some companies that did the other signals, and then they're going to try to merge all of that into a platform. Or if they're a new company, they're right out of the gate; they're going to be like, "Are we a platform?" Or how do we fit into this new platform world where all the data is going into one spot? + +So that's like a complete industry shift that probably would have happened anyways without OpenTelemetry, but OpenTelemetry is like this massive accelerant. + +**Adriana:** So you incentivize basically the unification of the signals—not just supporting the three signals, but actually intertwining them so that you can really get the most out of observability, really! + +**Ted:** Right, yeah! And I call the old version of it the three browser tabs of observability because calling it three pillars gives it too much credit. Like, there was some intentional structure there; that was never an intentional design! It’s like a terrible design! What should we do? We should silo all of the data and use our brains to try to figure out what connects to what. Like, that's obviously dumb! + +So obviously, we didn't design that; it just kind of accrued over time. And going from that to what I call the braid of data. So it is all this data, but it is being braided and connected in various ways. And you can think of the value that comes out of a braid, right? You can think of like the individual strands in a rope, and then you can think about a rope and how much more you can do with a rope than just pieces of straw. + +It's just so much stronger! + +**Adriana:** Right, exactly! **Ted:** And can you give folks just a high-level overview of what some of the updates that came out of the project update yesterday? -**Ted:** Yeah! I would say the super high level—and we didn't really cover this in the update, but—getting back to this industry sea change, right? Being at this moment where we're finally stabilizing tracing, logs, and metrics, right? We had the kind of last piece of that. There's a little bit of work that needs to be done related to resources, so if people don't know, resources are in a way like the fourth signal, right? Like when we, there's the data, there's the system, and there's like the structure of the system. The traces are kind of like the energy, right? That's the transactions; that's the stuff moving through the system. But then there's the actual stuff that it's moving through, and those are the resources—the Kubernetes clusters, the virtual machines, the application binaries, all of that stuff. When we started the project, there was this idea that came from OpenCensus that these resources were immutable compared to the lifespan of a service. When a service started, it was associated with some resources, and those resources did not change. +**Ted:** Yeah, I would say the super high level—and we didn't really cover this in the update but exclusive!—but getting back to this industry sea change, right, being at this moment where we're finally stabilizing tracing, logs, and metrics, right? + +We had the kind of last piece of that. There's a little bit of work that needs to be done related to resources. So if people don't know, resources are in a way like the fourth signal, right? When we—there's the data, there's the infrastructure of the system, and there's like the structure of the system. And so the traces are kind of like the energy, right? That's the transactions, that's the stuff moving through the system, but then there's the actual like stuff that it's moving through, and those are the resources—the Kubernetes clusters, the virtual machines, the application binaries—all of that stuff. + +And when we started the project, there was this idea that came from OpenCensus that these resources were immutable compared to the lifespan of a service. When a service started, it was associated with some resources, and those resources did not change. And this kind of made sense, yeah, because generally speaking, the resources you care about with server-side computing, they don't change. + +Now they do sometimes, but this is just like edge-casey enough that at Google, they kind of were just like, "Just going to ignore that." Like, "You can freeze a virtual machine, move it into a different data center, and then like turn it back on, right? And now all of its resources changed. It's running, it's the same computer program, and now it's running somewhere completely different." But in practice, we don't do that, right? -**Reys:** Right. +So maybe you could just ignore this requirement, and this was very convenient because you have this obnoxious thing with resources where you want to figure all of them out before you start sending data. You don't want to start sending partially indexed data, which is what would happen if you started emitting telemetry before you figured out your Kubernetes pod ID or whatever. -**Ted:** And this kind of made sense, yeah, because generally speaking, the resources you care about with server-side computing, they don't change. Now they do sometimes, but this is just like edge-casy enough that at Google, they kind of were just like, "Eh, we're just going to ignore that." +But figuring out some of these resources can take time. Yeah, you have to go query some API and like wait for the answer, and you don't want to delay the booting up of your app necessarily, right? -**Reys:** Right. +So what if one of these calls is failing? Right? Do you move forwards or is this gating? Turns out there's a lot of pernicious problems around actually gathering all of these resources and associating them. And the biggest problem is devs because it's annoying at startup to be like, "Well, whatever, we’ll do the dumb thing of we’ll just tack these resources on as they come back from their various APIs." -**Ted:** Like, you can freeze a virtual machine, move it into a different data center, and then like turn it back on, right? And now all of its resources changed! It's running; it's the same computer program, and now it's running somewhere completely different! But in practice, we don't do that, right? So maybe you could just ignore this requirement, and this was very convenient because you have this obnoxious thing with resources where you want to figure all of them out before you start sending data. You don't want to start sending partially indexed data, which is what would happen if you started emitting telemetry before you figured out your Kubernetes pod ID or whatever. But figuring out some of these resources can take time. +And then you end up with this situation where you have some partial data at the beginning. How do you prevent devs from making this mistake? Well, what if you made the resource API immutable where you could only set it once? Then they would try to do this, right? They would try to come back with some late-binding resource, and then the API would be like, "Oh, I can't." -**Reys:** Right. +Oh, this is annoying! I guess I'm forced to do it the right way. -**Ted:** You have to go query some API and like wait for the answer, and you don't want to delay the booting up of your app necessarily, so do you move forward? Or is this gating? It turns out there's a lot of pernicious problems around actually gathering all of these resources and associating them. And the biggest problem is devs, because it's annoying at startup to be like, "Well, whatever. We'll do the dumb thing of, we'll just tack these resources on as they come back from their various APIs." +**Adriana:** So it's not that resources are immutable by some intrinsic nature of what resources are; resources are mutable because that was a convenient way to force the Google devs to not make this stupid mistake! -**Reys:** Right. +**Ted:** Oh! And to protect them from themselves, to a certain extent! -**Ted:** And then you end up with this situation where you have some partial data at the beginning. How do you prevent devs from making this mistake? Well, what if you made the resource API immutable, where you could only set it once? Then they would try to do this, right? They would try to come back with some late binding resource, and then the API would be like, "Oh, I can't! Oh, this is annoying! I guess I'm forced to do it the right way!" So it's not that resources are immutable by some intrinsic nature of what resources are; resources are mutable because that was a convenient way to force the Google devs to not make this like stupid mistake. +**Adriana:** This is my claim! -**Reys:** Right. +**Ted:** But then you get to clients, right? And all of this was fine; this was just like fine. All swans are black. And then you get to client development, which we've been a little slow to get to, in part because almost all the engineers came from kind of a server-side background. -**Ted:** And to force them to do the extra effort figuring out how to resolve all of these resources at the beginning. So to protect them from themselves to a certain extent, this is my claim. +It was just a little bit late before the client-side people started showing up. But when they did, client-side computing is very different, right? When the application starts and stops is actually like pretty arbitrary. It's pretty arbitrary when you hit quit on your web browser or whatever; it doesn't really signify anything. -**Reys:** Right. +When the program boots or ends, but it goes through all these different states, right? Like it might be foregrounded or backgrounded; it might be on Wi-Fi or a cellular network, right? It might get put to sleep and then wake up in Hong Kong, you know? -**Ted:** So, but then you get to clients, right? And all of this was fine. This was just like fine; all swans are black. And then you get to client development, which we've been a little slow to get to in part because almost all the engineers came from kind of a server-side background. It was just a little bit late before the client-side people started showing up, but when they did, client-side computing is very different, right? When the application starts and stops is actually like pretty arbitrary. Like it's pretty arbitrary when you hit quit on your web browser or whatever; it doesn't really signify anything. When program boots or ends, but it goes through all these different states, right? Like it might be foregrounded or backgrounded, right? It might be on Wi-Fi or a cellular network, right? It might get put to sleep and then wake up in Hong Kong, you know? +And so all of these resources are changing over the lifespan of this application. Oops! Oops, indeed! Damn it! -**Reys:** Right. +So that's kind of where we're at with the project, where we finished our original mandate, but there's like a couple of squiggles around resources that are kind of like the last thing where it's like, "All right, we have to do this really annoying work." -**Ted:** And so all of these resources are changing over the lifespan of this application. Oops! Oops, indeed! Damn it! So that's kind of where we're at with the project, where we finished our original mandate, but there's like a couple of squiggles around resources that are kind of like the last thing where it's like, "All right, we have to do this really annoying work." This is going to be really annoying for me personally because this is the exact kind of thing where it's like we need to fix this to make it work for the clients, but we already told people it was immutable, so can we like take that back and say like some of these are immutable and some of these are not? Like, would that screw anybody up or not? Like, this is going to be very tricky. +This is going to be really annoying for me personally because this is the exact kind of thing where it's like we need to fix this to make it work for the clients, but we already told people it was immutable. So can we like take that back and say, "Like some of these are immutable and some of these are not?" -**Reys:** Right. +Like, would that screw anybody up or not? Like, this is going to be very tricky! And then we're going to have to come up with a solution, and there's going to be a bunch of community people being like, "But we said it was immutable!" And then we're like, "But clients need to be!" -**Ted:** And then we're going to have to come up with a solution, and there's going to be a bunch of community people be like, "But we said it was immutable!" And then we're like, "But clients need to be like, 'But I don't care because I'm a Go developer!'" You know? +Like, "But I don't care because I'm a Go developer!" You know? -**Reys:** Right. +So this is what I do! -**Ted:** Like, yeah, so this is what I do, okay? Is try to bring everyone together so we can move forward. But if we can get past that, now we are in the green field of the future where we are looking at all of the data that we haven't captured yet, and we want to figure out how to start bringing that data in. How do we bring profiling in? How do we bring source code commits, versioning, all of that in? Yeah, config files! That's the future! +Okay? Is try to bring everyone together so we can move forward. But if we can get past that, yeah, now we are in the green field of the future, where we are looking at all of the data that we haven't captured yet, and we want to figure out how to start bringing that data in. -**Reys:** That's exciting! +How do we bring profiling in? How do we bring source code commits, versioning, all of that in? Yeah, config files—that's the future! -**Ted:** Yeah! So we're, you know, at the five-year, five, six-year mark of the foundation. What do you think it's going to look like in another five years? +**Adriana:** That's exciting! -**Ted:** In another five years? Pretty much the same, but with more stuff tacked onto it. Sorry, I'm going to be very boring with that answer because I think we move slow, and I don't believe in 2.0's. No, it is just 1.0 forever. +**Ted:** Yeah! So we're, you know, at the five-year, five to six-year mark of the foundation. What do you think it's going to look like in another five years? -**Reys:** Right. +**Ted:** In another five years? Pretty much the same, but with more stuff tacked onto it. Sorry, I'm going to be very boring with that answer because I think we move slow, and I don't believe in 2.0s! No, it's just 1.0 forever! -[00:55:00] **Ted:** And so I just think it'll be, but there will be like a sea change that happens once you get enough of this data actually connected. There are certain it unlocks certain things, so I don't think you're going to see OpenTelemetry radically change in the way it looks, but five years out from now is enough time for the analysis tools to start to catch up with where the data is. So where I think you're going to see some amazing things is some amazing new analysis tools that are actually looking at all kinds of data that we can't look at today and are condensing that into like a helpful co-pilot for an operator trying to figure out their system. +And so I just think it'll be—but there will be like a sea change that happens once you get enough of this data actually connected. There are certain—it unlocks certain things. -**Reys:** That's exciting! +So I don't think you're going to see OpenTelemetry radically change in the way it looks, but five years out from now is enough time for the analysis tools to start to catch up with where the data is. So where I think you're going to see some amazing things is some amazing new analysis tools that are actually looking at all kinds of data that we can't look at today and are condensing that into like a helpful co-pilot for an operator trying to figure out their system! -**Ted:** Right on! And before we log off and get back into the real world, I have to learn about your pants. What is the story with these cat pants? Because they're fabulous! +**Adriana:** That’s exciting! -**Ted:** They have cats on them! You know these cats? +**Ted:** Right on! And before we, you know, log off and get back into the real world, I have to learn about your pants! What is the story with these cat pants? Because they're fabulous! -**Reys:** Do I know them? Are they your cats? +**Ted:** They have cats on them! You know, these cats? Do I know them? Are they your cats? -**Ted:** They are not my cats! I wish! I wish I had personally fabricated these pants, but I did not. The internet provided these pants! But I did know that these pants existed before I looked at them. I was like, "I want pants that have kittens and space on them!" And I knew the internet would give this to me. I had complete faith, and it took about 30 seconds. +**Adriana:** They are not my cats! I wish I had personally fabricated these pants, but I did not. The internet provided these pants! But I did know that these pants existed before I looked at them. I was like, "I want pants that have kittens and space on them!" And I knew the internet would give this to me. I had complete faith, and it took about 30 seconds! -**Reys:** Wow! The internet came through! +**Adriana:** Wow! The internet came through! -**Ted:** It came through! Yeah! You know, they can make me get out of bed; they can't make me get out of my pajamas. That's kind of my attitude. +**Ted:** It came through! Yeah, you know, they can make me get out of bed; they can't make me get out of my pajamas—that's kind of my attitude! -**Reys:** I love it! All right! What else would you like to share before we head back into the actual event that we're all here for? +**Adriana:** I love it! All right! Yeah, what else would you like to share before we head back into the actual event that we're all here for? -**Ted:** You know, I would just encourage anyone watching to, if you're an end user, please get involved in the developer experience SIG and give us feedback on your experience as an end user, in particular around installing the SDK and using the instrumentation APIs because we want to clean all that up now that things are stabilizing. So yeah, please get involved if that's what you're interested in. +**Ted:** You know, I would just encourage anyone watching, if you're an end user, please get involved in the developer experience SIG and give us feedback on your experience as an end user, in particular around installing the SDK and using the instrumentation APIs because we want to clean all that up now that things are stabilizing. -**Adriana:** And speaking of end users, Reys and myself work in the end user SIG. We do monthly events like OpenTelemetry in practice. We do end user interviews to learn more about your adoption and implementation process, you know, and get some feedback that we can share back with the SIGs. We'll have a QR code at the end that you can scan to get in touch with us on the OpenTelemetry SIG user channel on CNCF Slack. +So yeah, please get involved if that's what you're interested in. Speaking of end users, Adriana and myself work in the end user SIG. We do monthly events like OpenTelemetry in practice. We do end user interviews to learn more about your adoption implementation process, you know, and get some feedback that we can share back with the SIGs, and we'll have a QR code at the end that you can scan to get in touch with us on the OpenTelemetry SIG user channel on CNCF Slack. -**Ted:** And don't forget the survey that's open until the end of the week! +And don't forget the survey that's open until the end of the week! -**Adriana:** Oh, the docs survey! The docs survey! It ends Friday! Yes! If you've used OpenTelemetry in any way, shape, or form, and you have something to say about it, whether good or bad or just comments or questions, definitely please take the survey! We will link that in the show notes, I believe we can do that. +**Adriana:** Oh, the docs survey! The docs survey ends Friday! Yes! If you've used OpenTelemetry in any way, shape, or form, and you have something to say about it, whether good or bad or just comments or questions, definitely please take the survey! -**Ted:** That is the thing we can do! +We will link that in the show notes, I believe. We can do that! That is a thing we can do! -[00:58:00] **Adriana:** Yeah, and we should also, if you go to the OpenTelemetry socials on I think every social platform, we're on BlueSky, X, LinkedIn, and Mastodon, there should be some tweets also with links to the survey. +### [00:57:45] Call to action for audience engagement -**Ted:** Yes, and we are available on Slack at any time, and yeah, reach out to us; we'd love to hear from you. +**Ted:** Yeah! And we should also, if you go to the OpenTelemetry socials on, I think every social platform—we're on Bluesky, X, LinkedIn, and Mastodon—there should be some tweets also with links to the survey! -**Reys:** And I think we're good! +**Adriana:** Yes! And we are available on CNCF Slack at any time. Reach out to us; we’d love to hear from you! And I think we're good! ## Raw YouTube Transcript diff --git a/video-transcripts/transcripts/2024-12-17T22:14:29Z-humans-of-otel-kubecon-na-2024.md b/video-transcripts/transcripts/2024-12-17T22:14:29Z-humans-of-otel-kubecon-na-2024.md index 0a8e55a..579e091 100644 --- a/video-transcripts/transcripts/2024-12-17T22:14:29Z-humans-of-otel-kubecon-na-2024.md +++ b/video-transcripts/transcripts/2024-12-17T22:14:29Z-humans-of-otel-kubecon-na-2024.md @@ -10,20 +10,25 @@ URL: https://www.youtube.com/watch?v=TIMgKXCeiyQ ## Summary -In this video, a diverse group of tech professionals, including Hazel Weakly, Eromosele Akhigbe, Budha, Miguel Luna, Adriana Villela, David, Endre Sara, Braydon Kains, Christos, Reese Lee, and others, share their experiences and insights about OpenTelemetry, an open-source observability framework. They discuss how they became involved in the OpenTelemetry community, emphasizing its role in standardizing observability practices across various platforms and improving collaboration among engineers. Key themes include the importance of observability in software engineering, the benefits of using OpenTelemetry for troubleshooting, and the collective effort to establish common semantic conventions. The participants also reflect on their favorite OpenTelemetry signals, highlighting the significance of traces, metrics, and logs in understanding system behavior and enhancing user experience. +In this YouTube video, various software professionals, including Hazel Weakly, Eromosele Akhigbe, Budha, Miguel Luna, Adriana Villela, David, Endre Sara, Braydon Kains, Christos, Reese Lee, and others, discuss the significance and impact of OpenTelemetry in the field of observability. The participants share their personal journeys into the OpenTelemetry community, highlighting how their respective roles have evolved through involvement with this open-source project. Key points include the importance of standardizing observability practices, the benefits of collaboration among different vendors, and how OpenTelemetry helps in efficiently troubleshooting and understanding system behaviors. They elaborate on their favorite OpenTelemetry signals such as traces, logs, and metrics, emphasizing how these tools enhance user experience and operational insights within software systems. Overall, the conversation reflects a strong sense of community and shared goals among the contributors in advancing observability through OpenTelemetry. ## Chapters -00:00:00 Welcome and introductions -00:02:00 OpenTelemetry involvement stories -00:04:50 Contributions to OpenTelemetry -00:07:42 OpenTelemetry community impact -00:10:30 Observability definitions -00:14:00 Importance of user experience -00:17:30 Observability as understanding -00:20:00 Future of OpenTelemetry -00:23:00 OpenTelemetry as collaboration -00:25:22 Favorite OpenTelemetry signals +00:00:00 Introductions +00:00:22 Guest introduction: Eromosele +00:01:43 Guest introduction: Budha +00:02:04 Guest introduction: Miguel +00:02:35 Guest introduction: Adriana +00:03:06 Guest introduction: David +00:04:07 Guest introduction: Endre +00:05:10 Discussion about OpenTelemetry involvement +00:10:51 Discussion about observability +00:14:28 Definition of observability +00:25:19 Favorite OpenTelemetry signals + +## Transcript + +### [00:00:00] Introductions **Hazel:** Hey there. My name is Hazel Weakly and I have thoughts, lots of thoughts. They never stop thinking. And they never stop thinking. @@ -41,107 +46,129 @@ In this video, a diverse group of tech professionals, including Hazel Weakly, Er **Braydon:** My name is Braydon Kains. I'm a software developer at Google in the Google Cloud Org. I work for the Cloud Observability service and I mainly work on agents that customers install in their environments to collect telemetry signals and send them to Google Cloud. +### [00:01:43] Guest introduction: Budha + **Christos:** My name is Christos. I'm a software engineer at Elastic. I have been working mainly in observability over the past five years now and since last year I have been contributing mostly to the OpenTelemetry ecosystem. -[00:02:00] **Reese:** Hi, my name is Reese Lee and I am a Senior Developer Relations Engineer at New Relic. OpenTelemetry. I got into the project sort of almost accidentally, although I think at this point that's an answer that I give for everything. When I mean accidentally, it was I was looking for answers to questions that I had and more importantly, how do I teach other people to find answers to questions better and how do I continue to level up the teams that I worked with, the organizations that I worked with and in figuring out how to get people better at asking questions, getting answers, and learning from that? I finally stumbled onto OpenTelemetry. +### [00:02:04] Guest introduction: Miguel + +**Reese:** Hi, my name is Reese Lee and I am a Senior Developer Relations Engineer at New Relic. OpenTelemetry. So I got into the project sort of almost accidentally, although I think at this point that's an answer that I give for everything. When I mean accidentally, it was I was looking for answers to questions that I had and more importantly, how do I teach other people to find answers to questions better and how do I continue to level up the teams that I worked with, the organizations that I worked with and in figuring out how to get people better at asking questions, getting answers, and learning from that? I finally stumbled onto OpenTelemetry. + +### [00:02:35] Guest introduction: Adriana In March, I entered an internship called Outreachy and in Outreachy I was privileged to work on OpenTelemetry and I worked on building a logging bridge in Golang, and by the end of the internship I was able to build a logging bridge using OTel zerolog. +### [00:03:06] Guest introduction: David + How did I get involved with OpenTelemetry? This is a multi parter question or answer, I think in this case, because there were a couple of reasons why it caught my attention. Starting off with actually advocacy from our new group product manager who had recently joined and she was a big proponent of observability and OpenTelemetry specifically. I kind of had played around with OpenTracing and OpenCensus for a little while, but I hadn't really looked into OpenTelemetry. But once she came in I was a huge advocate for it and that got my attention. That was trigger number one. -Trigger number two was the fact that it was this open standard. So I think anything open standards to me is a no brainer. I've got a lot of time to invest in any sort of open standard that makes life easy. I think from a flexibility standpoint that's the way to go. So that was trigger number two. +Trigger number two was the fact that it was this open standard. So I think anything open standards to me is a no brainer. I've got a lot of time to invest in any sort of open standard that makes life easy. I think from a flexibility standpoint that's the way to go. So that was trigger number two. + +Trigger number three was actually when we started using OpenTelemetry. So we are an API management platform at Tyk. For us, OpenTelemetry was being used internally as well as externally. So internally we could already start seeing results in terms of how quickly and efficiently we were getting to troubleshoot problems and getting to the heart of issues. And not just limited to rest APIs but actually with GraphQL APIs as well, which you wouldn't have considered as a possible use case. But we were able to remediate some of those issues that we were facing with that. So that was sort of trigger number three. -Trigger number three was actually when we started using OpenTelemetry. So we are an API management platform at Tyk. For us, OpenTelemetry was being used internally as well as externally. So internally we could already start seeing results in terms of how quickly and efficiently we were getting to troubleshoot problems and getting to the heart of issues. And not just limited to REST APIs but actually with GraphQL APIs as well, which you wouldn't have considered as a possible use case. But we were able to remediate some of those issues that we were facing with that. So that was sort of trigger number three. +### [00:04:07] Guest introduction: Endre -And all of that collectively came together to say, hey, OpenTelemetry deserves attention. Initially I started as a product manager. It was a very interesting role because I started in a role where it was more about coordination rather than contributing directly. But I've been recently involved in the localization of the documentation. So that means translating the documentation, more specifically in my case among Spanish speakers. So, la traducción de la telemetria abierta. So, the traduction...the translation into the Spanish of the OpenTelemetry documentation. +And all of that collectively came together to say, hey, OpenTelemetry deserves attention. Initially I started as a product manager. It was a very interesting role because I started in a role where it was more about coordination rather than contributing directly. But I've been recently involved in the localization of the documentation. So that means translating the documentation, more specifically in my case among Spanish speakers. So, la traducción de la telemetria abierta. So, the traduction...the translation into the Spanish of the OpenTelemetry documentation. -[00:04:50] At my previous role, my manager at the time, as part of it, he encouraged me to actually join the OpenTelemetry community. And it was actually my first time ever contributing to open source and I never contributed to open source. I've been in tech for like over 20 years and my manager basically said, yeah, just attend a couple meetings. And my first meeting was for the OTel comms. And so that was kind of my gateway into OpenTelemetry. +### [00:05:10] Discussion about OpenTelemetry involvement -I started my career in embedded applications and I was doing eBPF tracing before that was even a thing. I then moved into Dropbox where all our telemetry was in-house before OpenTelemetry was mainstream and now on Monday I continue doing trace-based testing. I started to learn about OpenTelemetry. I realized that this is such an opportunity for the whole industry to actually commoditize and standardize how instrumentation is being done and to be able to use common semantic conventions so people can understand what's going on. So I got instantly excited and I started to work on it. +At my previous role, my manager at the time, as part of it, he encouraged me to actually join the OpenTelemetry community. And it was actually my first time ever contributing to open source and I never contributed to open source. I've been in tech for like over 20 years and my manager basically said, yeah, just attend a couple meetings. And my first meeting was for the OTel comms. And so that was kind of my gateway into OpenTelemetry. -First it was just a few test applications, then I played with and demoed to people on how to do instrumentation. But as we started our current company, from day one I said we have to make sure that we are properly instrumenting our software so that we can actually operate this as we get more customers for logs, metrics and disability testing, it has been helping us a lot. +I started my career in embedded applications and I was doing eBPF tracing before that was even a thing. And I then moved into Dropbox where all our telemetry was in-house before OpenTelemetry was mainstream and now on Monday I continue doing trace-based testing. I started to learn about OpenTelemetry. I realized that this is such an opportunity for the whole industry to actually commoditize and standardize how instrumentation is being done and to be able to use common semantic conventions so people can understand what's going on. -I got involved in OpenTelemetry because our team uses OpenTelemetry, namely the OpenTelemetry Collector, to support our customers collecting data off of their environments. When we had bugs and issues with OpenTelemetry in the past, there would be some light involvement from the team, but largely we would open an issue and sort of wait for it to get addressed. And I really wanted to change that within the group. And I wasn't the only one on our team who wanted to change that. So, you know, we all sort of started to make a more genuine effort to open issues that came with PRs. And that has generally moved our whole team forward into being more involved in OTel. +So I got instantly excited and I started to work on it. First it was just a few test applications, then I played with and demoed to people on how to do instrumentation. But as we started our current company, from day one I said we have to make sure that we are properly instrumenting our software so that we can actually operate this as we get more customers for logs, metrics and disability testing, it has been helping us a lot. -And I've ended up being much more involved in OTel to the point where now I'm a code owner on the Host Metrics Receiver, which is an important receiver to us, but I get to dedicate more time to making sure it's good for everyone and not just fixing our own problem. +I got involved in OpenTelemetry because our team uses OpenTelemetry, namely the OpenTelemetry Collector, to support our customers collecting data off of their environments. When we had bugs and issues with OpenTelemetry in the past, there would be some light involvement from the team, but largely we would open an issue and sort of wait for it to get addressed. And I really wanted to change that within the group. And I wasn't the only one on our team who wanted to change that. So, you know, we all sort of started to make a more genuine effort to open issues that came with PRs. And that has generally moved our whole team forward into being more involved in OTel. And I've ended up being much more involved in OTel to the point where now I'm a code owner on the Host Metrics Receiver, which is an important receiver to us, but I get to dedicate more time to making sure it's good for everyone and not just fixing our own problem. I was originally asked to contribute to the OpenTelemetry by helping with the Elastic Common Schema donation to the OpenTelemetry, specifically to the specification and the semantic conventions. And since then I have been more and more involved in other projects like the Collector. And right now I'm mainly focusing on the semantic conventions and the OpenTelemetry Collector, specifically the Collector contrib project. -[00:07:42] The way I got involved in OpenTelemetry was at New Relic. And at first my first experience with it was through some support tickets that we started to get around some of our customers who had adopted OpenTelemetry. And then I had a great opportunity to join our dedicated OpenTelemetry team at the time as a developer relations engineer. And this was back in November 2021. And I was able to integrate within the OpenTelemetry community pretty soon after that. And actually my previous manager, Sharr Creeden, she kind of spearheaded the work to build the End User Working Group at the time, and now we are the End User SIG. +The way I got involved in OpenTelemetry was at New Relic. And at first my first experience with it was through some support tickets that we started to get around some of our customers who had adopted OpenTelemetry. And then I had a great opportunity to join our dedicated OpenTelemetry team at the time as a developer relations engineer. And this was back in November 2021. And I was able to integrate within the OpenTelemetry community pretty soon after that. -OpenTelemetry has been really useful at my organizations that I've worked on because it's become something that you can tie into different vendors, tie into different tools, and into other intermediary ways. And the huge benefit of it for me is that I can take all these different bits of knowledge, not necessarily signals, but different bits of context from the company, tie it all together in a way that I can show people these answers to their questions, regardless of whether or not they're in engineering. And that is a new capability because previously engineering was kind of in its own bubble and increasingly it really can't continue to do that. +And actually my previous manager, Sharr Creeden, she kind of spearheaded the work to build the End User Working Group at the time, and now we are the End User SIG. OpenTelemetry has been really useful at my organizations that I've worked on because it's become something that you can tie into different vendors, tie into different tools, and into other intermediary ways. -And so OpenTelemetry has been super impactful for me for bringing our knowledge outside of engineering and bringing the outside knowledge into engineering. Things have become a lot more efficient internally. When I talk to our SRE teams, our DevOps teams, they're a lot happier when they're interacting or working with different elements of our platform stack. It's a lot easier to manage and handle it. Now when I talk about the end users, they can truly talk about the value of it. +And the huge benefit of it for me is that I can take all these different bits of knowledge, not necessarily signals, but different bits of context from the company, tie it all together in a way that I can show people these answers to their questions, regardless of whether or not they're in engineering. And that is a new capability because previously engineering was kind of in its own bubble and increasingly it really can't continue to do that. -And personally, I think just the advocacy side of things, I think has been really, really enriching for me to learn more about it. Being involved with the community in different ways. Earlier this year I had the privilege of putting together a mini conference called LEAP, which was the API Observability Conference, where a lot of the folks from the community were able to scroll, speak to the different areas and elements of OpenTelemetry, not just limited to, again, the engineering side of things, but also how decision makers could perceive the value of adopting something like OpenTelemetry within their organization. +And so OpenTelemetry has been super impactful for me for bringing our knowledge outside of engineering and bringing the outside knowledge into engineering. Things have become a lot more efficient internally. When I talk to our SRE teams, our DevOps teams, they're a lot happier when they're interacting or working with different elements of our platform stack. It's a lot easier to manage and handle it. -[00:10:30] It all started when Elastic, we decided to donate the Elastic Common Schema, which was a natural fit to the goals of OpenTelemetry, or standardizing observability and driving efficiency across getting telemetry data converged into a single standard. When I just started my career, there was no OpenTelemetry, so I had to figure out how to do traces and how to correlate them with metrics and how to do logs. But now all this effort has already been standardized by the community, so new engineers that are onboarding into OpenTelemetry have a much easier time than I have. +Now when I talk about the end users, they can truly talk about the value of it. And personally, I think just the advocacy side of things, I think has been really, really enriching for me to learn more about it. Being involved with the community in different ways. Earlier this year I had the privilege of putting together a mini conference called LEAP, which was the API Observability Conference, where a lot of the folks from the community were able to scroll, speak to the different areas and elements of OpenTelemetry, not just limited to, again, the engineering side of things, but also how decision makers could perceive the value of adopting something like OpenTelemetry within their organization. -In general, I think that the ability to be able to take signals from your application and to be able to use them to operate the environment to understand the behavior of the system is significantly easier with OpenTelemetry than it was with other proprietary instrumentations in the past. What I think is more interesting is what do you do with this data? Most of this information is being exposed to people in dashboards which are amazingly nicely presented, contextualized based on semantic conventions. +### [00:10:51] Discussion about observability -But I think that the biggest advantage is to be able to use software to reasonably data. The main way OpenTelemetry has helped me personally is really learning how to interact with a large community. I already had some experience with open source communities and there is this sort of general culture of like, you know, you do the work, you get a say in the project. That's pretty common in the open source world and I think that's fine. +It all started when Elastic, we decided to donate the Elastic Common Schema, which was a natural fit to the goals of OpenTelemetry, or standardizing observability and driving efficiency across getting telemetry data converged into a single standard. When I just started my career, there was no OpenTelemetry, so I had to figure out how to do traces and how to correlate them with metrics and how to do logs. But now all this effort has already been standardized by the community, so new engineers that are onboarding into OpenTelemetry have a much easier time than I have. -But OpenTelemetry has a very large... I really like working with the OpenTelemetry ecosystem in general because I believe that working with people from other companies, other teams, helps me personally as an engineer a lot because I see how other people do observability out there. So I keep learning a lot. So that's something that I really like and I believe in general my team is also really helped by this, by this fact and also for my job. I mean it's amazing because I really love open source, I really love working with open source projects. And yeah, I think that on a personal level it's really helpful. +In general, I think that the ability to be able to take signals from your application and to be able to use them to operate the environment to understand the behavior of the system is significantly easier with OpenTelemetry than it was with other proprietary instrumentations in the past. What I think is more interesting is what do you do with this data? Most of this information is being exposed to people in dashboards which are amazingly nicely presented, contextualized based on semantic conventions. But I think that the biggest advantage is to be able to use software to reasonably data. -OpenTelemetry has helped me personally in honestly really big way, in the sense that working in developer relations with OpenTelemetry, I've gotten to meet a lot of wonderful people which I talked about earlier. But as part of my role I get to submit topics to different events and part of that is being able to learn about all these different topics myself and being able to talk to people who are using it in production or trying it on themselves has been a really wonderful experience. +The main way OpenTelemetry has helped me personally is really learning how to interact with a large community. I already had some experience with open source communities and there is this sort of general culture of like, you know, you do the work, you get a say in the project. That's pretty common in the open source world and I think that's fine. But OpenTelemetry has a very large... -[00:14:00] My definition of observability, it is the process through which one develops the ability to ask meaningful questions, get useful answers and then act effectively on when you learn. So what I mean by that is it's not enough to be able to kind of figure out the answer. There's this process where you have to actually work on it over and over and over and you're developing a skill not just on a personal level, but on an organization level, on a group level, and in even broader an industry level. So as you continue to do that, continue to get those really, really useful answers and really, really meaningful questions that you can ask. You start to have this whole process of group learning that transcends the boundaries that we draw for ourselves and lets those boundaries become empowering rather than limiting. +I really like working with the OpenTelemetry ecosystem in general because I believe that working with people from other companies, other teams, helps me personally as an engineer a lot because I see how other people do observability out there. So I keep learning a lot. So that's something that I really like and I believe in general my team is also really helped by this, by this fact and also for my job. I mean it's amazing because I really love open source, I really love working with open source projects. -Observability Engineers are like the doctors of your system. So if something is going wrong in your system, you need us to be able to pinpoint where exactly or what exactly is wrong and how to solve whatever is wrong. So that's what observability means to me. +And yeah, I think that on a personal level it's really helpful. OpenTelemetry has helped me personally in honestly really big way, in the sense that working in developer relations with OpenTelemetry, I've gotten to meet a lot of wonderful people which I talked about earlier. But as part of my role I get to submit topics to different events and part of that is being able to learn about all these different topics myself and being able to talk to people who are using in production or trying it on themselves has been a really wonderful experience. + +My definition of observability, it is the process through which one develops the ability to ask meaningful questions, get useful answers and then act effectively on when you learn. So what I mean by that is it's not enough to be able to kind of figure out the answer. There's this process where you have to actually work on it over and over and over and you're developing a skill not just on a personal level, but on an organization level, on a group level, and in even broader an industry level. + +So as you continue to do that, continue to get those really, really useful answers and really, really meaningful questions that you can ask. You start to have this whole process of group learning that transcends the boundaries that we draw for ourselves and lets those boundaries become empowering rather than limiting. + +### [00:14:28] Definition of observability + +Observability Engineers are like the doctors of your system. So if something is going wrong in your system, you need us to be able to pinpoint where exactly or what exactly is wrong and how to solve whatever is wrong. So that's what observability means to me. What does observability mean to me? There is a technical answer to this, where it goes into the realm of perhaps monitoring, perhaps logging, and, you know, getting to the troubleshooting of all things. To me, it's all about understanding. It is essentially understanding the pulse of your platform that you have created. I work with APIs quite a lot, so everything underlying is all to do with API platforms. -So understanding the pulse of your API platform, the different components coming together and knowing exactly what's functioning, not functioning, the good, bad and ugly of it all, that, to me, is what observability is all about. So to be able to get to that part of the problem, to be able to know what's working, what's not working, and making decisions more effectively. +So understanding the pulse of your API platform, the different components coming together and knowing exactly what's functioning, not functioning, the good, bad and ugly of it all, that, to me, is what observability is all about. So to be able to get to that part of the problem, to be able to know what's working, what's not working, and making decisions more effectively. + +Monitoring means knowing answers to questions that you know you needed to ask. Observability means knowing questions to answers that you didn't know that you need to ask. To me, observability means the ability to get insights into your system. And for me, like, this was extremely transformative, because, like, there's so many times in my life where, you know, I was, like, debugging code, whether it was like my own code, like, as a developer or code in production, and not understanding, like, just looking through logs and not understanding, like, okay, but how does this relate to the bigger picture? -Monitoring means knowing answers to questions that you know you needed to ask. Observability means knowing questions to answers that you didn't know that you need to ask. To me, observability means the ability to get insights into your system. And for me, like, this was extremely transformative, because, like, there's so many times in my life where, you know, I was debugging code, whether it was like my own code, like, as a developer or code in production, and not understanding, like, just looking through logs and not understanding, like, okay, but how does this relate to the bigger picture? +Like, I have so many memories of, like, troubleshooting production issues, and it's like, oh, the system is slow. So you ask the person who's responsible for administering the app server, hey, can you check the logs? No, not my problem. You ask the DBA, no, no, it's not my problem. And then you ask whoever else, and you go down the whole line and, like, it's nobody's problem. And yet you're still seeing latency. -Like, I have so many memories of troubleshooting production issues, and it's like, oh, the system is slow. So you ask the person who's responsible for administering the app server, hey, can you check the logs? No, not my problem. You ask the DBA, no, no, it's not my problem. And then you ask whoever else, and you go down the whole line and, like, it's nobody's problem. And yet you're still seeing latency. And I feel like observability kind of like it. It uncloaks the whole thing because all of a sudden it exposes. Like, it exposes where the actual root cause is. And I think that's the magic and power of observability. +And I feel like observability kind of like it. It uncloaks the whole thing because all of a sudden it exposes. Like, it exposes where the actual root cause is. And I think that's the magic and power of observability. The most important thing in software engineering today is the user experience. And because our software is getting much more complex, it's getting harder to answer the question, how are my users experiencing my product? -The most important thing in software engineering today is the user experience. And because our software is getting much more complex, it's getting harder to answer the question, how are my users experiencing my product? And OpenTelemetry allows us to answer these difficult questions and provide us with visibility into our software. +And OpenTelemetry allows us to answer these difficult questions and provide us with visibility into our software. I think that probably the most obvious answer is to be able to collect signals. But I think that the real point of observability is to understand and reason about the behavior of the systems. Simply collecting data doesn't actually accomplish much. -[00:17:30] I think that probably the most obvious answer is to be able to collect signals. But I think that the real point of observability is to understand and reason about the behavior of the systems. Simply collecting data doesn't actually accomplish much. I think also with the becomes meaningful and valuable, and people are able to use this to drive actions, to drive decisions. Where do I need to improve reliability? Where do I need to improve the performance of my application? Where do I need to make architectural changes? I think observability is really serving that. Otherwise it's just a lake of data. +I think also with the becomes meaningful and valuable, and people are able to use this to drive actions, to drive decisions. Where do I need to improve reliability? Where do I need to improve the performance of my application? Where do I need to make architectural changes? I think observability is really serving that. Otherwise it's just a lake of data. -Observability means to me that you can tell what's going on. Computers are black boxes that understand what ones and zeros do. And being able as a human to understand what ones and zeros are doing at any given time, when a computer is blazing so fast, how would you ever be able to figure out what that means? So observability to me is the human version of understanding what a computer is doing. +Observability means to me that you can tell what's going on. Computers are black boxes that understand what ones and zeros do. And being able as a human to understand what ones and zeros are doing at any given time, when a computer is blazing so fast, how would you ever be able to figure out what that means? -So for me, observability is something that I have been working on since university, and it's a really important area because I think that what really matters when we are running systems, it's the way that you can observe your systems, you can know if your systems are doing good or not. And specifically, I'm coming from an infrastructure background, as I mentioned before. So for infrastructure specifically, it's really, really important when it comes to cost reduction. And this sort of stuff or how the whole system is working is an important piece that you cannot miss. +So observability to me is the human version of understanding what a computer is doing. So for me, observability is something that I have been working on since university, and it's a really important area because I think that what really matters when we are running systems, it's the way that you can observe your systems, you can know if your systems are doing good or not. -Observability to me means that I, as an end user of various applications and software programs, get to have a better experience because the companies that build these products, you know, assuming that they're using observability and being able to stay on top of issues that are happening in their code, it means I get to have a better experience as an end user. +And specifically, I'm coming from an infrastructure background, as I mentioned before. So for infrastructure specifically, it's really, really important when it comes to cost reduction. And this sort of stuff or how the whole system is working is an important piece that you cannot miss. Observability to me means that I, as an end user of various applications and software programs, get to have a better experience because the companies that build these products, you know, assuming that they're using observability and being able to stay on top of issues that are happening in their code, it means I get to have a better experience as an end user. -[00:20:00] OpenTelemetry to me is one really interesting approach towards building something that takes a very sort of capitalistic notion of companies need to be profitable, companies wanting to innovate, people wanting to compete, and people want to develop different solutions to things. And how do you wrap all that together in a project that's flexible enough to allow that competition, to allow those ideas to happen, and to allow this innovation to continue without limiting what's possible and without burdening the industry with the intermediate details of the evolution of that complex, the evolution of pursuing excellence. +OpenTelemetry to me is one really interesting approach towards building something that takes a very sort of capitalistic notion of companies need to be profitable, companies wanting to innovate, people wanting to compete, and people want to develop different solutions to things. And how do you wrap all that together in a project that's flexible enough to allow that competition, to allow those ideas to happen, and to allow this Innovation to continue without limiting what's possible and without burdening the industry with the intermediate details of the evolution of that complex, the evolution of pursuing excellence. OpenTelemetry is, I believe, the future of observability. In March, when I started doing research on OpenTelemetry, I discovered how big this can be and I decided that I was going to go in fully into OpenTelemetry. So I believe that it's the future of observability and everyone should take it. What does OpenTelemetry mean? To me that's an extension of that understanding. In a way it's the... Well, again, the technical answer to this would be, is the open standard that essentially powers distributed tracing. That's all fine. To me it's the extension of that understanding by creating a common language or framework, however you want to put it, that the different components and elements of your platform stack can unite together to speak to the health of your overall platform. -And that could go from the engineering standpoint all the way to the business standpoint. There are repercussions to both of those. So to me that's what OpenTelemetry as a technology brings, both from a business and a technical standpoint. But it's also about the community as well. It's sort of again the industry coming together and agreeing on a standard so that the life of SRE, DevOps organizations, tooling providers, end users, all of their lives are made a lot easier because by virtue of having an open standard, it means that your platform is a lot more flexible, you have a lot more freedom to evolve, to mature and actually be a bit more future ready. +And that could go from the engineering standpoint all the way to the business standpoint. There are repercussions to both of those. So to me that's what OpenTelemetry as a technology brings, both from a business and a technical standpoint. But it's also about the community as well. It's sort of again the industry coming together and agreeing on a standard so that the life of SRE, DevOps organizations, tooling providers, end users, all of their lives are made a lot more easier because by virtue of having an open standard, it means that your platform is a lot more flexible, you have a lot more freedom to evolve, to mature and actually be a bit more future ready. + +So that to me is the promise of flexibility and freedom that OpenTelemetry brings. For me it means a common language. So it's a place where we all, where users can made and at least understand that everything that we are going to collect is going to be collect in a similar way, with a similar mechanism. Also what we call things. So the semantic conventions we, we agree on common standards of what, what are we going to call things? So the telemetry is the same and it can be reusable. -So that to me is the promise of flexibility and freedom that OpenTelemetry brings. For me it means a common language. So it's a place where we all, where users can made and at least understand that everything that we are going to collect is going to be collected in a similar way, with a similar mechanism. Also what we call things. So the semantic conventions we agree on common standards of what we are going to call things. So the telemetry is the same and it can be reusable. +So yeah, so that's, that's OpenTelemetry for me. What does OpenTelemetry mean to me? To me, you know, it feels like home actually, because it's been like my home for the last like two and a half years. So it's been like really transformative in my life because it's like I said, was my gateway into like open source, into the CNCF community. -So yeah, so that's OpenTelemetry for me. What does OpenTelemetry mean to me? To me, you know, it feels like home actually, because it's been like my home for the last like two and a half years. So it's been like really transformative in my life because it's like I said, was my gateway into like open source, into the CNCF community. And so it takes on like a very personal flavor for me, just beyond like the, you know, the typical definition of OpenTelemetry, which is like this open standard for instrumenting your code. For me, it's just so much more than that. +And so it takes on like a very personal flavor for me, just beyond like the, you know, the typical definition of OpenTelemetry, which is like this open standard for instrumenting your code. For me, it's just so much more than that. It really is like this lovely community where we're working with different vendors from across the board and we're not enemies, we're all friends because we're working towards the same goal. -[00:23:00] It really is like this lovely community where we're working with different vendors from across the board and we're not enemies, we're all friends because we're working towards the same goal. OpenTelemetry is basically a way to standardize all the efforts, all the engineers that want to ask all the difficult questions about software. OpenTelemetry gives a way for multiple vendors to work together, to collaborate together and take the instrumentation as a given that is not a function of competition and really focus on adding value on how this information turns into an actionable insight. +OpenTelemetry is basically a way to standardize all the efforts, all the engineers that want to ask all the difficult questions about software. OpenTelemetry gives a way for multiple vendors to work together, to collaborate together and take the instrumentation as a given that is not a function of competition and really focus on adding value on how this information turns into an actionable insight. And I think that that is really where people, vendors, end users are expecting to innovate in. So OpenTelemetry is basically the enabler for vendors to focus on where the real value is. OpenTelemetry to me is a representation of the industry coming together and understanding, you know, what are we competing about really? Like what are, where are we really as different companies trying to fit in the market? -And I think we all sort of collectively understand that the signals themselves really aren't worth differentiating on. It's generally a net negative for everyone for us to not agree on this stuff. And if we can agree on the signal part, it just leaves everyone, all companies, more time to differentiate in the ways that are actually tangible in terms of how the data works. +And I think we all sort of collectively understand that the signals themselves really aren't worth differentiating on. It's generally a net negative for everyone for us to not agree on this stuff. And if we can agree on the signal part, it just leaves everyone, all companies, more time to differentiate in the ways that actually are tangible in terms of how the data works. -Having been involved in OpenTelemetry over the past year, I think that OpenTelemetry is a great place to learn things and meet other people that are really passionate about the whole observability area. And I think that consists of people that really like what they are doing and they are really good at this. So it's a great place for engineers to come together and work and share the observability space to evolve. +Having been involved in OpenTelemetry over the past year, I think that OpenTelemetry is a great place to learn things and meet other people that are really passionate about the whole observability area. And I think that consists of people that really like what they are doing and they are really good at this. -OpenTelemetry to me means a lot of things, you know, beyond the project and kind of the way it's helped different organizations, you know, move into and adopt open source throughout their stack. It's also such a huge, wonderful community. I really enjoy meeting the maintainers and getting to know the end users. I have really good relationships with a lot of the OpenTelemetry community people and that's what it means to me. +So it's a great place for engineers to come together and work and share the observability space to evolve. OpenTelemetry to me means a lot of things, you know, beyond the project and kind of the way it's helped different organizations, you know, move into and adopt open source throughout their stack. It's also such a huge, wonderful community. I really enjoy meeting the maintainers and getting to know the end users. I have really good relationships with a lot of the OpenTelemetry community people and that's what it means to me. -[00:25:22] My favorite OpenTelemetry signal. I'm going to cheat a little bit here and I'm going to say my favorite OpenTelemetry signal is the one that gives people the answer that they find most useful for the question that they find most meaningful. +### [00:25:19] Favorite OpenTelemetry signals -Traces. Traces are my favorite signal because, like, they give a full, you know, a full picture of everything going on in the system and you can easily spot on errors. +My favorite OpenTelemetry signal. I'm going to cheat a little bit here and I'm going to say my favorite OpenTelemetry signal is the one that gives people the answer that they find most useful for the question that they find most meaningful. -Favorite OpenTelemetry signal. This is a tough one. I think traces has to take the win at this point of time because again, just thinking about how things connect well. I'm also very, very keenly pursuing profiling. I think that's going to be potentially the winner in the next conversation we have because I think performance is a big area for a lot of organizations and especially when, as an API gateway, when we are working with different components, we have one part of the API platform stack to know if there are potential bottlenecks. +Traces. Traces are my favorite signal because, like, they give a full, you know, a full picture of everything going on in the system and you can easily spot on errors. Favorite OpenTelemetry signal. This is, this is a tough one. I think traces has to take the win at this point of time because again, just thinking about how things connect well. -Are we a bottleneck? Are there other elements that are potential bottlenecks there? How do we improve performance? How do we actually put our money where the numbers are, essentially? That's what profiling again, sort of promises to a point? +I'm also very, very keenly pursuing profiling. I think that's going to be potentially the winner in the next conversation we have because I think performance is a big area for a lot of organizations and especially when, as an API gateway, when we are working with different components, we have one part of the API platform stack to know if there are potential bottlenecks. Are we a bottleneck? Are there other elements that are potential bottlenecks there? How do we improve performance? How do we actually put our money where the numbers are, essentially? That's what profiling again, sort of promises to a point? -Because of the background, Elastic, I gotta say logs. But of course, you know, it's... The logs are, you know, they bring like deep contextual insights, but at the end of the day, you need them all. Like metrics are gonna let us know there is a problem. Tracing is gonna help us to understand where the problem is, and logs are gonna help us understand what the problem is. +Because of the background, Elastic, I gotta say logs. But of course, you know, it's. The logs are, you know, they bring like deep contextual insights, but at the end of the day, you need them all. Like metrics are gonna let us know there is a problem. Tracing is gonna help us to understand where the problem is, and logs are gonna help us understand what the problem is. -My favorite signal is traces because I fell in love with observability and OpenTelemetry because of traces. I would have to say that I got the most value out of tracing. But recently I started to correlate traces with metrics, and I think that is like the golden flow. +My favorite signal is traces because I fell in love with observability and OpenTelemetry because of traces. I would have to say that I got the most value out of tracing. But recently I started to correlate traces with metrics, and I think that is like the golden flow. I have been a huge fan of distributed tracing in general. I think it gives you the understanding of how big, like, services interact with each other. -I have been a huge fan of distributed tracing in general. I think it gives you the understanding of how big, like, services interact with each other. But I've been growing to like profiling. I think it gives interesting, exciting opportunities on how people understand even deeper how their systems behave, especially how their systems behave under different flow, different conditions, and to be able to adjust, improve their architectures and the scale of their systems to cater to future loads. +But I've been growing to like profiling. I think it gives interesting, exciting opportunities on how people understand even deeper how their systems behave, especially how their systems behave under different flow, different conditions, and to be able to adjust, improve their architectures and the scale of their systems to cater to future loads. My favorite OpenTelemetry signal right now is logs, because even though I'm fully immersed in OpenTelemetry now and I know what all three of the signals mean, I started on logs because logs are easy. It just makes so much sense and I understand where people are coming from, coming from with observability, second wave, you know, everything should just be trace or lot wide events. I understand the value of that, but I just feel like logs aren't ever going away. diff --git a/video-transcripts/transcripts/2024-12-20T07:03:14Z-otel-me-an-end-user-conversation-with-ariel-valentin.md b/video-transcripts/transcripts/2024-12-20T07:03:14Z-otel-me-an-end-user-conversation-with-ariel-valentin.md index 1f0454e..07f2401 100644 --- a/video-transcripts/transcripts/2024-12-20T07:03:14Z-otel-me-an-end-user-conversation-with-ariel-valentin.md +++ b/video-transcripts/transcripts/2024-12-20T07:03:14Z-otel-me-an-end-user-conversation-with-ariel-valentin.md @@ -10,288 +10,288 @@ URL: https://www.youtube.com/watch?v=vB9_SiTV5CI ## Summary -In this episode of "Oh Tell Me," hosted by Reys, a Senior Developer Relations Engineer at New Relic, the focus is on open telemetry and observability, featuring guest Ariel Valentin, a Staff Software Engineer at GitHub. The discussion revolves around Ariel's experiences with observability, the adoption of open telemetry at GitHub, and the challenges faced in transitioning from proprietary SDKs to open standards. Key topics include the importance of distributed tracing, the need for a shared vocabulary in observability, and the architectural landscape at GitHub, which utilizes a customized OpenTelemetry collector. Ariel shares insights on the impact of tracing on service monitoring, the growth in usage post-adoption, and the collaboration within the open telemetry community. The episode also emphasizes the importance of community involvement, encouraging viewers to participate in SIG meetings and contribute feedback. +In this episode of "Oh Tell Me," a Q&A session focused on end-user experiences with OpenTelemetry, host Reyes, a Senior Developer Relations Engineer at New Relic, interviews Ariel Valentin, a Staff Software Engineer at GitHub. The discussion centers around observability, specifically the adoption and challenges of using OpenTelemetry at GitHub. Ariel shares insights into GitHub's transition from proprietary SDKs to OpenTelemetry, emphasizing the importance of distributed tracing and collaboration within the community. They discuss the challenges of terminology in observability, the complexities of migrating systems, and the evolving architecture at GitHub that utilizes OpenTelemetry for tracing. The conversation also covers community engagement, contribution processes, and the significance of user feedback to improve open-source projects. Both participants encourage viewers to get involved in contributing to the community, whether through code, feedback, or participation in special interest groups (SIGs). ## Chapters -00:00:00 Welcome and intro -00:01:37 Guest introduction -00:03:00 Format overview -00:04:10 Warmup questions -00:05:40 Observability discussion -00:09:40 Adoption challenges -00:10:50 GitHub's OpenTelemetry adoption -00:12:10 Architecture landscape -00:15:00 Telemetry capture methods -00:19:01 Community contribution process +00:00:00 Introductions +00:01:01 Overview of the session +00:05:05 Guest introduction: Ariel Valentin +00:08:08 Discussion on distributed tracing +00:12:12 Challenges in observability terminology +00:18:18 GitHub's adoption of OpenTelemetry +00:24:24 Data dictionary and semantic conventions +00:32:32 Architecture landscape at GitHub +00:41:01 Discussion on community contributions +00:55:55 Closing remarks and future plans -**Reys:** Hello everyone! Welcome to a brand new episode of Oh Tell Me, an end user Q&A. If you have been to one of these sessions in the past, you might notice that this looks a little bit different and it has a bit of a different name. That's right, we have done some growing up since the last few times and we're very excited to debut this new look. But don't worry, almost everything else is going to be the same. We have a great interview here for you today to learn from. +## Transcript + +### [00:00:00] Introductions + +**Reys:** Hello everyone, welcome to a brand new episode of Oh Tell Me and End User Q&A. If you have been to one of these sessions in the past, you might notice that this looks a little bit different and it has a bit of a different name. That's right, we have done some growing up since the last few times, and we're very excited to debut this new look. But don't worry, almost everything else is going to be the same. We have a great interview here for you today to learn from. But before I introduce myself and our guest, I would love to know where everyone is connecting from. I am online from Portland, Oregon today and would love to see where people are from. It looks like someone is here from Brooklyn, New York, so thank you so much for joining out in New York. -[00:01:37] So my name is Reys. I am a senior developer relations engineer at New Relic. I also co-lead the Ner Sig, which is our cute little logo that you see up in the right corner. At the end of your Sig, we really focus on connecting directly with users and we are dedicated to helping the rest of the SIGs get feedback from end users so they can help improve the project. To that end, this is one of those events that we host to do that. +My name is Reys, I am a senior developer relations engineer at New Relic. I also co-lead the Ner Sig, which is our cute little logo that you see up in the right corner. At the end of your Sig, we really focus on connecting directly with users and we are dedicated to helping the rest of the sigs get feedback from end users so they can help improve the project. To that end, this is one of those events that we host to do that. We are going to have Ariel on. Ariel, if you would like to join us and introduce yourself. -**Ariel:** Hey, how's it going, everybody? I'm Ariel Valentin. I'm a Staff Software Engineer at GitHub, working on observability. Thanks so much, Rey, for inviting me here to chat with you today. It's an absolute honor to be the first in the new format. Also, a big shout out to everybody who's done work to try to get this new platform up and running, so thank you. +**Ariel:** Hey, how's it going everybody? I'm Ariel Valentin. I'm a Staff Software Engineer at GitHub working on observability. Thanks so much, Reys, for inviting me here to chat with you today. It's an absolute honor to be the first in the new format. Also, big shout out to everybody who's done work to try to get this new platform up and running, so thank you. -[00:03:00] **Reys:** Oh no, thank you for being here! I'm really excited. Just to get you and everyone up to speed on the format, it's similar to the ones if you've been on one of these before, but we're going to start with some warmup questions where we'll kind of massage Ariel into being comfortable, and then we'll hit him with some meaty questions. Then we will actually also get into questions more around the Open Telemetry community, where he'll have an opportunity to share feedback about his experiences with contributing, using the project, stuff like that. Then he'll get the chance to ask us questions as well in a little section we call "turn the tables." At the very end, if there are any audience questions—oh actually, scratch that—if you have questions that come up during our conversation, feel free to put them into the chat. If you're watching from YouTube, then just put them in the live chat, and then I think LinkedIn will also have its own chat, so put in your question there, and we will get to them as we can throughout the conversation. Then at the end, if you have questions at the end as well, then we will get to those. But yeah, if you have any questions that pop up as Ariel is telling us his story, we definitely want to get to those as they come in. +**Reys:** Oh no, thank you for being here! I'm really excited. Just to get you and everyone up to speed on the format, it's similar to the ones if you've been on one of these before, but we're going to start with some warmup questions where we'll kind of massage Ariel into being comfortable and then we'll hit him with some meaty questions. Then we will actually also get into questions more around the Open Telemetry community and where he'll have an opportunity to share feedback about his experiences with contributing, using the project, stuff like that. Then he'll get the chance to ask us questions as well in a little section we call "turn the tables." At the very end, if there are any audience questions—oh actually, scratch that—if you have questions that come up during our conversation, feel free to put them into the chat. If you're watching from YouTube, then just put them in the live chat, and then I think LinkedIn will also have its own chat, so put in your question there and we will get to them as we can throughout the conversation. At the end, if you have questions at the end as well, then we will get to those. But yeah, if you have any questions that pop up as Ariel is telling us his story, we definitely want to get to those as they come in. All right, so I think we are good to get started. Ariel, tell us a little bit about your role at the company. How did you get started with observability? How did you get started with Open Telemetry? -**Ariel:** Oh, those are great questions. I mean, I should have mentioned that I'm here in Austin, Texas—Sunny Austin, Texas—so you know where I am in the world. I think a lot of us started originally, and I don't think my experience is unique, is working with sort of like APM tools, which are vendor proprietary tools, generally speaking, that didn't have distributed tracing in place at that time. As the community evolved and Open Tracing became a standard, that was my first experience with working with distributed tracing was the Open Tracing specification and working and trying out different vendors, different experiences with Open Tracing. - -[00:05:40] By the time that I had gotten to GitHub, I was a champion for distributed tracing because I saw the power that was in there. Around that same time, folks were already moving towards developing and transitioning away from Open Tracing and Open Census to Open Telemetry, so I got really involved very early on in trying to spread the word and learning more and working with my team, which was the observability team, to start to adopt tracing more, to embrace it more, and to make Open Telemetry sort of our North Star for all of our telemetry signals. - -I feel like I keep saying the word telemetry over and over again. I'm going to have to find a synonym for that. +### [00:05:05] Guest introduction: Ariel Valentin -**Reys:** Yeah, me too! We're both going to stumble over Open Telemetry at some point or distributed tracing at some point. It's all right; this is a safe space. +**Ariel:** Oh, those are great questions. I mean, I should have mentioned that I'm here in Austin, Texas—Sunny Austin, Texas—so you know where I am in the world. I think a lot of us started originally, and I don't think my experience is unique, working with sort of like APM tools, which are vendor proprietary tools, generally speaking, that didn't have distributed tracing in place at that time. As the community evolved and Open Tracing became a standard, that was my first experience with working with distributed tracing was the Open Tracing specification and working and trying out different vendors, different experiences with Open Tracing. By the time that I had gotten to GitHub, I was a champion for distributed tracing because I saw the power that was in there. Around that same time, folks were already moving towards developing and transitioning away from Open Tracing and Open Census to Open Telemetry. I got really involved very early on in trying to spread the word and learning more, working with my team, which was the observability team, to start to adopt tracing more, to embrace it more, and to make OTEL sort of our North Star for all of our telemetry signals. -What do you think is a main challenge that most organizations face when it comes to observability? Is it not just understanding the value of distributed tracing? +I feel like I keep saying the word telemetry over and over again. I'm going to have to find a… yeah, me a lot of that. We're both going to stumble over Open Telemetry at some point or distributed tracing at some point. It's all right, this is a safe space. -**Ariel:** Yeah, because these are all sort of new concepts to folks, right? Folks have their different levels of understanding of the tools that are available to them, and there's all this vocabulary that you hear that's a little bit hard to parse through. Sometimes it's like, "Oh, when you say trace, do you mean like a trace log? Log level is at the trace level? When you say trace, do you mean the samples taken from a profiler?" There's a lot of this sort of language that, even though we're converging on a lot of this language and we have these dictionaries that are defined and published everywhere, there's still sort of like this hump that we have to get over or like a challenge that we have with trying to get everybody speaking the same language within the same context. Very similar to in domain-driven design, we have these bounded contexts where the same term means a different thing. That flows over into sort of our domain language when it comes to observability and SRE practices, and I'm sure many folks have faced those challenges as well. It's kind of like, "Let's all get on the same page about what we mean." +What? Okay, so you mentioned you became a champion for distributed tracing, and you know I think something that's interesting is, you know, we're so ensconced in the world of observability that we at least, I, you know, tend to presume everyone understands what that is. It's always surprising to me when someone's like, "What's distributed tracing?" -**Reys:** Oh, absolutely. What are currently some of the most interesting problems that you are facing in your role? +**Ariel:** Right. What do you think that's kind of a main challenge that most organizations face when it comes to observability? It's not just understanding the value of distributed tracing. -**Ariel:** Oh well, I mean one of those things is that transition, right? It's really hard for an organization like us, who's been around for a long time. I say a long time, but you know, whatever it is—10 years—where the system has grown and evolved. There have been acquisitions that have been brought together; there's disparate kind of backends where we're collecting this data. It's trying to transition from one way of doing things to a new way of doing things, right? So it's learning something new. That's always going to be a big challenge. Folks are trying to do their job every day; they're not trying to learn new SDKs or trying to learn new vocabulary or trying to learn how to be proficient in their backends. What they're trying to do is keep the system up and running and keep our customers happy, right? +**Reys:** Yeah, because these are all sort of new concepts to folks, right? Folks have their different levels of understanding of the tools that are available to them. There’s all this vocabulary that you hear that's a little bit hard to parse through. Sometimes it's like, oh, when you say trace, do you mean like a trace log? Log level is at the trace level? When you say trace, do you mean the samples taken from a profiler? There’s a lot of this sort of language that, even though we're converging on a lot of this language and we have these dictionaries that are defined and published everywhere, there's still sort of like this hump that we have to get over or like a challenge that we have with trying to get everybody speaking the same language within the same context, right? Very similar to in domain-driven design, we have these bounded contexts where the same term means a different thing. You know, that flows over into sort of our domain language when it comes to observability and SRE practices. I'm sure many folks have faced those challenges as well. It's kind of like, let's all get on the same page about what we mean. -I think those are some of the challenges that we all face. I know I face it, and I'm sure others do, which is making these transitions with the fewest pain points as possible, trying to avoid these pain points. There's so many more we can go on and on, Reys, but I think right now that's kind of like one of the biggest challenges for adoption overall. +### [00:08:08] Discussion on distributed tracing -[00:09:40] **Reys:** That's actually a great segue into the meaty section. What was the process like for GitHub to adopt Open Telemetry? +**Ariel:** Oh, absolutely. What are currently some of the most interesting problems that you are facing in your role? -**Ariel:** For us, it was the role of the advocate, right? I acted as an advocate and was a champion for Open Telemetry at the company. I was specifically brought in to help advance the mission of tracing, and it was something that I had pitched. There were other challenges that we faced too, which was, "Hey, let's get everybody building a data dictionary. Let's get everybody agreeing to the same language when it comes to what our attributes were going to be." As you can imagine, as a system of walls or acquisitions or other teams are rolled in, everybody has their own log attributes or their own metric attributes or whatever it is. +**Reys:** Oh well, I mean, one of those things is that transition, right? It's really hard for an organization like us who's been around for a long time. I say a long time, but you know, whatever it is—10 years—where the system has grown and evolved. There have been acquisitions that have been brought together. There's disparate kind of backends where we're collecting this data. It’s trying to transition from one way of doing things to a new way of doing things, right? It's learning something new. That's always going to be a big challenge. Folks are trying to do their job every day; they're not trying to learn new SDKs or trying to learn new vocabulary or trying to learn how to be proficient in their backends. What they're trying to do is keep the system up and running and keep our customers happy, right? Those are some of the challenges that we all, you know, I know I face it. I'm sure others do, which is making these transitions with the fewest pain points as possible, trying to avoid these pain points. There's so many more we can go on and on, Reys, but I think right now that's kind of like one of the biggest challenges for adoption overall. -[00:10:50] I said, "Look, here's a North Star. Here's semantic conventions from Open Telemetry. Let's anchor onto this and let's follow the rules around semantic convention so that we can all build up our own internal dictionary." That comes with its own challenges—schema migrations and trying to keep up to date with the spec and what the instrumentations are doing, right? As an advocate, I was also a lead on the rollout. One of the things I wanted to do was sunset all of our old SDKs and move over to our open-source SDKs. +**Reys:** That's actually a great segue into the meaty section. So what was the process like for GitHub to adopt Open Telemetry? -My team and I were all involved in Open Telemetry Ruby, for example, because we're a big Ruby shop. We got involved there to help the instrumentations get better and to help test different releases of the SDK and get that rolled down into our GitHub monolith. We were, like I said, very early adopters, ran into some challenges, and continued to give feedback to the community that way. I'm not only an end user, but like I said, I'm also a maintainer, so I'm playing multiple roles there, trying to help the community along. +**Ariel:** For us, it was the role of the advocate, right? I acted as an advocate and was a champion for OTEL at the company. I specifically was brought in to help advance the mission of tracing. It was something that I had pitched, and there were other challenges that we faced too, which was, hey, let's get everybody building a data dictionary. Let's get everybody agreeing to the same language when it comes to what our attributes were going to be, right? Because, as you can imagine, as a system evolves or acquisitions or other teams are rolled in, everybody has their own log attributes or their own metric attributes or whatever it is. I said, look, here's a North Star right here, here's semantic conventions from OTEL, let's anchor onto this and let's follow the rules around semantic conventions so that we can all build up our own internal dictionary. -**Reys:** That's a really interesting position to be in as an end user and contributor. I definitely want to circle back on that when we get to the OpenTelemetry community questions. Tell us about the architecture landscape at GitHub and the telemetry that you're capturing. +That comes with its own challenges, right? Schema migrations and trying to keep up to date with the spec and what the instrumentations are doing, right? As an advocate, I was also a lead on the rollout. One of the things I wanted to do was sunset all of our old SDKs and move over to our open-source SDKs. My team and I were involved in OTEL Ruby, for example, because we're a big Ruby shop. We got involved in there to help the instrumentations get better and to help test different releases of the SDK and get that rolled down into our GitHub monolith. We were very, like I said, very early adopters, ran into some challenges, and continued to give feedback to the community that way. I'm not only an end user, but like I said, I'm also a maintainer, so I'm playing multiple roles there, you know, trying to help the community along. -[00:12:10] **Ariel:** Sure! I've got this little slide that I put together in markdown, so it's not anything official. We can bring that up now so we can take a look at it. I imagine that for a lot of folks out there who are working with virtual machines where they're running systemd units or if they're running Kubernetes, we have different deployment styles here. +**Reys:** That's a really interesting position to be in as end user and contributor. I definitely want to circle back on that when we get to the community questions. Tell us about the architecture landscape at GitHub and the telemetry that you're capturing. -Some of our main workloads are running in Kubernetes, and the way that we've got everything set up, you can see here as part of our Kubernetes cluster, we run our own custom messaging graph. On every worker node, we're running a deployment of the OpenTelemetry collector, which we build using OCB. We have our own version of the OpenTelemetry collector that is specific to us. We chose that route because we wanted to ensure that we had the most secure build possible with the minimum number of dependencies, and we also wanted the ability to build our own custom processors so that we can address any issues that we might have that are specific to our needs. +### [00:12:12] Challenges in observability terminology -Inside of each of these pods, we also have a mesh sidecar. Ingress traffic is going to come into the OpenTelemetry collector over HTTP. If you have another application that's running your service, it's shooting over OTel traces over to the mesh and sending it to our OpenTelemetry collectors. From there, we generate span metrics and sample traces and send those off to a SaaS provider where we aggregate all this data. +**Ariel:** Sure. I've got this little slide that I put together in markdown, so it's not anything official. We can bring that up now so we can take a look at it. I imagine that for a lot of folks out there who are working with virtual machines where they're running systemd units or if they're running Kubernetes, we have different deployment styles here. Some of our main workloads are running in Kubernetes. The way that we've got everything set up, you could see here is as part of our Kubernetes cluster. We run our own custom meshing grass, and on every worker node, we're running a deployment of the OpenTelemetry collector, which we build using OCB. We have our own version of the OTEL collector that is specific to us. We chose that route because we wanted to ensure that we had the most secure build possible with the minimum number of dependencies. We also wanted the ability to build our own custom processors so that we can address any issues that we might have that are specific to our needs. -On each individual worker, we're running in this hybrid world right now. We're only leveraging OTel for traces and for generating span metric data. We're still living in a world where we're using non-OTel formats for collecting things like custom metrics, system metrics, and for collecting logs. On each one of our worker nodes, we have a metrics agent that can speak various different protocols, but mostly it's either using Open Metrics or it's using StatsD to collect data and aggregate it and send it off to the SaaS provider. +Inside of each of these pods, we also have a mesh sidecar. Ingress traffic is going to come into the OTEL collector over HTTP. If you have another application that's running your service, it's shooting over OT traces over to the mesh and sending it to our OTEL collectors, which we then from there generate span metrics and sample traces and send those off to a SaaS provider where we aggregate all this data. On each individual worker, we're running in this hybrid world right now. We're only leveraging OTEL for traces and for generating span metric data. We're still living in a world where we're using non-OTLP formats for collecting things like custom metrics, system metrics, and for collecting logs. -[00:15:00] In addition to that, on all of our worker nodes, we have Fluent Bit running, and that's what we're using to collect our logs. All of our logs end up getting streamed out through Azure Event Hubs, which are processed by a bunch of consumers and then sent off to our log store and search system. +We have on each one of our worker nodes a metrics agent, and that can speak various different protocols, but mostly it's either using OpenMetrics or it's using StatsD to collect data and aggregate it and send it off to the SaaS provider. In addition to that, on all of our worker nodes, we have Fluent Bit running, and that's what we're using to collect our logs. All of our logs end up getting streamed out through Azure Event Hubs, which are processed by a bunch of consumers and then sent off to our log store and search system. In addition to these Kubernetes workloads, we have our own virtual machines where we run systemd units. You have to think about, you know, like the Git file system services. Those are all running on systemd, and we have the metrics agent and Fluent running there. We don't yet have the OTEL collector deployed there, but that's where we want to get to. We want to get to a world where essentially our entire platform is running open-source software, the OpenTelemetry collector, and we're converging on OTLP for all these formats. -In addition to these Kubernetes workloads, we have our own virtual machines where we run systemd units. You have to think about the Git file system services. Those are all running on systemd, and we have the metrics agent and Fluent running there. We don't yet have the OpenTelemetry collector deployed there, but that's where we want to get to. We want to get to a world where essentially our entire platform is running open-source software, the OpenTelemetry collector, and we're converging on OTel for all these formats. So those services are also using StatsD, Open Metrics, and Fluent Bit's pulling things out of JournalD because that's where we're streaming all of our logs of the system journal. +Those services are also using StatsD, OpenMetrics, and Fluent Bit pulling things out of JournalD because that's where we're streaming all of our logs from the system journal. All that stuff again goes right through to the same channels, and it all gets aggregated in these places where we do our work. A little bit about some facts about sort of our trace data is that we've rolled out the OTEL SDKs for different programming languages. We have them for Ruby, Go, JavaScript, Node.js, .NET. We had some experimental Rust usage, but we've had to scale that back. We had some Java experiments that didn't pan out; we didn't roll those out to production. A lot of the stuff that we do, we started before any of the automatic instrumentation was available for some of these languages, so we're still deploying them through wrapper libraries. We maintain a set of wrapper libraries; you install those, it gives you the sort of the what we consider to be the minimal defaults that we require for our needs. We'll do periodic updates of those through Dependabot, right? As we roll out new versions of those things. -All of that stuff again goes right through to the same channels, and it all gets aggregated in these places where we do our work. A little bit about some facts about our trace data is that we've rolled out the OpenTelemetry SDKs for different programming languages. We have them for Ruby, Go, JavaScript, Node.js, .NET. We had some experimental Rust usage, but we've had to scale that back, and we had some Java experiments that didn't pan out; we didn't roll those out to production. +I feel like I've said a lot so far, and I don't know, Reys, if you had any follow-up questions for me. -A lot of the stuff that we do, we started before any of the automatic instrumentation was available for some of these languages, so we're still deploying them through wrapper libraries. We maintain a set of wrapper libraries. You install those; it gives you the sort of the what we consider to be the minimal defaults that we require for our needs, and we'll do periodic updates of those through Dependabot. As we roll out new versions of those things, I feel like I've said a lot so far, and I don't know, Rey, if you had any follow-up questions for me. +**Reys:** I actually have so many, but... -**Reys:** I actually have so many! +**Ariel:** Oh, okay, great. -**Ariel:** Oh, okay, great! +**Reys:** You mentioned you're leveraging OTEL for traces right now and you talked a little bit about the plans and some of the stuff that you tried. I was curious to find out more about— is that mainly because if you're primarily a Ruby shop and, you know, the SDKs, maybe the signals for metrics and logs aren't quite as mature yet? Is that the reason? -**Reys:** You mentioned you are leveraging OpenTelemetry for traces right now, and you talked a little bit about the plans and some of the stuff that you tried. I was curious to find out more about is that mainly because you're primarily a Ruby shop and the SDKs for metrics and logs aren't quite as mature yet? Is that the reason? +### [00:18:18] GitHub's adoption of OpenTelemetry -**Ariel:** That's one of the reasons. One of the other challenges that I didn't discuss when you asked about adoption challenges is even though OTel says, "Hey, this is what the signals look like; this is what the semantic attributes are like," there's a lag between the time that something is published or declared in the OTel spec to the time that vendors or open-source platforms are able to leverage those things. There's sort of like this feedback loop as we go through OTs and say, "Hey, we want to try this new thing out." +**Ariel:** That's one of the reasons. One of the other challenges that I didn't discuss when you asked about adoption challenges is, even though OTEL says, "Hey, this is what the signals look like, this is what the semantic attributes are like," there is a lag between the time that something is published or declared in the OTEL spec to the time that vendors or open-source platforms are able to leverage those things. There's sort of like this feedback loop as we go through OTLP and say, "Hey, we want to try this new thing out." We do a couple things in experimental languages. -[00:19:01] We do a couple things in experimental languages; we'll do this stuff, and it was a big enough challenge for us to start to roll tracing out that we didn't want to take yet another thing on, which was to say, "Okay, now we're going to switch over to native OTel metrics as well." Like you said, some of the SDKs are ahead of others. So with Ruby, we're just recently, with the help of our amazing maintainers, working diligently to get the metric SDK up to speed for Ruby. So that's something that wasn't available to us to use. +It was a big enough challenge for us to start to roll tracing out that we didn't want to take yet another thing on, which was to say, "Okay, now we're going to switch over to native OTLP metrics as well." Like you said, some of the SDKs are ahead of others. With Ruby, we're just recently, you know, with the help of our amazing maintainers, Schwan and AAA, they've been working diligently to get the metric SDK up to speed for Ruby, so that's something that wasn't available to us to use. I think that one of the biggest advantages that I see in tracing is the ability to generate span metrics from traces. It's like, hey, the fewer things that we have to impose on our users for them to try to figure out how to do, the better for us, right? -One of the biggest advantages that I see in tracing is the ability to generate span metrics from traces. It's like, "Hey, the less things that we have to impose on our users for them to try to figure out how to do, the better for us." +**Reys:** Are you using the span metrics connector? Is that what you're... -**Reys:** Are you using the span metrics connector? Is that what you're using? +**Ariel:** We're using a custom connector right now. I'd like to be able to get us to the point where we're using a span metrics connector, but we don't have egress in OTLP right now. We're still doing sort of like vendor proprietary formats for export. Where I want to get to is, you know, that's one of the biggest strengths of OTEL is that OTLP is that standard format that makes things portable for us. -**Ariel:** We're using a custom connector right now. I'd like to be able to get us to the point where we're using a span metrics connector, but we don't have egress in OTel right now. We're still doing sort of like vendor proprietary formats for export. Where I want to get to is, you know, that's one of the biggest strengths of OTel is that OTel is that standard format that makes things portable for us. +**Reys:** Absolutely, so that’s definitely sort of high on my list of things that I want to do. -**Reys:** Absolutely. So that sounds like the plan is to migrate your other signals over once those reach GA in languages. +**Ariel:** Okay, so it sounds like the plan is to migrate your other signals over once those reach GA in languages. -**Ariel:** Oh, certainly. And once I have more time on the calendar too, right? We have so many projects that we have to do as engineers and companies, and it's like, "Hey, where do we fit these in?" As you imagine, GitHub is constantly growing every day. We just hit 150 million users, so it's like just an amazing growth of the company. Shout out to all the people at GitHub who've been working so diligently to make this happen. We're here to support that volume growth and support our end users. I hope that answers your question. +**Reys:** Oh certainly, and you know once I have more time on the calendar too, right? We have so many projects that we have to do as engineers and companies, and it's like, hey, where do we fit these in, right? As you imagine, you know, GitHub is constantly growing every day. We just hit 150 million users, you know, so it's like just an amazing growth of the company. Shout out to all the people at GitHub who've been working so diligently to make this happen. We're here to support that volume growth and support our end users. I hope that answers your question. -**Reys:** Absolutely. I also want to mention again real quick for those who joined us a little bit later, feel free if you have questions that come up. You want to learn more about something that Ariel has gone over, feel free to pop it into the live chat of whatever platform you're watching from, whether it's YouTube or LinkedIn. Go ahead and pop the question in, and we will try to get to them throughout the show as you can. +**Ariel:** Absolutely. I also want to mention again real quick for those who joined us a little bit later: feel free if you have questions that come up, you want to learn more about something that Ariel has gone over, feel free to pop it into the live chat of whatever platform you're watching from, whether it's YouTube or LinkedIn. Go ahead and pop the question in and we will try to get to them throughout the show as you can. -I noticed on your second slide you mentioned probabilistic sampling, so it sounds like you're not doing any kind of tail sampling. +So I noticed on your second slide you mentioned probabilistic sampling, so it sounds like you're not doing any kind of tail sampling. -**Ariel:** No, not at the moment. Would you mind bringing that up again—the second slide? +**Ariel:** No, not at the moment. -**Reys:** Yeah, that's awesome. +**Reys:** Would you mind bringing that up again? The second slide? -**Ariel:** Yeah, so we started with probabilistic sampling. One of the hard things about tail sampling, as you can imagine, is that the traces all have to go to the same collector in order to make that decision if you're using the collector for tail sampling. Right now, we wanted to reduce that burden in complexity immediately. When we started with probabilistic sampling, some of the things that we want to do in the future is more advanced remote sampling so that we can have more fine-grain control of what we're looking at. We want to be able to leverage tail sampling rules, but as you know, that's very difficult to implement and a little bit difficult to scale in our case because we have about 2,000 collectors that are supporting everything right now across all of our fleet of like 14,000 hosts or something like that. I'm making that number up off the top of my head; I think that's the last number we had. +**Ariel:** Yeah, that's awesome. We started with probabilistic sampling. One of the hard things about tail sampling, as you can imagine, is that the traces all have to go to the same collector in order to make that decision if you're using the collector for tail sampling. Right now, we wanted to reduce that burden in complexity immediately. When we started with probabilistic sampling, some of the things that we want to do in the future is more advanced remote sampling so that we can have more fine-grained control of what we're looking at. We want to be able to leverage tail sampling rules, but as you know, that's very difficult to implement and a little bit difficult to scale in our case because we have about 2,000 collectors supporting everything right now across all of our fleet of like 14,000 hosts or something like that. I'm making that number up off the top of my head; I think that's the last number we had. Right now, we're doing about 26 million spans per second. Just yesterday during our peak times, we hit our all-time high of 32 million spans per second as we continue to grow our volume. -Right now, we're doing about 26 million spans per second. Just yesterday during our peak times, we hit our all-time high of 32 million spans per second as we continue to grow our volume. This is one of the challenges that we've had, and so on my list of all the things that I want to do, tail sampling is definitely on there. +This is one of the challenges that we've had, and so on my list of all the things that I want to do, tail sampling is definitely on there, you know what I mean? -**Reys:** At what point do you think you might get to the point where you could implement tail sampling? +**Reys:** At what point do you think you might get to the point where you could implement tail sampling? -**Ariel:** That would be more like when the processor has been more developed. I don't know how to answer that question because it's on the pile of the list of things that I want to do towards the bottom. +**Ariel:** It would be more like when the processor has been more developed. I don’t know how to answer that question because it's on the pile of the list of things that I want to do towards the bottom. -**Reys:** Got it. +**Reys:** Got it, got it. Although I am curious about the other... -**Ariel:** Although I am curious about the other. +**Ariel:** But that's okay! -**Reys:** But that's okay! That’s all right. +**Reys:** Questions that I want to get to as well. -**Ariel:** Questions that I want to get to as well. +**Ariel:** Sure. -**Reys:** Sure! Also, just going back, you mentioned trying some experiments in Java and Rust that didn't quite pan out, and I was just curious what it was that didn't work out or didn't meet your expectations. +**Reys:** Also, just going back, you mentioned trying some experiments in like Java and Rust that didn't quite pan out, and I was just curious, what was it that didn't quite work out or that didn't meet your expectations? -**Ariel:** Well, we've reduced the number—I'm sorry, all the JVM people out there. We've reduced the number of Java workloads that we're running, so those have been migrated over to different programming languages. That's one of the reasons why we didn't go through and say, "Oh, let me continue to roll out JVM work." We just didn't have the return on investment that we wanted to, you know, try. +### [00:24:24] Data dictionary and semantic conventions -Also, we don't have a lot of the same thing for Rust. We have a very limited set of applications that run Rust, and those are performing for performance reasons. They ran into some challenges with how it impacted their latency when they introduced the use of the SDK. I'm going to be honest with you: me not being a Rust expert and being able to get in there and get my hands dirty to try to help out with addressing some of those problems and reporting them upstream, that was something that we had to put on pause. Again, it's like one program versus all of these other services that are running in Go and Ruby and JavaScript that we need to pay attention to. +**Ariel:** Well, we've reduced the number—I'm sorry, all the JVM people out there—we've reduced the number of Java workloads that we've been running, so those have been migrated over to different programming languages. That's one of the reasons why we didn't go through and say, "Oh, let me continue to roll out JVM work." We just didn't have the return on investment that we wanted to try. Also, we don't have a lot of the same thing for Rust. We have a very limited set of applications that run Rust, and those performed for performance reasons. They ran into some challenges with how it impacted their latency when they introduced the use of the SDK. I'm going to be honest with you; me not being a Rust expert and being able to get in there and get my hands dirty to try to help out with addressing some of those problems and reporting them upstream, that was something that we had to put on pause because, again, it's like one program versus all of these other services that are running in Go, Ruby, and JavaScript that we need to pay attention to. -I think that's really where it was, where sort of like a critical mass of application services that use these programming languages, and it was like, "Hey, we have to focus our attention on those that are going to get us a higher return on investment for this." +**Reys:** Absolutely. Okay, so what was GitHub's observability tool before migrating to, I think you mentioned, you were using proprietary SDKs? -**Reys:** Absolutely. Okay, so what was GitHub's observability tool before migrating to OpenTelemetry? +**Ariel:** Yeah, I mean we were using proprietary SDKs for Open Tracing. Open Tracing was our—how we collected traces before. Generally speaking, I think I covered this in the first slide. GitHub is a huge StatsD metrics user still to this day. Logs are a big part of what we utilize here. Folks are looking at exception stack traces a lot of the times to try to understand where errors are coming from. They're looking at access log streams, and they were trying to piece together, "Hey, where's the request going and where is it slowing down?" I showed up with my, you know, magic tool, distributed tracing, and I'm like, "Hey, look, here it is on the flame graph or a cycle graph or here's a waterfall view of this thing." It's like, "Oh, that's pretty cool." People started looking into that as an additional tool in their toolbox for them to help try to debug things during an incident. -**Ariel:** Yeah, I mean, we were using a proprietary SDK for OpenTracing. OpenTracing was how we collected traces before, but generally speaking, GitHub is a huge StatsD metrics user still to this day. Logs are a big part of what we utilize here. Folks are looking at exception stack traces a lot of the times to try to understand where errors are coming from. They're looking at access log streams, and they were trying to piece together, "Hey, where's the request going, and where is it slowing down?" I showed up with my magic tool—distributed tracing. I'm like, "Hey, look, here it is on the flame graph or an icicle graph, or here's a waterfall view of this thing." It's like, "Oh, that's pretty cool!" People started looking into that as an additional tool in their toolbox for them to help try to debug things during an incident. +**Reys:** I hope that answers your question there. -I hope that answers your question there. +**Ariel:** Yes, and kind of along those lines, how have things changed since GitHub switched to OpenTelemetry? -**Reys:** Yes, and kind of along those lines, how have things changed since GitHub switched to OpenTelemetry? +**Ariel:** What I'll tell you is that just this year alone, we've seen a dramatic—I wish I had these statistics offhand—but we've seen a dramatic increase in the number of people using tracing and a number of services that have been instrumented. Part of that comes from the fact that I was like, "Hey everybody, we're all moving to this new SDK, so everybody install it and let Dependabot go ahead and do these installations on your apps." People started seeing the value of this investment as well, so they started to use it more. -**Ariel:** What I'll tell you is that just this year alone, we've seen a dramatic—I wish I had these statistics off hand—but we've seen a dramatic increase in the number of people using tracing and the number of services that have been instrumented. Part of that comes from the fact that I was like, "Hey, everybody, we're all moving to this new SDK, so everybody install it." Let Dependabot go ahead and do these installations on your apps, and people started seeing the value of this investment as well. +I want to say that when we started this adventure, only about, and I'm going to use the word services in air quotes, only about 80 services were instrumented, and now we're closer to about 300 of those services, which are effectively like Kubernetes deployment types or like systemd types, because as you imagine, our monolith has, you know, maybe a thousand services within it, so there are like subservices in there. The monolith itself has broken down into like eight services, like web UI, API, GraphQL, and background workers and stream processors and whatnot. That's why I put them in air quotes as these are services. -I want to say that when we started this adventure, only about 80—and I'm going to use the word services in air quotes—only about 80 services were instrumented, and now we're closer to about 300 of those services, which are effectively like Kubernetes deployment types or systemd types. As you can imagine, our monolith has, you know, maybe a thousand services within it, so there are like sub-services in there. But the monolith itself has broken down into like eight services—like web UI, API, GraphQL, background workers, stream processors, and whatnot. That's why I put them in air quotes as these are services. - -We saw that that was quite an increase in the number of services that came in. We were doing something like 5 million spans per second to now we're up to 32 million spans per second. It's just a huge increase in volume and usage. +We saw that that was quite an increase in the number of services that came in. We were doing something like 5 million spans per second to now we're up to 32 million spans per second. It's just a huge increase in volume in usage. **Reys:** Along with that volume increase in usage and data volume, how would you say that's impacted how your services are running and how quickly your team is able to debug issues as they arise? -**Ariel:** Part of it is an education thing. A lot of things that—I mentioned that I'm on the observability team, but really, we're two groups. One of the groups is called The Experience Team that works directly with teams. They operate in a role sort of like developer relations and advocacy, but also working on cost control and improving your experience, helping you with identifying new workflows and introducing them into your incident command experience. +**Ariel:** Part of it is an education thing. A lot of things that—so I mentioned that I'm on the observability team, but really we're two groups. One of the groups is called the Experience team that works directly with teams. They operate in a role sort of like developer relations and advocacy, but also working on cost control and improving your experience, helping you with identifying new workflows and introducing them into your incident command experience. Those teams set up sessions, education sessions, to bring folks on board, but we try to do them not during an incident because that puts some stress on people. -Those teams set up sessions, education sessions, to bring folks on board. We try to do them not during an incident because that puts some stress. What we try to do is, if an incident is happening, it's like, "Hey, here's some insights that we're gaining, and we'll share with you that we're seeing from the traces that you might not be able to see somewhere else." That has helped in a lot of cases where we couldn't exactly pinpoint what was going on. +What we try to do is, you know, if an incident is happening, it's like, "Hey, here's some insights that we're gaining," and we'll share with you that we're seeing from the traces that you might not be able to see somewhere else. That has helped in a lot of cases where we couldn't exactly pinpoint what was going on. But then there's also these spots where not every system has been instrumented, so we have to rely a lot on sort of like client metrics or client trace data and say, "Hey, look, this client is experiencing this problem. Can we take a look deeper at this other service that hasn't yet been instrumented?" -Then there are also these spots where not every system has been instrumented, so we have to rely a lot on sort of like client metrics or client trace data and say, "Hey, look, this client is experiencing this problem. Can we take a look deeper at this other service that hasn't yet been instrumented?" It's been sort of, I'm going to say, mixed results. Some teams have identified issues even before they go out to production, and other teams have been able to leverage it when it's like, "Oh, this, we're having an incident right now. Oh, here's the reason why this is failing." +It's been sort of, I'm going to say like this mixed results. Some teams have identified issues even before they go out to production, and other teams have been able to leverage it when it's like, "Oh, we're having an incident right now. Oh, here's the reason why this is failing." We've been able to identify bottlenecks and mistakes, like little coding mistakes. "Oh, I forgot to ack the message before I pulled it out of Kafka. Oh, I'm just retrying that same message millions of times," or "Oh, the client timeout doesn't match the server timeout, so the client times out and the server is continuing to churn along to try to do this request." We're identifying things like that that we couldn't identify before easily. -We've been able to do things like identify bottlenecks and mistakes, like little coding mistakes—"Oh, I forgot to ack the message before I pulled it out of Kafka. I'm just retrying that same message millions of times," or "Oh, the client timeout doesn't match the server timeout, so the client times out, and the server is continuing to turn along to try to do this request." We're identifying things like that that we couldn't identify before easily. +Those are some anecdotes. -**Reys:** Oh no, that's awesome! I think I will probably be asking you more about that because I think this is really interesting. One more question from our meaty section: what would prevent you from implementing better telemetry? +**Reys:** Oh no, that's awesome! I think I will probably be asking you more about that because I think this is really interesting. -**Ariel:** What would prevent me from doing that? I think it's because there's so much of the stuff that's in early stages. We were early adopters on a lot of things, but then there's a lot of stuff that's a lot more risky for us to try to roll out. For example, one of the things I'm really excited about when I came back from KubeCon is the continuous profiler. +One more question from our meaty section: what would prevent you from implementing better telemetry? -That's one of the things on my wish list that we'd be able to use today if we could. That's the thing that I would want to do, but because we're still in the early stages of the profiler and the specification, and there's still some churn with the data model, I think that there are—when it comes to the resource-intensive or sort of like introspective tools like that, we don't want to take that risk going a little bit too early to adopt those tools. +### [00:32:32] Architecture landscape at GitHub -Also, there's not a lot of support from our current vendors that will be able to leverage that. It's kind of like we would be experimenting with that to go to nowhere. That's kind of, you know, once vendors start to support the OpenTelemetry profiling format, data model, I should say, and once the profiler is a little bit more stable, I'd love to jump in there and be able to roll that out a little more widely. +**Ariel:** What would prevent me from doing that? I think it's because there's so much of the stuff that's in early stages. We were early adopters on a lot of things, but then there's a lot of stuff that's a lot more risky for us to try to roll out. For example, one of the things I'm really excited about when I came back from KubeCon is the continuous profiler. When you talk about one of the things on my wish list that we'd be able to use today if we could, that's the thing that I would want to do. But because we're still in the early stages of the profiler and the specification and there's still some churn with the data model, I think that there are, when it comes to the resource-intensive or introspective tools like that, we don't want to take that risk going a little bit too early to adopt those tools. -I think that a little bit of a bumpy road for us is still migrating Semantic conventions from pre-1.0 because we were early adopters of Semantic conventions, so we're at this pre-1.0 stage. Getting ourselves to migrate towards a 1.x version of Semantic conventions is another big challenge because we've already sent all of this data out to our backends. We have to figure out a way to upgrade it or to say, "Oh, this is version X." I kind of feel like a lot of the progress was stalled there for the moment. +Also, there's not a lot of support from our current vendors that will be able to leverage that. It's kind of like we would be experimenting with that to go to nowhere. That's kind of, you know, once vendors start to support the OTLP profiling format, data model I should say, and once the profiler is a little bit more stable, I’d love to jump in there and be able to roll that out a little more widely. I think that a little bit of a bumpy road for us still is migrating semantic conventions from pre-1.0 because we were early adopters of semantic conventions. We're at this pre-1.0 stage. Getting ourselves to migrate towards a 1.x version of semantic conventions is another big challenge because we've already sent all of this data out; it's in our backends. We have to figure out a way to upgrade it or to say, "Oh, this is version X." -**Reys:** If that answers your question. +I kind of feel like a lot of the progress was stalled there for the moment, so I don't know that there's any real solutions available that would say, "Okay, we can start migrating these schemas over and keep our user experience very high quality." -**Ariel:** Oh absolutely! I think that is another interesting topic—migrating some semantic conventions, so I might jump back if we have time later. But I do want to get to some of these community questions. +**Reys:** If that answers your question. -**Reys:** Yes, ma'am! +**Ariel:** Oh, absolutely. I think that is another interesting topic, migrating some semantic conventions. I might jump back if we have time later, but I do want to get to some of these community questions. -**Ariel:** Yeah, so one of the big things I think working in the community that you've seen, I've seen, is people want to contribute to the project, but they're not really sure where to get started. We have resources such as, you know, I think in the documentation we added getting started, and you have various SIG channels and CNCF Slack. I think we all have the problem of how do we reach these people because there are still so many. What was the contribution process for you and your team like to OpenTelemetry? +**Reys:** Yes, ma'am. -**Ariel:** For us, it was a short process with a lot of observation. The things that we looked for—myself and my teammates at the time—when we approached the community, we were looking at the code of conduct: what were the expectations of the code of conduct? We went through and actually looked at issues, previous PRs, the feedback that was in those PRs to see if the behaviors expected in the code of conduct were reflected in the PRs and in the issues. +**Ariel:** One of the big things I think, you know, working in the community that probably you've seen, you know, I've seen sure, is people want to contribute to the project, but they're not really sure where to get started. We have resources such as, you know, I think in the documentation we added, you know, getting started, and you have various SIG channels and CNCF Slack. I think we all have the problem of how do we reach these people because there's still so many. I don't know. So what was the contribution process for you and your team like to OpenTelemetry? -There's nothing worse than you wanting to be part of this community and finding that those two things don't match. To see that that happened, then we participated in the SIGs, which was very nice for us to be able and convenient for us in the U.S. because the Ruby SIG was in U.S. hours. We joined those meetings during the day and started to provide some feedback as end users and say, "Hey, look, we ran into this challenge." If we identified something that was a challenge for us, we would contribute a PR. +**Ariel:** For us, it was a short process with a lot of observation. The things that we looked for, myself and my teammates at the time when we approached the community, we were looking at the code of conduct, what were the expectations of the code of conduct. We went through and actually looked at issues, previous PRs, the feedback that was in those PRs to see if the behaviors expected in the code of conduct were reflected in the PRs and in the issues because there's nothing worse than you wanting to be part of this community and finding that those two things don't match. To see that that happened, then we participated in the SIGs, which was very nice for us to be able and convenient for us in the U.S. because the Ruby SIG was in U.S. hours. -The maintainers were very generous with their time, did their reviews, and we were able to get a couple of custom propagators merged. We got some instrumentations merged, and we identified other gaps for the instrumentations that we use because one of the biggest challenges, I think, as a maintainer is there's so much that you have to do. You've got the SDK, you've got the API, you've got documentation to do, and you have to have language-specific instrumentations. That's daunting, especially with all of the popular libraries that are out. +We joined those meetings during the day and started to provide some feedback as end users and say, "Hey, look, we ran into this challenge." If we identified something that was a challenge for us, we could contribute a PR. The maintainers were very generous with their time, did their reviews, and we were able to get a couple of custom propagators merged. We got some instrumentations merged. We identified other gaps for the instrumentations that we use because one of the biggest challenges, I think, as a maintainer is there’s so much that you have to do, right? You've got the SDK, you've got the API, you've got documentation to do, and you have to have language-specific instrumentations. That’s daunting, especially with all of the popular libraries that are out. -For us, we contributed back to the libraries that we use heavily. We have a subset of libraries that are very popular at GitHub and sort of blessed by their popularity. That's where we focused our energy and our contributions. Then we reached out to people. Anytime somebody comes to me and says, "Hey, I have this challenge with this thing. It's missing this attribute, or I'd really like it to work like this, or I've identified this bug," the first thing I do is say, "Let's open up an issue and let's work on this together. I'd love to review your PR." +For us, we contributed back to the libraries that we use heavily. We have a subset of libraries that are very popular at GitHub and sort of blessed by their popularity, so that's where we focused our energy and our contributions. Then we reached out to people. Anytime somebody comes to me and says, "Hey, I have this challenge with this thing. It's missing this attribute," or "I'd really like it to work like this," or "I've identified this bug," the first thing I do is say, "Let's open up an issue and let's work on this together. I'd love to review your PR." -Creating an environment that lowers the barrier to entry for folks to collaborate and submit contributions is important. I want them to feel like this is yours, this is ours—this is not mine, and I'm gatekeeping. We still have some standards for quality, obviously, and some expectations from you as a maintainer, but we try to make this a painless experience for you, at least in the Ruby community. That’s what the contribution process was like for me and getting involved in the community. +Creating an environment that lowers the barrier to entry to have folks collaborate and submit contributions, I want them to feel like this is yours, this is ours; this is not mine and I'm gatekeeping. We still have some standards for quality, obviously, and some expectations from you as a maintainer, but we try to make this a painless experience for you, at least in the Ruby community. That’s what the contribution process was like for me and getting involved in the community. -**Reys:** That's awesome! So you started basically joining the SIG meetings as an end user and sharing feedback and then finding information about how you can open an issue and then building custom stuff and helping the community or sorry, the Ruby SIG build out more of their components. +**Reys:** That's awesome. So you started basically joining the SIG meetings as an end user and sharing feedback and then finding information about how you can open an issue and then from there building custom stuff and helping the community, or sorry, the Ruby SIG build out more of their components. -**Ariel:** Okay, yeah, certainly. And like, you know, with the home for open source, right? For us, we open-sourced, for example, new database drivers in Ruby for MySQL. We had that instrumentation in-house before all of that became public, and as soon as we released that new driver, I went right to the SIG and said, "Ta-da, I have a donation for you!" It's this thing where there's just this spirit of collaboration and supporting open source that’s important to our mission. +**Ariel:** Okay. -**Reys:** Okay, so it sounds like you had a pretty positive experience from contributing to Ruby. Do you have any feedback for the SIG on ways to improve how things work based on your experiences with Ruby and then other SIGs? I know you mentioned also some other components that you've used, but we'll come back to that. +**Reys:** Yeah, certainly. With the home for open source, right, for us, you know, we open-sourced, for example, new database drivers in Ruby for MySQL. We had that instrumentation in-house before all of that became public, and as soon as we released that new driver, I went right to the SIG and said, "Tada! I have a donation for you." It's this thing where there's just this spirit of collaboration, right? This spirit of supporting open source that’s important to our mission. -**Ariel:** Yeah, for sure. Again, I want to thank everybody for their amazing work that they've done contributing to the collector in particular because we release our OCB build, or sorry, our custom build, every time a new version of the collector package rolls out. Every week, Dependabot is telling us, or every two weeks, it's telling us, "Hey, it's time to upgrade; a new release has rolled out." +**Ariel:** Okay, so it sounds like you had a pretty positive experience from contributing to Ruby. Do you have any feedback to the SIG on ways to improve how things work based on your experiences with Ruby and then other SIGs? I know you mentioned also some other components that you've used, but we'll come back to that. -We ran into some challenges where we identified some performance issues in the collector, and we worked with the team, provided profiles, provided feedback, and showed them our configurations, and they were so responsive to our concerns. They were happy to see that we were able to contribute back just in providing feedback, just giving them actual data of production workloads, which is very difficult to do as a maintainer. +**Reys:** Yeah, for sure. I want to thank everybody for their amazing work that they've done contributing to the collector in particular because we release our OCB build, or sorry, our custom build, every time a new version of the container package rolls out. Every week, Dependabot is telling us, or every two weeks, it's telling us, "Hey, it's time to upgrade. A new release has rolled out." We ran into some challenges where we identified some performance issues in the collector, and we worked with the team, provided profiles, provided feedback, showed them our configurations, and they were so responsive to our concerns. They were happy to see that we were able to contribute back just in providing feedback, just giving them actual data of production workloads, which is very difficult to do, right? -It's really hard for me to know, "Hey, what's going on in your customer's deployment or this user's deployment?" We can't replicate it in our own environment. That spirit of collaboration was really great, and that's the thing that I'd like to see happen in all the other SIGs. I have limited experience with other SIGs. +### [00:41:01] Discussion on community contributions -The other part was contributing to semantic conventions, and I contributed to semantic conventions, trying to introduce some new attributes, but there's just so much churn in that repository that it was hard for me to keep up with changes and deprecations. That was a little bit on me to be able to keep up and find out that the stuff I had submitted was deprecated in favor of something else. It's those kinds of challenges that I like—I have to find a better way for us to all be able to keep up with each other. +As a maintainer, it's really hard for me to know, "Hey, what's going on in your customer's deployment or this user's deployment?" We can't replicate it in our own environment, so that spirit of collaboration was really great. That's the thing that I'd like to see happen in all the other SIGs. I have limited experience with other SIGs. The other part was contributing to SemCom, and I contributed to SemCom trying to introduce some new attributes, but there's just so much churn in that repository that it was hard for me to keep up with changes and deprecations, right? -One suggestion I might say is, "Hey, look, a change happened. Here's an OTOP—a change happened in the spec that should automatically open issues for every maintainer repo and say, 'Here's a new thing that has changed. This is a new feature we expect that the SDK should support.'" That way, we're able to track it because right now, the way it works is, "Hey, we have somebody who's a representative at the SIG spec; they collect some data, they come back to us, and they tell us, 'Hey, we need to implement this other thing.'" Sometimes we're not diligent about project management, right? We're ICs who are doing this in our spare time, so we need to improve overall at being able to keep each other informed and tracking work. +That was a little bit on me to be able to keep up and find out that the stuff I had submitted was deprecated, for example, in favor of something else. It's those kinds of challenges that I like, I have to find a better way for us to all be able to keep up with each other. -**Reys:** Got it. So, making sure everyone's on the same page at the same time, which I can see obviously is a bit daunting. +**Reys:** Making sure everyone's on the same page at the same time, which I can see obviously is a bit daunting. -**Ariel:** Yeah. +**Ariel:** Yeah, so I know you mentioned some of the SIGs that you have worked with in the past. What would you say are the things you interact with most right now? -**Reys:** I know you mentioned some of the SIGs that you have worked with in the past. What would you say are the things you interact with most right now? +**Reys:** Right now, I almost exclusively participate in the Ruby SIG. I don't have a lot of interaction with other SIGs at the moment other than through an occasional issue or a discussion that’ll pop up or in Slack. I'll jump into a channel and say, "Hey, I've run into this problem," or "I have this question for y'all. Can you help me?" -**Ariel:** Right now, I almost exclusively participate in the Ruby SIG. I don't have a lot of interaction with other SIGs at the moment, other than through an occasional issue or a discussion that'll pop up, or in Slack, I'll jump into a channel and say, "Hey, I've run into this problem," or "I have this question for y'all. Can you help me?" +I've had limited interactions these days with other SIGs. Just recently, I'm meeting the end user SIG at KubeCon, so I'm going to probably get more involved in the end user SIG now that I have this special invitation from you. -I have limited interactions now these days with other SIGs. Just recently, I'm meeting the end user SIG at KubeCon, so I'm going to probably get more involved in the end user SIG now that I have this special invitation from you. +**Ariel:** Yes, oh, I'm so excited! -**Reys:** Oh, I'm so excited! +This brings us to our "turn the table" section where you get to ask me or anyone on the call questions, and we'll see if I can answer them or if anyone else can answer them. -**Ariel:** This brings us to our "turn the table" section where you get to ask me or anyone on the call questions, and we'll see if I can answer them or if anyone else can answer them. +**Ariel:** Okay, so for me, Rey, it's like that's how I got involved, but maybe you know a special sort of formula or the best way to have someone who's new get involved in particular in the end user SIG to provide feedback or how do they get involved in other SIGs that you've been involved in? -**Ariel:** Okay, so for me, Rey, it's like that's how I got involved, but maybe you know a special sort of formula or the best way to have someone who's new get involved, in particular in the end user SIG, to provide feedback, or how do they get involved in other SIGs that you've been involved in? +**Reys:** Oh for sure! There are a lot of different ways to contribute. I think a lot of people, when they think of contribute, they think it means, "Oh, I got to open PRs; I got to be writing code." That's not necessarily the case. There are so many different ways to provide feedback. You know, like one of the ways you mentioned was you just started out by going to a SIG meeting and sharing some stuff that you're seeing. That is an absolutely great way to contribute to the project, and that's what the end user SIG is all about. -**Reys:** Oh, for sure! So there are a lot of different ways to contribute, right? I think a lot of people, when they think of contribute, they think it means, "Oh, I got to open PRs. I got to be writing code." That's not necessarily the case. There are so many different ways you can provide feedback. One of the ways you mentioned was you just started out by going to a SIG meeting and sharing some stuff that you're seeing. That is an absolutely great way to contribute to the project, and that's what the end user SIG is all about. +One of the things we're all about is gathering feedback to give back to the appropriate SIG so we can help improve things. To that end, we do things like surveys, open Q&A interview sessions where we kind of dive deeper into the adoption implementation process and find out your pain points and specific feedback. You can contribute blogs. Maybe you love writing documentation. That's a great way. -One of the things we're all about is gathering feedback to give back to the appropriate SIG so we can help improve things. To that end, we do things like surveys, open solar Q&A interview sessions where we kind of dive deeper into the adoption implementation process and find out your pain points and specific feedback. You can contribute blogs. If you love writing, documentation is a great way. +Probably if you are not already part of the CNCF Slack instance, I would join. I just realized that I did not include that link, but we will have it in the show notes. Once you are a member of the CNCF Slack instance, you can scan the QR code and join the OTEL SIG end user channel, and we will be happy to direct you. If you have questions about your implementation, or you just kind of want to know, "Hey, where do I get started?" we're happy to help direct you. -If you are not already a part of the CNCF Slack instance, I would join, and I just realized I did not include that link, but we will have it in the show notes. Once you are a member of the CNCF Slack instance, you can scan the QR code and join the OTel Sig end user channel, and we will be happy to direct you if you have questions about your implementation or you just kind of want to know, "Hey, where do I get started?" +We currently have a survey going, which I also just realized I did not share the link. I'm so sorry, I'm going to see if I can get that right now. -We are happy to help direct you. We currently have a survey going, which I also just realized I did not share the link. I'm so sorry; I'm going to see if I can get that right now, Henrik. +Let's see. Yes, join the SIG meetings. Where can they access the calendar? It's through the community repository that has links to the calendar. -So yes, join the SIG meetings where they can access the calendar. It’s through the community repository; it has links to the calendar. +**Ariel:** Perfect. -**Ariel:** Perfect! +**Reys:** If there's like a specific language that you are already working in, check out the calendar to find out when the SIG meets and just hop on. When I first started, I was like, "I'm just going to check out a few different SIGs." I went to the Python SIG, and everyone there was so welcoming and so nice. I was so intimidated at first, and then I was like, "Oh wow, these people are so lovely." That's been my experience with almost, yeah, I mean, not almost but every SIG meeting that I've jumped into is everyone is so open to answering questions, and they want your help; they want to help you help them. -**Reys:** Yes, if there's a specific language that you are already working in, check out the calendar to find out when the SIG meets and just hop on. When I first started, I was like, "I'm just going to check out a few different SIGs." I went to the Python SIG, and everyone there was so welcoming and so nice. I was so intimidated at first, and then I was like, "Oh wow, these people are so lovely!" That's been my experience with almost every SIG meeting that I've jumped into. Everyone is so open to answering questions, and they want your help. They want to help you help them. +I think one of the things is just letting people know that it doesn't have to be contributions. Of course, if that's what you want to do and are able to, we would love that. But there are so many other different ways—feedback, documentation, blogs, doing interviews, surveys. -One of the things is just, you know, letting people know that it doesn't have to be contributions. Of course, if that's what you want to do and are able to, we would love that, but there are so many other different ways—feedback, donation, blogs, doing interviews, and surveys. +Every little bit counts, right? Every little bit helps. If you find yourself interested in trying to contribute back or you don't know where to get started or you feel just intimidated, just know that all of us, we do our best to make the most welcoming communities possible in our SIGs. I love that the end user SIG creates these opportunities for folks to come and just provide feedback—a space for them to say, "Hey, I'm having trouble with this," or "I like this," or "This was a great experience for me." -Every little bit helps, right? Every little bit counts. If you find yourself interested in trying to contribute back or you don't know where to get started or you feel just like intimidated, just know that all of us, we do our best to make the most welcoming communities possible in our SIGs. I love that the end user SIG creates these opportunities for folks to come and just provide feedback, a space for them to say, "Hey, I'm having trouble with this," or "I like this," or "This was a great experience for me." +This part has been very enjoyable for me to be able to share my personal experience and my team’s experience with you. I really hope that folks who are watching at home come on in; the water's fine! Don't worry; we have floaties, we got all kinds of gear to help you get started, right? -This part has been very enjoyable for me to be able to share my personal experience and my team's experience with you. I really hope that folks who are watching at home, hey, come on in; the water's fine! Don't worry! We have floaties, we have all kinds of gear to help you get started, right? +It's just the little things—every little contribution can help us. If your contribution is there to help you, it'll help out the community. -**Ariel:** Exactly! It's just the little things. Every little contribution can help us. If your contribution is there to help you, it'll help out the community. So come if you're having trouble with a specific problem. I guarantee you at least a dozen other people are having the same problem. If you don't understand something, I guarantee you probably at least a hundred other people don't understand it either. +So come if you're having trouble with a specific problem. I guarantee you at least a dozen other people are having the same problem. If you don't understand something, I guarantee you probably at least a hundred other people don't understand it either. -**Reys:** For sure! +**Ariel:** For sure! -**Ariel:** I really appreciate you answering my questions here. +**Reys:** I really appreciate you answering my questions here. -**Reys:** Oh, you are so welcome! I guess I kind of put this at right about time in case anyone has any last-minute questions. We'll hang out here for another minute and just kind of chitchat and give people an opportunity to share if they would like. Otherwise, I will just ask about any fun Christmas or New Year plans. +**Ariel:** Oh, you are so welcome. -**Ariel:** Yeah, we're going to be just spending it with family, so that's going to be really nice. We're going to be traveling, so that might be a little bit hectic. We have to travel; we're going to go visit my father, but we are traveling on Christmas Eve, so that should be fun, right? I'm going to show up there, and we'll be singing Christmas carols at the top of our lungs and waking up the neighbors as soon as we arrive. +I guess I kind of put this at right about time in case anyone has any last-minute questions. We'll hang out here for another minute and just kind of chit-chat and give people an opportunity to share if they would like. Otherwise, I will just ask about any fun Christmas and New Year plans. -**Reys:** Oh my gosh! How about yourself? +**Ariel:** Yeah, we're going to be just spending it with family, so that's going to be really nice. We're going to be traveling, so that might be a little bit hectic. We have to travel—we're going to go visit my father. But we are traveling on Christmas Eve, so that should be fun, right? I'm going to show up there, and we'll be singing Puerto Rican Christmas carols at the top of our lungs and waking up the neighbors as soon as we arrive. -**Ariel:** Whoa, whoa, whoa! What are these Puerto Rican Christmas songs, and where can I listen to them? +**Reys:** Oh my gosh, how about yourself? -**Reys:** Oh, okay! You can look at them on YouTube, on TikTok; they're all over the place. In Puerto Rico, folks and in other Caribbean islands will join and make something called the "parranda" and have a "paranda," which is a party basically. It's like a Katamari ball, so you go from house to house knocking on doors, and then you start singing these songs. +**Ariel:** Whoa, whoa, whoa, whoa, what are these Puerto Rican Christmas songs, and where can I listen to them? -The songs are like, "Hey, open up the door! I'm here to say hello!" "Oh, you turned the lights on; I can see you in there! Let me in!" You have to feed the guests and give them beverages and stuff. What you do is you take those people whose home you invaded, and we call it an "asalto," which is like—we're showing up uninvited. We break into your home and then we bring you and add you to the "parranda," and we move to the next house, and we come and, you know, we'll sing and stuff like that. +**Reys:** Oh, okay! You can look at them on YouTube, on TikTok; they're all over the place. In Puerto Rico, folks and in other Caribbean islands, people will join and make something called a traya and have a paranda, which is a party basically. It's like a Katamari ball, so you go from house to house knocking on doors, and then you start singing these songs. The songs are like, "Hey, open up the door, I'm here to say hello. Oh, you turn the lights on, I can see you in there, let me in." You have to feed the guests and give them beverages and stuff. What you do is you take those people whose home you invaded—that we call an asalto, which is like we show up and we don't want to say the direct translation is more like you've been stu'd, we're sticking you up, but we break into your home and then we bring you and add you to the Katamari ball and we move to the next house. We sing and stuff like that, so it's really great. It's a lot of fun, and so we're looking forward to doing that with our family. -It's really great; it's a lot of fun, and so we're looking forward to doing that with our family! +**Reys:** That sounds so fun! What did you say that was called? -**Ariel:** That sounds so fun! What did you say that was called? +**Ariel:** Uh, the traya, the asalto—those are the songs that we will sing. -**Reys:** The "parranda" and the "asalto." +**Reys:** That sounds lovely, honestly! That sounds fantastic! -**Ariel:** That sounds lovely! Honestly, that sounds fantastic! +**Ariel:** It's a lot of fun; it's a lot of fun. I'm so excited! You have to share pictures! -**Reys:** It's a lot of fun! I'm so excited! You have to share pictures! +**Reys:** I will! -**Ariel:** I will, thank you! Maybe videos too—who knows what kind of trouble we can get ourselves into! +**Ariel:** Maybe videos too, who knows what kind of trouble we can get ourselves into. -**Reys:** Yes! I would love to have a video of you singing one of these songs! +**Reys:** Yes, oh, would love to have a video of you singing one of these songs. -**Ariel:** There are videos of me on the internet singing songs, but not now! +**Ariel:** There are videos of me on the internet singing songs, but not now! -**Reys:** What are you going to be doing this holiday season? +**Reys:** Rey, what are you going to be doing this holiday season? **Ariel:** I am going to be at home busting out hopefully a bunch of house projects that have been on my to-do list for three years. **Reys:** Nothing like it, right? You're going to get to the tail sampling project, right? -**Ariel:** Oh my gosh! I have! But I'm really excited about it because I moved into this house about three years ago, and it's like, you know, 85% there. There are still—I just—don't look in the cabinets. That's a project; I need to organize all the inside stuff. +**Ariel:** Oh my gosh, I have. But I'm really excited about it because, yeah, I moved into this house about three years ago, and it's like, you know, 85% there. There are still just—don't look in the cabinets! That's a project I need to organize all the inside stuff. -**Ariel:** That's the thing I tell people all the time: "Hey, don't look on the inside of the repositories. You may not know; you may not like how the way things are arranged, but you know where things are, so that's totally fine." +**Ariel:** And that's the thing I tell people all the time—it's like, "Hey, don't look on the inside of the repositories! You may not know, you may not like the way things are arranged, but you know where things are, so that's totally fine!" -**Reys:** Well, all right! Thank you so much, Ariel! You were a fantastic guest! I'm really excited to learn more. As I said, I did have some follow-up questions, so I'm sure I'll be slacking you to learn more about some of your adoption processes as well as some of the other components you mentioned you had used, such as the OCB. +**Reys:** Well, all right, thank you so much, Ariel. You were a fantastic guest. I'm really excited to learn more. As I said, I did have some follow-up questions, so I'm sure I'll be slacking you to learn more about some of your adoption processes as well as some of the other components you mentioned you had used, such as the OCB. -But yeah, if anyone wants to reach out to either of us, you can find us in the CNCF Slack's OTel SIG end user channel. Again, this link is up here for you to scan, and I believe we will have the information in the show notes as well, which will be posted, I think, right after this. +If anyone wants to reach out to either of us, you can find us in the CNCF Slack's OTEL SIG end user channel. Again, this link is up here for you to scan, and I believe we will have the information in the show notes as well, which will be posted, I think, right after this. Thank you so much for joining us, and we look forward to seeing you all again next time! -**Ariel:** Yeah, thank you for having me again, Rey! This was really awesome. Thanks everybody for watching, wherever you are and whenever you are! I hope to see y’all in a SIG room sometime. +### [00:55:55] Closing remarks and future plans + +**Ariel:** Thank you for having me, Reys. This was really awesome. Thanks everybody for watching, wherever you are and whenever you are. I hope to see y’all in a SIG room sometime! -**Reys:** Oh yes! And happy everything to everyone! Adios! +**Reys:** Oh yes, and happy everything to everyone! Adios! ## Raw YouTube Transcript diff --git a/video-transcripts/transcripts/2025-01-31T05:58:14Z-cfp-writing-q-a-livestream.md b/video-transcripts/transcripts/2025-01-31T05:58:14Z-cfp-writing-q-a-livestream.md index 07d74c2..b9f3d33 100644 --- a/video-transcripts/transcripts/2025-01-31T05:58:14Z-cfp-writing-q-a-livestream.md +++ b/video-transcripts/transcripts/2025-01-31T05:58:14Z-cfp-writing-q-a-livestream.md @@ -10,224 +10,456 @@ URL: https://www.youtube.com/watch?v=6SmO4yKjmCs ## Summary -In this YouTube video, Dan hosts a panel discussion about effectively writing conference proposals (CFPs) and preparing for speaking engagements, particularly within the context of open telemetry and observability. Panelists include Adriana Vela, Henrik Rexed, Josh, and Ree, who share their insights on choosing topics, overcoming imposter syndrome, and enhancing presentation skills. Key points include the importance of submitting unique and relevant topics, the value of end-user stories in presentations, and strategies for engaging audiences. The panelists also discuss the common experience of rejection when submitting CFPs, emphasizing the need to manage expectations and learn from feedback. They highlight their own conference experiences, including the challenges of live demos and the significance of networking at events. The video encourages viewers to participate in the open telemetry community and share their knowledge through speaking opportunities. +In this YouTube video, Dan hosts a panel discussion featuring experts in the technology conference arena, including Adriana Vela, Henrik Rexed, Josh, and Ree, who share insights about submitting proposals for speaking at conferences, known as CFPs (Call for Proposals). The panelists discuss various topics such as selecting the right conferences, crafting unique and engaging submission titles, and overcoming imposter syndrome when applying to speak. They emphasize the importance of sharing personal user experiences, especially in the observability and open telemetry fields, and provide tips for preparing for talks, including rehearsal techniques and the value of feedback. Throughout the conversation, they highlight the normalcy of rejection in the conference speaking circuit and encourage aspiring speakers to persist and refine their proposals. The panelists also share memorable anecdotes from their speaking experiences, including technical difficulties and the importance of connecting with audiences. Overall, the video serves as a resource for those looking to enhance their conference speaking skills and navigate the CFP process effectively. ## Chapters -00:00:00 Welcome and introduction -00:02:20 Meet the panelists -00:04:30 What is a CFP? -00:06:50 Finding conferences to speak at -00:09:17 Motivation for speaking -00:12:51 Overcoming imposter syndrome -00:15:12 Importance of unique topics -00:18:10 End user stories value -00:22:40 Choosing topics for CFP -00:29:36 Hot topics in observability +00:00:00 Introductions +00:01:16 What is a CFP? +00:05:30 Finding conferences to speak at +00:09:40 Reasons to speak at conferences +00:12:44 Overcoming imposter syndrome +00:17:24 Importance of user stories +00:22:04 Choosing unique topics +00:30:56 Hot topics in observability +00:39:38 Preparing for your talk +00:44:28 Handling rejection and feedback -**Dan:** Hello everybody, good morning, good afternoon, good evening. I'm Dan, and as part of the Open Telemetry and End User SIG, today I have the absolute pleasure of hosting this panel on end users. We normally focus on Open Telemetry — I mean, it's in the name. We host end users telling us how they're adopting observability best practices across the industry. If you want to know more, you can follow the QR code on the screen and you'll find how to get in touch. +## Transcript -However, this time we're going to be getting some tips and tricks from true veterans in the tech conference arena. We'll be talking about how to write good CFPs, what topics to choose, how to prepare for a talk, or even why going through the whole process of applying to speak at a conference is well worth it. So if you have recently applied to speak at KubeCon or Observability Day in London or another conference, it doesn't matter if you made it or not; I think these experts will have some advice that you can take home and use in your future talks. +**Dan:** Hello everybody, good morning, good afternoon, good evening. I'm Dan, and as part of the OpenTelemetry end user SIG today, I have the absolute pleasure of hosting this panel. We normally focus on OpenTelemetry, I mean, it's in the name. We host end users telling us how they're adopting observability best practices across the industry. If you want to know more, you can follow the QR code on the screen and you'll find how to get in touch. -[00:02:20] We want this to be an interactive panel, so if you're watching live on YouTube or LinkedIn and you have any questions related to the topics that we're discussing, please drop them in the chat, and I will try to incorporate them into the discussion as best I can. Okay, so it's time to meet our panelists. If you're watching live on YouTube or LinkedIn, I would love to know where you're watching from, so drop a comment in the chat and tell us where you're watching from. Myself, I'm in Edinburgh, Scotland, where it is currently 5:02 PM. +### [00:01:16] What is a CFP? -Let's meet our first panelist, Adriana. Hello! Can you tell us where you're connecting from and a little bit about yourself? +However, this time we're going to be getting some tips and tricks from true veterans in the tech conference arena. We'll be talking about how to write good CFPs, what topics to choose, how to prepare for a talk, or even why going through the whole process of applying to speak at a conference is well worth it. So if you have recently applied, maybe to speak at KubeCon or Observability Day in London or another conference, it doesn't matter if you made it or not. I think these experts will have some advice that you can take home and use in your future talks. -**Adriana:** Hey Dan, nice to see you! My name is Adriana Vela. I am connecting from Toronto, Canada. It is noon here, and I work alongside Dan as one of the maintainers of the Open Telemetry End User SIG. Thank you very much! +We want this to be an interactive panel, so if you're watching live on YouTube or LinkedIn and you have any questions related to the topics that we're discussing, please drop them in the chat, and I will try to incorporate them into the discussion as best I can. -**Henrik:** Hello, Henrik here. Hey, pleasure to be here. My name is Henrik Rexed, as it looks like it's written at the bottom of my video. I'm based in the south of France, the beautiful sunny south of France, even if it's still 10-15 degrees at the moment Celsius. It's 6:03 PM local time in France. Otherwise, I'm trying to be involved in a lot of observability topics, so Tag Observ, and I try to give a hand as well in the End User SIG. Thank you very much. +Okay, so it's time to meet our panelists. If you're watching live on YouTube or LinkedIn, I would love to know where you're watching from, so drop a comment in the chat and tell us where you're watching from. Myself, I'm in Edinburgh, Scotland, where it is currently 5:02 PM. -**Josh:** Hello, yes, thanks for having me! I'm Josh. I've been a developer advocate and product manager in all kinds of things around Open Telemetry, and I'm connecting from Brussels, where I'm here for FUM, starting on Saturday. Looking forward to that! I like the European representation here. +Let's meet our first panelist, Adriana. Um, hello! Can you tell us where you're connecting from and a little bit about yourself? -**Ree:** Hi everyone, I'm so excited to be here! I am joining from Vancouver, Washington, not to be confused with Vancouver, BC. It's like 10 minutes north of Portland. I work in developer relations at New Relic, and I also work with these lovely folks in the Open Telemetry community, primarily as part of the End User SIG. +**Adriana:** Hey Dan, nice to see you! My name is Adriana Vela. I am connecting from Toronto, Canada. It is noon here, and I work alongside Dan as one of the maintainers of the OpenTelemetry end user SIG. Thank you very much. -[00:04:30] **Dan:** Thanks to everyone for being here! So let's start with the first question from me, which is going to be directed to Adriana. What is a CFP in the first place? Can you tell us more about it? +**Henrik:** Hello, Henrik here! Hey, pleasure to be here. My name is Henrik Rexed, as it looks like it's written at the bottom of my video. I'm based in the south of France—the beautiful sunny south of France—even if it's still 10-15 degrees at the moment Celsius. It’s 6:03 PM local time in France. Otherwise, I’m trying to be involved in a lot of observability topics, so I'm involved in observability and I also try to give a hand in the end user SIG as well. Thank you very much. -**Adriana:** Okay, CFP, if I got this right, stands for Call for Proposals or Call for Papers, depending on the context. It's basically a request for a proposal for a talk in the context of conferences, where you basically have an idea for a talk, and you give the details for said talk. Depending on what conference you're applying to, the details will vary from conference to conference. For example, if anyone's ever applied to SREcon, it is a song and a dance to apply to SREcon because I feel like they really want you to hash out all of the details of your proposal ahead of time compared to, say, KubeCon, where you can be a little bit more high-level. So yeah, that is CFP in a nutshell. I don't know if anyone else wants to chime in. +**Josh:** Hello, yes! Thanks for having me. I'm Josh. I've been a developer advocate and product manager in all kinds of things around OpenTelemetry, and I'm connecting from Brussels where I’m here for FUM, starting on Saturday. Looking forward to that. I like the European representation here. -**Henrik:** For me, a CFP is the opportunity to be in a conference to meet people and to network, because at the end, in my role, to be able to reach out to that conference is a great excuse to talk. For me, a CFP is like, "Oh, maybe I will have some miles with my air because I'm an Air France member if I'm accepted." That would be great! +**Ree:** Hi everyone! I'm so excited to be here! I am joining from Vancouver, Washington—not to be confused with Vancouver, BC—right? It's like 10 minutes north of Portland. I work in developer relations at New Relic, and I also work with these lovely folks in the OpenTelemetry community, primarily as part of the end user SIG. -**Dan:** Okay, so we'll move to the next one. If you're new to conferences, you're probably thinking, "Okay, so how do I find conferences to talk at?" Josh, can you tell us a bit more about how you go about finding what conferences you want to apply to speak at? +**Dan:** Thanks to everyone for being here! Right, so let's start with the first topic—the first question from me, which is going to be directed to Adriana. What is a CFP in the first place? Can you tell us more about it, starting with the tough questions? -**Josh:** Absolutely! A really good way is to use the aggregator sites that a lot of the conferences will post their CFPS on. I think the two big ones would be Session Eyes and Paper Call. And then what's the third one? Anyone can help me out with the name of that third one? +**Adriana:** Okay, a CFP, if I got this right, stands for Call for Proposals or Call for Papers, depending on the context. It’s basically a request for a proposal for a talk in the context of conferences, where you basically have an idea for a talk and you give the details for said talk. Depending on what conference you're applying to, the details will vary from conference to conference. -**Adriana:** Pre-Talks! +### [00:05:30] Finding conferences to speak at -[00:06:50] **Josh:** Pre-Talks is the third one! You can kind of check out those or Skedge, right? Those are not necessarily going to be specific to the area that you want to speak at. Of course, for the CNCF, you can look at all the CNCF events; that's going to be relevant for open topics. There are also a lot of aggregators. I think we can maybe add some of those to this resources list after this is done. There are some GitHub repos and some Airtable databases that are maintained fairly well that just sort of keep an up-to-date list of all of the ongoing CFPS sorted by their due dates. +For example, if anyone's ever applied to SREcon, it is a song and a dance to apply to SREcon because I feel like they really want you to hash out all of the details of your proposal ahead of time compared to, say, KubeCon, where you can be a little bit more high level. So yeah, that is CFP in a nutshell. I don't know if anyone else wants to chime in. -**Dan:** And then from those, do you normally choose by the topic of the conference? +**Henrik:** For me, a CFP is the opportunity to be in a conference, to meet people, to network. Because at the end, in my role, to be able to reach out to that conference is a great excuse to talk. For me, a CFP is like, "Oh, maybe I will have some miles with my air because I'm an Air France member if I'm accepted." That would be great. -**Josh:** Sometimes it's... I think the topic helps narrow down the initial list, and then from there... Oh, maybe Henrik has something to say on this. +**Dan:** Okay, so we'll move to the next one. I think, you know, if you're new to conferences, you're probably thinking, "Okay, so how do I find conferences to talk at?" Josh, can you tell us a bit more about how you go about finding what conferences you want to apply to speak at? + +**Josh:** Absolutely! A really good way is to use the aggregator sites that a lot of the conferences will post their CFPs on. I think the two big ones would be SessionEyes and PaperCall. And then, what's the third one? Anyone can help me out with the name of that third one? Oh, Pre-Talks! Pre-Talks is the third one. + +So yeah, you can check out those or Skedge. Those are not necessarily going to be specific to the area that you want to speak at. Of course, for the CNCF, you can look at all the CNCF events; that's going to be relevant for open topics. There are also a lot of aggregators. I think we can maybe add some of those to this resources list after this is done. + +But yeah, there are some GitHub repos and some Airtable databases that are maintained fairly well that just sort of keep an up-to-date list of all of the ongoing CFPs sorted by their due dates. + +**Dan:** Nice! And then from those, do you normally choose by the topic of the conference, or do you just— I mean, I rather like have everyone talk about OpenTelemetry all the time everywhere in every conference. Is that something that you do normally, or how do you choose? + +**Josh:** Sometimes, I think the topic helps narrow down the initial list, and then from there—oh, maybe Henrik has something to say on this. **Henrik:** No, no, I'm changing the display. -**Josh:** I forget what I was saying about that. Oh yeah! The topic narrows down the list, but especially if your topic is as broad as DevOps or even observability, that's not going to narrow it down that much. There are still a lot of conferences, so then you have to pick: where am I okay to travel? Do I like to travel far, like Henrik and rack up those miles, or do I want to maybe get my start somewhere that I can drive to and I don't have to get a hotel? Right? And how big will the conference be? I think there's a really big difference between speaking at a single-track conference as a first-time speaker versus speaking at a multi-track conference. I think we'll maybe get into that in some of the other questions, but yeah, I think just the topic alone is not enough to narrow it down. +**Josh:** I forget what I was saying about that. Oh yeah, so the topic, I think, narrows down the list. But especially if your topic is as broad as DevOps or even observability, that's not going to narrow it down that much. There are still a lot of conferences. So then you have to pick, like, "Where am I okay to travel? Do I like to travel far like Henrik and rack up those miles, or do I want to maybe get my start somewhere that I can drive to and I don't have to get a hotel?" + +And how big will the conference be? I think there's a really big difference between speaking at a single track conference as a first-time speaker versus speaking at a multi-track conference. I think we'll maybe get into that in some of the other questions, but yeah, I think just the topic alone is not enough to narrow it down. + +**Dan:** Nice, that's cool. Alright, so we've found the conference and we've got some resources there that people can go and check out to find where to speak. I guess a question for Ree: what makes you want to speak? Why do you apply to speak at conferences? + +**Ree:** One thing that I kind of found out early on was because my job is so busy, it can sometimes be hard to find time to learn something new or something maybe more niche. So if I can submit a topic and it gets accepted on something that I want to learn more about, then that automatically gives me the time and bandwidth to work on that topic and learn about it. + +### [00:09:40] Reasons to speak at conferences + +That's one reason, and now that I've been doing it a while, just the benefits from the physical interactions of being able to be in the same space as people who want to learn about these things and having conversations about it. One, I'm able to learn more, and then also there are a lot of other people that also want to learn the same thing that I wanted to learn. It just becomes like this really cool collective, I guess, and you get to meet a lot of cool people that way too. + +And you know, not just for professional networking, but also some have become personal friends. It’s just really great—a great community exposure. + +**Adriana:** I was going to say, sometimes, you know, for me personally, speaking is almost like a challenge. I love speaking in front of audiences, but I also find it a little bit terrifying, so this kind of forces me to get past that discomfort. + +But another thing that I wanted to point out on why to speak, especially for those of us who are in underrepresented groups in tech, I think the more of us in underrepresented groups that go out and do talks, the more we can show those folks in underrepresented groups that we exist and we can empower them to go out and speak as well. I think that's so, so important. + +**Henrik:** Just to add as well, I think all of us, we all work in technical environments, and sometimes we think that what we are working on is quite normal and nothing special, but there's always some things to learn. If you like the community and you want to share your work, you want to share your experience, you share your journey from being a complete beginner to a complete expert, and the advice that you can share, I think it's the best opportunity because sharing a talk is a lot of sharing knowledge and education to the community. I think it's an amazing opportunity for all. + +**Josh:** And as well, like for someone that's not like a DevRel, for example, you know, what motivates you? + +**Henrik:** Well, no, so I am a time devil, so I get the same excuse as Ree, right? Like if I get a talk accepted, that's brownie points for my boss, and it justifies my existence a little bit. But for people who don’t have that motivation, right, there's another thing that DevRels actually do that's really, really important, right, which is bring feedback back from the community. + +So that's another thing if you're an engineer on an engineering team—like it can get a little bit insular in our bubbles—and so going and talking at a conference is a chance to like kind of kick the tires on some ideas with some people that are new to you and gather that feedback from the community as well. + +**Dan:** I think that's it as well. Sometimes you forget that what you're doing might be something that is ahead of the curve, and you want to give back. + +### [00:12:44] Overcoming imposter syndrome + +That's great! Okay, moving on, let's say that you know basically I'm convinced that I want to talk, but sometimes I think a lot of us suffer from imposter syndrome and you think that you're not good enough to talk about a certain topic. So I'll go back to you, Adriana. How do you get past that imposter syndrome and that sense of like, "I don't know enough to talk about a topic"? + +**Adriana:** I feel like you just have to force yourself past it—like just force yourself to do it anyway. Because you know what? If you don't do it, someone else is going to do it, so why not you, right? + +And I think a lot of times—I think this is advice that someone gave me early on when I started applying for talks, which is like you don't have to be the expert. I think Ree touched on this too—you don't necessarily have to be the expert; you can use this as an opportunity to force yourself to learn something about a topic. + +The other thing is, like, "Ah, I have to be the expert to be able to speak intelligently about it," but there's something to be said for having kind of a newbie's point of view as well, because we are so much more relatable that way when we talk about our experiences as newbies. + +There are so many people who are like, you know, new to things, and to show people that you're human and you're not like some perfect being that's up on this pedestal, I think it makes it super relatable and more fun too for people to learn. + +**Henrik:** Adriana, maybe I don't know about you, but I have not gotten to the point where I don't have a little bit of imposter syndrome or stage fright, right? Like it just never goes away completely. + +**Adriana:** I think I'm with you. + +**Henrik:** Yeah, you really do just have to push through it because it's always there. But I think it's the talk, like you mentioned before; it's the opportunity to learn and to be more expert on that side. But even if you're not the maintainer or the main contributor of the project or whatever topics you decided to present, I think if you have a way of presenting that is super interesting and brings a lot of entertainment, people will probably listen more to you than to someone who is very boring on stage. + +You don't need to be the full expert; it's just sharing a passion in a very funny way or in your way, in fact, and that brings value for the audience. + +**Adriana:** That’s such a good point! I've seen talks where, you know, subject matter experts who were really, really knowledgeable about the subject but they weren't necessarily presenting it in the most absorbable way. -**Dan:** Nice! That's cool. All right, so we found the conference and we've got some resources there that people can check out to find where to speak. I guess, you know, a question for Ree: what makes you want to speak? Why do you apply to speak at conferences? +And I've seen people who were newer to the subject, you know, do really creative things that—for me, I was able to be like, “Oh my gosh, yes, this makes complete sense.” So I think that is a really good point. You can bring your own perspective into things, and the way you share your knowledge will, I guarantee you, resonate with somebody in the audience. -[00:09:17] **Ree:** One thing that I kind of found out early on was, because my job is so busy, it sometimes can be hard to find time to learn something new or, you know, something maybe more niche. So if I can submit a topic and it gets accepted on something that I want to learn more about, then that automatically gives me the time and bandwidth to work on that topic and learn about it. So that's one reason. And now that I've been doing it a while, just the benefits from the physical interactions of being able to be in the same space as people who want to learn about these things and having conversations about it — I'm able to learn more. There are a lot of other people that also want to learn the same thing that I wanted to learn, and so it just becomes like this really cool collective, I guess. You get to meet a lot of cool people that way too, and not just for professional networking, but also some have become personal friends. It's just really great; it's a great community exposure. +**Ree:** Yeah, absolutely! I will also say that this is why Ree and I have cats on our slides. -**Adriana:** I was going to say, sometimes, you know, for me personally, speaking is almost like a challenge. I love speaking in front of audiences, but I also find it a little bit terrifying. This kind of forces me to get past that discomfort. Another thing I wanted to point out, especially for those of us who are in an underrepresented group in tech, I think the more of us in underrepresented groups that go out and do talks, the more we can show those folks in underrepresented groups that we exist, and we can empower them to go out and speak as well. I think that's so, so important. +**Josh:** You mean that you bought cats just for conferences? -**Henrik:** Just to add as well, I think all of us, we all work in technical environments, and sometimes we think that what we are working on is quite normal and nothing special, but there's always something to learn. So if you like the community and you want to share your work, you want to share your experience, you share your journey from being a complete beginner to being a complete expert, and the advice that you can share, I think it's the best opportunity. Sharing a talk is a lot of sharing knowledge and education with the community. I think it's an amazing opportunity for all. +**Ree:** Oh, so like Ree's cat Taco is like the most photogenic cat ever, and so we just like feature her on a bunch of our slides whenever we do talks together. She's a pretty short hair, she's ridiculous! -**Josh:** Yeah, and as well, like for someone that's not a dev advocate, for example, you know, what motivates you? +**Dan:** I think we talked about—go ahead. -**Henrik:** Well, I'm a dev, so I get the same excuse as Ree, right? If I get a talk accepted, that's brownie points for my boss, and it justifies my existence a little bit. But for people who don't have that motivation, there's another thing that devs actually do that’s really, really important, right? Which is bring feedback back from the community. So that's another thing. If you're an engineer on an engineering team, it can get a little bit insular in our bubbles. Going and talking at a conference is a chance to kick the tires on some ideas with some people that are new to you and gather that feedback from the community as well. +**Adriana:** No, go ahead! -**Dan:** I think that's it as well. Sometimes you forget that what you're doing might be something that is ahead of the curve, and you want to give back. That's great. +**Dan:** I was just going to add, if the conference organizers chose your talk, like, right? Don't be afraid to submit, and then if you get chosen, you got chosen! They want you to talk about it more than anyone else. -[00:12:51] Moving on, they say that, you know, now basically I'm convinced that I want to talk, but sometimes, you know, I think a lot of us suffer from imposter syndrome, and you think that you're not good enough to talk about a certain topic. So I'll go back to you, Adriana. How do you get past that imposter syndrome and that sense of, "I don't know enough to talk about a topic?" +**Henrik:** That is so true! I think we talked a bit about end users and, you know, I think in OpenTelemetry and observability, for example, you may think as an end user, or you know, people will want to know about the latest thing that maintainers have been working on in OpenTelemetry or the latest thing that this particular vendor is delivering. -**Adriana:** I feel like you just have to force yourself past it — just force yourself to do it anyway. Because you know what, if you don't do it, someone else is going to do it. So why not you, right? A lot of times, I think this is advice that someone gave me early on when I started applying for talks, which is like you don't have to be the expert. I think Ree touched on this too: you don't necessarily have to be the expert; you can use this as an opportunity to force yourself to learn something about a topic. Or the other thing is, like, I have to be the expert to be able to speak intelligently about it, but there's something to be said for having kind of a newbie's point of view as well. We're so much more relatable that way when we talk about our experiences as newbies because there are so many people who are new to things. To show people that you're human and you're not some perfect being up on this pedestal, I think it makes it super relatable and more fun for people to learn. +But I think there's a lot of value from users telling their stories as well. Do you agree with that, Henrik, for example? I think you talked about that previously. -**Henrik:** I think it's important to note that I have not gotten to the point where I don't have a little bit of impostor syndrome or stage fright. It just never goes away completely; I think I'm with you. +### [00:17:24] Importance of user stories -**Adriana:** You really do just have to push through it because it's always there. But I think it's the talk, like you mentioned before; it's the opportunity to learn and to be more of an expert on that side. But even if you're not the maintainer or the main contributor of the project or whatever topic you decided to present, I think if you have a way of presenting that is super interesting, brings a lot of entertainment, and people will probably listen more to you. If you are someone who is very boring on stage, you don't need to be the full expert; it's just sharing a passion in a very funny way or in your way, in fact, and that brings value for the audience. +**Henrik:** I think having vendors presenting will always bring their angle, and I think having a user sharing their story has a different angle and a journey or an experience that is always super interesting. It's like a book where you have an adventure, and you just follow the adventure. -[00:15:12] **Josh:** That's such a good point! I've seen talks where you put subject matter experts who are really, really knowledgeable about the subject, but they weren't necessarily presenting it in the most absorbable way. I've seen people who were newer to the subject do really creative things that, for me, I was able to say, "Oh my gosh, yes, this makes complete sense!" I think that is a really good point. You can bring your own perspective into things, and the way you share your knowledge will resonate with somebody in the audience. +So I think it brings lots of value to the audience. Sometimes even more value to have someone like me, like a DevRel, going on stage because I will not have necessarily this experience and this journey. I will bring the same topic from a different angle. -**Ree:** I will also say that this is why Ree and I have cats on our slides! +So I think a user has a real experience story; I think it's the best one from my perspective. -**Adriana:** You mean that you bought cats just for conferences, or you had cats in the beginning? +**Josh:** Absolutely! So we recently organized the Open Source Analytics Conference at my company, and we participated in organizing it. At the beginning of the process, right, choosing the talks, we specifically separated out all the end-user talks because we didn't have enough of them. We wanted to make sure that they were well represented in the schedule. -**Ree:** Oh, so like Ree's cat Taco is like the most photogenic cat ever, and so we just feature her on a bunch of our slides whenever we do talks together. She's a pretty short hair; she's ridiculous. Real animal! +So we definitely gave preferential treatment to those, and the reason for that is everything that Henrik said, right? Like other people want to get these stories that they can follow along with and relate to, that don't have that vendor spin necessarily. -**Dan:** I think we talked about end users and how, you know, in Open Telemetry and observability, for example, you may think as an end user, or you know, people will want to know about the latest thing that maintainers have been working on in Open Telemetry or the latest thing that this particular vendor is delivering. But I think there's a lot of value from users telling their stories as well. Do you agree with that, Henrik, for example? I think you talked about that previously. +**Dan:** I think we got that basically from one of our audience saying that, yeah, end users just give a different weight to what they're saying, right? Because they are using it in production; they're using it in their systems. -**Henrik:** Yeah, I think having vendors presenting will always bring their angle, and I think having a user share their story offers a different angle and journey or an experience that is always super interesting. It's like a book where you have an adventure, and you just follow the adventure. I think it brings a lot of value to the audience. Sometimes, even more value to have someone like me, like a dev, going on stage because I won't necessarily have this experience and this journey. I will bring the same topic from a different angle, so I think a user has a real experience story. I think it's one of the best ones from my perspective. +So yeah, definitely! Okay, I think we can move on to another topic, which is the topic selection. So we've got an idea that we want to speak at a conference, and then how do you actually decide what topics to go through? I guess, you know, what topics to put in that CFP? And I will ask that to Ree. -[00:18:10] **Josh:** Absolutely! We recently organized the Open Source Analytics Conference at my company, where we participated in organizing it. At the beginning of the process, right, choosing the talks, we specifically separated out all the end-user talks because we didn't have enough of them, and we wanted to make sure that they were well represented in the schedule. We definitely gave preferential treatment to those. The reason for that is for everything that Henrik said, right? Other people want to get these stories that they can follow along with and relate to that don't have that vendor spin necessarily. +**Ree:** This is an interesting question because when I first submitted my very first CFP to KubeCon in—was it for KubeCon EU 2022? I think OpenTelemetry was still pretty new as a topic at KubeCon at that time. -**Dan:** I think we got that basically from one of our audience, saying that end users just give a different weight to what they're saying because they are using it in production; they're using it in their systems. +For like the first couple years, I was having pretty good success getting my topics selected, and I've found, you know, in the last year, it's been getting harder. One, because there are more people talking about it, but also because there are more and more things being covered. -Okay, I think we can move on to another topic, which is the topic topic! So we've got, you know, we've got an idea that we want to speak at a conference, and then how do you actually decide what topics to go through? I guess, you know, what topics to put in that CFP? I will ask that to Ree. +So I think it's been interesting to kind of figure out like, "Oh, now it's time to get really niche." I mean, there's still a place for intro-level talks, but I'm kind of in that space where I'm trying to figure out, you know, what are interesting aspects and topics related to OpenTelemetry and observability in general to submit for. -**Ree:** This is an interesting question because when I first submitted my very first CFP to KubeCon in 2022, I think Open Telemetry was still pretty new as a topic at KubeCon at that time. For the first couple of years, I was having pretty good success getting my topics selected. I've found that in the last year, it's been getting harder, one because there are more people talking about it, but also there are more and more things being covered. I think it's been interesting to figure out now it's time to get really kind of niche. I mean, there's still a place for intro-level talks, but I'm kind of in that space where I'm trying to figure out what are interesting aspects and topics related to Open Telemetry and observability in general to submit for. +**Adriana:** I did, yeah! I was going to say on that same vein because, yeah, like Ree said, it's getting harder and harder, especially in the observability space. -**Adriana:** I did, yeah! I was going to say on that same vein because, yeah, like Ree said, it's getting harder and harder especially in the observability space. It's interesting too because I think some of the folks on this panel have reviewed CFPS as well for KubeCon. I've done a number of CFP reviews for KubeCon, and it's interesting to see what topics come up over and over and over. I will tell you AI comes up a lot. It's almost to the point where you're like, "Oh my God, not another freaking AI CFP for the love of God!" They're not unique because here's the deal: there's nothing wrong with submitting a CFP on AI, but make sure it's unique at this point because a lot of the stuff that's out there is different permutations of the same thing. It's really about what's your unique take on it. Also, we were talking about end-user stories; I think those are still extremely useful and relevant, but again, after a while, you start seeing a lot of the same end-user stories. So again, when you're submitting an end-user story, what is special? What makes you a snowflake when you're telling your end-user story? +And then it's interesting too because I think some of the folks on this panel have reviewed CFPs as well for KubeCon. I've done a number of CFP reviews for KubeCon, and it's interesting to see what topics come up over and over and over. I will tell y'all, AI comes up a lot! And it's almost to the point where you're like, "Oh my God, not another freaking AI CFP, for the love of God!" -The other thing I would say is some folks try to submit project updates as part of a CFP for KubeCon or whatever, and it's like, dude, a project update does not automatically get you accepted! Save that for the relevant venue, because I think there's special project update sessions, for example. Those are a couple of my pet peeves. I don't know if anyone else wants to chime in. +And they're not unique because, like, here's the deal, right? There's nothing wrong with submitting a CFP on AI, but make sure it's freaking unique at this point because a lot of the stuff that's out there is like different permutations of the same thing. -**Josh:** I agree with that, right? Like, how do you be trendy without being too trendy? +And so it's really about like, "What's your unique take on it?" And also, like, you know, we were talking about end-user stories. I think those are still extremely useful and relevant, but again, after a while, you start seeing a lot of the same end-user stories. -[00:22:40] **Adriana:** I see in the chat there's a thing about people not understanding the basics of OpenTelemetry. When you were speaking about trying to find your niche, do other people call it OTel or is it just me? +### [00:22:04] Choosing unique topics -**Josh:** I love it! OTel! +So again, when you're submitting an end-user story, what is special? What makes you a snowflake when telling your end-user story? The other thing I would say is like some folks try to submit project updates as part of, like, you know, a CFP for KubeCon or whatever, and it's like, "Dude, a project update does not automatically get you accepted." -**Adriana:** Yes! +Like, save that for the relevant venue because I think there are special project update sessions, for example. So those are like a couple of my pet peeves. I don't know if anyone else wants to chime in. -**Josh:** Anyways, maybe at KubeCon, right? That's true, you need to have something unique and novel to bring to a KubeCon conference. But outside of our bubble, right? Open Telemetry is still a very niche topic. I've had my introduction to Open Telemetry, right? The smaller regional conferences are clamoring for those introductory topics. +**Josh:** I agree with that, right? Like, how do you be trendy without being too trendy? I see in the chat there's a thing about how people don't understand the basics of OpenTelemetry. -**Ree:** Introductory talks on these niche topics that are our niche, right? +And Ree, when you were speaking about trying to find your niche, do other people call it "Ollie" or is it just me? -**Josh:** Yeah, and in my case, I usually try to avoid those introduction talks because I think many people will do the same, so it's just a matter of being different or bringing a different angle. When the people review your talk, then you get an interesting angle that brings value to the community, in my perspective. +**Adriana:** I love it! Ollie, yes! -**Dan:** I do talks that are very expensive in terms of preparation because I love benchmarking. I think I like to have those studies where you show numbers, you show things. It's like an experience, and then you show that to the stage because I think it brings another value because sometimes you don't show that in the normal track. Having that difference, you probably have more chance to be selected. +**Josh:** Anyways, maybe at KubeCon, right? That's true, like you've got to have something unique and novel to bring to a KubeCon conference. -**Dan:** I guess, you know, related to this, I think the introductory topics and the more in-depth topics sometimes, you know, when you're applying to speak at a conference, you don't know what level of detail you want to go through either in the CFP or in the talk itself. Is it good to—how do you know your audience, and how do you know what level of depth you want to apply? Ree, how do you normally go about it? +But outside of like our—again, getting outside of our bubble, right? OpenTelemetry is still a very niche topic, and I've had my introduction to OpenTelemetry, right? Like the smaller regional conferences are clamoring for those introductory topics, introductory talks on these niche topics that are niche, right? -**Ree:** I still find that tough sometimes. Sometimes I'll start with like, "Okay, I kind of want to be more high-level about this topic," but in order for someone to come to this topic, they might need, you know, XYZ knowledge. So even though it's more high-level on that specific thing, it might be considered like an intermediate, if that makes sense. It really depends. I'll also try to consider how much, you know, am I talking about something newer, you know, that was more newly developed, so there's less info about it, or is it something that's been around for a while? I don't have a really good answer; I still find it tough honestly. I try to stick to either beginner or intermediate level in general because of the topics that I typically choose to do. +**Henrik:** In my case, I usually try to avoid those introduction talks because I think many people will do the same. So it's just a matter of being different or bringing a different angle. -**Josh:** The thing also is, it's a personal judgment as well. Sometimes you say, "Oh, I think it's beginner," or "I think it's..." I have no idea; it's complicated. +Then when the people will review your talk, they will probably find it interesting. That will bring value to the community, in my perspective. I do talks that are very extensive in terms of preparation because I love benchmarks. -**Ree:** What may seem basic to you might be very advanced to others or the opposite, right? +I think I like to have those studies where you show numbers, you show things—it's like an experience. Then you show that to the stage because I think it brings another value because sometimes you don't show that in a normal track. -**Dan:** Okay, Henrik, I'm going to go back to you because I know that you're everywhere in Open Telemetry as well. I know that there are a lot of hot topics in Open Telemetry. What are the hot topics that you recommend people be writing about right now? I know that it doesn't have to be Open Telemetry; I know that we all love OTel, but what are the sort of hot topics that you would recommend thinking about writing about or talking about? +And I think by having that difference, you probably have more chance to be selected. -**Henrik:** What I would suggest is that, by the way, AI is not valid; it is! In general, you have to look at previous conferences and what has been presented and covered so far. If you start to say, "I want to do Open Telemetry on instrumentation," there's plenty of them out there; there's plenty of video out there, and then there's no hot topic in general. Usually, I try to bring the problem statement. For example, how do I sample properly? How do I optimize? Because there is a big concern about observability being expensive, so how can I control that cost? Going through those directions, I think also there is another trend that I think is really important that people need to be educated a bit more on: sustainability. How can we make it green, make it better? Talking about AI is not going to be green because you're going to consume more resources. So maybe having another approach where you say, "We need to be good citizens of the world and save energy," I think that's a pretty good one. Also, I think there are new projects coming in. For example, today, as we speak, Open Telemetry profiling — there are a few talks out there in KubeCon, and it's going to be more and more popular. So there's a big chance that profiling will bring new problems and challenges, so covering them could also be interesting. Try to follow what happens in the industry and think about, "If I start digging this, I may have problems, so how can I resolve them and maybe share that solution or this approach with a larger group in the community?" +**Dan:** Nice! I guess, you know, related to this, I think the introductory topics and the more in-depth topics sometimes, you know, when you're applying to speak at a conference, you don't know what level of detail you want to go through either in the CFP or in the talk itself. -**Adriana:** I just add, Henrik, you're really good at this, right? You mentioned sampling, and I saw that video in my feed, and I was like, "Oh, that's a great topic! I'm going to check that out later when I have time." +Is it good to—how do you know your audience, and how do you know what level of depth you want to apply? Ree, how do you normally go about it? -**Dan:** I think we have some hot topics here. Any other takes on what some of the hot topics are? +**Ree:** I still find that tough sometimes. Sometimes I'll start with, like, "Okay, I kind of want to be more high level about this topic," but in order for someone to have to come to this topic, they might need, you know, XYZ knowledge. -**Ree:** I would like to add something. You know, I think piggybacking on what Henrik was saying on the topics of sustainability, there are some really cool CNCF projects out there on tech sustainability, and I feel like this is a super hot topic for the year just because we're seeing a lot of stuff around wacky environmental things happening, right? Like an increase in forest fires, bizarre temperature swings. We're inherently in an industry that is contributing to the problem. Writing talks about how we can use technology to lessen the problem, I think, can be really compelling and very timely. There's a hot topic for y'all for anyone considering it. +So even though it's more high level on that specific thing, it might be considered like an intermediate, if that makes sense. So it really depends, and I'll also try to consider, like, how much—am I talking about something newer that was like more newly developed, so there's less info about it? Or is it something that's been around for a while? -I would also mention, we talk a lot about KubeCons, and I think Josh made a point earlier about there being a lot of conferences where things like Open Telemetry are still kind of not super well known. Especially like a lot of these open-source conferences, like Scale, for example, I think just added an observability track this year. FUM I think would probably be another great one. State of Open Con would be another great one where we probably don't have enough talks on observability. Getting into those sort of more niche conferences, I think, would be a good place to start, especially if you're looking to do a talk on observability. +I don't have a really good answer; I still find it tough, honestly. I try to stick to either beginner or intermediate level in general because of the topics that I typically choose to do. + +**Josh:** And the thing also is it's a personal judgment as well. Sometimes you say, "Oh, I think it's beginner," or "I think it's—I have no idea; it's complicated." + +What may seem basic to you might be very advanced to others or the opposite, right? + +**Dan:** Yeah, yeah, exactly! Do you have any other tips there, Josh, to try to gauge the audience? + +**Josh:** I have a quick anecdote. So yeah, it’s hard to get it right. I gave one talk that was an introductory talk, but I was trying to make it a little more advanced. I thought, "Oh, this seems like a fairly technical audience. I'd been having conversations with people at lunch. Everybody here is going to know what eBPF is," so I didn’t mention it. + +And then the next speaker after me says, "Raise your hand if you know what eBPF is," and me and the one person I was talking to raised our hands. + +**Henrik:** Oh, well, awesome! + +**Josh:** So yeah, it's hard to get it right; it’s hard. + +**Dan:** Okay, Henrik, I'm going to go back to you because I know that you're everywhere in OpenTelemetry as well. I know that there are a lot of hot topics in OpenTelemetry, but what are the hot topics that you recommend people be writing about right now? + +I know that it doesn't have to be OpenTelemetry; I know that we all love OpenTelemetry, but what are the sort of hot topics that you would recommend thinking about writing about or talking about? + +**Henrik:** I think what I would suggest is that—by the way, AI is not valid; it is. I think in general, you have to look at previous conferences and what has been presented and covered so far. + +If you start to say, "I want to do OpenTelemetry on instrumentation," there are plenty of those out there, plenty of videos out there, and then there's no hot topic in general. + +So usually, I try to bring the problem statement. For example, how do I sample properly? How do I optimize? Because there is a big concern about observability being expensive, so how can I control that cost? + +Going through on that direction, I think also there is another trend that I think is really important that people need to be educated a bit more about: it's sustainability. + +How can we make it green and better? Talking about AI is not going to be green because you're going to consume more resources. So maybe having another approach where you say, "Oh, we need to be good citizens of the world and save energy," I think that's a pretty good one. + +And also, I think there are new projects that come in. For example, today, as we speak, OpenTelemetry profiling—there are a few talks out there in KubeCon, and it's going to be more and more popular. + +So there's a big chance that profiling will bring new problems and new challenges, so covering them could also be interesting. Try to follow what happens in the industry and think about, "Oh, if I start digging this, I may have problems, so how can I resolve them and maybe share that solution or this approach to a larger group or to the community?" + +**Ree:** I’d just add, Henrik, you're really good at this, right? Like you mentioned sampling, and I saw that video right in my feed, and I was like, "Oh, that's a great topic. I'm going to check that out later when I have time." + +So yeah, if Henrik's talking about it, he can only be one place at once, right? So maybe that's an interesting thing for you to put your own spin on somewhere else. + +**Dan:** Yeah, I think so. Absolutely! Any other takes on, you know, what are some of the hot topics? + +**Ree:** I would like to add something. So, you know, I think piggybacking on what Henrik was saying on the topics of sustainability, there's some really cool CNCF projects out there on tech sustainability. + +And I feel like this is a super hot topic for the year just because we're seeing a lot of stuff around just wacky environmental things happening—right? Like increases in forest fires, bizarre temperature swings. + +And we're inherently in an industry that is contributing to the problem, so writing talks about how we can use technology to lessen the problem, I think, can be really compelling and very timely. + +So there's a hot topic for y'all for anyone considering it. I would also mention, you know, we talk a lot about KubeCon, and I think Josh made a point earlier about like there's a lot of conferences where like things like OpenTelemetry are still like kind of, you know, not super well-known. + +### [00:30:56] Hot topics in observability + +So especially like a lot of these open-source conferences, like SCALE for example, I think just added an observability track this year. FUM, I think, would probably be another great one. + +State of Open Con, another great one where we probably—there's probably not enough talks on observability, so getting into those sort of more niche conferences, I think would be a good place to start, especially if you're looking to do a talk on observability. **Dan:** Oh yeah, all things open as well. -**Ree:** Yes! +Well, Ree mentioned in our—yeah, so right, I'm going to take one question from the audience. Thank you so much for your questions! And again, you know, if you've got any questions on what we're talking about, drop them in the chat on YouTube or LinkedIn. + +And the question is, should we go for a catchy title, a clickbaity title when you write your CFP, or should you do something more perhaps descriptive and accurate? + +Josh, what do you think about that? Do you normally go for something like that? + +**Josh:** I almost always do a clickbaity title, or at least I did. That was always sort of my way. But I've only been doing this for a couple of years, and someone who's been doing it for longer than me told me it's a pendulum, right? It swings. + +What the conference organizers are looking for is going to swing back and forth almost like a cultural zeitgeist. There are times where they want it to be super, super specific, and we're not in one of those times right now. + +Maybe we're swinging in that direction, but for me right now, it does feel very much like the clickbaity titles are in, and maybe we'll all get tired of them because of the AI topic, and we'll be like, "No, you need to tell me exactly what's in your talk so that I know that it's not a surprise AI talk," and that'll be where we're at a year from now. + +Who knows? + +**Adriana:** I think if you can come up with a catchy title that also will give the audience an idea of what to expect or like what your topic is about, that is a great way to go. I also do really like just clear, straightforward titles as well. + +So I think they both have their place. And you know, the content for sure is going to matter more. I say because you could have a great catchy title and then the abstract is kind of, you know, maybe doesn't fully flush out the idea. + +So I think the content is still king, but yeah, the title is important as well. + +**Henrik:** I think when I'm preparing my talks, usually I try to bring the technical aspects and then try to find a funny angle or an analogy to a movie or video games or whatever. + +And then in the title, I try to bring that fun angle because at the end, I think that could be more attractive. Because at the end, when you have the schedule and you see a title that sounds more fun, then maybe more people will join your session for some reason. + +So I think the title is as important as the abstract from my perspective. + +**Ree:** I think the title is important, especially for the audience, right? + +**Dan:** Yeah, and I think as well—go ahead, go ahead! + +**Ree:** Oh, I was going to say I do generally agree with you on that. The only thing I would caution again is, A, make sure your titles don't sound like they were generated by ChatGPT; and B, make sure that your titles aren't so pop culture where it ends up alienating potentially the CFP reviewers. + +Like, I don't know, if someone's making a Game of Thrones reference in a talk title, I'll be like, "I have no idea what you're talking about," and I'm kind of super put off by it. + +So I would just caution—it's a fine line to trade. + +**Dan:** Nice! I guess, you know, that's as well part of—there are a couple of questions from YouTube that are related to that, basically to knowing your audience, right? + +So who's going to be reading your title, your CFP? So how do you know that? How do you know, like, how to approach the audience in a way that they would understand or engage with? And I'll ask that to Henrik. + +**Henrik:** It's a very, very good question. I usually go to KubeCon and to KubeCon or open-source friendly conferences, so I know that the people that will be at that conference have at least a technical background. + +So then I know that my talk would not be perceived as too technical. But yeah, if you go to a conference that is more salesy, then yes, you may have to adjust and say, "Okay, what are the types of personas that will be in this conference?" + +Do I have the normal audience that I'm talking to in general, or should I reduce the complexity of my talk so that people can follow the idea behind this talk? + +**Dan:** Cool! Right, I'm going to move on to another topic, which is like the post-acceptance. So you got accepted—hooray! So what happens now? How do you train? How do you rehearse? What can you give people to get ready for the day? I know it was a very wide question, so I'm going to start with Ree. How do you like prepare for it? + +**Ree:** Well, so when I got my first talk—so funny story! My very first CFP that I submitted ever was also my first one that was ever accepted, which was huge and terrifying. My manager at the time, she actually hired—she got me a few lessons, virtual lessons, with a public speaking coach, and I found that that was super helpful. + +It was just three virtual sessions, and I came away with techniques that I still use. But I also know people who are great speakers who haven’t gone through that professional training. + +I say it's at least worth it to, if you can, if you have the resources to, I recommend it. But if not, there are also, you know, YouTube videos you can watch. + +And of course, rehearse as much as you can so that you feel comfortable and it feels natural when you're like presenting. But I'm sure other people will have their tips. + +**Henrik:** Any other tips or tricks to basically get to the training and rehearsing stage? + +**Henrik:** I usually don't—I do just the rehearsal, not a full fresh rehearsal. I'm just checking that my talk is basically respecting the duration, because sometimes I put a lot of content, and then I realize, "Oh maybe too much here." + +So I need to maybe figure out by timing yourself, and then you know that when you will be on stage, there's a big chance that you will take probably more time. + +So I had a few minutes to be always in time. I think that's the best thing. But what I usually do—I'm trying to put a lot of graphics, make it a nicer experience. + +I put—I mean, that's my style, so I like to make a lot of graphics and make it a smoother experience for the audience, so that's something that I prepare a lot in the background. + +**Dan:** I'm going to take now a question from the audience, and I think I’m going to choose—I'm going to pick Josh to answer this question. If a conference offers multiple formats, for example, lightning talk or a deep dive, and you know, your talk could fit more than one, should I spam the organizers with multiple responses or is there a better way? + +**Josh:** In my opinion, that's something that will work if you know the organizer and can reach out to them directly, right? Like, or if they're having office hours and you can talk to them about it and you can get that feedback, that's fine. + +### [00:39:38] Preparing for your talk + +But I do think that if you're a first-time speaker at the conference or sort of unknown to the conference, you have to sell the talk. You have to be a little bit more confident in the talk that you're proposing because they're going to be evaluating you based on the talk more than how they know you as a speaker. + +**Adriana:** See anybody else? + +**Adriana:** I wanted to just mention something. Especially depending on the conference, some conferences don't have limits as to how many talks you can submit. + +And so sometimes, you know, especially if you're like a newbie speaker, go for it and just see how far you can take it. But other conferences, like KubeCon, where there is a limit on the number of sessions, I would caution against submitting basically two versions of the session where it's like one's a lightning talk and the other one is a longer-form talk. + +I think you probably have a better chance of just submitting two unique topics because you never know. Also, I will say, don't be afraid to recycle submissions at other conferences. + +It doesn't always have to be a unique topic—super, super important! It saves you a lot of mental energy because putting together a talk is a lot of work. + +**Henrik:** I would just add to the fact that for the reviewers, if you have a talk and usually you design your abstract for a 30-minute talk, for example, then the reviewer would expect some details. + +So then you say, "Okay, so he's going to cover this and that; it makes sense in 30 minutes because it's going to be well covered." And then if you do the same CFP in a lightning talk, then you say, "Wow, how is it going to cover that in five or ten minutes?" + +So I think you need to adjust, of course, the way you're going to present the abstracts. I would definitely recommend to do, like Adriana mentioned, to have two different submissions—one for the lightning talks and one for the normal track. + +**Dan:** Cool! Okay, I think it's time to start closing thoughts. And I think one of the—well, not quite there yet, but one of the questions that people normally get is, "I applied to many conferences," and then you get rejected. + +How normal is it to get rejected? How do you manage your expectations for getting your CFPs accepted? + +**Josh:** Yeah, it's tough, isn't it? So Josh, do you want to go? Give us your thoughts. + +**Josh:** Sure! I think, again, getting advice from mentors is great. Someone told me to expect a 10% acceptance rate, so if you want to talk at one conference a year, that means you need to submit ten. + +And I think that holds even for experienced speakers to some extent, as well as new, at least from what I've seen. I don't know how you all feel about that ratio, but you know, that's interesting because I haven't actually done like a statistical analysis myself. + +But it is absolutely normal to get rejected. You know, you think about conferences; some conferences get, especially the bigger ones, they get thousands—sometimes, but usually, like, you know, at least tens of dozens or hundreds of submissions. + +And just because they rejected it, you know, this time doesn’t mean that they won’t next time. I know people who've submitted the same one a few times to the same conference before it got accepted. + +So sometimes it’s just timing; sometimes it’s just, you know, they just have a lot of topics that were just really, really good. And sometimes too, you can also ask for feedback from the conference. + +KubeCon, for example, they're pretty good about sharing anything. If they get feedback on your proposal, they'll be happy to share it. + +**Adriana:** Cool! + +**Ree:** I also wanted to add, like, it’s okay to mourn your rejection because, you know, especially when you're really invested in a topic and you think, "I've got such a great chance of getting this in," and you get rejected. + +You got like a great title, and you’re just, "Ah!" Yeah, it's like it’s okay to mourn and take that time to mourn. + +And then as Ree said, like if possible, ask for feedback, resubmit it to that same conference or another conference, or like in the case of KubeCon, there are so many across the globe—submit it to a different KubeCon. + +You know, like if it didn’t get in for NA, submit for EU. There’s Open Source Summit NA, there’s Open Source Summit EMEA. And also, like take some time to reflect on your CFP and see if there are like areas where you see improvement. + +### [00:44:28] Handling rejection and feedback + +Like sometimes I’ll like write out a CFP and then I’ll like, you know, it gets rejected. I resubmit it for another conference, and I'm like, "Oh yeah, I can kind of see why they didn’t like that." + +And just make a few tweaks. -**Dan:** Right, I'm going to take one question from the audience. Thank you so much for your questions. Again, if you've got any questions on what we're talking about, drop them in the chat on YouTube or LinkedIn. The question is: should we go for a catchy title, a clickbaity title, when you write your CFP, or should you do something more perhaps descriptive and accurate? Josh, what do you think about that? +**Dan:** Yeah, agree! Well, thank you so much! I think now I've got one last question for you, and I think as—well, if everyone's been following the Golden Globes and the Academy Awards, you'll see that we've got quite a lot of horror films coming up and the awards. -**Josh:** I almost always do a clickbaity title, or at least I did. That was always sort of my way. But I've only been doing this for a couple of years, and someone who's been doing it for longer than me told me, "Actually, it's a pendulum, right? It swings." What the conference organizers are looking for is going to swing back and forth almost like a cultural zeitgeist. There are times when they want it to be super, super specific, and we're not in one of those times right now. Maybe we're swinging in that direction, but for me right now, it does feel very much like the clickbaity titles are in, and maybe we'll all get tired of them because of the AI topic, and we'll be like, "No, you need to tell me exactly what's in your talk so that I know that it's not a surprise AI talk." That'll be where we're at a year from now. +So I've got a question for you: I think you've been to many conferences, and I would like to hear some of the conferences that you've been to, because I think we didn't cover that in the intro. -I think if you can come up with a catchy title that also gives the audience an idea of what to expect or what your topic is about, that is a great way to go. I also do really like just clear, straightforward titles as well. I think they both have their place, and the content for sure is going to matter more, I say. Because you could have a great catchy title, and then the abstract is kind of, you know, maybe doesn't fully flesh out the idea. The content is still king, but yeah, the title is important as well. +And I would like to know, you know, some of the conferences that people could go and rewatch your videos, but also like if you've got a quick anecdote or something that failed that was a bit of a horror story, right? -**Henrik:** When I prepare my talks, I usually try to bring the technical aspects and then try to find a funny angle or an analogy to a movie, to video games, to do whatever. Then in the title, I try to bring that fun angle because at the end, I think that could be more attractive. When you have the schedule and you see a title that sounds more fun, then maybe more people will join your session for some reasons. +So the mic doesn't work, or you know, something that doesn't work. Can you tell us about it? And I'll go for Ree first because she's got her hands raised. -**Adriana:** I think the title is as important as the abstract, from my perspective. +**Ree:** My very first talk that I did—it was about tail sampling in OpenTelemetry. I had a live demo, and everything was working. I checked it right before my talk; everything was working. -**Josh:** I think, going back to what you just said, it's important to note that some titles sound like they were generated by ChatGPT. +I got up to the stage, started the demo—it didn't work! And I still don't know what it was because I got it to work a few minutes later, and I know there's some people that still remember that too. -**Ree:** Yes! +So anyways, that's my—I have other ones too, but that one is very—sticks closely to my heart. -**Josh:** And B, make sure that your titles aren't so pop culture where it ends up alienating potentially the CFP reviewers. Like, I don't know if someone is making a Game of Thrones reference in a talk title. I'll be like, "I have no idea what you're talking about," and I'm kind of super put off by it. +**Henrik:** What was my worst experience? I would say it happens once where I had to connect my laptop. The screen mirroring didn't work, and then I suddenly did the demo and ended up having nothing on my screen. -**Dan:** So I would just caution, it's a fine line to tread. +And you're like this, trying to type and looking at—and it's like a nightmare. I was saying, "Okay, the demo is going to be a nightmare; impossible to achieve." -**Dan:** And I guess that's part of—there are a couple of questions from YouTube which are related to that, basically to knowing your audience, right? Who's going to be reading your title, your CFP? So how do you know that? How do you know how to approach the audience in a way that they would understand or engage with? I'll ask that to Henrik. +And so then I think the demo—I tried to skip it very fast, but I was not very happy. People didn't see so much because I tried to react as fast as possible to avoid having that blank effect where stress was coming up to me. -**Henrik:** It's a very, very good question. I usually go to KubeCons and to Open Source-friendly conferences, so I know that the people that will be in that conference have at least a technical background. I know that my talk would not be perceived as too technical. But yeah, if you go to a conference that is more salesy, then yes, you may have to adjust and say, "Okay, what are the type of personas that will be in this conference? Do I have the normal audience that I'm talking to in general, or should I reduce the complexity of my talk so that people can follow the idea behind this talk?" +But yeah, at the end, I was unhappy because I said, "My performance was just not normal and not acceptable," so I had to improve that. -**Dan:** Cool. Right, I'm going to move on to another topic, which is the post-acceptance. So you got accepted — hooray! So what happens now? How do you train? How do you rehearse? What can you give people to get ready for the day? I know that was a very wide question, so I'm going to start with Ree. How do you prepare for it? +But yeah, where can we see you next? Where can you— -**Ree:** Well, so when I got my first talk, funny story: my very first CFP that I submitted ever was also my first one that was ever accepted, which was huge and terrifying. My manager at the time actually hired a—well, she got me a few lessons, virtual lessons, with a public speaking coach, and I found that that was super helpful. It was just three virtual sessions, and I came away with techniques that I still use. But I also know people who are great speakers who haven't gone through that professional training. I say it's at least worth it if you can, if you have the resources to. I recommend it, but if not, there are also YouTube videos you can watch. Of course, rehearse as much as you can so that you feel comfortable and it feels natural when you're presenting. +**Henrik:** I was rejected for KubeCon, so I've been crying. -**Josh:** Any other tips or tricks to basically get to the training rehearsing stage? +**Ree:** It happens to the best! -**Henrik:** I usually don't. I do just the rehearsal, not a full-fledged rehearsal. I'm just checking that my talk is basically respecting the duration because sometimes I put a lot of content in, and then I realize, "Oh, maybe too much here." So I need to time myself, and then you know when you will be on stage, there's a big chance that you will take probably more time. So I had a few minutes to be always on time. I think that's the best thing. But what I usually do, I'm trying to put a lot of graphics, make it nicer. I put a lot of effort into making a smoother experience for the audience, so that's something that I prepare a lot in the background. +**Henrik:** It does! So yeah, the next I have a virtual conference that—I don't know when it's going to happen in a few weeks I'm presenting a talk, and then I submitted lots of talks for the season. -**Dan:** I'm going to take now a question from the audience. I think I'm going to choose—I'm going to pick Josh to answer this question. If a conference offers multiple formats, for example, lightning talks or a deep dive, should I spam the organizers with multiple responses, or is there a better way? +So a lot of KubeCons and KubeCon China and Japan, so we'll see. I didn't have the feedback yet, so cross the fingers. But otherwise, if you want to watch any content from my end, you can find it on KubeCon Europe, KubeCon North America, and then I have plenty of other KubeCons where I have presented talks last year. -**Josh:** In my opinion, that’s something that will work if you know the organizer and can reach out to them directly, right? Or if they're having office hours and you can talk to them about it and you can get that feedback, that's fine. But I do think that if you're a first-time speaker at the conference or sort of unknown to the conference, you have to sell the talk. You have to be a little bit more confident in the talk that you're proposing because they're going to be evaluating you based on the talk more than how they know you as a speaker. +**Josh:** Awesome! Do you want to tell us your horror story? -[00:29:36] **Ree:** Yeah, I wanted to just mention something. Especially depending on the conference, some conferences don't have limits as to how many talks you can submit. Sometimes, you know, especially if you're a newbie speaker, go for it and just see how far you can take it. But other conferences, like KubeCon, where there is a limit on the number of sessions, I would caution against submitting two versions of the same session where one's a lightning talk and the other one's a longer-form talk. I think you probably have a better chance of just submitting two unique topics because you never know. Also, I will say, don't be afraid to recycle submissions at other conferences. It doesn't always have to be a unique topic. Super, super important! It saves you a lot of mental energy because putting together a talk is a lot of work. +**Josh:** Yeah, it's actually quite similar to Henrik's. It had to do with screen mirroring and not working, and this is actually also like recent. My very first talk—the audience could only see half of my slides. -**Henrik:** I would just add to the fact that for the reviewers, if you have a talk and usually you design your abstract for a 30-minute talk, for example, then the reviewer would expect some details. Then you say, "Okay, so he’s going to cover this and that, and it makes sense in 30 minutes because it's going to be well covered." If you do the same CFP in a lightning talk, then you say, "Wow, how is it going to cover that in five or ten minutes?" I think you need to adjust, of course, the way you're going to present the abstracts. +I could see all of my slides, but none of my notes. So my lesson that I took away from that is I never rely on my notes ever again. -**Ree:** I would definitely recommend doing, like Adriana mentioned, to have two different submissions: one lighter for the lightning talks and one bigger for the normal track. +**Henrik:** Plus one! -**Dan:** Cool! Okay, I think it's time to start closing thoughts. One of the questions that people normally get is: I apply to many conferences, and then you get rejected. How normal is it to get rejected? How do you manage your expectations for getting your CFPs accepted? Josh, do you want to give us your thoughts? +**Josh:** Plus one to that! You have to have it all in your head; you have to be able to just talk about the thing for 20 minutes with no notes and no slides if it comes down to it. -**Josh:** Sure! I think someone, again, you know, getting advice from mentors is great. Someone told me to expect a 10% acceptance rate. If you want to talk at one conference a year, that means you need to submit 10. I think that holds even for experienced speakers to some extent as well as new, at least from what I've seen. I don't know how you all feel about that ratio, but it is absolutely normal to get rejected. You think about conferences; some conferences get, especially the bigger ones, thousands, sometimes, but usually at least tens or dozens or hundreds of submissions. Just because they rejected it this time doesn't mean that they won't next time. I know people who've submitted the same one a few times to the same conference before it got accepted. Sometimes it's just timing; sometimes it's just, you know, they just have a lot of topics that were really good. Sometimes too, you can also ask for feedback from the conference. KubeCon, for example, they're pretty good about sharing feedback on your proposal; they'll be happy to share it. +I think not to scare anyone else off from doing this, right? Like you can do that. Actually, it's not as hard as it sounds. -**Ree:** I also wanted to add, it's okay to mourn your rejection because especially when you're really invested in a topic, and you think, "I've got such a great chance of getting this in!" and you get rejected. You got a great title, and you're just... +But yeah, that was my worst horror story. Although I would say, like, I feel like at least something small goes wrong pretty much every time. -**Josh:** Yes! +**Adriana:** And then where you can find me? -**Ree:** Yeah, it's okay to mourn and take that time to mourn. As Ree said, if possible, ask for feedback. Resubmit it to that same conference or another conference or, like in the case of KubeCons, there are so many across the globe. Submit it to a different KubeCon. You know, if you didn't get in for NA, submit for EU. There's Open Source Summit NA, there's Open Source Summit EMEA, and also take some time to reflect on your CFP and see if there are areas where you see improvement. Sometimes I'll write out a CFP, and then I'll realize, "Oh, yeah, I can kind of see why they didn't like that," and just make a few tweaks. +**Ree:** A lot of DevOps days—I’m really big on those, and I love that the KCDs are starting to adopt that community format and sort of the open discussion format more. I really love those conferences, and I love to get the combination of like the people who are coming from all over, like you would see at the bigger conferences, and then just also the local representatives of the tech companies that are employers in that area. -**Dan:** I agree! +**Henrik:** KCD in Edinburgh! -Well, thank you so much! I think now I've got one last question from you, and I think, as well, if everyone's been following the Golden Globes and the Academy Awards, you'll see that we've got quite a lot of horror films coming up in the awards. I've got a question for you. I think you've been to many conferences, and I would like to hear some of the conferences that you've been to because I think we didn't cover that in the intro. I would like to know some of the conferences that people could go and rewatch your videos, but also if you've got a quick anecdote or something that failed that was a bit of a horror story — right? The mic doesn't work or, you know, something that doesn't work. Can you tell us about it? I'll go for Ree first because she's got her hands raised. +**Ree:** Yeah, that is a KCD in Edinburg! -**Ree:** My very first talk that I did, it was about tail sampling in Open Telemetry. I had a live demo, and everything was working. I checked it right before my talk; everything was working. I got up on the stage, started the demo, and it didn't work! I still don't know what it was because I got it to work a few minutes later, and I know there are some people that still remember that too! So anyways, that's my—I have other ones too, but that one sticks close. +**Dan:** Yeah, that is a KCD in Edinburgh! You can find out this year; I’m not sure the CFP yet. Is it when is it open? -**Henrik:** What was my worst experience? I would say it happened once where I had to connect my laptop, and the screen mirroring didn't work. I suddenly did the demo, and then you end up having nothing on your screen, and you're like this, trying to type and looking at it. It's like a nightmare! I was saying, "Okay, the demo is going to be a nightmare; it's impossible to achieve!" So then I think the demo, I tried to skip it very fast, but I was not very happy. People didn't see so much because I tried to react as fast as possible to avoid having that blank effect where stress is coming up to you. At the end, I was unhappy because I said, "Ah, my performance was just not normal and not acceptable!" I had to improve that. +**Ree:** It's in October, so I'm not sure when the—okay! But yeah, everyone in Edinburgh is really excited about it. I mean, I'm not sure the normal person in the street would be, but I am! -But yeah, where can we see you next? +**Adriana:** I have a few conferences that I've spoken at. We've spoken at KubeCon North America and EU a few times together. We've spoken at Observability Day North America and EU together. -**Henrik:** I was rejected for KubeCon, so I've been crying! So now I think I have no tears anymore. +I've spoken at Platform Engineering Day most recently in North America. We've done All Things Open together. We did Open Source Summit North America and EU coming up. -**Ree:** It happens to the best! +And I got in for Observability Day EU as well. And then I did a DevOps Days in Montreal, and most recently one of my favorites was a KCD BUU where I got to give a keynote. -**Henrik:** It does! So yeah, the next I have a virtual conference that I don't know when it's going to happen in a few weeks. I'm presenting a talk, and then I submitted lots of talks for the season. I have a lot of KCDs and KubeCon China and Japan. We'll see; I didn't have the feedback yet, so cross your fingers. But otherwise, if you want to watch any content from my end, you can find it on KubeCon Europe, KubeCon North America, and I have plenty of other KCDs where I have presented talks last year. +Unfortunately, there's no recording of the keynote, but I have a blog post version of the talk if anyone wants to check out my Medium channel. -**Josh:** I think it's quite similar to Henrik’s. My very first talk, the audience could only see half of my slides. I could see all of my slides, but none of my notes. My lesson that I took away from that is I never rely on my notes ever again! Plus one to that! You have to have it all in your head. You have to be able to talk about the thing for 20 minutes with no notes and no slides if it comes down to it. I think not to scare anyone else off from doing this, right? You can do that; it's not as hard as it sounds. But yeah, that was my worst horror story. Although I would say I feel like at least something small goes wrong pretty much every time. +**Dan:** Awesome! Well, thanks everybody so much for—I mean, you are really the dream team. So really, like all the veterans from all the, you know, conference doers. -And then where you can find me? A lot of DevOps Days; I'm really a big fan of those. I love that the KCDs are starting to adopt that community format and sort of the open discussion format more. I really love those conferences, and I love to get the combination of the people who are coming from all over like you would see at the bigger conferences, and then just also the local representatives of the tech companies that are employers in that area. +You can see that, you know, from the amount of conferences that you've been to. So thank you so much for your tips and tricks and your advice. I think, you know, it's been really, really useful. -**Adriana:** I think I know what my horror story is! I think I had a similar experience where I had a demo that was not working, and the audience could see it! But I think for me, the worst is when you give a talk, and you're like, "H, that wasn't my best work." You feel like you're a little bit off. Sometimes my voice is a little shaky or I forget my talking points. I think that's getting rattled by that, and it's interesting because you come off a talk, and people are like, "Oh, that was so great," and you're like, "Oh, I sucked!" You kind of just have to get over yourself and tell yourself it happens, and it'll happen again, and you just have to be okay with it. I would also suggest when you're practicing talks, practice. As you're practicing, if you stumble, just power through it because that can happen in real life. +But I think that's all we had time for. And I just want to say thanks to the panelists and thanks to the people that asked questions in the audience. I think we didn't get to cover all the questions, but we are always open in the end user SIG. You can come to the CNCF Slack, you can go to the resources that are here basically, and get in touch with us. -As far as conferences that I've spoken at, and I'm just sending Henrik a link to my YouTube channel because I have a playlist of all my talks. Recently, I've done a bunch of talks; we've spoken at KubeCon North America and EU a few times together. We've spoken at Observability Day North America and EU together. I've spoken at Platform Engineering Day most recently in North America. We've done All Things Open together. We did Open Source Summit, and I'm speaking at KubeCon EU coming up in London. I got in for Observability Day EU as well. I did DevOps Days in Montreal, and one of my favorites was KCD BUU, where I got to give a keynote. Unfortunately, there's no recording of the keynote, but I have a blog post version of the talk if anyone wants to check out my Medium channel. +I think we would love to hear more about your, well, in general anything about OpenTelemetry, but also about topics like this one—how do you go and talk about OpenTelemetry everywhere? -**Dan:** Awesome! Well, thanks everybody so much. You are really the dream team—really all the veterans from all the conference dos. You can see that from the amount of conferences you've been to! Thank you so much for your tips, tricks, and advice. I think it's been really, really useful. But I think that's all we had time for. I just want to say thanks to the panelists and thanks to the people that asked questions in the audience. I think we didn't get to cover all the questions, but we are always open in the End User SIG. You can come to the CNCF Slack; you can go to the resources that are here, basically, and get in touch with us. I think we'd love to hear more about your—well, in general, anything about Open Telemetry, but also about topics like this one: how do you go and talk about Open Telemetry everywhere? +So yeah, with that, thank you very much, and see you at a conference! Get the questions that were unanswered in the Slack channel! -With that, thank you very much, and see you at a conference! Get the questions that were unanswered in the Slack channel. +**Ree:** Yes! Right! Thank you! Bye-bye! -**Dan:** Thank you! Bye-bye! +**All:** Bye! ## Raw YouTube Transcript diff --git a/video-transcripts/transcripts/2025-02-26T15:40:58Z-what-is-otel-otel-for-beginners-the-javascript-journey.md b/video-transcripts/transcripts/2025-02-26T15:40:58Z-what-is-otel-otel-for-beginners-the-javascript-journey.md index 7b9c883..52bd960 100644 --- a/video-transcripts/transcripts/2025-02-26T15:40:58Z-what-is-otel-otel-for-beginners-the-javascript-journey.md +++ b/video-transcripts/transcripts/2025-02-26T15:40:58Z-what-is-otel-otel-for-beginners-the-javascript-journey.md @@ -10,58 +10,66 @@ URL: https://www.youtube.com/watch?v=iEEIabOha8U ## Summary -In the video "OTel for Beginners," Lisa Jung, a member of the OpenTelemetry (OTel) Communication and End User Special Interest Groups, shares her journey as a newcomer to OTel and aims to guide viewers through the basics of using this open-source framework, particularly through a JavaScript lens. Lisa first explains what OTel is and emphasizes the importance of observability in modern software systems, likening a non-observable system to a "black box." She discusses how OTel helps to convert this black box into a "glass box" by generating, collecting, managing, and exporting telemetry data, which is crucial for understanding system performance. The video also addresses the issue of vendor lock-in in observability, illustrating how OTel provides a universal standard that allows for easier transitions between different observability backends, thus reducing the costs associated with switching vendors. Lisa encourages viewers to explore various resources for learning OTel, including documentation, the official OTel YouTube channel, and community support through the CNCF Slack Channel. The next episode will focus on starting the JavaScript journey with OTel. +In this introductory video titled "OTel for Beginners," Lisa Jung, a member of the OpenTelemetry (OTel) Communication and End User Special Interest Groups (SIGs), shares her journey of learning about OTel and aims to guide others through the process. The video explains the concept of observability in modern software systems, highlighting its importance for understanding and diagnosing issues within complex infrastructures. OTel is presented as an open-source framework that enables the generation, collection, management, and export of telemetry data, helping to avoid vendor lock-in by establishing a standard for data transmission across different observability backends. Lisa emphasizes the benefits of using OTel, such as reduced costs and simplified vendor transitions, and provides resources for viewers to begin their own OTel journey, particularly focusing on JavaScript in upcoming episodes. The video encourages viewers to engage with the OTel community for support and further learning. ## Chapters -00:00:00 Welcome and intro -00:00:30 Observability explanation -00:01:00 Importance of telemetry data -00:02:00 OTel framework overview -00:03:46 Vendor lock-in discussion -00:04:10 USB-C analogy -00:05:00 Switching vendors challenges -00:06:00 OTel as an open standard -00:07:00 Benefits of using OTel -00:08:00 Resources for getting started +00:00:00 Introductions +00:00:40 What is observability? +00:01:15 Importance of telemetry data +00:02:00 Introduction to OTel +00:03:30 Vendor lock-in explained +00:04:15 Benefits of OTel +00:07:15 OTel community resources +00:08:00 Getting started with OTel +00:08:00 OTel documentation overview +00:09:00 Upcoming JavaScript Journey episode + +## Transcript + +### [00:00:00] Introductions **Lisa:** Hi! Welcome to OTel for Beginners. My name is Lisa Jung and I'm a member of the OTel Communication SIG and End User SIG. I recently began my journey with OTel and as a newbie, I had a tough time figuring out how to get started. I want you to have a different experience, so I'll learn alongside you and share what I'm learning through the series. Depending on which programming language you're working with, you'll follow a language-specific journey. In this series, I'll go through the JavaScript journey to get you started with OTel. -[00:00:30] Before we get our hands dirty, let's talk about what OTel is and why you should consider using it and the resources to get started. OTel stands for OpenTelemetry and it plays an important role in observing your system. So let's talk about observability first, then delve into how OTel fits into all of this. +### [00:00:40] What is observability? + +Before we get our hands dirty, let's talk about what OTel is and why you should consider using it, and the resources to get started. OTel stands for OpenTelemetry and it plays an important role in observing your system. So let's talk about observability first, then delve into how OTel fits into all of this. Our modern software systems can consist of complex, multi-layered, and distributed systems with many interdependencies. A system without observability is like a black box; we have no idea what's going on inside, so if something goes wrong, it's going to be more difficult and more time-consuming to solve the problem. + +### [00:01:15] Importance of telemetry data -[00:01:00] Our modern software systems can consist of complex, multi-layered, and distributed systems with many interdependencies. A system without observability is like a black box; we have no idea what's going on inside, so if something goes wrong, it's going to be more difficult and more time-consuming to solve the problem. With observability, we turn this black box into a glass box. As a matter of fact, it helps you collect the data necessary to visualize and understand what's going on in your system. +With observability, we turn this black box into a glass box. As a matter of fact, it helps you collect the data necessary to visualize and understand what's going on in your system. How do we make this possible? First, you have your infrastructure or applications that you want to observe. You'll collect data from it and send the data to the observability backend of your choosing. Then connect the backend to a visualization frontend where you can query and use the data that you're interested in. The most common types of data collected for observability are metrics, logs, and traces. These are known as telemetry data. -How do we make this possible? First, you have your infrastructure or applications that you want to observe. You'll collect data from it and send the data to the observability backend of your choosing. Then connect the backend to a visualization front end where you can query and use the data that you're interested in. The most common types of data collected for observability are metrics, logs, and traces. These are known as telemetry data. +### [00:02:00] Introduction to OTel -[00:02:00] Getting the telemetry data into the backend is an important part of understanding your infrastructure or applications, and this is where OTel comes in. OTel is an open-source framework. Using OTel, you can add software to your applications or systems to generate telemetry data. This process is known as instrumentation. Then it collects, manages, and exports telemetry data to an observability backend and the database for storage. Different aspects of OTel make this process possible, and we're going to talk about that in more detail as we go through the JavaScript Journey. For now, remember that OTel is focused on the generation, collection, management, and export of telemetry data. +Getting the telemetry data into the backend is an important part of understanding your infrastructure or applications, and this is where OTel comes in. OTel is an open-source framework. Using OTel, you can add software to your applications or systems to generate telemetry data. This process is known as instrumentation. Then it collects, manages, and exports telemetry data to an observability backend and the database for storage. Different aspects of OTel make this process possible, and we're going to talk about that in more detail as we go through the JavaScript Journey. For now, remember that OTel is focused on the generation, collection, management, and export of telemetry data. You can easily instrument your applications or systems, no matter their language, infrastructure, or runtime environment, and the storage and visualization of telemetry are intentionally left to other tools. -You can easily instrument your applications or systems, no matter their language, infrastructure, or runtime environment, and the storage and visualization of telemetry are intentionally left to other tools. So why should we consider using OTel? Before we answer this question, let's talk about something that almost all of us have experienced. +### [00:03:30] Vendor lock-in explained -Now, if we dig through our drawers at home, we could probably find a bunch of cables of different types. Why? Because the devices we own come from different vendors, and each vendor has a specific type of cable and port to charge or connect your gadget. We're all familiar with buying multiple products from the same vendor and investing in additional accessories and apps specifically designed for that product. Before we know it, we get so used to using the line of products that switching over to another vendor could get pretty difficult. +So why should we consider using OTel? Before we answer this question, let's talk about something that almost all of us have experienced. Now, if we dig through our drawers at home, we could probably find a bunch of cables of different types. Why? Because the devices we own come from different vendors, and each vendor has a specific type of cable and port to charge or connect your gadget. We're all familiar with buying multiple products from the same vendor and investing in additional accessories and apps specifically designed for that product. Before we know it, we get so used to using the line of products that switching over to another vendor could get pretty difficult. -[00:03:46] The product from another vendor may operate differently, which will take some time to learn and get used to. It would also cost us more money because the additional accessories or apps we invested in don't work with a product from a new vendor. This is a problem known as vendor lock-in, where the costs of switching vendors are so high that customers feel stuck with what they have. Then things are slowly changing with USB-C ports becoming the universal standard. +### [00:04:15] Benefits of OTel -[00:04:10] Let's take the newest model of phones as an example. It doesn't matter if you're using an iPhone or an Android. You could charge or connect many of these phones with a USB-C cable because a universal standard has been set. Users can use the same cable regardless of how many times they charge their phones. OTel has similar implications in observability as a USB-C does for phones. +The product from another vendor may operate differently, which will take some time to learn and get used to. It would also cost us more money because the additional accessories or apps we invested in don't work with a product from a new vendor. This is a problem known as vendor lock-in, where the costs of switching vendors are so high that customers feel stuck with what they have. Then things are slowly changing with USB-C ports becoming the universal standard. Let's take the newest model of phones as an example. It doesn't matter if you're using an iPhone or an Android. You could charge or connect many of these phones with a USB-C cable because a universal standard has been set. Users can use the same cable regardless of how many times they charge their phones. -Let's do a quick review. We have our infrastructure or applications we want to observe. We want to collect telemetry data and send it to an observability backend for storage, so this data can be queried and visualized with an observability frontend. Say you have three observability backends to choose from: vendors A, B, and C. Each vendor often has proprietary instrumentations, agents, and or collectors. +OTel has similar implications in observability as a USB-C does for phones. Let's do a quick review. We have our infrastructure or applications we want to observe. We want to collect telemetry data and send it to an observability backend for storage, so this data can be queried and visualized with an observability frontend. Say you have three observability backends to choose from: vendors A, B, and C. Each vendor often has proprietary instrumentations, agents, and or collectors. Now, let's say you picked vendor A. You're using their proprietary instrumentations, agents, or collectors and sending the data to their backend. -[00:05:00] Now, let's say you picked vendor A. You're using their proprietary instrumentations, agents, or collectors and sending the data to their backend. But your needs change down the road; you want to switch your backend to vendor B. But switching vendors is not as simple as you might think. You can't just send the existing data from vendor A to B, because vendor B requires its own instrumentation, agents, or collectors, so you can't accept data from vendor A's proprietary instrumentation. +But your needs change down the road; you want to switch your backend to vendor B. But switching vendors is not as simple as you might think. You can't just send the existing data from vendor A to B because vendor B requires its own instrumentation, agents, or collectors, so you can't accept data from vendor A's proprietary instrumentation. Now your development team has to change their instrumentation, possibly to a new proprietary instrumentation of vendor B. Now imagine doing this for thousands of Linux machines or dozens of applications. Is the cost of money, time, and effort worth the benefits of changing vendors? -Now your development team has to change their instrumentation, possibly to a new proprietary instrumentation of vendor B. Now imagine doing this for thousands of Linux machines or dozens of applications. Is the cost of money, time, and effort worth the benefits of changing vendors? As you could see with this setup, the cost of switching vendors can get so high that customers can be effectively locked in by their choices. +As you could see with this setup, the cost of switching vendors can get so high that customers can be effectively locked in by their choices. Now, imagine if all vendors accepted the same standard to send or receive telemetry data, similar to many phone companies accepting USB-C as a standard. When you switch vendors, you don't need to learn proprietary instrumentations, agents, or collectors each time. You just need one technology and learn a single set of APIs and conventions associated with it. Whatever data you generate with this technology is yours. You could send the data to any observability backend that accepts the standard, so you could easily switch vendors with substantially reduced cost, time, and effort. -[00:06:00] Now, imagine if all vendors accepted the same standard to send or receive telemetry data, similar to many phone companies accepting USB-C as a standard. When you switch vendors, you don't need to learn proprietary instrumentations, agents, or collectors each time. You just need one technology and learn a single set of APIs and conventions associated with it. Whatever data you generate with this technology is yours. You could send the data to any observability backend that accepts the standard, so you could easily switch vendors with substantially reduced cost, time, and effort. +### [00:07:15] OTel community resources -[00:07:00] This is exactly what OTel does for you. It creates an open standard, a set of guidelines, rules, or specifications to send and receive data. There's quite an incentive for the vendors to accept OTel. Many customers now prefer OTel to avoid vendor lock-in. They want to learn a single set of APIs and conventions rather than having to learn a new one every time they change a vendor. They also want to own their data and send the data to any observability backend that accepts these standards. +This is exactly what OTel does for you. It creates an open standard, a set of guidelines, rules, or specifications to send and receive data. There's quite an incentive for the vendors to accept OTel. Many customers now prefer OTel to avoid vendor lock-in. They want to learn a single set of APIs and conventions rather than having to learn a new one every time they change a vendor. They also want to own their data and send the data to any observability backend that accepts these standards. Now, the vendors benefit from accepting OTel as well. Customers may already be familiar with using OTel. There's a vibrant OTel community that serves as a great resource because of that. Accepting OTel helps the vendors reduce their support and implementation costs. -Now, the vendors benefit from accepting OTel as well. Customers may already be familiar with using OTel. There's a vibrant OTel community that serves as a great resource because of that. Accepting OTel helps the vendors reduce their support and implementation costs. On top of that, there is even more drive for innovation. Vendors are receiving the same data, so they have to innovate to stand out from the competitors. +### [00:08:00] Getting started with OTel -As you can see, there are many benefits to accepting open standards, and you can take advantage of these benefits by using OTel. Now that we covered what OTel is and why you should use it, let's go over the resources to get started. The links to all the resources are included in the description of the video. +On top of that, there is even more drive for innovation. Vendors are receiving the same data, so they have to innovate to stand out from the competitors. As you can see, there are many benefits to accepting open standards, and you can take advantage of these benefits by using OTel. Now that we covered what OTel is and why you should use it, let's go over the resources to get started. The links to all the resources are included in the description of the video. The best place to get started is the OTel documentation. The documentation is continuously being improved, so the page may look different by the time you watch this video. -[00:08:00] The best place to get started is the OTel documentation. The documentation is continuously being improved, so the page may look different by the time you watch this video. So use a link in the description to check out the latest page. The first place in the doc you should start with is the Language APIs and SDKs. As I mentioned earlier, your OTel journey will differ depending on the programming language you're working with. +So use a link in the description to check out the latest page. The first place in the doc you should start with is the Language APIs and SDKs. As I mentioned earlier, your OTel journey will differ depending on the programming language you're working with. Select the language of your choice and you should end up on a page that lists all the resources to get started. Next, we have the official OTel YouTube channel. You'll find helpful videos along with the OTel for Beginners series on this channel. -Select the language of your choice and you should end up on a page that lists all the resources to get started. Next, we have the official OTel YouTube channel. You'll find helpful videos along with the OTel for Beginners series on this channel. Last but not least, as you start your OTel journey, you'll come across a lot of questions. We have a huge community of OTel users on Slack. +### [00:09:00] Upcoming JavaScript Journey episode -So join the CNCF Slack Channel, post your questions on the OpenTelemetry channel, and connect with other community members. Again, check the description of this video to access all these resources. In the next episode, we'll talk about how to get started with a JavaScript Journey, so stay tuned for that. Thank you for watching, and I'll see you in the next episode. +Last but not least, as you start your OTel journey, you'll come across a lot of questions. We have a huge community of OTel users on Slack. So join the CNCF Slack Channel, post your questions on the OpenTelemetry channel, and connect with other community members. Again, check the description of this video to access all these resources. In the next episode, we'll talk about how to get started with a JavaScript Journey, so stay tuned for that. Thank you for watching, and I'll see you in the next episode. ## Raw YouTube Transcript diff --git a/video-transcripts/transcripts/2025-03-21T05:53:37Z-otel-me-with-jerome-johnson-cal-loomis.md b/video-transcripts/transcripts/2025-03-21T05:53:37Z-otel-me-with-jerome-johnson-cal-loomis.md index 117a0c2..ebdbd67 100644 --- a/video-transcripts/transcripts/2025-03-21T05:53:37Z-otel-me-with-jerome-johnson-cal-loomis.md +++ b/video-transcripts/transcripts/2025-03-21T05:53:37Z-otel-me-with-jerome-johnson-cal-loomis.md @@ -10,134 +10,170 @@ URL: https://www.youtube.com/watch?v=DrD35XxTDsY ## Summary -In this episode of "Hotel Me," hosts Ree and Adriana, broadcasting from Vancouver and Toronto respectively, engage in a discussion about observability and OpenTelemetry with guests Cal and Jerome from Relativity. The conversation dives into Relativity's journey as one of the early adopters of OpenTelemetry, exploring their tech stack, architecture, and the complexities of managing a vast number of services and telemetry data. They share insights on how they transitioned from a bespoke telemetry system to OpenTelemetry, the challenges faced, and the cultural shifts within their organization towards a more observability-driven approach. The guests discuss their experience with deploying custom OpenTelemetry collectors, their strategies for data management, and the importance of standardizing telemetry data for downstream consumers like AI divisions and management reporting. They also provide feedback for the OpenTelemetry project, emphasizing the need for improved documentation and better integration of semantic conventions. The episode concludes with audience questions regarding data consumption and auto instrumentation, highlighting the ongoing evolution in telemetry practices. +In this episode of "Hotel Me," hosts Ree and Adriana, broadcasting from Vancouver and Toronto respectively, engage with guests Cal and Jerome from Relativity, a company that provides a SaaS platform for legal e-discovery. They explore Relativity's journey with OpenTelemetry, discussing their tech stack and the challenges they faced as early adopters. Cal, an architect, and Jerome, part of the observability team, share insights into their observability platform, which currently manages over 1500 services and processes 10-15 terabytes of telemetry data daily. They highlight the importance of adopting OpenTelemetry to streamline their telemetry processes and improve data management. The conversation touches on the evolution of their implementation, the significance of auto-instrumentation, and the cultural shifts within their organization to embrace observability. They also provide feedback on the OpenTelemetry project, emphasizing the need for better standardization in telemetry data schemas and the importance of community discussions around semantic conventions. The episode concludes with an invitation to the audience for further engagement and questions. ## Chapters -00:00:00 Welcome and intro -00:01:50 Guest introductions -00:04:10 Relativity overview -00:06:00 Tech stack insights -00:09:50 OpenTelemetry adoption discussion -00:12:30 Collector setup explanation -00:15:00 Managing collector fleet -00:20:00 Observability team structure -00:21:50 Implementation evolution -00:30:00 Data consumption and management +00:00:00 Introductions +00:01:47 Guest introduction: Cal and Jerome +00:06:00 Discussion about Relativity's tech stack +00:10:08 Why they adopted OpenTelemetry +00:18:14 Managing the fleet of collectors +00:22:30 Observability team structure +00:29:32 Evolution of OpenTelemetry implementation +00:35:06 Cultural changes in observability +00:39:36 Feedback on OpenTelemetry project +00:46:48 Audience Q&A session -**Ree:** Hello. Welcome everyone to a new episode of Hotel Me, formerly known as the end user Q&A. I'm very excited to be here. If you have been here before, I am Ree and I have Adriana here with me. I am coming to you live from Vancouver, Washington, not BC. Adriana, where are you coming from? +## Transcript + +### [00:00:00] Introductions + +**Ree:** Hello. Welcome everyone to a new episode of Hotel Me, formerly known as the end user Q&A. I'm very excited to be here. If you have been here before, I am Ree and I have Adriana here with me. I am coming to you live from Vancouver, Washington, not BC. Adriana, where are you coming from? **Adriana:** I am coming to you live from Toronto, Canada. -**Ree:** So, we're at opposite ends of the coast, which is pretty— +**Ree:** So, we're at opposite ends of the coast, which is pretty... + +**Adriana:** We are. + +### [00:01:47] Guest introduction: Cal and Jerome + +**Ree:** Yeah. And so, for those of you who are familiar with the flow, hang tight. We are going to bring our guests on pretty soon. For those of you who are new, welcome. So, how this is going to go is we are going to bring on our two special guests from an end user organization on shortly. We're going to have a quick introduction. They'll introduce us to their tech stack and how long they've been using open telemetry. This one's gonna be a really interesting one because they were actually one of the first guests we talked to when we started this segment back in 2022. So, I'm pretty excited to follow up with them and see, you know, how far they've come in their journey, what kind of struggles they've had along the way and what they've figured out. And then we'll get into some time for them to give us some feedback about the project. We should also have some time at the end for any audience members to ask questions. If you do have questions that come up as we go, feel free to pop them in the chat, whether you're on LinkedIn or YouTube, and we will get to them as appropriate during the conversation. If not, we will wait until the end. And with that, I believe we can bring our guests on. Oh, and Adriana, do you want to talk about our resources just a little bit? + +**Adriana:** Oh, yeah. So, our resources, I believe this will... I'm going to scan the QR code myself because I don't remember what this does. This will take us to open telemetry.io. And we have a page in the hotel site all about our end user SIG. If you want to learn more about the end user SIG, we welcome you to join. I believe there's a link on there also for joining our Slack channel on CNCF Slack. So, if you're already on CNCF Slack, please come find us. We welcome everyone. People come along to ask questions. We might not have all the answers, but we can definitely direct you to where you need to be. Oh, and also we would love to know where you are listening from. So feel free to put that in the chat as well so we can... we just like to know where people are listening from. + +**Ree:** And so the guests we have for you today are Jerome Johnson and Cal from Relativity. + +**Jerome:** Hello. + +**Cal:** Hi everyone. + +**Jerome:** Howdy howdy. + +**Ree:** Hello. We would love to learn about you and can you tell us a little bit about your role in the company and kind of what Relativity is focused on? + +**Cal:** You can go ahead, Cal. Alphabetical order and whatnot. + +**Jerome:** Oh, okay. All right. Excellent. Very democratic. + +**Cal:** Yeah. So, a little bit about Relativity. Relativity is a company that has a predominantly a SaaS product that provides services to the legal community. So, we automate essentially all the processes around legal e-discovery. So, exchange of documents and all of that. It is a SaaS platform. We have a lot of companies around the world that actually use it and we, as observability, are really tuned toward how our SaaS project is actually working at any particular point in time. As far as me, I started with Relativity about four, almost five years ago now. I was brought in basically to upgrade the way we do telemetry and as Ree mentioned, we were probably one of the first ones to have a chat with you folks but also one of the first adopters of open telemetry in production. So, we were very, very early adopters of open telemetry. I'm an architect here. Then I'll let Jerome introduce himself. + +**Jerome:** Hello. As Cal said, we're on the observability team. My name is Jerome. I've been at the company for about 10 years or so now. But I've only been on the observability team, specifically the one that's dealing with open telemetry for a little over a year now. So hopefully I can bring some perspective on how easy it is to use from the internal side and how nice it is. So looking forward to chatting it up with you guys. + +### [00:06:00] Discussion about Relativity's tech stack + +**Ree:** Cool. That's so exciting. Well, I think this is a great opportunity for us to start digging in. Can either of you or both of you give us some insights into your tech stack and architecture? + +**Cal:** Yeah, so I could talk about that a bit. Like I said, we were an early adopter of open telemetry. What we had before that was an entirely bespoke, I would say, system for collecting telemetry from a lot of different sources. In addition to that, we did use New Relic at the time, but we also had a lot of other vendors in there too. So, it was kind of a nightmare from an operational standpoint to actually do any troubleshooting and debugging because you had to go to lots of different sources to sort of put all that telemetry together in your head. At the time when I was hired, the company made a decision to try to unify everything. We looked around to see what would be the best platform for doing that. The one that checked all the boxes, even though it was really immature, was open telemetry. So that was the one which we pointed to do that. Now internally we have a very complicated platform. Maybe this is time to bring up the diagram on what we actually do now with open telemetry. We have 1500 plus different services running on our platform and that is a huge mix of things. We are predominantly a Windows and .NET shop. So most of that is actually Windows and .NET. But we have a mix of basically every other language in the world. So we have a very complicated tech stack and we need to pull in the telemetry from all those different things. Most of the complexity in our observability platform, and I'll probably mention at various points, so if I say RELI, that is our observability platform that's built over open telemetry. Most of the complexity is just being able to pull those signals in from lots of different sources, and you see most of them here. We really prefer that people use the open telemetry SDKs now and we push that as hard as we can. But we still have people using our old platform which is relatively APM, which we translate into open telemetry now. We have things like SNMP. We pull logging from lots and lots of different places and that's probably the part which is most difficult and most heterogeneous in terms of the platform itself. We have a whole fleet of open telemetry collectors that we run in 20 different regions around the world. Each one of those is load balanced. So we're running anywhere between, depending on the time of day and all that, between 200 and probably 800 individual collectors around the world to pull all this telemetry in. Then we send that to various data stores for operations. This is really New Relic at this point. That's where most of our engineers go to actually see what the telemetry is doing. We have a subset of people who use LaunchDarkly and use that for future releases. So we send some set of telemetry there. We actually archive all of our telemetry for compliance reasons for a year and for forensic analysis and stuff like that. We also have a reporting platform that uses the Azure blob storage basically as its back end to make that reporting for managers and things like that. This makes it look simpler than it is. It's a lot more gnarly than this diagram lets on. But we do have a large scale. I mean 20 plus regions around the world, 1500 services all feeding into this is a fairly complicated platform. -**Adriana:** We are. Yeah. +### [00:10:08] Why they adopted OpenTelemetry -[00:01:50] **Ree:** And so, for those of you who are familiar with the flow, hang tight. We are going to bring our guests on pretty soon. For those of you who are new, welcome. So, how this is going to go is we are going to bring on our two special guests from an end user organization shortly. We're going to have a quick introduction. They'll introduce us to their tech stack and how long they've been using open telemetry. This one's gonna be a really interesting one because they were actually one of the first guests we talked to when we started this segment back in 2022. So, I'm pretty excited to follow up with them and see how far they've come in their journey, what kind of struggles they've had along the way, and what they've figured out. And then we'll get into some time for them to give us some feedback about the project. We should also have some time at the end for any audience members to ask questions. If you do have questions that come up as we go, feel free to pop them in the chat, whether you're on LinkedIn or YouTube, and we will get to them as appropriate during the conversation. If not, we will wait until the end. And with that, I believe we can bring our guests on. Oh, and Adriana, do you want to talk about our resources just a little bit? +**Ree:** Dang, that was really cool actually. Questions. So many questions. I do actually have a lot of follow-up questions, but I also want to get into why open telemetry. When exactly, I know you mentioned, you know, kind of back in 2022 your team was early adopters. How did you come upon open telemetry and why did you decide like, oh this makes sense for us? -**Adriana:** Oh, yeah. So, our resources, I believe this will—I'm going to scan the QR code myself because I don't remember what this does. This will take us to—this takes us to open telemetry.io. We have a page in the hotel site all about our end user SIG. If you want to learn more about the end user SIG, we welcome you to join. I believe there's a link on there also for joining our Slack channel on CNCF Slack. So, if you're already on CNCF Slack, please come find us. We welcome everyone. People come along to ask questions. We might not have all the answers, but we can definitely direct you to where you need to be. Oh, and also, we would love to know where you are listening from. So feel free to put that in the chat as well so we can—we just like to know where people are listening from. +### [00:35:06] Cultural changes in observability -**Ree:** And so the guests we have for you today are Jerome Johnson and Cal—columnist from Relativity. +**Cal:** Yeah. So, you know, I think the thing which sort of ticked all the boxes for us... We came from a place where we were having to maintain our own observability libraries. Given the fact that we were .NET and then trying to diversify into lots of different languages meant that we were having to translate that thing again and again and again. On top of that, we also had... Well, we were using New Relic at the time and we had a lot of the New Relic agents in the mix as well. So, we had this weird mix of our own libraries for some stuff. We had a mix of things from New Relic with the agents and all of that kind of stuff. We had a lot of people who were writing to New Relic directly from their code as well as to other sources. One of the things which was really complicated on our side was every time we wanted to move something or we wanted to change something, it meant code changes. That's something we wanted to avoid going forward. We wanted to standardize on something which was ideally open source that we could include in our services one time and maintain that and then change where we put things but didn't have to change the code every time. We were really looking for something that would give us stability in terms of the codebase but allow us the flexibility to send things to other places as we needed to do that and open telemetry was the only one which ticked those boxes for us at the time. The only worry we really had at the point was that it was a new project. There was a lot of risk in going that direction. We decided in the end it wasn't an easy decision. We actually discussed this a long time internally but we decided that it was worth the risk to go ahead and adopt that because it looked like it was going to tick the boxes for what we wanted to do there. Basically, it's to isolate basically our code base from what we do with the telemetry afterwards, which was sort of the selling point for what we wanted to do there. -**Jerome:** Hello. +**Jerome:** Oh yeah, just to add on to that, I can give somewhat of a perspective to what it was like before we had open telemetry. I was serving on the performance team for a good number of years and it was very challenging to do analysis of specific applications just because, like Cal had mentioned, it was kind of the wild west on how we handled metrics. Performance metrics and monitoring are somewhat look similar, a little different, but enough to where every time we had to analyze a specific application, we would have to do a deep dive on a set application like what telemetry are you guys putting out? Oh, you're not putting out any at all. Then we'd have to dig into the code and that would really muck up how much time we could actually drill down into what the core problem of said application performance perspective was. Usually, it would take half the time we would do an analysis to try to understand what the product was and then actually coming up with a solution would be the latter 50 or 40%. Since joining observability, and even though Cal showed a simplified version of our RELI stack, and it's a bit more complicated once you drill down into it. I was amazed just how much cleaner it is to understand what Relativity is doing as a whole. -**Cal:** Hi everyone. Howdy howdy. +**Ree:** That's very cool. As a follow-up question, one of the things that we are always curious about in the SIG is collector setup. What kind of... you mentioned, I believe Cal mentioned that you guys manage a fleet of collectors. How do you manage that fleet of collectors? Can you give us an idea of how many collectors? Is there a particular setup that you use? We would love to know. -**Ree:** We would love to learn about you. Can you tell us a little bit about your role in the company and kind of what Relativity is focused on? +**Cal:** Yeah. So, you know, I think in terms of how things are usually deployed, I think you call it a gateway setup. All of our collectors are basically in a gateway setup. People send their telemetry to us, we process it and send it on. Over the years, we've had more or less complicated pipelines, but in general, we have essentially a single set of collectors with a common configuration that handles all of that telemetry. In terms of the numbers of things, we are running over 20 plus regions. Each region has somewhere between, depending on the load, between 10 and probably 50 to 100 instances themselves. So we're running at any point in time somewhere between 200 and 500 individual collectors. These are all running in Kubernetes. We are very focused on compute and Kubernetes. That's where almost all these things are running. In terms of the deployment itself, we actually handle all of our deployments via Helm charts. We use Helm charts basically to deploy these all out to the Kubernetes clusters. We do take pains to make sure that essentially all of our configurations are isolated. So our collectors, except in very few cases, do not reach out to external services. We manage the configurations all locally to make sure that they are as available and robust as possible. That's kind of how we do all of that. -**Cal:** You can go ahead, Jerome. Alphabetical order and whatnot. +**Jerome:** In terms of the pipelines themselves, one of the other reasons why we went with open telemetry is we have some internal attribute enrichments that we do specifically to our product and the way we use things. We actually implement those inside of the collector itself. We have some custom processors which enhance the data in various ways. We actually validate it. We actually drop data which contains things which shouldn't be there, for instance. We do a lot of validation on the fly as things go through the collectors themselves before we write it out to our various data stores. I think that hit most of the points you're asking for. Did I miss any of the things you're asking? -[00:04:10] **Jerome:** Oh, okay. All right. Excellent. Very democratic. Yeah. So, a little bit about Relativity. Relativity is a company that has predominantly a SaaS product that provides services to the legal community. So, we automate essentially all the processes around legal e-discovery, so the exchange of documents and all of that. It is a SaaS platform. We have a lot of companies around the world that actually use it, and we as observability are really tuned toward how our SaaS project is actually working at any particular point in time. As far as me, I started with Relativity about four—almost five years ago now. I was brought in basically to upgrade the way we do telemetry, and as Ree mentioned, we were one of the probably the first ones to have a chat with you folks but also one of the first adopters of open telemetry in production. So we were very, very early adopters of open telemetry. I'm an architect here, and then I'll let Jerome introduce himself. +**Ree:** I think you got most of them. And yeah, actually one follow-up question I had was, since you mentioned you deploy your collectors in Kubernetes via Helm charts, have you used the open telemetry operator to manage any of your collectors? Is that something you played with? -**Jerome:** Hello. As Cal said, we're on the observability team. My name is Jerome. I've been at the company for about 10 years or so now, but I've only been on the observability team, specifically the one that's dealing with open telemetry, for a little over a year now. So hopefully I can bring some perspective on how easy it is to use from an internal side and how nice it is—so looking forward to chatting it up with you guys. +**Cal:** We have not. The open telemetry operator sort of came out after we were already doing stuff. In fact, we don't, but not for any particularly good reason. -[00:06:00] **Ree:** Cool. That's so exciting. Well, I think this is a great opportunity for us to start digging in. Can either of you or both of you give us some insights into your tech stack and architecture? +**Jerome:** Okay. Fair enough. -**Cal:** Yeah, so I could talk about that a bit. Like I said, we were an early adopter of open telemetry. What we had before that was an entirely bespoke, I would say, system for collecting telemetry from a lot of different sources. In addition to that, we did use New Relic at the time, but we also had a lot of other vendors in there too. So, it was kind of a nightmare from an operational standpoint to actually do any troubleshooting and debugging because you had to go to lots of different sources to sort of put all that telemetry together in your head. At the time when I was hired, the company made sort of a decision to try to unify everything. We looked around to see what would be the best platform for doing that. The one that sort of checked all the boxes, even though it was really immature, was open telemetry. So that was the one which we pointed to to do that. Now internally we have a very complicated platform. Maybe this is time to bring up the diagram on what we actually do now with open telemetry. We have 1500 plus different services running on our platform, and that is a huge mix of things. We are predominantly a Windows and .NET shop, so most of that is actually Windows and .NET. But we have a mix of basically every other language in the world. So we have a very, very complicated tech stack and we need to pull in the telemetry from all those different things. Most of the complexity in our observability platform—and I'll probably mention at various points—if I say RELI, that is our observability platform that's built over open telemetry. Most of the complexity is just being able to pull those signals in from lots of different sources, and you see most of them here. We really prefer that people use the open telemetry SDKs now, and we push that as hard as we can. But we still have people using our old platform, which is relatively APM, which we translate into open telemetry now. We have things like SNMP. We pull logging from lots and lots of different places, and that's probably the part which is most difficult and most heterogeneous in terms of the platform itself. We have a whole fleet of open telemetry collectors that we run in 20 different regions around the world. Each one of those is load balanced. So we're running anywhere between, depending on the time of day and all that, between 200 and probably 800 individual collectors around the world to pull all this telemetry in. Then we send that to various data stores for operations. This is really New Relic at this point. So that's where most of our engineers go to actually see what the telemetry is doing. We have a subset of people who use LaunchDarkly and use that for future releases. So we send some set of telemetry there. We actually archive all of our telemetry for compliance reasons for a year and for forensic analysis and stuff like that. We also have a reporting platform that uses the Azure blob storage basically as its backend to make that reporting for managers and things like that. This makes it look simpler than it is. It's a lot more gnarly than this diagram lets on. But we do have a large scale—I mean, 20 plus regions around the world, 1500 services—all feeding into this is a fairly complicated platform. +### [00:18:14] Managing the fleet of collectors -[00:09:50] **Ree:** Dang, that was really cool actually. Questions. So many questions. I do actually have a lot of follow-up questions, but I also want to get into why open telemetry. How, like when exactly—I know you mentioned, you know, kind of back in 2022, your team was early adopters. How did you come upon open telemetry and like why did you decide, like, "Oh, this makes sense for us?" +**Cal:** It's that we had... consider it sort of tech debt. We had the Helm chart and all of that beforehand. So, we've just continued doing that and we're kind of used to managing things that way. We actually will have an internal shift in our deployment tooling coming up for the organization. So, that might be an opportunity to switch over how we're actually managing things. But at the moment, yeah, we're not... we don't have any experience with the open telemetry operator at the moment. -[00:12:30] **Cal:** Yeah. So, you know, I think the thing which sort of ticked all the boxes for us—we came from a place where we were having to maintain our own observability libraries, and given the fact that we were .NET and then trying to diversify into lots of different languages meant that we were having to translate that thing again and again and again. On top of that, we also had—well, we were using New Relic at the time and we had a lot of the New Relic agents in the mix as well. So we had this weird mix of our own libraries for some stuff. We had a mix of things from New Relic with the agents and all of that kind of stuff. We had a lot of people who were writing to New Relic directly from their code as well as to other sources. One of the things which was really complicated on our side was every time we wanted to move something or we wanted to change something, it meant code changes. That's something we wanted to avoid going forward. So we wanted to standardize on something which was ideally open source that we could include in our services one time and maintain that and then change where we put things but didn't have to change the code every time. So we were really looking for, you know, something that would give us stability in terms of the codebase but allow us the flexibility to send things to other places as we needed to do that. Open telemetry was the only one which sort of ticked those boxes for us at the time. The only worry we really had at that point was that it was a new project. There was a lot of risk in going that direction. We decided in the end—it wasn't an easy decision. We actually discussed this a long time internally, but we decided that it was worth the risk to go ahead and adopt that because it looked like it was going to tick the boxes for what we wanted to do there. Basically, it's to isolate basically our codebase from what we do with the telemetry afterwards, which was sort of the selling point for what we wanted to do there. +**Ree:** Got it. And another similar question. Since you do manage a fleet of collectors, have you played around with the opamp protocol for doing that? -**Jerome:** Oh yeah, just to add on to that, I can give somewhat of a perspective to what it was like before we had open telemetry. I was serving on the performance team for a good number of years, and it was very challenging to do analysis of specific applications just because, like Cal had mentioned, it was kind of the wild west on how we handled metrics. Performance metrics and monitoring are somewhat look similar, a little different, but enough to where every time we had to analyze a specific application, we would have to do a deep dive on that application. Like, "What telemetry are you guys putting out?" "Oh, you're not putting out any at all." Then we'd have to dig into the code, and that would really muck up how much time we could actually drill down into what the core problem of said application from a performance perspective was. Usually, it would take half the time we would do an analysis. It would be like trying to understand what the product was and then actually coming up with a solution would be the latter 50 or 40%. So since joining observability, even though Cal showed a simplified version of our RELI stack—and like you said, it's a bit more complicated once you drill down into it—I was amazed at just how much cleaner it is to understand what our Relativity is doing as a whole. Take that with what you want. +**Cal:** We've not again. Again, we've tried very hard to make our fleet essentially standalone. So even for configurations and things like that, we don't actually reach out to anything else. We really deployed sort of static config maps for that. That was basically a choice that we made just because we have such a large deployment and because it's distributed so widely, we wanted to make sure we were as resilient against networking problems as possible. So even for things like configuration changes, we push out new configs for those things rather than doing something that... -**Ree:** That's very cool. Now as a follow-up question, one of the things that we are always curious about in the SIG is collector setup. What kind of—you mentioned, I believe, Cal mentioned that you guys manage a fleet of collectors. How do you manage that fleet of collectors? Can you give us an idea of how many collectors? Is there a particular setup that you use? We would love to know. +**Jerome:** Got it. -[00:15:00] **Cal:** Yeah. So, you know, I think in terms of how things are usually deployed, I think you call it a gateway setup. All of our collectors are basically in a gateway setup. People send their telemetry to us, we process it, and send it on. Over the years, we've had more or less complicated pipelines, but in general, we have essentially a single set of collectors with a common configuration that handles all of that telemetry. In terms of the numbers of things, we are running over 20 plus regions. Each region has somewhere between, depending on the load, between 10 and probably 50 to 100 instances themselves. So we're running at any point in time somewhere between 200 and 500 individual collectors. These are all running in Kubernetes. We are very focused on compute and Kubernetes. So that's where almost all these things are running. In terms of the deployment itself, we actually handle all of our deployments via Helm charts. We use Helm charts basically to deploy these all out to the Kubernetes clusters. We do take pains to make sure that essentially all of our configurations are isolated. Our collectors, except in a very few cases, do not reach out to external services. We manage the configurations all locally to make sure that they are as available and robust as possible. So that's kind of how we do all of that. In terms of the pipelines themselves, one of the other reasons why we went with open telemetry is we have some internal attribute enrichments that we do specifically to our product and the way we use things. We actually implement those inside of the collector itself. So we have some custom processors which enhance the data in various ways. We actually validate it. We actually drop data which is not—contains things which shouldn't be there, for instance. So we do a lot of validation on the fly as things go through the collectors themselves before we write it out to our various data stores. I think I think that hit most of the points you're asking for. Did I miss any of the things you're asking? +**Cal:** It's not to say that's a bad idea. It's just it's a choice that we made. -**Ree:** I think you got most of them. And yeah, actually, one follow-up question I had was, since you mentioned you deploy your collectors in Kubernetes via Helm charts, have you used the open telemetry operator to manage any of your collectors? Is that something you played with? +**Ree:** Of course. That makes sense. What about maintaining the collector pipeline? Who is there a dedicated team? Is that just whoever is on the observability team or how do you maintain the local configurations and the pipelines? -**Cal:** We have not. The open telemetry operator sort of came out after we were already doing stuff. In fact, we don't—but not for any particularly good reason. No. +**Cal:** Yeah, so we do have an observability team. We are split between Poland and here. Our other big engineering office is actually in Krakow, Poland. So we have two observability teams split between the two regions. Each one has I think around five people at the moment. But collectively we are responsible for maintaining those pipelines and that configuration. Almost, I would say almost entirely we are responsible for that. There are a few areas where we share responsibility with certain places where we pull in configurations. One of them is our Kubernetes team. They actually define part of our configuration for which metrics we actually pull into the global system rather than just keeping it local to their Prometheus instances. But by and large, we are responsible for that entire configuration. We try to avoid shared responsibility there just because we want to make sure that shared responsibility always means no one's responsible. So we try to really avoid that. -**Ree:** Okay. Fair enough. +**Ree:** As you know, the project has matured, and it sounds like you've got a pretty good setup that is working well for you guys right now. How has the implementation evolved over the years, you know, as things have become GA? Have you added additional instrumentation signals? How has that looked as you've kind of grown alongside the project? -**Cal:** So, it's that we had—you can consider it sort of tech debt. You know, we had the Helm chart and all of that beforehand. So we've just continued doing that, and we're kind of used to managing things that way. We actually will have an internal shift in our deployment tooling coming up for the organization. So that might be an opportunity to switch over how we're actually managing things. But at the moment, yeah, we don't have any experience with the open telemetry operator at the moment. +**Cal:** Yeah. So, it's actually an interesting question in the sense that the thing that surprised me overall is that the deployment that we have hasn't really changed in terms of its architecture since the beginning. It's been the same architecture the entire time. Now, we've made tweaks here and there and we've simplified things, you know, as new features have come out and that kind of stuff, but by and large, it's stayed the same. And that I think is amazing given the fact that it was really sort of pre-alpha when we started. So we've gone through the alpha, beta, you know, and things are starting to become stable now and we've not really noticed any really heavy changes that we've needed to make in the platform. -**Ree:** Got it. Got it. And I have another similar question. Since you do manage a fleet of collectors, have you played around with the opamp protocol for doing that? +### [00:22:30] Observability team structure -**Cal:** We've not. Again, we’ve tried very hard to make our fleet essentially standalone. So even for configurations and things like that, we don't actually reach out to anything else. We really deployed sort of static config maps for that. That was basically a choice that we made just to—because we have such a large deployment and because it's distributed so widely, we wanted to make sure we were as resilient against networking problems as possible. So even for things like configuration changes, we push out new configs for those things rather than doing something that— +**Jerome:** Now that being said, I mean we have added new signals. So at the beginning, of course, it was metrics and traces that were there at the beginning. Logs came in afterwards. We had logs coming into our system even earlier than that. We had some of our own receivers to do some of that before it was an official signal. So we've sort of folded those changes into the platform as they come along and it hasn't really been terribly painful in doing that. As far as the community goes, the alpha, beta, stable kind of thing, we've not seen really any negative consequences from that. We've had to update on occasion. But by and large, those updates are pretty easy. -**Ree:** Got it. Got it. Do you have a—it's not to—not to say that's a bad idea. It's just it's a choice. It's a choice that we made. +**Jerome:** And I mean, I've been doing a lot of the following for the collector. We have our own build of the collector which, of course, we try to keep in sync with the external one and I've been doing quite a lot of that so you can probably give some feedback on how difficult that process is and where we've run into issues. -**Cal:** Of course. Of course. That makes sense. What about maintaining the collector pipeline? Is there a dedicated team? Is that just whoever is on the observability team, or how do you maintain the local configurations and the pipelines? +**Jerome:** Oh yeah, I'm sure. I mean, for the most part, and I don't know if this speaks to open telemetry or the automation we put around it, but as far as just updating the collectors, it's been a relatively painless process. I mean, sure, there have been a few changes that we've had to kind of team just file down on as a team and like, hey, what path do we want to go down? But as far as just pushing out those changes, they've been pretty simple, especially with how elaborate and detailed the white papers or the change logs have been out of open telemetry just explicitly telling, hey, these libraries or modules you've been using, we're introducing a breaking change or an API is going to be deprecated. But overall, it's been quite an easy experience and getting more people in our team to actually utilize that process has been fairly easy as well. -[00:20:00] **Jerome:** Yeah, so we do have an observability team. So we are split between Poland and here. Our other big engineering office is actually in Krakow, Poland. So we have two observability teams split between the two regions. Each one has I think around five people at the moment. But collectively we are responsible for maintaining those pipelines and that configuration. Almost, I would say almost entirely, we are responsible for that. There are a few areas where we share responsibility with certain places where we pull in configurations. One of them is our Kubernetes team. They actually define part of our configuration for which metrics we actually pull into the global system rather than just keeping it local to their Prometheus instances. But by and large, we are responsible for that entire configuration. We try to avoid shared responsibility there just because we want to make sure that, you know, shared responsibility always means no one's responsible. So we try to really avoid that. +**Ree:** I have a follow-up actually on building your collectors because I think out of all the people we've talked to, I think you guys might be the ones that have built your own collector, which makes me very excited. Did you... in terms of building your own collector, what was your experience around that like with as far as the documentation provided on the open telemetry site? Because that's another thing that we want to hear from practitioners of OTEL is how usable is the documentation and have you noticed an improvement in the documentation? So even building your own collector, how useful was that documentation? Did you have to do a little extra Sherlock Holmes-ing to figure stuff out? And also, were you able to build a nice streamlined process for building the collector that... like did you, I guess, have to go outside of the confines of what was already documented to do that? -**Ree:** So, as you know, the project has matured, and it sounds like you've got a pretty good setup that works well for you guys right now. How has the implementation evolved over the years, you know, as things have become GA? Have you added additional instrumentation signals? How has that looked as you've kind of grown alongside the project? +**Cal:** Yeah. So I'm not sure I'm going to be able to provide necessarily documentation feedback on the documentation, but the process as a whole... I mean, one of the things we started with was we had to start with our own build of the open telemetry collector mainly because we do have custom receivers, we have custom processors, we have custom exporters as well. We kind of needed that at the beginning to cover some of our use cases, so we were forced basically at the beginning to build our own collector, right? At the beginning, it was very painful. It was basically us cloning the entire repository and then making the changes we needed to build it. Once the collector builder came along, that made that whole process so much easier and it was a great addition to that process. Right now, basically, we have moved to a situation where we have our own repository. Basically, we have the single config file for the collector builder that just tells us what we pull in from contrib and from the main open telemetry collector, and the only thing we have in our repo at this point is our own custom stuff. Once the builder was there, everything since then has been really, really easy and smooth to build our own stuff. We did that right at the beginning, so before there was documentation. So, it's worked since then. I can't say whether the current documentation is good there or not, but we use the builder and it works seamlessly for us. -[00:21:50] **Cal:** Yeah. So, yeah, it's actually an interesting question in the sense that the thing that surprised me overall is that the deployment that we have hasn't really changed in terms of its architecture since the beginning. It's been the same architecture the entire time. Now, we've made tweaks here and there, and we've simplified things, you know, as new features have come out and that kind of stuff, but by and large, it's stayed the same. I think that is amazing given the fact that it was really sort of pre-alpha when we started. So we've gone through, you know, we sort of gone through the alpha, beta, you know, and things are starting to become stable now, and we've not really noticed any really heavy changes that we've needed to make in the platform. Now, that being said, I mean, we have added new signals. So at the beginning, of course, it was metrics and traces that were there at the beginning. Logs came in afterwards. We had logs coming into our system even earlier than that, so we had some of our own receivers to do some of that before it was an official signal. We've sort of folded those changes into the platform as they come along, and it hasn't really been terribly painful in doing that. As far as the community goes, the alpha, beta, stable kind of thing, we've not seen really any negative consequences from that. We've had to update on occasion, but by and large, those updates are pretty easy. Jerome has been doing a lot of the following for the collectors. We have our own build of the collector, which of course we try to keep in sync with the external one, and Jerome has been doing quite a lot of that, so you can probably give some feedback on how difficult that process is and where we've run into issues. +**Jerome:** And just as a follow-up, because you mentioned you built your own components, how was that experience of building your own components? Was that tricky? What was the... because I would say personally it feels a little bit lacking in the open telemetry documentation for building your own receivers, processors, exporters kind of thing. How was that for you? -**Jerome:** Oh yeah, I'm sure. I mean, for the most part, I don't know if this speaks to open telemetry or the automation we put around it, but as far as just updating the collectors, it's been a relatively painless process. I mean, sure, there have been a few changes that we've had to team up and just file down on as a team and like, "Hey, what path do we want to go down?" But as far as just like pushing out those changes, they've been pretty simple, especially with how elaborate and detailed the white papers or the change logs have been out of open telemetry, just explicitly telling, "Hey, these libraries or modules you've been using—here's an introduction to breaking changes or an API that's going to be deprecated." But overall, it's been quite an easy experience, and getting more people on our team to actually utilize that process has been fairly easy as well. +**Cal:** Yeah. I think in terms of the team itself, I think the biggest hurdle there was learning Go. We weren't... so I think from the team standpoint that was sort of the biggest initial obstacle for that. All of us I think have some good experience with Go now so I don't think that's an obstacle anymore. In terms of how to do it, we again were doing this very early. So it came at a time when it was basically, well, let's take one which we know works, copy it over and then make the changes we need to do to make our own stuff, right? Since then, the documentation has gotten much better and it's much easier to start with those things and get them done. But you know, I must say the process of just copying something that works and moving it over and making the changes you need, that also worked pretty easily as well. For an experienced programmer, I don't think it's a huge barrier to do those things. Now, there are some things which I would love to have better documentation on, like in terms of internal telemetry, how to hook in better to the internal telemetry and that kind of stuff. We sort of worked it out by reverse engineering what was there, but some of the things like that would be much more helpful if they were better documented, for instance. -**Ree:** I have a follow-up actually on building your collectors because I think out of all the people we've talked to, I think you guys might be the ones that have built your own collector, which makes me very excited. In terms of building your own—did you have—what was your experience around that, like with as far as the documentation provided on the open telemetry site? Because that's another thing that we want to hear from practitioners of OTEL is how usable is the documentation, and have you noticed an improvement in the documentation? Even building your own collector, how useful was that documentation? Did you have to do a little extra Sherlock Holmesing to figure stuff out? Also, were you able to build a nice streamlined process for building the collector that—did you have to go outside of the confines of what was already documented to do that? +**Ree:** Oh, I was just going to say yes, I remember that when we were catching up a bit yesterday and we definitely want to get into more of the feedback that you have for the project. I know you talked a little bit about the issue with conventions for internal data schemas. So if you want to go into a little bit more about that and anything else that you would like to see improvements in or feature requests, things like that. -**Cal:** Yeah. So, I'm not sure I'm going to be able to provide necessarily documentation feedback on the documentation, but the process as a whole—I mean, one of the things we started with was we had to start with our own build of the open telemetry collector mainly because we do have custom receivers, we have custom processors, we have custom exporters as well, and we kind of needed that at the beginning to cover some of our use cases. So we were forced basically at the beginning to build our own collector, right? At the beginning, it was very painful. It was basically us cloning the entire repository and then making the changes we needed to build it. Once the collector builder came along, that made that whole process so much easier, and it was a great addition to that process. Right now, basically we have moved to a situation where we have our own repository, and basically we have the single config file for the collector builder that just tells us what we pull in from contrib and from the main open telemetry collector, and the only thing we have in our repo at this point is our own custom stuff. Once the builder was there, everything since then has been really, really easy and smooth to build our own stuff. We did that right at the beginning, so before there was documentation, so it's worked since then. I can't say whether the current documentation is good there or not, but we use the builder and it works seamlessly for us. +### [00:29:32] Evolution of OpenTelemetry implementation -**Jerome:** And just as a follow-up because you mentioned you built your own components. How was that experience of building your own components? Was that tricky? What was the—because that’s definitely something that I would say personally feels a little bit lacking in the open telemetry documentation for building your own receivers, processors, exporters, that kind of thing. How was that for you? +**Cal:** Yeah. So, first let me preface this by, you know, I think probably from what I've said already, I am 100% in with open telemetry and I think it's a great platform and what the maintainers are doing is fantastic both on the documentation level and the code level, right? So that's all great. And this isn't feedback necessarily in terms of the actual collector implementations or open telemetry project as a whole, but I think it's sort of feedback on I think where we're going in our own journey and probably that has some bearing on other people as well. As we're generating more and more telemetry and it's becoming more and more useful for people, we have a lot of downstream consumers now. So we have an entire AI division who's now looking at our telemetry. We have people managers who are trying to do reporting off of this and getting standard data is becoming more and more important. One of the things we built initially and that one of the reasons why we had to have a custom processor is we do have standards for the attributes which come into the system. So we enforce a sort of a global schema on everyone even though that's extensible. Now how that fits in with the semantic conventions, how that fits in with the elastic common schema, in general how this fits in with sort of data contracts and how you have contracts between the producers of the data and the consumers of it. All those things are things we're thinking very hard about right now and we don't have good ideas on how to apply that across the board and good ways and the level at which we need to do that. I think discussions about the semantic conventions and the ECS and how to do things like data contracts with open telemetry data I think is going to become more and more important. I see that in our own organization and it'd be good to come up with common themes there and common tooling and common ideas. A little more convergence I think within the community there would go a long way. -**Cal:** Yeah. So I think in terms of the team itself, I think the biggest hurdle there was learning Go. We weren’t—so I think from the team standpoint that was sort of the biggest initial obstacle for that. All of us, I think, have some good experience with Go now, so I don't think that's an obstacle anymore. In terms of how to do it, we again were doing this very early. So it came at a time when it was basically, "Well, let's take one which we know works, copy it over, and then make the changes you need to make our own stuff." Since then, the documentation has gotten much better, and it's much easier to sort of start with those things and get them done. But you know, I must say the process of just copying something that works and moving it over and making the changes you need also worked pretty easily as well. For an experienced programmer, I don't think it's a huge barrier to do those things. Now, there are some things which I would love to have better documentation on, like in terms of internal telemetry—how to hook in better to the internal telemetry and that kind of stuff. We sort of worked it out, you know, by reverse engineering what was there, but some of the things like that would be much more helpful if they were better documented, for instance. +**Jerome:** In terms of other feedback the tech stack, I think in general is really solid and the way things are managed, I think is great. So I don't have a whole lot of feedback there. -**Ree:** Thanks. Oh, I was just going to say, yes, I remember that when we were catching up a bit yesterday, and we definitely want to get into more of the feedback that you have for the project. I know you talked a little bit about the issue with conventions for internal data schemas. So if you want to go into a little bit more about that and anything else that you would like to see improvements in or feature requests, things like that. +**Ree:** Yeah, so I think it's mainly looking forward and seeing how we can sort of use the technological base that we have and come up with better ways of standardizing the data which is produced, making it more useful for downstream consumers I think is the main thing I would like to see discussed more. -[00:30:00] **Cal:** Yeah. So, first let me preface this by saying I think probably from what I've said already, I am 100% in with open telemetry, and I think it's a great platform. What the maintainers are doing is fantastic, both on the documentation level and the code level, right? So that's all great. This isn't feedback necessarily in terms of the actual collector implementations or open telemetry project as a whole, but I think it's sort of feedback on I think where we're going in our own journey, and probably that has some bearing on other people as well. As we're generating more and more telemetry and it's becoming more and more useful for people, we have a lot of downstream consumers now. So we have an entire AI division who's now looking at our telemetry. We have people managers who are trying to do reporting off of this, and getting standard data is becoming more and more important. One of the reasons we had to have a custom processor is we do have standards for the attributes which come into the system. So we enforce a sort of a global schema on everyone, even though that's extensible. Now how that fits in with the semantic conventions, how that fits in with the Elastic Common Schema, in general, how this fits in with sort of data contracts and how you have contracts between the producers of the data and the consumers of it. All those things are things we're thinking very hard about right now, and we don't have good ideas on how to apply that across the board and good ways and the level at which we need to do that. I think discussions about the semantic conventions and the ECS and how to do things like data contracts with open telemetry data I think is going to become more and more important. I see that in our own organization, and it would be good to come up with sort of common themes there and common tooling and common ideas. A little more convergence, I think, within the community there I think would go a long way. In terms of other feedback, the tech stack, I think in general, is really, really solid, and the way things are managed I think is great. So I don't have a whole lot of feedback there. +**Adriana:** Right on, thank you. And I think we have a bit of time to catch up on some audience questions. So let's see. One of them is from Doug on YouTube and Doug wants to know, with regards to collectors, their dream is that cloud providers will offer as a boring text similar to an SMTP or SFTP. Is Doug crazy to want that? -**Ree:** Yeah, so I think it's mainly looking forward and seeing how we can sort of use the technological base that we have and come up with better ways of standardizing the data which is produced, making it more useful for downstream consumers, I think is the main thing I would like to see discussed more. +**Cal:** We... the first question we ask any vendor now is do they support OTLP? So, you know, we treat that as our standard protocol and it's a negative check mark against someone if they're not supporting it already. We're pushing all of our vendors and various tech products and whatever to support that as much as possible. I don't think that that's a pipe dream. I think pushing people in that direction is a good thing. As I said, we are very heavily a Microsoft shop, so we use Azure very much and the feedback we've consistently given them is we want Azure monitor and those types of things to produce open telemetry signals because we don't want to have to do that translation ourselves. -**Adriana:** Right on. Thank you, and I think we have a bit of time to catch up on some audience questions. So let's see, one of them is from Doug. Doug on YouTube wants to know, with regards to collectors, their dream is that cloud providers will offer as a boring text similar to an SMTP or SFTP. Is Doug crazy to want that? +**Adriana:** Thank you. And Manas, I see you have a question about OPM tutorials. We will respond directly with some resources for you in the LinkedIn chat. In the meantime, there was another question. Did you ever feel that there were classes of telemetry data that open telemetry didn't collect well and you needed to add another collection method such as eBPF? -**Cal:** We—the first question we ask any vendor now is do they support OTLP? So, you know, we treat that as our standard protocol, and it's a negative check mark against someone if they're not supporting it already. So we're pushing all of our vendors and various tech products and whatever to support that as much as possible. I don't think that that's a pipe dream. I think pushing people in that direction is a good thing. As I said, we are very heavily a Microsoft shop, so we use Azure very much, and the feedback we've consistently given them is we want Azure Monitor and those types of things to produce open telemetry signals because we don't want to have to do that translation ourselves. +**Cal:** We've had to come up with some custom receivers and things like that that weren't covered beforehand. One of the early things we did was there was no way to sort of provide a web hook into open telemetry at the beginning. There is now. So that's one of our legacy receivers that we've had. That was something that was not difficult to write but something that was sort of lacking. In terms of other classes of telemetry, I would say that one of the biggest gaps we've had internally as an organization is front-end telemetry. That's something that we're actually closing now. We actually have a project currently going on basically to collect more of our front-end telemetry with the open telemetry SDK, the JavaScript one. So that was something that we internally found difficult to do for many reasons and we're now sort of closing that gap. In terms of other types of telemetry, that's been difficult. I don't think we've had any real issues. Most of the things that we've had to collect if it's not OTLP or things like Splunk, Hack, or Fluent Bit and things like that where we've been able to get things into our collectors fairly easily. -**Adriana:** Thank you. And Manas, I see you have a question about OPM tutorials. We will respond directly with some resources for you in the LinkedIn chat. In the meantime, there was another question. Did you ever feel that there were classes of telemetry data that open telemetry didn't collect well and you needed to add another collection method, such as eBPF? +**Adriana:** Perfect. Thank you. And I think our time is just about up. Were there any last... well, not last parting words? -**Cal:** We've had to come up with some custom receivers and things like that that weren't covered beforehand. One of the early things we did was there was no way to sort of provide a web hook into open telemetry at the beginning. There is now. So that's one of our legacy receivers that we've had, and that was something that was not difficult to write but something that was sort of lacking. In terms of other classes of telemetry, I would say that one of the biggest gaps we've had internally as an organization is front-end telemetry. That's something that we're actually closing now. We actually have a project currently going on basically to collect more of our front-end telemetry with the open telemetry SDK—the JavaScript one. So that was something that we internally found difficult to do for many reasons, and we're now sort of closing that gap. In terms of other types of telemetry that's been difficult, I don't think we've had any real issues. Most of the things that we've had to collect, if it's not OTLP or things like Splunk, Hack, or Fluent Bit and things like that, we've been able to get things into our collectors fairly easily. +**Cal:** No, just thanks for inviting us. I think is the last thing from our side. If anyone wants to reach out or whatever, I am in the Slack so feel free to reach out to me directly if you have questions or whatever. Happy to discuss what we've done in detail if it's helpful. -**Adriana:** Perfect. Thank you. And I think our time is just about up. Were there any last—well, not last parting words? +**Adriana:** Thank you so much for offering that and thank you so much Cal and Jerome for being on and sharing your journey with us. We will link to our hotel SIG end user channel as well, so you can find all of us in there. You can also reach out to Cal directly via the CNCF Slack as well. And so we do have a link, Manas, for you that we'll put up here in a second that links to OTE documentation. -**Cal:** No, just thanks. Thanks for inviting us, I think is the last thing from our side. If anyone wants to reach out or whatever, I am in the Slack, so feel free to reach out to me directly if you have questions or whatever. Happy to discuss what we've done in detail if it's helpful. +**Ree:** Oh, well, we might have time for one more quick question. This is kind of... what do you think about auto instrumentation? -**Ree:** Thank you so much for offering that, and thank you so much, Cal and Jerome, for being on and sharing your journey with us. We will link to our hotel SIG and user channel as well, so you can find all of us in there. You can also reach out to Cal directly via the CNCF Slack as well. And so we do have a link, Manas, for you that we'll put up here in a second that links to OPE documentation, and—oh, well, we might have time for one more quick question. This is kind of—what do you think about auto instrumentation? +**Cal:** Me personally, I think it's a great thing. We use it where we can. In fact, for the front-end stuff, we are planning to use some of that auto instrumentation there. The downside we've seen, we have tried to use it and we do use it on the .NET side in the .NET SDK for HTTP requests and things like that. The problem we've run into there is volume. It tends to be difficult to sort of scale down to exactly which ones you want. So the auto instrumentation tends to be overly broad sometimes, but that's the only sort of downside we've seen with it. In general, we think it's a great thing and if it works for you, then yeah, definitely use it. -**Cal:** Me personally, I think it's a great thing. We use it where we can, and in fact, for the front-end stuff, we are planning to use some of that auto instrumentation there. The downside we've seen—we have tried to use it, and we do use it on the .NET side in the .NET SDK for HTTP requests and things like that. The problem we've run into there is volume. So it tends to be difficult to sort of scale down to exactly which ones you want. The auto instrumentation tends to be overly broad sometimes, but that's the only sort of downside we've seen with it. So in general, we think it's a great thing, and if it works for you, then yeah, definitely use it. +### [00:46:48] Audience Q&A session -**Adriana:** Awesome. Thank you. Well, Ree and I would like to thank you again so much for being on here. This has been really great, and I'm sure I'll have myself and other people will have more questions to learn more about open telemetry implementation since you guys were early adopters. We do have a couple things we want to share real quick before we head out. Hotel Community Day is coming up on June 25th, I believe, and we are currently accepting talk proposals. So, if you would like to submit a talk, we definitely encourage you to submit a topic for open telemetry day. It's going to be in Colorado as part of Open Source in Denver. It's a collocated event of Open Source Summit North America in Denver, Colorado. +**Adriana:** Awesome. Thank you. Well, Adriana and I would like to thank you again so much for being on here. This has been really great and I'm sure I'll have myself and other people will have more questions to learn more about open telemetry's implementation since you guys were early adopters. We do have a couple things we want to share real quick before we head out. Hotel community day is coming up on June 25th, I believe, and we are currently accepting talk proposals. So, if you would like to submit a talk, we definitely encourage you to submit a topic for open telemetry day. It's going to be in Colorado as part of open source in Denver. It's a collocated event of Open Source Summit North America in Denver, Colorado. -**Ree:** Yes. So if you want to come out—well, I guess that's too late to ski, but the beautiful mountains, I guess. +**Ree:** Yes. So if you want to come out, well, I guess that's too late to ski, but the beautiful mountains I guess. -**Adriana:** Yes. Also, if you are going to—or if you are already in Europe and you're planning to head out to KubeCon EU in London in two weeks—less than two weeks. +**Adriana:** Yes. Also, if you are going to or if you are already in Europe and you're planning to head out to KubeCon EU in London in two weeks, less than two weeks, we would love to see you there. Adriana and I will be speaking at the event and we have a blog post about the event that outlines all the open telemetry specific talks. The recordings, if you're not able to make it, will be available on the CNCF YouTube channel I think about one to two weeks after, depending. -**Ree:** Yes. +**Ree:** Yeah, they're usually pretty fast. Sometimes they get it within a couple of days. So just keep an eye out. -**Adriana:** We would love to see you there. Ree and I will be speaking at the event, and we have a blog post about the event that outlines all the open telemetry specific talks. The recordings, if you're not able to make it, will be available on the CNCF YouTube channel, I think about one to two weeks after, I think—depending. +**Adriana:** Yes. So if you can make it in person, you are definitely welcome to check out the recordings on YouTube. -**Ree:** Yeah, they're usually pretty fast. Sometimes they get it within a couple of days, so just keep an eye out. +**Ree:** And I believe that's it. Do you want to mention also if you are at KubeCon EU, we are going to be hanging out at the hotel observatory off and on. So if you're there, come say hi. The observatory is always lots of fun because there's usually tons of hotel contributors, maintainers, etc. hanging out. So it's great to have ad hoc conversations with folks. There's also going to be, if you're a KubeCon hotel project updates, so I strongly encourage folks to join that as well if they want to see what's new and exciting in hotel. That's always a fun session to attend. I think it's usually like half an hour that it's slated for. -**Adriana:** Yes, so if you can make it in person, you are definitely welcome to check out the recordings on YouTube. And I believe that's it. Do you want to mention also if you are at KubeCon EU, we are going to be hanging out at the hotel observatory off and on. So if you're there, come say hi. The observatory is always lots of fun because there's usually tons of hotel contributors, maintainers, etc. hanging out. It's great to have ad hoc conversations with folks. There's also going to be—if you're a KubeCon hotel project updates, I strongly encourage folks to join that as well if they want to see what's new and exciting in hotel. That's always a fun session to attend. I think it's usually like half an hour that it's slated for. +**Adriana:** And also if you made this recording but you have a friend who couldn't make it, you can replay on our LinkedIn. Just share this same link for the live recording with your friends or this will also be available on our hotel YouTube channel. So tell your friends. -**Ree:** And also, if you made this recording but you have a friend who couldn't make it, you can replay on our LinkedIn—just share the same link for the live recording with your friends, or this will also be available on our hotel YouTube channel. So tell your friends—officially tell your friends. +**Ree:** Officially tell your friends. -**Adriana:** Tell your friends. +**Adriana:** Tell your friends. -**Ree:** All right. Thank you all so much, and we will see you next time. Bye. +**Ree:** All right. Thank you all so much and we will see you next time. Bye. ## Raw YouTube Transcript diff --git a/video-transcripts/transcripts/2025-04-04T01:10:32Z-humans-of-otel-streaming-live-from-kubecon-with-austin-parker-marylia-gutierrez.md b/video-transcripts/transcripts/2025-04-04T01:10:32Z-humans-of-otel-streaming-live-from-kubecon-with-austin-parker-marylia-gutierrez.md index 38c8304..c00facd 100644 --- a/video-transcripts/transcripts/2025-04-04T01:10:32Z-humans-of-otel-streaming-live-from-kubecon-with-austin-parker-marylia-gutierrez.md +++ b/video-transcripts/transcripts/2025-04-04T01:10:32Z-humans-of-otel-streaming-live-from-kubecon-with-austin-parker-marylia-gutierrez.md @@ -10,308 +10,478 @@ URL: https://www.youtube.com/watch?v=EL0UkhvFAmY ## Summary -In this live stream of "Humans of OA," host Reese, a senior developer relations engineer at New Relic, is joined by colleagues Adriana Vila and Marilia, a staff engineer at Grafana Labs. The discussion centers around their experiences with OpenTelemetry, including Marilia's journey from an end user to a significant contributor. They explore the importance of contributor roles within the open-source community, the recent release of a database monitoring tool, and the significance of localization efforts to make technical content accessible in multiple languages. Additionally, they touch upon the evolving role of AI in observability and OpenTelemetry, emphasizing its potential to reduce toil in software development and improve documentation practices. The conversation highlights the collaborative nature of the tech community, ongoing projects, and upcoming events like the OpenTelemetry Community Day. +In this live stream of "Humans of OA," host Reese, a senior developer relations engineer at New Relic, along with colleagues Adriana and guest Marilia, a staff engineer at Grafana Labs, discuss various aspects of open telemetry and its community. They explore Marilia's journey from an end user to a contributor, her experience with observability at Cockroach Labs, and her current role in maintaining components within the open telemetry project. Key topics include the significance of semantic conventions for databases, the importance of localization in documentation, and the roles within the open source community, such as contributor, triager, approver, and maintainer. Additionally, they highlight the growth of the open telemetry community and its ongoing projects, including the evolution of logging infrastructure and the introduction of AI in enhancing observability. The discussion wraps up with a mention of upcoming community events, including the OpenTelemetry Community Day, and the vibrant atmosphere of the CubeCon EU. ## Chapters -00:00:00 Welcome and intro -00:00:30 Guest introduction: Marilia -00:02:00 Database monitoring talk -00:03:30 Semantic conventions discussion -00:05:11 Contributor roles overview -00:07:40 Localization importance -00:09:30 Localization SIG details -00:11:50 Contributor experience SIG -00:14:42 Guest introduction: Austin Parker -00:16:40 OpenTelemetry project updates +00:00:00 Introductions +00:05:50 Guest introduction: Marilia +00:06:58 Discussion about localization +00:12:40 Overview of contributor roles +00:19:00 Importance of semantic conventions +00:24:42 Updates on OpenTelemetry projects +00:30:24 Upcoming OpenTelemetry Community Day +00:32:18 Discussion about AI's impact on observability +00:34:34 Project updates and growth of OpenTelemetry +00:35:17 Closing remarks and acknowledgments + +## Transcript + +### [00:00:00] Introductions **Reese:** Hello, welcome to a live stream of humans of OA. We are coming to you live from what had a little bit of subtitle issues, but we are here and we're so excited. My name is Reese. I am a senior developer relations engineer at New Relic and I am joined here by my lovely industry colleague Adriana and Marilia who is our first guest. -**Adriana:** Hey everyone. Thanks for tuning in. My name is Adriana Vila. I work alongside Reese. +**Adriana:** Hey everyone, thanks for tuning in. My name is Adriana Vila. I work alongside Reese. -[00:00:30] **Marilia:** Hi everyone. My name is Marilia. I am a staff engineer at Grafana Labs and also in OpenTelemetry. I work in a few different groups. I am a maintainer for the contributor experience and I'm an approver for... +**Marilia:** Hi everyone. My name is Marilia. I am a staff engineer at Grafana Labs and also in open telemetry. I work in a few different groups. I am a maintainer for the contributor experience and I'm an approver for... -**Reese:** I was earlier about all the different parts that you're involved in and we're really curious because you started as an end user and then eventually started contributing. +**Reese:** I know you mentioned earlier about all the different parts that you're involved in and we're really curious because you started as an end user and then eventually started contributing. -**Marilia:** Yeah. And so we would love to... on my prior job I was responsible for the observability. I used to work at Cockroach Labs and I was the manager for the observability there. So I started learning about OpenTelemetry specifically because there we did would be able to use that. Maybe someone want to see some of those things outside of the database. So we created like some endpoints here and there, but it was like very basic stuff. +**Marilia:** Yeah. And so we would love to... On my prior job I was responsible for the observability. I used to work at Cockroach Labs and I was the manager for the observability there. So I started learning about hotel specifically because there we would be able to use that like maybe someone want to see some of those things outside of the database. So we created some endpoints here and there but it was very basic stuff. -**Reese:** So this team is to work only on OpenTelemetry. So your focus is just OpenTelemetry. +**Reese:** So this team is to work only on open telemetry. So your focus is just open telemetry. **Marilia:** So this is my day-to-day. I can have fun and just be on it. -**Reese:** That's awesome. I didn't realize that. That's so cool. And what would that new like Java and .NET and I want to make sure that all SDKs are on the same level and the other two were a little here. What are the things that we can add? +**Reese:** That's awesome. I didn't realize that. That's so cool. And what would that be like with Java and .NET? I want to make sure that all SDKs are on the same level. + +**Marilia:** The other two were a little... Here, what are the things that we can add? So we have an owner for the PostgreSQL plugin that I was already touching. That's awesome. -**Marilia:** So we have owner for the Postgres plugin that I was already like touching and there. That's awesome. And actually you gave a talk yesterday about database monitoring. +**Reese:** And actually you gave a talk yesterday about database monitoring. -[00:02:00] **Marilia:** Exactly. So one of the other big ones. So fresh news. We just this week released the release candidate two. So that means by the end of the month we're going to mark it as stable. So people can really start using it without any concerns. +**Marilia:** Exactly. So one of the other big ones. So fresh news, we just this week released the... Was it the release candidate two? So that means by the end of the month we're going to mark it as stable. So people can really start using it without any concerns. **Reese:** You mentioned that you're in the semantic convention SIG for databases and there's like a general semantic convention SIG, correct? What is it about the database semantic convention SIG that spun off? -**Marilia:** Yeah. So usually you have the general semantic convention and on that one you have sometimes people from all of the other semantics that can give updates or discuss this particular name of the metric. And spend an hour when you have a lot of people on the code they're not related to databases, so they don't have any input to give. It might be a waste of time. So it's kind of like you spin up like different semantic conventions for each of like, oh we want to focus on database, we want to focus on HTTP, we want to focus on RPC. So now you can bring people that have experience in that area and talk about it. The same way you can think about if you would have a SIG for SDK, you have so many languages. So we have now one SIG for each language. So it's kind of like the same idea. +**Marilia:** Yeah. So usually you have the general semantic conversion and on that one you have sometimes people from all of the other semantics that can give updates or discuss this particular name of the metric and spend an hour when you have a lot of people on the code that are not related to databases, so they don't have any input to give. It might be a waste of time. So it's kind of like you spin up different semantic conventions for each of like... Oh, we want to focus on database, we want to focus on the HTTP, we want to focus on RPC. So now you can bring people that have experience in that area and talk about it. + +**Reese:** The same way you can think about if you would have a SIG for SDK, you have so many languages. So we have now one SIG for each language. So it's kind of like the same idea. + +**Marilia:** Yeah. Help keep it focused and easy to prioritize. And then we can always have the updates that we can bring to the main semantic one because when they generate a version, it's for all semantics. It's not as specific for the database. So you just merge that repo and whenever a new version comes in, it might be bringing updates for the database, might be from the HTTP, and so on. So we just can bring the update to the main ones like, "Hey, this one we're marking as stable." We have these updates and things like that. + +**Reese:** And then you mentioned, and I'm glad you were able to rattle off the long list of titles that you have in the community because I was like, "I am not going to remember all these," but they're so cool. -**Reese:** Help like keep it like the work focused and easy to prioritize. +**Marilia:** Um, so there's like different titles. There's an approver, then a maintainer. Okay, perfect. And then could you explain to us briefly what each... What's the distinction between each role? -[00:03:30] **Marilia:** Yeah. And then we can always have the updates that we can bring to the main semantic one because when they generate a version, it's for all semantics. It is not as specific for the database. So you just merge that repo and whenever a new version comes in, it might be bringing updates for the database, might be from the HTTP, and so on. So we just like can bring the update like to the main ones like, hey this one we're marking as stable. We have these updates and things like that. +**Marilia:** Yeah. So you started first as a contributor. So you can become a contributor not being a member of CNCF at all. So when you do a couple of contributions, you ask to become a member which is something I think everybody should do if you are doing because it's very easy to join. You just have to open a PR and say like, "Hey, I want to become an official member of CNCF and then OpenTelemetry." -**Reese:** And then you mentioned, and I'm glad you were able to rattle off the long list of titles that you have in the community because I was like I am not going to remember all these but they're so cool. So there's like different titles. There's approver, then approver, then maintainer. +**Reese:** Then you are officially a contributor. -**Marilia:** Okay perfect. And then could you explain to us briefly like what each, what's the distinction between each role? +**Marilia:** Then you pick something that is more interesting to you. So for example, I really like this language or I really like this specific component. Start joining the SIGs just to learn more about it, see how you can help. And when you start really helping out, start reviewing PRs, putting comments, creating issues for things that you mind and with time you get the status of triager, which is somebody that is helping when a lot of issues are coming up. You help exactly triage those issues and see things that make sense, things that actually they were not using correctly or there are actual bugs. -[00:05:11] **Marilia:** Yeah. So you started first as a contributor. So you can become a contributor not being a member of CNCF at all. So when you do a couple of contributions, you ask to become a member, which is something I think everybody should do if you are doing it because it's very easy to join. You just have to open a PR, say like, hey I want to become an official member of CNCF and then OpenTelemetry and then you are officially a contributor. Then you pick something that is more interesting to you. So for example, I really like this language or I really like this specific component. Start joining the SIGs just to learn more about it, see how you can help. And when you start really helping out, start reviewing PRs, putting comments, creating issues for things that you mind. And with time you get the status of triager, which is somebody that is helping when a lot of issues come up. You help exactly triage those issues and see like things that make sense, things that actually they were not using correctly or there are actual bugs. Then from that, you can become an approver. When you approve a PR, your approval actually counts because people can approve, but to be able to merge you need to have people with permission giving the approval and being able to merge. So the approver now has this, whatever we approve it counts and somebody who has the permission can just merge it in. And then you have the final one, which is maintainer. So these are the people that can actually merge, usually the ones creating the releases, creating like road maps, but they do get input from everybody that is contributing. +**Reese:** Then from that you can become an approver. + +**Marilia:** That when you approve a PR, your approval actually counts because people can approve but to be able to merge, you need to have people with permission giving the approval and being able to merge. So the approver now has this... Whatever we approve, it counts and somebody who has the permission can just merge it in. And then you have the final one which is maintainer. So these are the people that can actually merge. Usually, they are the ones creating the releases, creating roadmaps, but they do get input from everybody that is contributing. **Reese:** That's so exciting. That is really cool. Thanks for the overview. -**Marilia:** Yeah. The other thing that we wanted to ask, you know, you mentioned that you're working in the Portuguese localization. Talk a little bit about that, like talk about why it's important to have localization, how you got involved with that, specifically how that's been. +### [00:05:50] Guest introduction: Marilia + +**Marilia:** Yeah. Um, the other thing that we wanted to ask... You know, you mentioned that you're working in the Portuguese localization. Talk a little bit about that. Talk about why it's important to have localization, how you got involved with that specifically, how that's been. + +**Marilia:** Yeah. So I think it's really important because it's a barrier, like the language. And it's already hard for you to completely learn something completely new like on tech, but imagine if you don't even know the language. A lot of other countries, their first language is not English. It's not always you're going to find this documentation and it's not easily accessible for everybody. So actually, I started a while ago. I created my own blog post and I was like, "Everything that I'm creating there, I'm going to create it." And where people that actually met when I did like conference days in Brazil and there was the bad group. + +### [00:06:58] Discussion about localization + +**Marilia:** So I started like, "Oh, I'm going to help them out." I like to review a lot of things and the good thing is a lot of things in English that you learn, like just experience living, so I can help out with the translation, the localization, and also decisions like what are the things that you don't translate at all because we're going to say tracer. We don't translate that. That makes sense because at the same time, you don't want to translate everything because then when they're going to search those words, they don't exist anywhere. -**Marilia:** Yeah. So I think it's really important because it's a barrier, like the language. And it's already hard for you to like completely learn something completely new like on tech, but imagine if you don't even know the language. And a lot of other countries like their first language is not English. It's not always you're going to find this documentation and not easily accessible for everybody. So actually I started like a while ago. I created like my own blog post and I was like everything that I'm creating there, I'm going to create and where people that actually met when I did like conference days in Brazil and there was the bad group. So I started like, oh I'm going to help them out. So I like to like review a lot of things and the good things a lot of things in English that you learn like just experience living. So I can help out with like the translation, the localization, and also decisions like what are the things that you don't translate at all because we're going to say tracer. We don't, that makes sense because at the same time you don't want to translate everything because then when they're going to search those words don't exist anywhere. +**Reese:** But we have a lot of... -**Reese:** Yeah, but yeah, I think it's like really good to just break this barrier and bring a lot of people they would have no idea. +**Marilia:** Yeah, but yeah, I think it's really good to just break this barrier and bring a lot of people who would have no idea. -**Marilia:** Yeah, that's so great. That's a lot of work. I guess in some ways has it like improved your Portuguese as a result? +**Reese:** Yeah, that's so great. That's a lot of work. Um, I guess in some ways has it improved your Portuguese as a result? -[00:07:40] **Marilia:** So yeah, it's always funny because like living outside for so long and sometimes you, I'm going to give a talk like in Portuguese and I was like here so I know the right thing to use but it's always always a challenge. But we also try because there's not only Portuguese. We also have like people working on the Spanish, French, and Japanese. So do completely different than the French know. We also have this thing like which is like similar to the same. So we have the documentation one, but we have different groups. So every time a docs page comes out, you like the localization SIG translates. +**Marilia:** So, yeah, it's always funny because living outside for so long and sometimes I'm going to give a talk like in Portuguese and I was like here, so I know the right thing to use, but it's always a challenge. But we also try because there's not only Portuguese. We also have people working on the Spanish, French, and Japanese. -**Reese:** Yeah. So that is the idea. So right now that people are accessing more things like that. So we start with those ones and like the same thing for SDKs. I see which ones people are like downloading more. +**Reese:** So do completely different from the French, you know? -**Marilia:** So we do have something to... because part of the when you do a localization there is a new kind of like tag that you had to put in what was the commit that you translated. Because it's really hard for people to know that somebody did an update somewhere. You had to really be paying attention. So the idea is to have something that would tell us, hey those are behind just catch up and things like that. So we were always up to date. +**Marilia:** We also have this thing which is similar to the same. So we have the documentation one but we have different groups. So every time a docs page comes out, you like the localization, we translate. Yeah. So that is the idea. So right now that people are accessing more things like that. So we start with those ones and like the same thing for SDKs. I see which ones people are like downloading more. -**Reese:** Oh, sorry. Go ahead. +**Reese:** So we... -**Marilia:** I was curious because when I was asking you all your different roles, that was like the first time I'd heard of that specifically because I know we have translated some of our documentation, but I didn't realize there were specific teams working on these languages. +**Marilia:** Some come in and do updates on the original one. So we do have something to... Because part of when you do localization, there is a new kind of tag that you had to put in what was the commit that you translated. Okay? Because it's really hard for people to know that somebody did an update somewhere. You had to really be paying attention. So the idea is to have something that would tell us, "Hey, those are behind. Just catch up," and things like that. -**Marilia:** So the localization SIGs, they are only translating documentation or what all involves? +**Reese:** So we were always up to date. -[00:09:30] **Marilia:** So like the official page like the OpenTelemetry.io. All the pages there are the ones. If you look at the now at the top that didn't have anything. Now if you're looking at the page, the top you have a select there that you can change the language that you want. +**Marilia:** Oh, sorry. Go ahead. -**Reese:** So whenever the page is translated now you have... +**Reese:** I was curious because when I was asking you, you know, all your different... That was like the first time I'd heard of that specifically because I know we have like translated some of our documentation but I didn't realize there were like specific teams working on these languages. -**Marilia:** So cool, learned something new today. I then joke that I have a PR with the translation to the Japanese even though I don't speak any Japanese and sometimes it goes wrong there too. +**Marilia:** Um, so the localization SIGs, they are only translating documentation or what all involves? -**Reese:** Uh-huh. +**Marilia:** The official page like open telemetry.io, all the pages there, if you look at the top that didn't have anything, now if you're looking at the page at the top, you have a select there that you can change the language that you want. So whenever the page is translated, now you have... -**Marilia:** So it's a way to catch up. So like when they was like, "Okay, so the..." I was like, "Oh, let me see if any other languages translated the same thing wrong." Async or something. +**Reese:** So cool! Learned something new today. -**Reese:** Okay. +**Marilia:** I then joke that I have a PR with the translation to the Japanese even though I don't speak any Japanese and sometimes it goes wrong there too. -**Marilia:** Yeah, that's pretty big. +**Reese:** Uh-huh. -**Marilia:** And then it was good that it was just like a word and I translated and I put a PR and I tagged the reviewers for Japanese. I was like, I don't speak any Japanese. I put this thing that I think is the right translation. Like there was a part for the translation, but we even like looking for we want to add also more examples because we don't have a lot of examples. If there is something like for example you never did, so you try it yourself and you might for the future ones that are coming that they know like more examples, more things to try on. +**Marilia:** It's a way to catch up. + +**Reese:** So like... And then it was like, "Okay, so the..." I was like, "Oh, let me see if any other languages translated the same thing wrong." Async or something. + +**Marilia:** Okay. Yeah, that's pretty big. + +**Reese:** And then it was good that it was just like a word and I translated and I put a PR and I tagged the reviewers for Japanese. I was like, "I don't speak any Japanese. I put in this thing that I think is the right translation." Like there was a part for the translation, but we even like looking for... We want to add also more examples because we don't have a lot of examples. + +**Marilia:** If there is something, for example, you never did, so you try it yourself and you might for the future ones that are coming, that they know like more examples, more things to try out on. **Reese:** Yeah, man. This is so exciting. I'm learning all kinds of new things. -**Marilia:** I know. Yeah. +**Marilia:** I know. -**Marilia:** So yeah, so many things that we're like when people ask like how do I start, like so many of you're going to be able to find somewhere something that is interesting. There's always something for you, right? +**Reese:** Yeah, so many things that we're like when people ask like, "How do I start?" like so many of... You're going to be able to find somewhere something that is interesting. There's always something for you, right? -**Reese:** Yeah, it's really hard and just try joining just like listening. But at the same time, keep in mind that open source is different than a day-to-day job because sometimes what I see people find things was like how I found the things that like, oh, they need help with this, they need help with that. And then the things that I already have priorities like from even coming like from, hey can you take a look at this up for grabs? A few repos have them, so we have things like for example contribute fest that we did here and a lot of them got tagged with contribute fest, but not all of them got actually worked on. +**Marilia:** Yeah, it's really hard. Just try joining, just like listening. But at the same time, keep in mind that open source is different than a day-to-day job because sometimes what I see people find things was like, "How I... I found the things that like, oh, they need help with this, they need help with that," and then the things that I already have priorities like from even coming like, "Hey, can you take a look at this?" -**Marilia:** So those are the ones that you know and you're going to find something and people in the community are very helpful like tag them on Slack, on the issue itself, they are happy to give you guidance as well. +**Reese:** Up for grabs a few repos have them. -[00:11:50] **Reese:** Yeah, and I'll do a shameless plug for the end user SIG as well. We are always happy to have contributors. You can find more information in our show notes. Part of like the end user SIG. The other thing I want to ask is the contributor experience. So if you want to contribute like what is missing? So for example, one of the projects that I did that was so like how do I start? I don't know how to set up locally. There was no like what is the dependency that you need. So there was like nothing there. So I worked like as a mentor for the AI program template that all repos could use and started creating like the PR so people have at least the basic things that they need to just at least start having things. +**Marilia:** So we have things like for example contribute fest that we did here and a lot of them got tagged with contribute fest, but not all of them got actually worked on. So those are the ones that you know and you're going to find something and people in the community are very helpful like tag them on Slack, on the issue itself. They are happy to give you guidance as well. -**Marilia:** Oh, that's amazing. +**Reese:** Yeah, and I'll do a shameless plug for the end user SIG as well. We are always happy to have contributors. You can find more information in our show notes. -**Reese:** And really quickly, actually, I want to ask about the contributor experience SIG because that one's a relatively new SIG, right? +**Marilia:** Part of like the end user SIG. The other thing I want to ask is the contributor experience. So if you want to like, "I want to contribute, like what is missing?" So for example, one of the projects that I did that was so like, "How do I start? I don't know how to set up locally." There was no like, "What is the dependency that you need?" So there was like nothing there. -**Marilia:** Yes. +**Reese:** So I worked like as a mentor for the AI program template that all repos could use and started creating the PR so people have at least the basic things that they need to just at least start having things. -**Reese:** Can you tell us a little bit about that SIG, how it got started, and how you became a member? +**Marilia:** Oh, that's amazing. -**Marilia:** It was because we saw this need of people complaining that they don't know how to start. They don't always understand like the path for like triager or like how like there is nobody there to guide. Okay, if nobody is going to be there for you, are you able to find all the documentation that you need to be able to follow the path that you want? Or like we didn't even know the challenges people were having a few of the things and we do like also surveys to find out people like how are you feeling like being a contributor and we started tackle the things that were the most concerned. Like documentation was the top one, so this is why I started that project and container almost right away. +### [00:12:40] Overview of contributor roles + +**Reese:** And really quickly, actually, I want to ask about the contributor experience SIG because that one's a relatively new SIG, right? + +**Marilia:** Yes. Can you tell us a little bit about that SIG, how it got started, and how you became a member? + +**Marilia:** Was it because we saw this need of people complaining that they don't know how to start? They don't always understand like the path for like triager or like how... Like there is nobody there to guide. Okay, if nobody is going to be there for you, are you able to find all the documentation that you need to be able to follow the path that you want? Or like we didn't even know the challenges people were having a few of the things and we do like also surveys to find out people like how are you feeling like being a contributor and we started tackling the things like were the most concerned like documentation was the top one. So this is why I started that project and came in almost right away. **Reese:** What are some of the upcoming things for the contributor experience? -**Marilia:** So for this one, I just finished this part like this month for the template. So my next meeting is going to be actually picking up what is going to be the next one. But we do have like issues open that is just helping out like clarify things to people. And if anyone like also if you're listening, if you have something that is really like bothering you and you need, feel free to like message us or like open an issue directly on the repo there. We can prioritize because we have a list there, but we are just prioritizing ourselves because we don't have necessarily feedback things but react like thumbs up or really want this kind of thing and we will be able to put it in first. +**Marilia:** So for this one, I just finished this part, like this month for the template. So my next meeting is going to be actually picking up what is going to be the next one. But we do have like issues open that is just helping out like clarify things to people. And if anyone like also if you're listening, if you have something that is really like bothering you and you need, feel free to like message us or like open an issue directly on the repo there. We can't prioritize because we have a list there, but we are just prioritizing ourselves because we don't have necessarily feedback things, but react like thumbs up or really want this kind of thing and we will be able to put it in first. **Reese:** Awesome. And I think we're almost ready for our next guest. -[00:14:42] **Marilia:** Yeah, I think we're so I guess we will continue our conversation. I guess as we get ready to bring on our next guest experience, they will be able to share like people just joining and people that have a lot of experience will be able to guide you as well just as I join the calls. +**Marilia:** Yeah, I think we're... So I guess we will continue our conversation. I guess as we get ready to bring on our next guest experience, they will be able to share like people just joining and people that have a lot of experience will be able to guide you as well just as I join the calls and you're going to be able to... + +**Austin:** I'm just going to get the surprise out of the way. Uh, community manager for OpenTelemetry and we are so excited to have him on. + +**Reese:** How am I still community manager? -**Reese:** And they are Austin Parker. I'm just going to get the surprise out of the way. Community manager for OpenTelemetry and we are so excited to have him on. +**Austin:** Wait, are you no longer the community manager? What? You're still community manager? -**Austin:** How am I still community manager? +**Austin:** Yeah, I think I am. If you go and check the repo, I probably am. I mean, I do both jobs still. So, I'm a member of the OpenTelemetry governance committee. I also have been a maintainer on the OpenTelemetry website, uh, OpenTelemetry demo. I've been working in a working group, I guess you could say, like where we combine OpenCensus maintainers and OpenTracing maintainers where we, you know, came in and worked with some people. -**Reese:** Wait, are you no longer the community manager? +**Reese:** What have you seen as like the biggest changes since things started? -**Austin:** What? You're still community manager? +**Austin:** When we first came to KubeCon, um, zero day event, you know, ask people how many people are using OpenTelemetry and it's like every hand in the room is going up, right? Like the growth of... -**Reese:** Yeah, I think I am. If you go and check the repo, I probably am. I mean, I do both jobs still. So, I am a member of the OpenTelemetry governance committee. I also have been a maintainer on the OpenTelemetry website, OpenTelemetry demo. I've working group I guess you could say like where we combine OpenCensus maintainers and OpenTracing maintainers where we, you know, came in and worked with some people. +**Reese:** Yeah, which was 2022. -**Reese:** What have you seen as like the biggest changes since things started? +**Austin:** Um, the project updates room, you know, the project was given like a really small room. It was like pretty packed. Yeah. And over the years I've seen that room like get bigger and bigger. -**Austin:** Like when we first came to KubeCon, zero day event, you know, ask people who's how many people are using OpenTelemetry and it's like every hand in the room is going up, right? Like the growth of OpenTelemetry... +**Reese:** Yeah, I know they probably like a 200-seat room and there are still people standing in the back, right? -**Reese:** Which was 2022. +**Austin:** Um, there was also a lot of, you know, there was a keynote yesterday that was actually the thing that I've been most excited about the growth of OpenTelemetry isn't necessarily but the opportunities that we see other people kind of taking and running with, right? -[00:16:40] **Austin:** The project updates room, you know, the project was given like a really small room. It was like pretty packed. Yeah. And over the years I've seen that room like get bigger and bigger. +**Reese:** Um, projects like Percy’s new vendors, new commercial solutions built on top of OpenTelemetry have been really exciting to see. -**Reese:** Yeah. I know they probably like a 200 seat room and there are still people standing in the back, right? +**Austin:** Um, and I think, you know, that that's the sort of stuff that you... -**Austin:** There was also a lot of, you know, there was a keynote yesterday that was actually the thing that I've been most excited about. The growth of OpenTelemetry isn't OpenTelemetry necessarily, but the opportunities that we see other people kind of taking and running with, right? Projects like Percy’s, new vendors, new commercial solutions built on top of OpenTelemetry have been really exciting to see. +**Reese:** Yeah, definitely. -**Reese:** And I think, you know, that that's the sort of stuff that you... +**Austin:** Yeah. It's been exciting to see it grow. I remember even like my first KubeCon was Detroit and there was, uh, there was an open observability day and then there was an OpenTelemetry unplugged with KubeCon North America and EU, which I'm super stoked about. -**Austin:** Yeah. Yeah, definitely. Yeah. It's been exciting to see it grow. I remember even like my first KubeCon was Detroit and there was an Open Observability Day and then there was an OpenTelemetry unplugged with KubeCon North America and EU, which I'm super stoked about. +**Reese:** Um, what? -**Reese:** What super busy too. We have hundreds and hundreds of people showing up for those. +**Austin:** Super busy too. We have hundreds and hundreds of people showing up for those. -**Austin:** Yeah. It was great. +**Reese:** Yeah, it was great. -**Reese:** Yeah. Thank you. I mean, I feel like it's also one of those things where it's kind of like you can't really do three tracks, right? Like that's a bit like two is good. Seems like a good number, but there's observability talks, you know, throughout the week at KubeCon. I don't know. This used to be the Kubernetes club, right? This is the operations, the SRE kids. And now, you know, security has a big presence in the CNCF. +### [00:32:18] Discussion about AI's impact on observability -**Austin:** I think what we're seeing... +**Austin:** Yeah. I mean, I feel like it's also one of those things where it's kind of like you can't really do three tracks, right? Like that's a bit like two is good. Seems like a good number, but there's observability talks, you know, throughout the week at KubeCon. I don't know. This used to be the Kubernetes club, right? This is the operations, uh, the SRE kids. And now, you know, security has a big presence in the CNCF. -**Austin:** What I think is important is you in a lot of ways if you think about how a lot of technology that we still rely on today, you know, was built and maintained and came right. Maybe someone, maybe someone and if that's you, more power to you. +**Reese:** I think what we're seeing... -**Reese:** To America, we might need your help. +**Austin:** What I think is important is you in a lot of ways if you think about how a lot of technology that we still rely on today, you know, was built and maintained and came right, maybe someone... Maybe someone and if that's you, more power to you. Um, to America, we might need your help. -**Austin:** Things became the de facto standards through whatever, you know, combination of factors happened, right? Like I think what's cool about C4 vendor, you happen to use or so on and so forth. +**Austin:** Um, things became the de facto standards through whatever, you know, combination of factors happened, right? -**Reese:** Now the downside is, you know, in 25 years my kid's going to copy me and be like, "What the, what's this OpenTelemetry crap dad?" +**Reese:** Like I think what's cool about C4 vendor you happen to use or so on and so forth. -**Austin:** Don't metaverse OpenTelemetry now. +**Austin:** Now the downside is, you know, in 25 years my kid's going to copy me and be like, "What the what's this OpenTelemetry crap, dad?" -**Reese:** Oh man. +**Reese:** Don't metaverse OpenTelemetry now. -**Austin:** Also speaking of the future, yeah, we were curious about how do you think AI might impact OpenTelemetry? +**Austin:** Oh man, also speaking of the future, yeah, we were curious about how do you think AI might impact OpenTelemetry? How it might impact or have an impact or... -**Austin:** How it might, how it might impact or have an impact or... +### [00:19:00] Importance of semantic conventions -**Austin:** Yeah. There's a lot of ways. I think one thing I was, so it's funny 'cause I was just talking to someone about this. A project I've been working on kind of on the side is how to make better docs for LLMs because the documentation that we write for human beings is really good for human beings and LLMs are not human beings. They are something else. So you need to kind of give them documentation in a way that is similar to how you would give it to a human but distinct enough that it's not, you know, it's trickier than just saying like, oh go read this web page, right? +**Austin:** Yeah. Uh, there's a lot of ways. I think one thing I was... So, it's funny because I was just talking to someone about this. Um, a project I've been working on kind of on the side is how to make better docs for LLMs because the documentation that we write for human beings is really good for human beings and LLMs are not human beings. They are something else. -**Reese:** 'Cause the web page has a lot of stuff that's for humans. +**Reese:** So you need to kind of give them documentation in a way that is similar to how you would give it to a human but distinct enough that it's not, you know, it's trickier than just saying like, "Oh, go read this web page," right? -**Austin:** We care about things like font sizes and colors and we care a lot about sort of the organization of information and we want to have pages that are single concept, right? So that you can focus on like what is the goal of this page. But then LLMs, there's a distinction you need. You can, like one interesting thing if you think about writing a document, you know, writing documentation is you're not supposed to use, even if there's a more specific word, like use more words, not less, right? You have to kind of hit it at whatever reading level you expect people to be at, even for that kind of stuff. But with LLMs, actually, they don't have that disadvantage, right? Like you can give it a very large flowery word and if it's the most specific word, that's actually helpful because of the way the semantic search works. +**Austin:** Because the web page has a lot of stuff that's for humans. We care about things like font sizes and colors and we care a lot about sort of the organization of information and we want to have pages that are single concept, right? So that you can focus on like what is the goal of this page and but then LLMs, there's a distinction you need. -**Reese:** Precision in language is actually really important. +**Austin:** Um, you can... One interesting thing if you think about writing a document, you know, writing documentation is you're not supposed to use um, even if there's a more specific word like use more words not less, right? You have to kind of hit it at whatever reading level you expect people to be at. -**Austin:** So you wind up needing documentation that is the same documentation you give to humans because the concepts all need to match but is organized and structured in a different way, is much longer, has kind of everything in one big chunk, is optimized for the amount of tokens and the semantic values of those tokens. But the advantage of doing this right, of thinking about how do I give the LLM documentation is that LLMs are remarkable, you know, especially if you're using them as part of like AI-assisted coding, are remarkable at reducing toil. +**Reese:** Even for that kind of stuff, but with LLMs, they actually don't have that disadvantage, right? -**Austin:** One of the things that I think, you know, all of us here have been working in observability for years and what is like the one thing nobody wants to do? Nobody wants to do a migration, right? No, you go tell someone like, "Oh, here's the new thing." They have to rewrite all this instrumentation, all these logs, all this stuff. They just be like, "Thanks, I'll pass." It's 'cause it's a toil. It doesn't make sense, right? The existing stuff works well enough. It's maybe it could be better, but you know, we're not going to go dedicate however many engineers' lives for three months to rewrite all of our logging statements. +**Austin:** Like you can give it a very large flowery word and if it's the most specific word, that's actually helpful because of the way the semantic search works. -**Reese:** But what if you just have an AI do it, right? The AI doesn't care. It doesn't complain. If you give it good instructions and good rules, it's able to, you know, make good decisions. And it's not like you're replacing human effort, right? You're not replacing the programmer. You're not replacing the people that are responsible for maintaining the system. You're easing the burden of modernizing what they're trying to do. +**Reese:** Um, precision in language is actually really important. -**Austin:** And so you're actually benefiting those people a lot 'cause even with improvements in AI-powered anomaly detection, whatever, right? Like we've been doing AI in observability for a while. LLMs just advance it a little bit, maybe a lot, we'll see. But even non-AI stuff, right? If you think about traditional sort of anomaly detection and heuristic-based detection, it's still machine learning. LLM machine learning. It's all machine learning. We've been doing machine learning for a while. And when you get to a certain size and complexity of your systems, you have to have it because there's just too much data for humans to process and go through. +**Austin:** So you wind up needing documentation that is the same documentation you give to humans because the concepts need to all match but is organized and structured in a different way, is much longer, has kind of everything in one big chunk, is optimized for the amount of tokens and the semantic values of those tokens. -**Reese:** So let's figure out how we can, you know, that to me is like the impact of AI on observability and on OpenTelemetry is we can make it easier for the AI, make it easier for your observability tooling to understand OpenTelemetry and interpret what's happening in your system better and at the end of the day give you more time to do... +**Reese:** But the advantage of doing this right, of thinking about how do I give the LLM documentation is that LLMs are remarkable, you know, especially if you're using them as part of like AI-assisted coding, are remarkable at reducing toil. + +**Austin:** One of the things that I think, you know, all of us here have been working in observability for years and what is like the one thing nobody wants to do? Nobody wants to do a migration, right? + +**Reese:** No, you go tell someone like, "Oh, here's the new thing," they have to rewrite all this instrumentation, all these logs, all this stuff. They just be like, "Thanks, I'll pass." It's because it's a toil. It doesn't make sense, right? The existing stuff works well enough. + +**Austin:** It's maybe it could be better, but you know, we're not going to go dedicate however many engineers' lives for three months to rewrite all of our logging statements. But what if you just have an AI do it, right? + +**Reese:** The AI doesn't care. Doesn't complain. + +**Austin:** If you give it good instructions and good rules, it's able to, you know, make good decisions. + +**Reese:** Um, and it's not like you're replacing human effort, right? You're not replacing the programmer. You're not replacing the people that are responsible to maintain the system. You're easing the burden of modernizing what they're trying to do. + +**Austin:** And so you're actually benefiting those people a lot because even with, you know, improvements in AI-powered anomaly detection, whatever, right? + +**Reese:** Like we've been doing AI in observability for a while. + +**Austin:** LLMs just advance it a little bit, maybe a lot, we'll see. But even non-AI stuff, right? If you think about traditional sort of anomaly detection and heuristic-based detection, it's still machine learning. LLM machine learning. It's all machine learning. + +**Reese:** We've been doing machine learning for a while. + +**Austin:** And when you get to a certain size and complexity of your systems, you have to have it because there's just too much data for humans to process and go through. + +**Reese:** So let's figure out how we can... You know, that to me is like the impact of AI on observability on OpenTelemetry is we can make it easier for the observability tooling to understand OpenTelemetry and interpret what's happening in your system better and at the end of the day give you more time to do... **Austin:** We had talked about OpenTelemetry project updates, which took place yesterday, right? -**Reese:** That sounds right. I'm losing track. I'm losing... +**Reese:** That sounds right. I'm losing track. + +**Austin:** Yeah, it is. It was yesterday. + +**Reese:** Milk? That doesn't sound like a good bagel. + +**Austin:** I wish it said bagel clock. This is probably a great... This is probably a great visual bit for the... + +**Reese:** And it's been... It attracted my attention ever since I came up here and now it's like I've been obsessed. I've been waiting for the appropriate moment to drop the bagel clock into the conversation. + +**Austin:** You did it. + +**Reese:** There you go. This is what we call commitment to the bit. + +**Austin:** Yes. + +**Reese:** Project update. Um, yeah, project update. + +**Austin:** So, um, can you give folks a, uh, OpenTelemetry right now is the second or first biggest project in the CNCF, depending on how you count it, but contributing across, you know, dozens and dozens of repositories. + +**Reese:** We're maintaining APIs, SDKs, tools in, you know, dozen plus languages. A lot's going on, right? -**Austin:** Yeah, it is. It was yesterday. +**Austin:** At this point, the project is really too big almost to have kind of a single narrative of whatever we're doing, but there's a few areas that we wanted to focus on. -**Reese:** Milk. That doesn't sound like a good bagel. I wish it said bagel clock. +**Reese:** We're starting to see a lot of great adoption of OpenTelemetry by the sort of broader community outside of... -**Austin:** This is probably a great visual bit for the... +**Austin:** Just so we're starting to see more CNCF projects natively integrate OpenTelemetry. -**Reese:** And it's been... it attracted my attention ever since I came up here and now it's like I've been obsessed. I've been waiting for the appropriate moment to drop the bagel clock into the conversation. +**Reese:** Um, we're starting to see... -**Austin:** You did it. There you go. This is what we call commitment to the bit. +**Austin:** Are integrating OpenTelemetry into their frameworks and into the, in Dino's case, into the runtime itself, right? So if you're writing a JavaScript app and you're using Dino, you pass in a config, you don't have to do anything, which is great. That's the vision for the project. -**Reese:** Yes. I update. +### [00:24:42] Updates on OpenTelemetry projects -**Austin:** Yeah, project update. So, OpenTelemetry right now is the second or first biggest project in the CNCF, depending on how you count it, but contributing across, you know, dozens and dozens of repositories. We're maintaining APIs, SDKs, tools in, you know, dozen plus languages. A lot's going on, right? And at this point, the project is really too big almost to have kind of a single narrative of whatever we're doing, but there's a few areas that we wanted to focus on. +**Reese:** So um, beyond that, you know, beyond the kind of growth we're seeing in adoption, we're seeing a few, you know, longer-term projects that we are proceeding along. -**Austin:** We're starting to see a lot of great adoption of OpenTelemetry by the sort of broader community outside of just OpenTelemetry. We're starting to see more CNCF projects natively integrate OpenTelemetry. We're starting to see are integrating OpenTelemetry into their frameworks and into the, in Dino's case, into the runtime itself, right? So if you're writing a JavaScript app and you're using Dino, you pass in a config, you don't have to do anything, which is great. That's the vision for the project. +**Austin:** So one thing that we've been working on over the past six to eight months, we've had a lot of progress there. We have a system-level profiler that is being worked on. It uses eBPF and other various technologies to let you use profiling across your services on a node. That is still in alpha like it's not done. It's not ready, but pretty soon that should be ready for people to start banging on. -**Reese:** So beyond that, you know, beyond the kind of growth we're seeing in adoption, we're seeing a few, you know, longer-term projects that we are proceeding along. +**Reese:** Another important thing we're doing is that we are evolving our logging infrastructure or logging APIs. -**Austin:** So one thing that we've been working on over the past six to eight months, we've had a lot of progress there. We have a system-level profiler that is being worked on. It uses eBPF and various other technologies to let you use profiling across your services on a node. That is still in alpha, like it's not done. It's not ready, but pretty soon that should be ready for people to start banging on. +**Austin:** So tradition originally we would just bridge to your existing logging API because there's a lot of those. There's log4j, there's various facades in Go and .NET and wherever, but one of the pieces of feedback we were going out and finding these sort of consistent metadata across services and libraries and domains is people need... People needed a structured way to emit structured events, right? -**Austin:** Another important thing we're doing is that we are evolving our logging infrastructure or logging APIs. So traditionally we would just bridge to your existing logging API because there's a lot of those. There's Log4j, there's various facades in Go and .NET and wherever, but one of the pieces of feedback we were going out and finding these sort of consistent metadata across services and libraries and domains is people need a structured way to emit structured events, right? Things like that you and I would probably call the LOG. Some people would call it an event. +**Reese:** Things like that you and I would probably call the LOG. Some people would call it an event. -**Reese:** And one thing I have learned is that the third rail of observability is talking about logging at all because people are very, very precious about what the word logs means to them. +**Austin:** And one thing I have learned is that the third rail of observability is talking about logging at all because people are very, very precious about what the word logs means to them. -**Austin:** I've noticed. +**Reese:** I've noticed. -**Reese:** Oh, true. Yeah. It's you don't mess with people's logs. +**Austin:** Oh, true. -**Austin:** Yes. So what we've kind of come to realize through this whole process is that we need some sort of API level answer to that and we're pretty close to have, you know, we have some OTAs and some specs in flight on this, but the idea is that there will be an OpenTelemetry logging API that will exist to let you emit structured events. +**Reese:** Yeah. You don't mess with people's logs. -**Austin:** A structured event is really just a fancy way of saying a structured log that has a known schema, right? In the same way that semantic conventions in OpenTelemetry let you apply schemas to your telemetry, to your logs, or sorry, to your metrics and traces. You'll be able to say, hey, here's a client-side ROM event, or here's a, you know, out of memory exception, or any of the various things that can happen. You'll be able to say, "Hey, here's a generative AI prompt, for example." And what we'll do is you'll be able to either take that and use it like you would use a span event today and bundle it in with the span or you'll be able to emit it separately through the log record signal and then have your backend either stitch them together or process them independently or do whatever, right? Like once it's out of our hands, we don't really care what you do with it. +**Austin:** Yes. So what we've kind of come to realize through this whole process is that we need some sort of API-level answer to that and we're pretty close to have... You know, we have some OTAs and some specs in flight on this, but the idea is that there will be an OpenTelemetry logging API that will exist to let you emit structured events. -**Austin:** But that's probably the two big in-flight things I would say. Beyond that, a lot of work is happening on other things. Stabilizing the collector, stabilizing various other SDKs and APIs. Shout out to our JavaScript SIG which just released JS SDK 2.0. My understanding is this fixes a lot of problems that people have had with especially with bundling it and things around ESM modules. I don't quite know what all that is. It sounds very scary. The JS devs assure me it's very important, but seriously I think it's actually a really good sign of the health of that project, right? That they have been able to get enough feedback about like, hey, these are the decisions that worked and didn't work to create a 2.0. And then for an end user, I was actually talking to someone Monday, Sunday at Cloud Native Rejects about this who maintains an integration into OpenTelemetry into his company's product and he was like, oh yeah, the migration was like five minutes, right? +**Austin:** And a structured event is really just a fancy way of saying a structured log that has a known schema, right? In the same way that semantic conventions in OpenTelemetry let you apply schemas to your telemetry, to your logs, or sorry, to your metrics and traces. -**Reese:** Wow. +**Reese:** You'll be able to say, "Hey, here's a client-side ROM event," or "Here's a, you know, out of memory exception," or any of the various things that can happen. -**Austin:** Because the API and the SDK are independent, so an SDK change is really very, you know, it's not a huge deal, it's not a lot that you have to do to take those updates. So that's something that for obviously other maintainers may or may not decide to do it, but we're, you know, it certainly seems that a lot of maintainers are thinking about, well maybe it's a good idea to go back. It's been five, six years, right? Like you can learn a lot, you get a lot of great feedback over that time and there's things that we would probably do differently in every language if we had a do-over. So thanks to the OpenTelemetry architecture, you can kind of get that do-over, which is cool. +**Austin:** Um, you'll be able to say, "Hey, here's a generative AI prompt, for example." And what we'll do is you'll be able to either take that and use it like you would use a span event today and bundle it in with the span or you'll be able to emit it separately through the log record sign through the logging signal and then have your backend either stitch them together or process them independently or do whatever, right? -**Reese:** That's great. How are we on time? I'm not sure 'cause we have... +**Reese:** Like once it's out of our hands, we don't really care what you do with it. -**Austin:** Are we on time, producer? +**Austin:** But that's probably the two big in-flight things I would say. + +**Reese:** Beyond that, um, a lot of work is happening on other things. Stabilizing the collector, stabilizing various other SDKs and APIs. + +**Austin:** Um, shout out to our JavaScript SIG which just released JS SDK 2.0. + +**Reese:** Know which, uh, my understanding is this fixes a lot of problems that people have had with especially with bundling it and things around ESM modules. + +**Austin:** I don't quite know what all that is. It sounds very scary. + +**Reese:** The JS devs assure me it's very important. + +**Austin:** But seriously, I think it's actually a really good sign of the health of that project, right? That they have been able to get enough feedback about like, "Hey, these are the decisions that worked and didn't work to create a 2.0." + +**Reese:** Know, and then for an end user, I was actually talking to someone Monday, um, Sunday at Cloud Native Rejects about this who maintains an integration into OpenTelemetry into his company's product. And he was like, "Oh yeah, the migration was like five minutes," right? + +**Austin:** Wow! + +**Reese:** Because the API and the SDK are independent, so an SDK change is really very... You know, it's not a huge... It's not a lot that you have to do to take those updates. + +**Austin:** So that's something that for obviously other maintainers can't may or may not decide to do it, but we're, you know, it certainly seems that a lot of maintainers are thinking about, "Well, maybe it's a good idea to go back. It's been five, six years, right? Like you can learn a lot, you get a lot of great feedback over that time." + +**Austin:** And there's things that we would probably do differently in every language if we had a do-over. + +**Reese:** So thanks to the OpenTelemetry architecture, you can kind of get that do-over, which is cool. + +**Austin:** That's great. + +**Reese:** Um, how are we on time? I'm not sure because we have... + +**Austin:** Are we on time, producer? **Reese:** Okay. Well, I guess this could be a good opportunity for us to plug some upcoming stuff like OpenTelemetry Community Day. -**Austin:** Yes, OpenTelemetry Community Day in Denver, Colorado coming up this summer. June 25th or 26th. It's the same week as Open Source Summit. Go look on the web. The CFP is closed unfortunately, but we will be, we should be announcing the schedule on that here pretty soon. Even if you aren't planning on, you know, plan on speaking, highly recommend everyone that's in the US, North America to come out to that. It's going to be great. +**Austin:** Yes, OpenTelemetry Community Day in Denver, Colorado coming up this summer. Um, June 25th or 26th. It's the same week as Open Source Summit. Go look on the web. Um, the CFP is closed, unfortunately, but we will be, uh, we should be announcing the schedule on that here pretty soon. -**Austin:** And we are also, no promises, but we're trying to do a community day in Europe this year. So stay tuned. If not this year, definitely next year. +### [00:30:24] Upcoming OpenTelemetry Community Day -**Reese:** Is it going to be part of Open Source Summit EU or that would be the plan, but nothing's set in stone. But we've definitely, one of the fun facts at the project update is that about 50% of our contributors are actually not in the US in OpenTelemetry and we've seen now that's US and then everywhere else, right? So, but if you look at, if you break it down by like region, so you go like North America, EMIA, APAC, other, then we like the line for EMIA is just like doing this and the US one is kind of doing this a little bit. +**Reese:** Um, even if you aren't planning on, you know, plan on speaking, highly recommend everyone that's in the US, North America to come out to that. It's going to be a great... -**Austin:** So they're starting to get closer and closer together. But we've definitely, you know, one of the things I always love coming to KubeCon EU is we have so many, you know, our user community is so vibrant here. We have so many maintainers whose work is just fantastic and we really want to support our European end user and contributor community. So we're very strongly going to be figuring out how to do a European community day. +**Austin:** And we are also, no promises, but we're trying to do a community day in Europe this year. So stay tuned. If not this year, definitely next year. -**Reese:** Awesome. Oh, that's awesome. +**Reese:** Is it going to be part of Open Source Summit EU or that... That would be the plan, but nothing's set in stone. -**Austin:** Yeah. I always love the vibe at the KubeCon EU. It's just very vibrant. And I think at the keynote, they mentioned this was the biggest KubeCon so far, 12,000. +**Austin:** Um, but we've definitely... One of the fun facts at the, um, project update is that about 50% of our contributors are actually not in the US in OpenTelemetry. -**Reese:** Yeah. Over 13,000 people. +**Reese:** Um, and we've seen now that's US and then everywhere else, right? -**Austin:** Over 13,000. That's the number I heard. Damn, that is wild. +**Austin:** So, but if you look at... If you break it down by like region, so you go like North America, EMEA, APAC, other, then we like the line for EMEA is just like doing this and the US one is kind of doing this a little bit. -**Reese:** Yeah. No, they... I mean I... +**Reese:** So they're starting to get closer and closer together. -**Austin:** And it's like how's your KubeCon? And they say it's like, "Oh, it's my first." I'm like, "You're going to have that. Bring lots of water. Bring lots of water." +**Austin:** But we've definitely, you know, one of the things I always love coming to KubeCon EU is we have so many, you know, our user community is so, you know, so vibrant here. -**Reese:** But I love that we're still that new people are still coming into this community, right? That we're really cool to see the growth of, you know, this community, right? Like to see it expand, to see it bring in new people, right? +**Reese:** Um, we have so many maintainers whose work is just fantastic and we really want to support our European end-user and contributor community. -**Austin:** Yeah. Oh my god. There's a ton of people that we know that all the three of us here know that careers, friendships, right? Friendships like that's given people the opportunity to really, you know. +**Austin:** So we're very strongly going to be figuring out how to do a European Community Day. -**Reese:** Yeah. +**Reese:** Awesome. + +**Austin:** Oh, that's awesome. + +**Reese:** Yeah. I always love the vibe at the KubeCon EU. It's just very vibrant. + +**Austin:** And I think, um, at the keynote, um, they mentioned this was the biggest KubeCon so far. + +**Reese:** 12,000. + +**Austin:** Yeah, over 13,000 people. + +**Reese:** Over 13,000. That's the number I heard. -**Austin:** That's neat. +**Austin:** Damn, that is wild. -**Reese:** Yeah. Also, it feels like a family reunion every time we're together, right? +**Reese:** Yeah. No, they... I mean, I... -**Austin:** Yeah. No, it's great. It's like a family reunion with like 10,000 people. +**Austin:** And it's like, how's your KubeCon? And they say it's like, "Oh, it's my first." I'm like, "You're going to have that. Bring lots of water. Bring lots of water." -**Reese:** Yeah, a bunch of batteries and then we get on the other side. They have the retail technology exhibition. +**Reese:** But I love that we're still... That new people are still coming into this community, right? That we're really cool to see the growth of, you know, this community, right? -**Austin:** Yep. They actually get a red carpet. Very loud burgers walking around which was exciting. It was definitely unexpected. +**Austin:** Like to see it expand, to see it bring in new people, right? -**Reese:** Yeah, I know. We were walking yesterday and we're like well the tube it was like there's like two stops. +**Reese:** Like, oh my god. There's a ton of people that we know that all the three of us here know that careers, friendships, right? Friendships like that's given people the opportunity to really, you know... -**Austin:** Yeah, there's a station either end. +**Austin:** Yeah. + +**Reese:** Um, that's neat. + +**Austin:** Yeah. Also, it feels like a family reunion every time we're together, right? + +**Reese:** Yeah. No, it's great. It's like a family reunion with like a 10,000. + +**Austin:** Yeah. + +**Reese:** So a bunch of batteries and then we get on the other side. They have the retail technology exhibition. + +**Austin:** Yep. They actually get a red carpet. + +**Reese:** Very loud bugers walking around which was, um, exciting. It was definitely unexpected. + +**Austin:** Yeah, I know. We were walking yesterday and we're like, well the tube... It was like there's like two stops. + +**Reese:** Yeah, there's a station at either end. + +**Austin:** Yeah. **Reese:** Yeah. -**Austin:** And I heard that it's actually faster if you time it right to take the tube. +**Austin:** And I heard that, um, it's actually faster if you time it right to take the tube. -**Reese:** If you hit... if you hit the... +**Reese:** If you hit... If you hit the... **Austin:** It also depends on where you start from, but if you're from door to door, it's definitely faster. -**Reese:** Yeah, it's nice. You just tap in and out. +**Reese:** Yeah, it's nice. You just tap in and out. + +**Austin:** So... -**Austin:** Oh, I know. +**Reese:** Oh, I know. -**Reese:** Yeah, it's really card. Discovered that. +**Austin:** Yeah, it's really... -**Austin:** Oh, is that better than the... +**Reese:** Discovered that. -**Reese:** I just use Google Pay. +**Austin:** Oh, is that better than the... + +**Reese:** I just use Google Pay. **Austin:** Yeah, you can use your phone. **Reese:** Yeah. -**Austin:** Yeah. Works well. +**Austin:** Yeah, yeah. + +**Reese:** It works well. + +**Austin:** Yeah, we have that in Toronto. I mean, we have, uh, tap in New York. They finally, um, it's Omni now. + +**Reese:** Omny instead of... + +**Austin:** This for another episode. + +### [00:34:34] Project updates and growth of OpenTelemetry + +**Reese:** Um, live stream for you from KubeCon EU. Sorry guys, it's been a long week. + +**Austin:** Um, thank you so much for joining us. We will have, um, our lovely guests. + +**Reese:** Um, again, I'm Reese. This is Adriana. Thank you so much, Austin, for being here and Marilia for being here earlier. + +**Austin:** And also shout out to our behind the scenes. + +**Reese:** Um, but he is the one producing all this for you. All of our streams. -**Reese:** Yeah, we have that in Toronto. I mean, we have tap in New York. They finally... it's Omni now. Omny instead of... +**Austin:** Yes. -**Austin:** This for another episode. +### [00:35:17] Closing remarks and acknowledgments -**Reese:** Live stream for you from KubeCon EU. Sorry guys, it's been a long week. Thank you so much for joining us. We will have our lovely guests. Again, I'm Reese. This is Adriana. Thank you so much Austin for being here and Marilia for being here earlier. And also shout out to our behind the scenes. He is the one producing all this for you, all of our streams. Yes. Henrik said from Dana Trace. Thank you so much and we will see you next time. +**Reese:** Thank you so much and we will see you next time. ## Raw YouTube Transcript diff --git a/video-transcripts/transcripts/2025-05-01T06:00:52Z-otel-me-with-eromosele-akhigbe.md b/video-transcripts/transcripts/2025-05-01T06:00:52Z-otel-me-with-eromosele-akhigbe.md index 92f3132..36187c1 100644 --- a/video-transcripts/transcripts/2025-05-01T06:00:52Z-otel-me-with-eromosele-akhigbe.md +++ b/video-transcripts/transcripts/2025-05-01T06:00:52Z-otel-me-with-eromosele-akhigbe.md @@ -10,216 +10,182 @@ URL: https://www.youtube.com/watch?v=KzqY4roXhHs ## Summary -In the latest episode of "O Tell Me," host Adriana Vila, along with co-host Victoria, welcomes guest Mole Aigbe, a developer advocate at Step Security and an OpenTelemetry community member. The discussion revolves around Mole's journey into open source through the Outreachy program, which aims to support underrepresented communities in tech. Mole shares his initial hesitations about contributing to open source, his learning process in Golang, and the challenges faced while creating an OpenTelemetry exporter. He emphasizes the importance of mentorship and community support, particularly from figures like Adriana and his mentors, who encouraged him throughout his journey. The conversation highlights the value of community, advocacy for open source, and the resources that helped Mole, such as YouTube channels and blog posts. The episode wraps up with mentions of upcoming events and initiatives within the OpenTelemetry community. +In the latest episode of "O tellme," host Adriana Vila, along with co-host Victoria, engages in a deep conversation with guest Mole Aigbe, a Developer Advocate at Step Security and an OpenTelemetry community member. They discuss Mole's journey into open source through the Outreachy program, which supports underrepresented communities in tech. Mole shares insights about his experiences learning Go, contributing to the OpenTelemetry project, and building a log exporter for the OpenTelemetry Collector. Key points include the challenges of navigating documentation, the importance of mentorship, and the supportive nature of the OpenTelemetry community. They also emphasize the value of advocacy and personal connection in learning and contributing to open source. The session concludes with announcements about upcoming events and resources available for those interested in OpenTelemetry. ## Chapters -00:00:00 Welcome and introductions -00:02:30 Outreachy program overview -00:05:20 Application process discussion -00:10:00 Internship duration and mentors -00:11:30 Contributions to OpenTelemetry -00:14:02 Securing first job experience -00:15:50 Learning resources and methods -00:20:01 Documentation feedback and suggestions -00:25:00 Experience creating first PR -00:30:00 Building an OpenTelemetry exporter +00:00:00 Introductions +00:01:52 Guest introduction: Mole Aigbe +00:03:44 Discussion about Outreachy program +00:06:02 Mole's application process for Outreachy +00:10:16 Importance of mentorship +00:14:06 Mole's contributions to OpenTelemetry +00:20:02 Discussion about learning Go +00:25:12 Creating first PR experience +00:30:48 Building OpenTelemetry exporter +00:46:00 Community advocacy and contributions -**Adriana:** Hey everyone, welcome to the latest edition of O tellme. My name is Adriana Vila. I am one of the maintainers of the open telemetry enduser SIG. Super excited to have folks joining us here, and please feel free in the chat to just say where you're joining from. I'm joining from Toronto, Canada. I have some very special folks here joining today. First of all, I have Victoria who is co-hosting with me. So, Victoria, why don't you introduce yourself? +## Transcript -**Victoria:** Hi everyone. I'm Victoria. I'm a user experience designer. I'm currently interning with the Prometheus project as an LFM. My project has got me interfacing with a lot of hotel members, which is how I learned about this program. I'm looking forward to interviewing and learning all about his experience. +### [00:00:00] Introductions -**Adriana:** Awesome. And now it's time to introduce our guest for the day. +**Adriana:** Hey everyone, welcome to the latest edition of O tellme. My name is Adriana Vila. I am one of the maintainers of the OpenTelemetry End User SIG. Super excited to have folks joining us here, and please feel free in the chat to just say where you're joining from. I'm joining from Toronto, Canada. I have some very special folks here joining today. First of all, I have Victoria who is co-hosting with me. So, Victoria, why don't you introduce yourself? -**Mole:** Hello everyone. It's so nice to be here. Thank you so much, Adriana. My name is Mole Aigbe. I am a developer advocate at step security. I'm also an open telemetry member. I was a former algi intern and that's where I got introduced to open telemetry, and I'm so excited to share my experience with hotel here today. Thank you for having me. +**Victoria:** Hi everyone. I'm Victoria. I'm a user experience designer. I'm currently interning with the Prometheus project as an LFM. My project has got me interfacing with a lot of OpenTelemetry members, which is how I learned about this program. I'm looking forward to interviewing and learning all about his experience. -[00:02:30] **Adriana:** Amazing. Yeah, super excited. Well, you mentioned outreachy in your intro. Why don't you tell folks a little bit about the outreachy program? +**Adriana:** Awesome. And now it's time to introduce our guest for today. -**Mole:** Okay. So, the outreach program is an initiative that aims to help people from underrepresented communities, you know, marginalized, looked down on communities, get into open source. They encourage them to put their first leg into open source and they also give some kind of incentive to assist them, right? To get their first leg into the open source community at large. So, that is what outreachy is. It's not just for open source; it's for both open source and open science. They have projects that cover biological things, projects that cover geomaps, you know, different diverse projects—not just open source projects. But my love, since I'm a tech guy, I love open source, and that's where I got to know about open source and the amazing open telemetry community. +### [00:01:52] Guest introduction: Mole Aigbe -**Adriana:** Did you know about open source before you applied for outreachy, or was outreachy your awareness into open source then? +**Mole:** Hello everyone. It's so nice to be here. Thank you so much, Adriana. My name is Mole Aigbe. I am a developer advocate at Sematext. I'm also an OpenTelemetry member. I was a former Outreachy intern, and that's where I got introduced to OpenTelemetry. I'm so excited to share my experience with OpenTelemetry here today. Thank you for having me. -[00:14:02] **Mole:** Okay. So, yeah. I always knew that open source existed, right? I always knew that, okay, there's something out there in the world called open source that some cool dudes, very smart, very intelligent, are doing awesome work in, right? But I just never believed that someone like me could contribute to open source because I was like, bro, what do you know? You do not work in Microsoft. You don't work in Google. You don't have that knowledge, man. You don't have experience. So just chill when you're skilled enough, maybe you can start playing in the big leagues, right? So open source was something I remember, you know, several times I would use YouTube to learn how to contribute to open source, and then I would try to watch some videos, I’d get scared and then I'd run away, right? But outreachy provided that platform. We had mentors that actually directed and showed us the step-by-step process on how to make your first contribution. And then once you make your first contribution, you realize that wait a second man, it's not that hard. It's all in your head, right? And once you make that first contribution, you just keep making and making, and then it just became so cool. So, yeah, that's how I got to learn about open source. I knew about open source, but outreachy gave me the platform to take that first step into open source. +**Adriana:** Amazing. Yeah, super excited. Well, you mentioned Outreachy in your intro. Why don't you tell folks a little bit about the Outreachy program? -**Adriana:** Yeah, really interesting. Could you tell us more about the application process? What was it like? +**Mole:** Okay. So, the Outreachy program is an initiative that aims to help people from underrepresented communities, marginalized, looked down upon communities, get into open source. They encourage them to get into open source to put their first leg into open source. They also give some kind of incentive to assist them, right, to get their first leg into the open source community at large. So that is what Outreachy is. It's not just for open source; it's for both open source and open science. They have projects that cover biological things, projects that cover geomaps, you know, different diverse projects, not just open source projects. But my love, since I'm a tech guy, I love open source, and that's where I got to know about open source and the amazing OpenTelemetry community. -[00:05:20] **Mole:** Okay. So first of all, I almost did not apply for outreachy because I also had that whole mentality that, okay man, this is open source. This is for the cool kids. I'm not a cool kid, right? So I was like, you know, my very good friend was like, no, you can do it. Just apply. What's the worst that can happen? They tell you no, you go back, you cry, and you're fine. So, I decided to take that step. The application process is that you have to write like five essays on why you believe that you should be awarded the internship. Because, like I said, the internship is not about they’re not looking for very skilled people. They're looking to actually support, you know, people that come from the minorities of society, like people that you would hear about on a regular day, right? Like people that are looked down on, you know, that people don’t even like—the big leagues don’t even look at in society. So that's their goal, right? Those are the people they are trying to support. So, you can be quote unquote very smart or high and mighty, and they won’t accept you. But you know, if you feel like man, who will hear my voice, no one cares about me, I’ll say go for outreachy. They care. The people actually sit down and read people's essays to see and to look for those kinds of people that they know they can support. So, I know people have heard a lot of things about job applications that they use one AI tracker to just wipe people's applications, but I can tell you from experience that outreachy is not like that. Real people actually read your essays and you can tell your story, and through your story, you know, you might get a chance to walk into and begin your own open source journey. +**Adriana:** Yeah. So, were you aware of open source before you applied for Outreachy, or was Outreachy like your awareness into open source then? -[00:11:30] **Mole:** So you have to write five essays. Once you write the five essays, you get into the first phase. There are actually two stages. The first phase is the essay phase where you have to prove that you need the internship. The second phase is where you actually have to now do the work. In doing that work, I’ll say there were a million times I wanted to give up because I did not know Golang. That's the language that most cloud-native applications are written in. I remember in December 2023, two months before I was learning about DevOps. So I just in that February I started learning Golang. I was trying to learn open telemetry; I was trying to learn Golang; and I was trying to contribute—all in one month. So it was a whole lot. But you know, I always say shout out to Henrik and Adriana. Their content really, you know, ramped me up to speed on how to first understand the project and then make some reasonable contributions. I was able to get like 70 hours merged, and then my mentors were like, "You're so cool. Come on board." My mentors, by the way, Jurassi and Yuri are the coolest guys out there. Shout out to them. I love them so much with all my heart, and I'm always grateful to them for the forever they will be part of my tech journey. I always mention their names everywhere I go, you know, as the ones that helped me like a baby and trained me up to be this man that I am now. +### [00:03:44] Discussion about Outreachy program -**Adriana:** That's like a summary of the application process. +**Mole:** Okay. So, yeah. I always knew that open source existed, right? I always knew that, okay, there's something out there in the world called open source that some cool dudes, very smart, very intelligent, are doing awesome work in, right? But I just never believed that someone like me could contribute to open source because I was like, "Bro, what do you know? You do not work in Microsoft. You don't work in Google. You don't have that knowledge, man. You don't have experience. So just chill when you're skilled enough, maybe you can start playing in the big leagues." So open source was something I remember several times I would use YouTube to learn how to contribute to open source, and then I'd try to watch some videos, I'd get scared, and then I'd run away, right? But Outreachy provided that platform for us. We had mentors that actually directed and showed us the step-by-step process on how to make your first contribution. Once you make your first contribution, you realize that, wait a second, man, it's not that hard. It's all in your head, right? Once you make that first contribution, you just keep making and making, and then it just became so cool. So, yeah, that's how I got to learn about open source. I knew about open source, but Outreachy gave me the platform to take that first step into open source. -**Mole:** Yeah, that's so great. And you know, I can't underscore enough the importance of programs like outreachy because they give opportunities to people who otherwise wouldn't necessarily have gotten the opportunity. And especially like nowadays, I'm going to get on my little soap box and talk about the fact that so many DEI programs are getting cut and there's so much, I don't know, there's like global hate towards DEI programs. So it's so nice to have a reminder that programs like outreach exist. They benefit folks. They elevate folks who are awesome who we wouldn't necessarily have known about. I love that you got so much out of the program. I love that the program exists. I wanted to ask you, how long was your internship? +**Adriana:** Amazing. Yeah, really interesting. Could you tell us more about the application process? What was it like? -[00:10:00] **Mole:** Okay, so the internship was for three months. But if I want to combine the whole application process, everything together was like six months, right? But officially, the internship is just for three months. +### [00:06:02] Mole's application process for Outreachy -**Adriana:** Awesome. You talked a little bit about your mentors. How do you get paired up with a mentor? +**Mole:** Okay. So first of all, I almost did not apply for Outreachy because I had that whole mentality that, okay, man, this is open source. This is for the cool kids. I'm not a cool kid, right? So I was like, you know, but my friend, my very good friend, she was like, "No, you can do it. Just apply. What's the worst that can happen? They tell you no, you go back, you cry, and you're fine." So I decided to take that step. I applied. The application process is like this: you have to write five essays on why you believe that you should be awarded the internship because, like I said, the internship is not about looking for very skilled people. They're looking to actually support people from the minorities of society, people that you would hear about on a regular day, right? Like people that are looked down on, that people don't even like, the big leagues don't even look at in society. So that's their goal, right? Those are the people they are trying to support. You don't have to be quote-unquote very smart or high and mighty for them to accept you. But if you feel like, man, who will hear my voice? No one cares about me, I'll say go for Outreachy. They care. The people actually sit down and read people's essays to see and look for those kinds of people that they know that they can support. I know people have heard a lot of things about job applications that they use one AI tracker to just wipe people's applications, but I can tell you from experience that Outreachy is not like that. Real people actually read your essays, and you can tell your story, and through your story, you might get a chance to walk into and begin your own open source journey. -**Mole:** Okay, so when you pick a project, every project has mentors for the project, right? So when I picked the project open telemetry, it was Jurassi and Yuri that actually volunteered to be mentors for a particular project, right? And so they became my mentors. +### [00:46:00] Community advocacy and contributions -**Adriana:** Cool. I would like to ask—I know that hotel is a very big project—what specific areas did you contribute to? +You have to write five essays. Once you write the five essays, you get into the first phase, and then there's actually two stages. The first phase is the essay phase, where you have to prove that you need the internship, and the second phase is where you actually have to now do the work. In doing that work, I'll say there were a million times I wanted to give up because I did not know Golang. That's the language that most cloud-native applications are written in. In December 2022, that's two months before I was learning about DevOps, I started learning Golang. I was trying to learn OpenTelemetry. I was trying to learn Golang, and I was trying to contribute all in one month. It was a whole lot. But I always say shout out to Henrik and Adriana. Their content really ramped me up to speed on how to first understand the project and then make some reasonable contributions. I was able to get like 70 hours merged, and then I also spoke at an event, and my mentors were like, "You're so cool. Come on board." My mentors, by the way, Jurassi and Yuri, are the coolest guys out there. Shout out to them. I love them so much with all my heart, and I'm always grateful to them. They will forever be part of my tech journey. I always mention their names everywhere I go, you know, as the ones that helped me like a baby and trained me up to be this man that I am now. So, yeah, that's like a summary of the application process. -**Mole:** Okay. So the, I'll go before the actual internship. I remember scrolling through the different repos on GitHub and I was like, oh my god, where do I even start from? I found solace in the collector contrib. In outreachy, the project that I worked on was building a logging bridge in Golang, and by the end of the internship, I was able to build the zero log bridge for Golang. So those are the projects that I'm working on, and currently I still contribute to the collector contributions. +**Adriana:** Yeah, that's so great. And I can't underscore enough the importance of programs like Outreachy because they give opportunities to people who otherwise wouldn't necessarily have gotten the opportunity. Especially like nowadays, I'm going to get on my little soapbox and talk about the fact that so many DEI programs are getting cut, and there's so much, I don't know, there's like global hate towards DEI programs. It's so nice to have a reminder that programs like Outreach exist. They benefit folks. They elevate folks who are awesome and who we wouldn't necessarily have known about. And so, I love that you got so much out of the program. I love that the program exists. I wanted to ask you, how long was your internship? -**Adriana:** Wow! But like it's funny to see how you went from not knowing about school to completing your internship to building something that the community is making. That's huge. Could you share with us how the outreachy internship helped you secure your first job? +**Mole:** Okay, so the internship was for three months. But if I want to combine the whole application process, everything together was like six months, right? But officially, the internship is just for three months, you know, so that's how long the internship was. -**Mole:** Okay. So what happened was I had done a lot of work and then I was writing articles for Signals. Signals is a very cool collab observability platform that also supports open telemetry natively. I think my former boss, Otus, the CEO of Semantex, saw some of the articles that I wrote and, you know, had a conversation with me that, oh hey man, you worked on open telemetry, right? I'm like yeah, I did. What did you do? I showed him what I did. We got on a call, I did some demos on how auto instrumentation works. He was like, wow, this is so cool. I was like, yeah, it's so cool. Oh, open telemetry is so cool. And he was like, okay, they want to integrate their platform with open telemetry and am I up for it? I'm like, okay, sure, I'm up for it. Let's go. And at Semantex, what I was working on was building an exporter for their platform so that they can receive data from open telemetry data into their platform directly using the open telemetry collector. So that was the project I worked on while at Semantex, and it was very stressful but it was very cool. +### [00:10:16] Importance of mentorship -**Adriana:** That's amazing. I think it really underscores like two things. First of all, outreachy kind of gave you the tools to contribute to open source, contribute to open telemetry, but then you did another awesome thing, which is putting yourself out there, writing these blog posts, which is so important. It's so hard sometimes to put ourselves out there. And you know, I think we all in tech suffer from impostor syndrome at some point or another. Maybe constantly feel it comes in waves for me. And you just have to push past that impostor syndrome and do it anyway. Mega kudos to what you're doing. Now you've made a few references about learning open telemetry, learning Golang. What did you use for—can you dig a little bit deeper on how you learned open telemetry? +**Adriana:** Awesome. And you talked a little bit about your mentors. How do you get paired up with a mentor? -[00:15:50] **Mole:** Okay, yeah. I'll start from my first introduction to open telemetry. I just Googled open telemetry, entered, and I saw the documentation. I was like, oh, this looks so cool. And then I tried to read the docs and my brain wanted to blow. So I went to—I'm more of a visual learner, right? So I went to YouTube and I typed "what is open telemetry?" and Henrik, he is the owner of the channel—is it observable?—and I was just watching a lot of his videos at that time. I would say those videos gave me a lot of context because how I learn is I like specific information before general information. So I used his videos a lot to ramp up my knowledge on open telemetry. And then our wonderful co-host here, Adriana—her blogs are so awesome. I read a lot of her blogs at that time just to ramp up because the documentation was very overwhelming for me. After reading those two things, I then began to go to the docs and actually narrow down on what I was looking for. I really knew what I was looking for because documentation sometimes can be very overwhelming because there's so much information packed in the documentation and it's trying to talk to different classes of people—both beginners, intermediate, and advanced people. So beginners sometimes always feel overwhelmed right from those documentation. But I'll say, when it came to running the open telemetry demo, the docs did a good job. +**Mole:** Okay, so when you pick a project, every project has mentors for the project, right? When I picked the OpenTelemetry project, it was Jurassi and Yuri that actually volunteered to be mentors for a particular project, right? So they volunteered to be mentors for the OpenTelemetry project at that time, and that's how they became my mentors. -**Adriana:** Wow, that's great. Personally, I'm more of a reader. I prefer to consume text-based resources than watching. +**Adriana:** Cool. I would like to ask, I know that OpenTelemetry is a very big project. What specific areas of OpenTelemetry did you contribute to? -**Mole:** Cool. I'm with you on that. +**Mole:** Okay. Before the actual internship, I remember scrolling through the different repos on GitHub, and I was like, "Oh my god, where do I even start from?" I found solace in the collector contrib. The project that I worked on was building a logging bridge in Golang, and by the end of the internship, I was able to build the Zero Log Bridge for Golang. That's the project that I'm working on, and currently, I still contribute to the collector contributions. -**Adriana:** That's interesting to know. Good for you. Of all these resources that you mentioned, the YouTube videos, which of them would you say was the most useful for you to learn about? +**Adriana:** Wow. But it's funny to see how you went from not knowing about school to completing your internship to building something that the community is making. That's huge. Could you share with us how the Outreachy internship helped you secure your first job? -**Mole:** So you want me to pick one? If you tell me to pick one, I'm very biased. I always go for videos. So, if you tell me to pick one, I'll go for Henrik's channel. Hands up. +### [00:14:06] Mole's contributions to OpenTelemetry -**Adriana:** I mean, Henrik produces amazing content, so I can definitely vouch for that. - -**Mole:** Yep. So, for video, I'll say Henrik's channel, Is it observable? If you haven't, go and check it out. It's awesome. And then for text, Adriana's blog on Medium. I don't know if she's hearing me right now, but Adriana's blog, yeah. So, you should also check that out. - -**Adriana:** That's cool. - -**Mole:** Yeah. - -**Adriana:** Now, you said you have a preference for the videos, right? Is there a reason why? - -**Mole:** Okay. So how I reason is I'm a very imaginative thinker. I can sit down and just go on and just be imagining a lot of things. I like pictures, visual things because if I'm just reading a bunch of text of a concept I haven't understood yet, I get bored. I will just easily get bored and zone off. So, if I'm watching a video, you know, the sound of the person, the tone, the infographics, the pictures, you know, just keeps me excited to keep going. Before I know it, I've tricked my brain into learning a new concept, right? So, yeah. So that's just what works for me. I've observed it over time. - -[00:20:01] **Adriana:** That's awesome. Yeah, makes sense. The next question that we were wondering about, you mentioned that you use the hotel docs as a reference. One of the goals of these sessions that we have like with hotel me is also to provide feedback to the hotel community on areas for improvement. Is there anything that you found about the third-party documentation more useful than the official hotel docs and what would you do to improve the docs experience if it were up to you as a new learner? - -**Mole:** Okay. So, sorry, this is an issue with multiple questions. The first question is what made me prefer the docs or maybe leave the docs? - -**Adriana:** Yeah. Why did you choose third-party documentation over the hotel docs for starters? - -**Mole:** Okay. Well, because number one, there’s no video documentation yet. That's why. - -**Adriana:** Fair. +**Mole:** Okay. What happened was I had done a lot of work, and then I was writing articles for Sematext. My former boss, Otus, the CEO of Sematext, saw some of the articles that I wrote and had a conversation with me. He said, "Oh hey man, you worked on OpenTelemetry, right?" I was like, "Yeah, I did." He asked, "What did you do?" I showed him what I did. We got on a call, and I did some demos on how auto-instrumentation works. He was like, "Wow, this is so cool." I was like, "Yeah, it's so cool. OpenTelemetry is so cool." He said, "Okay, we want to integrate our platform with OpenTelemetry, and am I up for it?" I was like, "Okay, sure, I'm up for it. Let's go." What I was working on was building an exporter for their platform so that they can receive data from OpenTelemetry directly using the OpenTelemetry Collector. That was the project I worked on while at Sematext, and it was very stressful, but it was very cool in the end. So, that's how I got my first job. The contributions I did and the demos I did about OpenTelemetry got me my first job. -**Mole:** But also, when I'm reading someone's content, right? Like I'm learning your content or someone else out there, I am not just reading. I can feel human connection. You can say something like, "Ah, I got stuck at this." This is what I had to do, and then this got solved. But in a documentation, it's just man, it's like reading a textbook, you know? Do this, do this, do this, do that, move on, you know? But when you're writing your content, you can say, "Oh, do this. If this does not work, try this." So, I'd say that's why I preferred. When I entered the documentation, I got overwhelmed because I was scrolling through a lot of things. I didn't understand what instrumentation was, and documentation has so many links that link you to so many things, and before you know, you're lost in the rabbit hole of the documentation. So, I’d be reading something, I'm like, what is instrumentation? I'll see a link, click the link, and I'm going to a different entire part, and then before you know, I'm like, where am I? What am I learning? So, I got overwhelmed. +### [00:20:02] Discussion about learning Go -**Mole:** What made me go back to the docs? I already had a foundation. So I was not overwhelmed when I saw what instrumentation was or when I saw what a collector was. So imagine someone that is new that does not know anything about a collector trying to learn open telemetry. They're seeing open telemetry. What is open telemetry? You can't really define open telemetry without the four components: instrumentation, collector, and you’re like, what? What is an open telemetry operator? Oh my gosh, they're seeing Kubernetes—like where am I, you know? So, I already got that foundational knowledge. I could now go and head for the docs. Because now I already knew what I was looking for in the documentation. So, let's say I want to do a particular task, I can just search for that particular part in the docs and focus on just that side. +**Adriana:** That's amazing. And I think, you know, it really underscores two things. First of all, Outreachy kind of gave you the tools to contribute to OpenTelemetry, but then you did another awesome thing, which is putting yourself out there, writing these blog posts, which is so important. It's so hard sometimes to put ourselves out there. And we, I think we all in tech suffer from impostor syndrome at some point or another, maybe constantly. It comes in waves for me. You just have to push past that impostor syndrome and do it anyway. Mega kudos to what you're doing. Now, you've made a few references about learning OpenTelemetry and learning Golang. What did you use for learning? -**Adriana:** And what would you do to improve the doc experience? +**Mole:** Okay, yeah. I'll start from my first introduction to OpenTelemetry. I just Googled "OpenTelemetry" and saw the documentation. I was like, "Oh, this looks so cool." Then I tried to read the docs, and my brain wanted to blow up. I'm more of a visual learner, right? I went to YouTube and typed "What is OpenTelemetry?" and Henrik, the owner of the channel, was there. I watched a lot of his videos at that time, and I would say those videos gave me a lot of context. I like specific information before general information, so I used his videos a lot to ramp up my knowledge on OpenTelemetry. Then our wonderful co-host here, Adriana, her blogs are so awesome. I read a lot of her blogs at that time just to ramp up because the documentation was very overwhelming for me. After reading those two things, I began to go to the docs and actually narrow down on what I was looking for. I really knew what I was looking for because documentation can sometimes be very overwhelming. -**Mole:** I’d say I'd make video documentation for those who do follow the hotel content. We do have our first video, like instructional video, on the hotel YouTube channel. There will be more to come. You can also satisfy both at the same time. +**Adriana:** Wow, that's great. Personally, I'm more of a reader; I prefer to consume text-based resources than watching videos. -**Adriana:** That is a super fair point. It's interesting too, like what you were saying, that basically you use the hotel docs for more of a directed search, right? Like first understand stuff from people's experiences and then, okay, I want to learn about this, I will go into the docs to learn about that. +**Mole:** I'm with you on that. -**Mole:** That’s the approach that has worked for me to learn everything and that’s what I’ll keep using. By the way, Adriana, I started learning Kubernetes—just a side note. +**Adriana:** Good for you. Of all these resources that you mentioned, which of them would you say was the most useful for you to learn about OpenTelemetry? -**Adriana:** Oh, right on! Good luck! +**Mole:** If you tell me to pick one, I'm very biased. I always go for videos. So, if you tell me to pick one, I'll go for Henrik's channel. Hands down. -**Mole:** Thank you! - -[00:25:00] **Adriana:** Yeah. Could you tell us what your experience was like creating your first PR? - -**Mole:** Ah, okay. We're here. So, creating my first PR, I’ll say it was a whole experience, right? After watching some videos, reading some documentation, I think I did this for two weeks. I was just learning a whole lot and then I was like, you know what? You need to make your first PR. You can't waste any more time. I went to the repos and I was like, okay, so which repo do I pick? I think that was the first issue I had. Open telemetry has a bunch of repos, so I was like, which one do I go for? For a while, I was just lost. Do I contribute to NodeJS? Do I know Node? I don't know Node. Let me go to collector. I don't know—I'm like, okay. - -**Mole:** Then, what I started doing was I started reading other people's PRs. I would literally go to an issue that is closed, go under the PR, you know, because when an issue is closed, the link to the PR that closed the issue will always be there. I would literally go and be studying their code. That's why I did a lot. I don't think I've said this anywhere before, but that's why I did a lot. I would go and literally be studying people's PRs, studying their code, studying how they framed things. Okay, this is how you write this in Go because remember I was still learning Go at this point. So, it was like a—I was learning so much in one month because I was studying people's code, I was doing a course at the same time, right? - -**Mole:** After studying people's PRs for a while, I gained some confidence. I found an issue that someone had done something similar. That's the beautiful thing about open source. Sometimes you want to update a bunch of components at the same time. I saw a similar issue to one of the PRs I had studied and I was like, I can tackle this. I asked, “Hello, my name is this. Can I please be assigned to this issue?” I didn’t think the person would answer me, and he was like, "Oh yeah, sure, why not? Go ahead." I think his username is MX-Psi. I don't know his real name. - -**Mole:** He was like, "Why not go ahead?" I tried it out. I made mistakes, obviously, because I was new to Go. I didn't understand the whole lint check thing. But he, you know, like I said, the open telemetry community is so wonderful. They do not look down on you. They don't look down on where you are as long as you're honest, right? Don't come and prove to be a senior developer when you're actually not. Just come clean. I told the guy, man, I haven't done this before. Please help my life. I went to his DMs and he was like, "Okay, do this, do this, do this." At the end of, you know, I think after two days, it got merged. I was like, "Yay, my first PR!" And then from there, it was just a full journey, right? It was a full journey, you know, from there. Really, really nice journey. - -**Adriana:** I have to agree with you that like that first PR getting merged is like the most glorious experience ever. But I will tell you, like every time I have a PR merged, I do like a little happy dance at home. It's like, oh my god, I'm up for something. By the way, I gotta say hats off to you in your approach to working on issues too, studying the PRs and the code behind the PRs. You have the heart and soul of a software engineer. It's like you were made for this. That's amazing. - -**Adriana:** I think this is a good lead-in to our next area that we want to talk about, which is—I believe you mentioned earlier that you did some work on creating an open telemetry exporter for the collector. Do talk about that. What was that experience like? +**Adriana:** I mean, Henrik produces amazing content, so I can definitely vouch for that. -[00:30:00] **Mole:** Okay, I have a lot of funny experiences. Sorry. When I got the project with my former company, Semantex, I was like, okay, let's do this, right? How hard can it be? So I went to the documentation and there was no documentation on how to create an exporter, right? The only documentation was how to create a receiver, and I was trying to understand it, and it wasn't working. One thing I realized later that I didn't know then was that there's not really a standard way of creating an exporter. Well, there’s a template you have to follow, but there’s no standardized way because exporters depend on your vendor back end. So at Semantex, we were receiving logs over influx 5 protocol and we were receiving metrics over elastic bulk index format, right? So I had to build an exporter that would convert hotel data to these two sources. +**Mole:** Yep. For video, I'll say Henrik's channel, and for text, Adriana's blog on Medium. Adriana's blog—if she's hearing me right now—should also be checked out. -**Mole:** What I did, like I did for my other PRs, was I started studying other people's exporters. I studied the influx DB exporter because that way our metrics would look like. Then I studied the elastic search exporter. I did a lot of study. I tried to break down a lot of things with the help of my assistant—great assistant. I tried to break it down. I tried to fit things in our own context. So how do I take this part from here, put it here, remove this part, you know? It was a whole trial and error. There was really no content that I could look to; okay, this is what assisted me in this journey. Oh, sorry, pardon me on that. I remember the open telemetry collector builder. +**Adriana:** Yeah, that's cool. Now, you said you have a preference for the videos, right? Is there a reason why? -**Mole:** While learning how to build the exporter, I had to use the OCB, the open telemetry collector builder, and that's when I came across Bind Plane's content on YouTube. They have a full series on open telemetry collector—a very wonderful series. I enjoyed every bit. That's why I learned how to use the open telemetry collector builder. The OCB is very fun. It's so cool, man. It's like you can build your own custom collector, you know, with whatever component you want. It's cool. It's like play-doh, right? You can build as you want, play with it as you want, connect things, join things, break things, and you know, it's fine. +**Mole:** Yeah. I'm a very imaginative thinker. I can sit down and just imagine a lot of things. I like visual things because if I'm just reading a bunch of text of a concept I haven't understood yet, I get bored. If I'm watching a video, the sound of the person, the tone, the infographics, the pictures just keeps me excited to keep going. Before I know it, I've tricked my brain into learning a new concept. -**Mole:** I learned how to use the OCB. After lots of trial and error, I got my first metric into Semantex, and it was an exciting feeling, right? After a while again, I got the logs in, and then awesome. Then I built the exporter. That was the summarized version of the experience of building the exporter. +**Adriana:** That's awesome. The next question we were wondering about: you mentioned that you use the OpenTelemetry docs as a reference. One of the goals of these sessions is to provide feedback to the OpenTelemetry community on areas for improvement. Is there anything you found that you found third-party documentation more useful than the official OpenTelemetry docs? What would you do to improve the docs experience if it were up to you as a new learner? -**Adriana:** It's really interesting. Believe me, I'm sharing in your excitement right now. I can only imagine. So for our next question, could you tell us a bit about learning Go to be able to work on this hotel exporter that you just talked about? +**Mole:** Sorry, this is an issue with multiple questions. The first question is what made me prefer third-party documentation over the OpenTelemetry docs? Well, because number one, there's no video documentation yet. That's why. Also, when I'm reading someone's content, I can feel a human connection. In documentation, it's just like reading a textbook—do this, do this, do that, move on. When you're writing your content, you can say, "Oh, do this. If this does not work, try this." -**Mole:** Learning Go has been a very interesting journey because I did not learn Go the normal way, right? The normal way is just watching a course and just building some interesting projects, you know, maybe to-do app stuff. I learned Go the hard way because I was actually learning it and had to apply it directly into real-life projects that go into production, right? I remember I had a lot of help from the people I worked with at Semantex—Bora and Akshhat, really cool guys, part of my team. They were so—I’d make mistakes, and they would be like, no, this is not the right way to do this. I know this is the YouTube way, this is the course way, but in production, this would break, this would fail. +When I entered the documentation, I got overwhelmed because I was scrolling through a lot of things, then I didn't understand what instrumentation was. The documentation has so many links that lead to so many things, and before you know it, you're lost in the rabbit hole of the documentation. I would be reading something, and then I'd see a link and click it, and before you know it, I was like, "Where am I? What am I learning?" So I got overwhelmed. -**Mole:** I say I learned Go hands-on, you know, contributing to things, right? It was a very interesting journey. I actually had a course from Code Cloud that gave me a lot of context. I could understand code; if you give me any Golang code, I can know okay this is what this is, what this is, this is a struct, this is an interface, this is what this does, this is what it should do. But then I had a lot of building things in real life. I feel like that’s really the best way to learn, you know, although it’s very stressful. You feel very dumb sometimes because some things—I remember sometimes at night I’d literally be up by 2:00 a.m. and the test kept failing, and I’m like please just run, please just run. I’d literally be praying. I’d be praying over and over and over again. Finally, it would run, and I’d be like yay, let’s go! Another milestone achieved! +What made me go back to the docs is I already had a foundation. I was not overwhelmed when I saw what instrumentation was or what a collector was. I already knew what I was looking for in the documentation. So, if I wanted to do a particular task, I could just search for that part in the docs and focus on that side. -**Adriana:** Cool. Throughout our conversation, you’ve talked a lot about the resources that helped you to learn open telemetry, that helped you to learn Golang, helped you to build the auto exporter. Now, could you tell me what plan you had for documenting this exporter that you built so that other people could use it? +What would I do to improve the doc experience? I'd say I'd make a video documentation. There are two different kinds of learners—visual learners and text-based learners. Satisfying both at the same time would be great. -**Mole:** What do you mean document it? Do you mean like you want me to document my journey or how I built the exporter? Can you clarify documentation for me? +**Adriana:** That is a super fair point. It's interesting too that you mentioned using the docs for more directed searches. -**Adriana:** Documentation for the exporter itself, for those who would be using it. +**Mole:** Exactly! -**Mole:** Okay. So, I think when you're contributing an exporter, there's actually a template of documentation that you have to do. I’ll answer your question from the standpoint because I think Adriana and I spoke about this last time. Even if I document it, and she actually put an idea that I should write an article on how it's not really possible to show you how to build an exporter because I tried. I actually sat down after building the exporter, and I was like, okay, how can I help the community with this? Even if I show you how I did it, it's not going to be useful to you because your context might be different, right? +### [00:25:12] Creating first PR experience -**Mole:** So what I would say is maybe the article I can write on or maybe the documentation will be: don't give up. Try, try, try again. Even when it fails, keep trying. It's going to work. Another thing I'll say is study other code. Studying other people's code—like I’ll say that’s one thing that has helped me in my whole coding journey and has really accelerated my speed. I always studied people's code and understood the patterns in the code. From understanding those patterns, I can now create my own, you know, find how to remove parts and then join to what I'm looking for. +**Adriana:** Could you tell us what your experience was like creating your first PR? -**Adriana:** That's great. It sounds like a great plan. +**Mole:** Creating my first PR was a whole experience. After watching some videos and reading some documentation, I did this for two weeks. I was like, "You need to make your first PR; you can't waste any more time." I went to the repos and was like, "Okay, which repo do I pick?" I was lost. I would go to an issue that was closed, go under the PR. I would literally go and study their code. That's why I did a lot. -**Mole:** Because I guess from the sounds of it, like the other two components—receivers and processors—I think there is some documentation around creating those because it's more formulaic, right? Whereas exporters are not. But the thing that's in common is like what's the thought process that you need for creating it. +After studying people's PRs for a while, I gained some confidence. I found an issue that was similar to one of the PRs I had studied, and I asked, "Can I please be assigned to this issue?" The person replied, "Yeah, sure, go ahead." I made mistakes obviously because I was new to Golang, but they were very supportive. At the end of two days, I got my first PR merged, and I was like, "Yay, my first PR!" From there, it just became a full journey. -**Adriana:** But not just this is the exporter template, this is step by step. +**Adriana:** I have to agree with you that getting that first PR merged is like the most glorious experience ever. I do a little happy dance at home every time it happens. -**Mole:** Now, yeah, I think that would be such a great blog post to write because I think, you know, again, it all goes back to what you were saying too about some of these third-party sources being about personal experiences and having that human touch, human connection. I think that would be so beneficial. It would also be interesting—I'm just giving an idea—to put in the docs something along the lines of like why is it that we don't have a template for creating open telemetry exporters? Because that was like, I remember at some point I was looking into that as well, and I'm like, I don't get it. Why is there no template for this? +**Mole:** Exactly. -**Mole:** Yeah, absolutely. There’s no official template. I think it’s when you start and then you speak to the maintainers, they tell you, oh no, no, no, this should be here, this shouldn’t be here. I’d be like, okay, let’s change things up, right? +**Adriana:** Hats off to you in your approach to working on issues too, like studying the PRs and the code behind them. You have the heart and soul of a software engineer. It's like you were made for this. -**Adriana:** Yeah, I think that’s great. +**Mole:** Thank you! -**Mole:** That’s great feedback. +**Adriana:** I believe you mentioned earlier that you did some work on creating an OpenTelemetry exporter for the collector. What was that experience like? -**Adriana:** The other thing that you mentioned that I want to dig into is the open telemetry collector builder. Specifically, what was your experience around that? Because I’ve played around with the builder; you’ve played around with the builder. Was it as easy as you thought it would be? If not, what would you say is some feedback that you would like to give on what can be done to make it easier? +**Mole:** I have a lot of funny experiences. When I got the project with Sematext, I was like, "Okay, let's do this, right? How hard can it be?" I went to the documentation, and there was no documentation on how to create an exporter. The only documentation was on how to create a receiver, and it wasn't working. I realized later that there's no standardized way of creating an exporter because exporters depend on your vendor backend. -**Mole:** Okay. You know, in my journey—and I’m going to be very honest here—in my journey with learning to try to get the understanding of the exporter, I didn’t really get any good information from the documentation. I’ll say for a while I just did not look at the docs anymore. I just went to YouTube and was doing the research. That’s why I found Bind Plane, right? They did a whole video on how to build your own custom collector—a 45, 46-minute video—and that basically showed me, okay, yeah, this works, this can do this. +### [00:30:48] Building OpenTelemetry exporter -**Mole:** I think also the readme files—some of the open telemetry collector builder are more documented in the readme file than well at that time than the documentation itself. So I think I got more value from the readme files at that time than the actual documentation. I don't know if it has been updated yet. I'll probably check, but yeah. +In Sematext, we were receiving logs over InfluxDB protocol and metrics over Elastic bulk index format. I had to build an exporter that would convert OpenTelemetry data to these two sources. Like I did for my other PRs, I started studying other people's exporters. I studied the InfluxDB exporter and the Elastic Search exporter, trying to fit things into our context. -**Adriana:** I did just get a PR merged in the hotel docs about—because when I was doing it, I’m like, okay, I built this DRO—how the hell do I containerize it? And there was no documentation around that. I had to ask, and then I wrote a blog post about it. But I’m like, it should be in the docs too. I just had that PR merged, I think today or yesterday. +I learned how to use the OpenTelemetry Collector Builder, and I came across BindPlane's content on YouTube. They have a full series on the OpenTelemetry Collector that was very helpful. After a lot of trial and error, I got my first metric into Sematext, and it was an exciting feeling. -**Mole:** That’s cool. That’s so cool. +**Adriana:** That's really interesting. I can only imagine. For our next question, could you tell us a bit about learning Golang to be able to work on this OpenTelemetry exporter? -**Adriana:** Yeah. So that was my experience with the open telemetry collector. +**Mole:** Learning Golang has been a very interesting journey. I did not learn Golang the normal way, just watching a course and building projects. I learned Golang the hard way because I was applying it directly to real-life projects that went into production. I had a lot of help from my team at Sematext. They were like, "No, this is not the right way to do this." -**Mole:** Awesome. +I got a course from CodeCloud, which gave me a lot of context. When I learned to contribute, it was stressful, but I feel like that's the best way to learn. Although it's very stressful, you'd feel dumb sometimes because things would fail. I remember being up at 2:00 a.m., trying to get my code to run, praying for it to work. When it finally did, I'd be like, "Yay, let's go!" -**Adriana:** I think I have just one more question. Could you tell us a bit about the open telemetry community? What’s your first impression of the community? How have you been able to collaborate with the community from the time you contributed to it as an intern and beyond that? +**Adriana:** Throughout our conversation, you've talked a lot about the resources that helped you learn OpenTelemetry, Golang, and build the auto exporter. Now, could you tell me what plan you had for documenting this exporter for those who would be using it? -**Mole:** Okay, I love the open telemetry community so much. My first experience with them was, I introduced myself: hello guys, I am—so I used to be introverted in some sort of way, but the people don’t really believe it because I’m very outspoken. But I used to be introverted, right? I was kind of shy because I was like, okay, I’m a newbie; I don’t really know much about stuff; I don’t know anything about open source. Who’s going to listen to me, right? I was like hello, my name is Mole from Nigeria. Nice to meet you all. And people said hi back. People waved back. Oh, nice to meet you, welcome to the open telemetry community. I was like, oh, that’s so cool. +**Mole:** When you're contributing an exporter, there's a template of documentation that you have to follow. Even if I document how I built the exporter, it might not be useful to someone else because their context might be different. So, I would say the article I could write is more about not giving up and trying again. -**Mole:** When I stumbled into a lot of like—back to my mentors, right? So Jurassi was my major mentor for the outreach internship. We had a lot of calls, right? In those calls, he just kept encouraging me that I know this is tough, but don’t worry, you’re going to get it. Just keep pushing, you know, keep trying. I believe in you. He’d come and say, I believe in you, I know you can do this. +Another thing I'd say is to study other code. Studying people's code has really accelerated my speed. Understanding those patterns allows me to create my own. -**Mole:** It was like a collective effort. I’d go to some people's DMs and I’d be like I’d be crying that please help me. I don’t understand this issue. How do I solve it? You’ve already assigned it to me. I tried this; it did not work. They’d be like, "Okay, you know what? Try this. Make this change." I remember when I wrote my first blog, my first blog post on open telemetry, and I had already read some of Adriana’s blog posts. I was like I’m going to send a DM to Adriana because I like to hear her thoughts on my blog post. I was like hello Adriana, hi, I’m Mole. Please can you review my blog post for me? She replied, and she reviewed my blog post. I was like wow, she doesn’t know me from anywhere, you know? We are not in the same continent or in different tribes, and she’s actually willing to assist, right? +**Adriana:** That would be a great blog post to write. It would be beneficial to share those experiences and practical advice. -**Mole:** I would say the open telemetry community is very loving. I can’t really vouch for a lot of open source communities, but I can vouch for the open telemetry community. They’re really cool and it’s a very nice community to join, by the way. I’m now a member of the open telemetry community. You can also text me. I’m very cool if you have questions about it, and she can text me. I’m cool. They are also cool, right? The community is loving, nice, really caring, and I love it. But then when I got to KubeCon and I met everyone in person, it was cooler. I met Adriana in person. I met Henrik. I met Tras and some other really awesome people. It was really nice. +**Mole:** Absolutely. -**Adriana:** I’m going to completely agree with everything you just said because I just recently joined the community too, and everyone has been amazing. Plus one to that; Adriana is very cool. It’s been nice getting to interact with her. +**Adriana:** You also mentioned the OpenTelemetry Collector Builder. What was your experience around that? Was it as easy as you thought it would be? If not, what would you say is some feedback that you would like to give on what can be done to make it easier? -**Adriana:** One more question before we go. Have you made any recent—I believe as an open telemetry community member now, you have to contribute to the open telemetry community. Have you had any recent contributions to the community that you’d like to talk about? +**Mole:** In my journey, I didn’t get good information from the documentation. For a while, I just did not look at the docs anymore. I went to YouTube and did research. I found BindPlane, and they did a video on how to build a custom collector. I got more value from the README files than the actual documentation at that time. -**Mole:** Yeah. Okay. So what I have been doing majorly right now is—when I came back from KubeCon, I think when I started learning the whole open telemetry thing, I realized that open telemetry was not a topic that was spoken about a lot in my country locally, you know, in Africa. Kubernetes is very popular here; everyone knows Kubernetes, right? So, open telemetry is not really heard a lot. I started advocating a lot about it. I think that’s how I even slided into my developer advocate role. +**Adriana:** That’s great feedback. I just had a PR merged in the OpenTelemetry docs about containerizing a collector because there was no documentation around that. -**Mole:** Currently, I’m a developer advocate. I’m not a full software engineer anymore. I started attending conferences, talking about open telemetry, showing people how to get started with open telemetry, how to get into open source, you know, because I experienced the dramas, the struggles, you know? I understand when people come to me and tell me I’m so confused about open source; how do I start? I can also give them practical steps I took to get started. I would say I’ve been doing more of that, and then I’m actually still—the exporter has not been built fully, but I’m still pushing it into the open telemetry collector contrib. So, I’d say that’s the—aside from related to code, those are the contributions that I’ve been making, but then I’ve been doing more of advocacy about open telemetry in my country. +**Mole:** That’s cool! -**Adriana:** That’s amazing. And I think that makes a really good point as well because contributing to open telemetry isn't just about the PRs that you're making; it's also about the advocacy, the bringing that awareness. I think that's great. +**Adriana:** So, I think this wraps up the main Q&A. If anyone from the audience has any questions, we are happy to take your questions. -**Adriana:** I think this wraps up the main Q&A, and we do have a few minutes left. If anyone from the audience has any questions that they would like to ask at this point, we are happy to take your questions. +**Mole:** I feel like we need some music. -**Mole:** I feel like we need some music. This is where you take out the guitar for this. +**Adriana:** This is where you take out the guitar for this. -**Adriana:** Dang it! Who’s the music? Is that you? Was that—I think that was Henrik, our amazing producer. +**Mole:** Dang it. Who had the music? -**Mole:** Well, it doesn’t look like we have any questions. So, what I will say is we’ll wrap up, but before we go, a few quick announcements, promotions. First of all, for anyone who is planning on attending OpenSource Summit in Denver, Colorado, there’s also going to be as a collocated event, there’s an open observability con and hotel community day that is going on. The CFPs for hotel community day are closed, but open observability con CFPs are still open. That’s going to be at the end of June. Feel free to submit CFPs to attend if you’re in the area or if you’re planning on attending. OpenSource Summit also is like lots of fun. It’s like KubeCon lite. It’s all the awesome people who attend without the flash, so it’s tons of fun. +**Adriana:** I think that was Henrik, our amazing producer. -**Adriana:** The other thing, for anyone who is interested, as I mentioned, I’m one of the maintainers of the hotel end user SIG. Victoria is a newly joined member of the hotel end user SIG. If you’re interested in joining us, seeing what we’re up to, Henrik just flashed a link for our GitHub repo. We do a bunch of things like these types of events. We have hotel me. We have hotel in practice, where if anyone who's watching is interested, if you have a topic—an observability hotel topic that you’d like to present on, maybe you’re testing out a topic for a conference for a talk, want to flesh it out, use us. Just DM us on the hotel end user SIG Slack and just let us know if you’re interested in doing a presentation for hotel in practice. +**Adriana:** It doesn't look like we have any questions, so what I will say is we'll wrap up. But before we go, a few quick announcements. -**Adriana:** We also run surveys in collaboration with the various hotel SIGs. Stay tuned; there will be more surveys heading your way. The surveys are a great way for us to get feedback from our user community on ways to improve the SIGs. That information is super important, super valuable to the community. Thank you to the folks who have responded to the surveys. Thank you to the folks who have engaged in the hotel end user SIG. +For anyone who is planning on attending Open Source Summit in Denver, Colorado, there's also going to be, as a collocated event, an Open Observability Con and OpenTelemetry Community Day. The CFPs for OpenTelemetry Community Day are closed, but Open Observability Con CFPs are still open. -**Adriana:** Also, for those who aren’t able to attend and for those who were able to attend, tell your friends that this video will be posted on our hotel YouTube channel. It’s hotel-official on YouTube. You can catch all of our previous hotel in practice and hotel me sessions. We do have our first how-to hotel video on the hotel YouTube channel, so for those who are visual learners, this is your opportunity to check that out. +The other thing for anyone who is interested: as I mentioned, I'm one of the maintainers of the OpenTelemetry End User SIG. If you're interested in joining us and seeing what we're up to, Henrik just flashed a link for our GitHub repo. We do a bunch of things like these types of events. -**Adriana:** Thank you so much, Mole, for joining and for Victoria for co-hosting with me today. This has been lots of fun. I always enjoy having these conversations, and we will see everyone at the next event. +Thank you so much, Mole, for joining, and Victoria, for co-hosting with me today. This has been lots of fun. I always enjoy having these conversations, and we will see everyone at the next event. -**Mole:** Sure. Thanks, Adriana. +**Mole:** Thanks, Adriana. -**Victoria:** Thank you. +**Victoria:** Thank you! **Mole:** Thanks, Adriana. Thanks, Victoria. It was nice meeting you. diff --git a/video-transcripts/transcripts/2025-05-02T18:18:11Z-even-more-humans-of-opentelemetry-kubecon-eu-2025.md b/video-transcripts/transcripts/2025-05-02T18:18:11Z-even-more-humans-of-opentelemetry-kubecon-eu-2025.md index 43c0296..aa23dfe 100644 --- a/video-transcripts/transcripts/2025-05-02T18:18:11Z-even-more-humans-of-opentelemetry-kubecon-eu-2025.md +++ b/video-transcripts/transcripts/2025-05-02T18:18:11Z-even-more-humans-of-opentelemetry-kubecon-eu-2025.md @@ -10,116 +10,128 @@ URL: https://www.youtube.com/watch?v=eZ3OrhxUAmU ## Summary -In this YouTube video, several experts in the field of observability and OpenTelemetry come together to share their experiences and insights. Notable participants include Marylia Gutierrez from Grafana Labs, Adriel Perkins from Liatrio, Jamie Danielson from Honeycomb, and Alolita Sharma from Apple, among others. The discussion revolves around the significance of OpenTelemetry as a community-driven, vendor-neutral standard for observability, which allows for better data collection, sharing, and interoperability across different systems and platforms. Participants reflect on their journeys in OpenTelemetry, emphasizing the importance of observability in understanding complex systems and improving engineering practices. They also discuss their favorite telemetry signals, highlighting the roles of metrics and tracing in diagnosing system issues and enhancing performance. Overall, the video showcases the collaborative spirit of the OpenTelemetry community and the critical role observability plays in modern software development. +In this YouTube video, various experts in observability and OpenTelemetry discuss their experiences, roles, and insights regarding the importance of observability in software engineering. Key participants include Marylia Gutierrez, Adriel Perkins, Jamie Danielson, and Alolita Sharma, among others, who share their backgrounds in the field. They emphasize the significance of OpenTelemetry as a collaborative, vendor-neutral standard that simplifies the process of collecting and analyzing telemetry data. Discussion points include the evolution of OpenTelemetry, the value of different telemetry signals like metrics and tracing, and the community aspect that fosters collaboration across various companies. The speakers highlight how observability helps in understanding complex systems, improving software development, and enhancing operational efficiency. ## Chapters -00:00:00 Introductions of speakers -00:02:40 First involvement with OpenTelemetry -00:06:31 Observability definitions and insights -00:10:30 Personal experiences with observability -00:12:39 Importance of OpenTelemetry -00:15:30 Community and collaboration in OpenTelemetry -00:18:36 OpenTelemetry protocol significance -00:19:20 Working on OpenTelemetry Collector -00:21:40 Favorite telemetry signals discussion -00:24:00 Closing thoughts on observability +00:00:00 Introductions +00:01:18 Discussion on OpenTelemetry +00:03:54 Guest introduction: Jamie Danielson +00:05:30 Guest introduction: Alolita Sharma +00:07:22 Observability definition and importance +00:09:06 Discussion on incident management +00:10:50 Observability in cloud-native infrastructure +00:13:00 OpenTelemetry community and collaboration +00:15:36 OpenTelemetry as a vendor-neutral standard +00:20:48 Discussion on telemetry signals: metrics vs. traces -**Marylia:** My name is Marylia Gutierrez. I'm a staff software engineer at Grafana Labs. And I also work on a few different groups on OpenTelemetry. +## Transcript -**Adriel:** My name is Adriel Perkins. I'm a principal engineer at a consulting company in the United States called Liatrio. We're both an end user, but I'm also a contributor in the OpenTelemetry project. I'm co-lead of the CI/CD SIG, with Dotan Horvitz, a CNCF Ambassador. And we work in the Collector as well as the specification repository. +### [00:00:00] Introductions -**Hanson:** My name is Hanson Ho and I do Android stuff. Specifically observing the Android stuff, at Embrace. +**Marylia:** My name is Marylia Gutierrez. I'm a staff software engineer at Grafana Labs. And I also work on a few different groups on OpenTelemetry. -**Jamie:** My name is Jamie Danielson. I'm an engineer at Honeycomb, and I work on instrumentation libraries and OpenTelemetry. +**Adriel:** My name is Adriel Perkins. I'm a principal engineer at a consulting company in the United States called Liatrio. We're both an end user, but I'm also a contributor in the OpenTelemetry project. I'm co-lead of the CI/CD SIG., with Dotan Horvitz, a CNCF Ambassador. And we work in the Collector as well as the specification repository. -**Mikko:** I’m Mikko Viitanen. I work as a product manager for Dynatrace. Then I'm also a maintainer in the OTel Demo App. +**Hanson:** My name is Hanson Ho and I do Android stuff. Specifically observing the Android stuff, at Embrace. -**Damien:** I am Damien Matheiu, and I do many things at OpenTelemetry. I am a maintainer for OpenTelemetry Go. I'm also a code owner for everything profiling on the Collector. And I'm an approver for the eBPF profiler. +**Jamie:** My name is Jamie Danielson. I'm an engineer at Honeycomb, and I work on instrumentation libraries and OpenTelemetry. -**Jacob:** My name is Jacob Aronoff. I am the CTO at Omelet. We're a new startup that does observability, telemetry pipelines, and OpenTelemetry very generally. +**Mikko:** I’m Mikko Viitanen. I work as a product manager for Dynatrace. Then I'm also a maintainer in the OTel Demo App. -[00:02:40] **Alolita:** Hi, everyone. I'm Alolita Sharma, and I lead AIML at Apple, for observability engineering, and observability infrastructure. I started in OpenTelemetry. I had actually been working on the observability world for a little while, and eventually you end up finding out about OpenTelemetry, and you find that this is, like, a really cool way to work and do not have dependency. My first involvement with OpenTelemetry was actually, I got asked to look at an observability enterprise solution, and that was when I discovered OpenTelemetry, and specifically the Collector. This discovery was because we had a lot of different data sources from different places, and we wanted to centralize them so that you could get a holistic view. So that was my like, first introduction when I was finding OpenTelemetry and its Collector. And I said, this stuff is really, really, really cool. +### [00:01:18] Discussion on OpenTelemetry -**Hanson:** Embrace looked to see how it could better serve the community. So OpenTelemetry was an obvious thing to look at being an open framework folks contribute to Common Standard. So when I saw it, I was like, this is great. This is kind of what we need. So we had a proprietary SDK collecting proprietary signals sending to our own servers. With OpenTelemetry, we were able to expand where we send this data to open source open Collectors, without changing, you know, a ton of stuff in the guts of the SDK. +**Damien:** I am Damien Matheiu, and I do many things at OpenTelemetry. I am a maintainer for OpenTelemetry Go. I'm also a code owner for everything profiling on the Collector. And I'm an approver for the eBPF profiler. -**Jamie:** So when I started looking at it, I was like, hey, look at mobile. Mobile's great. And at that point, not a lot of people were looking at mobile world of OpenTelemetry with a few exceptions. And I think now even like since a year or so that I've been involved, things have grown a lot, and a lot more interest. So, I'm really happy to get involved. +**Jacob:** My name is Jacob Aronoff. I am the CTO at Omelet. We're a new startup that does observability, telemetry pipelines, and OpenTelemetry very generally. -**Adriel:** When I started at Honeycomb in 2021, the team that I was on started working on OpenTelemetry more, and working through observability and instrumentation libraries. So I started working on the Collector and a little bit of Java before sort of settling into OpenTelemetry JavaScript. And so I've been there, you know the last three years, becoming an approver and more recently a maintainer of the project. +**Alolita:** Hi, everyone. I'm Alolita Sharma, and I lead AIML at Apple, for observability engineering and observability infrastructure. I started in OpenTelemetry. I had actually been working on the observability world for a little while, and eventually, you end up finding out about OpenTelemetry, and you find that this is, like, a really cool way to work and do not have dependency. My first involvement with OpenTelemetry was actually, I got asked to look at an observability enterprise solution, and that was when I discovered OpenTelemetry, and specifically the Collector. This discovery was because we had a lot of different data sources from different places, and we wanted to centralize them so that you could get a holistic view. So that was my first introduction when I was finding OpenTelemetry and its Collector. And I said, this stuff is really, really, really cool. Embrace looked to see how it could better serve the community. So OpenTelemetry was an obvious thing to look at being an open framework folks contribute to Common Standard. So when I saw it, I was like, this is great. This is kind of what we need. So we had a proprietary SDK collecting proprietary signals sending to our own servers. With OpenTelemetry, we were able to expand where we send this data to open source open Collectors, without changing, you know, a ton of stuff in the guts of the SDK. So when I started looking at it, I was like, hey, look at mobile. Mobile's great. And at that point, not a lot of people were looking at the mobile world of OpenTelemetry with few exceptions. And I think now, even like since a year or so that I've been involved, things have grown a lot, and a lot more interest. So, I'm really happy to get involved. -**Damien:** I started with OpenTelemetry around three years ago, and I started doing small contributions to the OTel Demo App. And I found that’s a great place to learn the basics of instrumentation and a little bit of the Collector, configuration and, and kind of I found the Demo App actually provides a little bit of everything around code and hands on. So I find it well. +### [00:03:54] Guest introduction: Jamie Danielson -**Mikko:** I was working at a different company in an observability team, and I was rather frustrated because I had a hard time. I was convinced already that we should not just be using logs, but we should also do tracing and therefore use OpenTelemetry. And it was very hard to convince some of our folks in engineering there at the time. And so kind of as a New Year resolution in 2022, I decided that I would start watching the OpenTelemetry Go repository and start contributing. And one thing led to another. And, a year later, I got a full time job working on OpenTelemetry. +**Jamie:** When I started at Honeycomb in 2021, the team that I was on started working on OpenTelemetry more, and working through observability and instrumentation libraries. So I started working on the Collector and a little bit of Java before sort of settling into OpenTelemetry JavaScript. And so I've been there, you know, the last three years, becoming an approver and more recently a maintainer of the project. -**Jacob:** I started my OTel journey at Lightstep, my former employer. I started on the telemetry pipelines team, working with some amazing contributors to the OTel ecosystem. I started on the Kubernetes side of things with the OpenTelemetry Operator, working on upgrading the target allocator, which does horizontally sharding of, scrape targets for Prometheus, for the Collector. Very technical, but very important part of the ecosystem. +**Damien:** I started with OpenTelemetry around three years ago, and I started doing small contributions to the OTel Demo App. And I found that’s a great place to learn the basics of instrumentation and a little bit of the Collector, configuration, and kind of I found the Demo App actually provides a little bit of everything around code and hands-on. -[00:06:31] **Damien:** I got started with OpenTelemetry more than six years ago now. And I was at AWS at that time, and I've been always very involved in open source projects all the way from the beginning Linux. And, in my journey in the cloud native world, as I was building platform services at AWS, we decided to build out a new generation of Kubernetes native services. And, it was really exciting that we took the opportunity to get involved, as a team in all the new shiny of open source observability projects. And, of course, OpenTelemetry was at the forefront. This is right after OpenTracing and OpenCensus got together and combined to form OpenTelemetry, and it was an exciting change to now see this beautiful brand new project with all contributors from both two projects combining together and getting new contributors like me involved. +**Damien:** I was working at a different company in an observability team, and I was rather frustrated because I had a hard time. I was convinced already that we should not just be using logs, but we should also do tracing and therefore use OpenTelemetry. And it was very hard to convince some of our folks in engineering there at the time. And so kind of as a New Year resolution in 2022, I decided that I would start watching the OpenTelemetry Go repository and start contributing. And one thing led to another. And, a year later, I got a full-time job working on OpenTelemetry. -**Marylia:** Observability, to me, means that you are able to find the things that you didn't even know that you wanted to know, because you have a lot of information. But just having information doesn't mean anything, if you don't know how to interpret. So is a as like they say that it actually comes from like mechanic engineering. That was just trying to understand like the system where you can just extrapolate this for everything. So finding a way to understand everything from your system can even bring to your life observability, understand what is going on. +### [00:05:30] Guest introduction: Alolita Sharma -**Adriel:** What does observability mean to me personally? It enables me to find things out that I didn't know and improve. And I've always been someone who loves continuously learning and continuous improvement and observability is the thing that helps me do that, because there's a lot of things that I don't know. I think the more that I find out that I do now, the more I realize there's more things I don't know. +**Mikko:** I started my OTel journey at Lightstep, my former employer. I started on the telemetry pipelines team, working with some amazing contributors to the OTel ecosystem. I started on the Kubernetes side of things with the OpenTelemetry Operator, working on upgrading the target allocator, which does horizontally sharding of scrape targets for Prometheus, for the Collector. Very technical, but very important part of the ecosystem. -**Hanson:** So observability has really helped me to, like, discover that. Both the technical level of like various different applications and services. But it's also helped me do that in the socio-technical aspect. Right. So, the telemetry that I've been able to find and discover as part of just like software development lifecycle has enabled me to be a better engineer. And so it's just help for discovery in general for myself. +**Adriel:** I got started with OpenTelemetry more than six years ago now. And I was at AWS at that time, and I've been always very involved in open source projects all the way from the beginning Linux. And in my journey in the cloud native world, as I was building platform services at AWS, we decided to build out a new generation of Kubernetes native services. And it was really exciting that we took the opportunity to get involved, as a team in all the new shiny open source observability projects. And, of course, OpenTelemetry was at the forefront. This is right after OpenTracing and OpenCensus got together and combined to form OpenTelemetry, and it was an exciting change to now see this beautiful brand new project with all contributors from both two projects combining together and getting new contributors like me involved. -**Jamie:** Observability. I mean, to boil Hazel Weekly's definition, it's about asking questions and getting answers, specifically ones that you didn't really think needed to ask initially, then doing something with that. Being able to act and learn from your data. It's not just telemetry. It's about understanding your system through data. +### [00:07:22] Observability definition and importance -**Mikko:** Observability means having insights into your system. It's about understanding how your applications work, how your systems are working, and having visibility into things you don't even know necessarily are important until something comes up. So being able to find, you know, those unknown unknowns in your services and be able to make sense of things that maybe don't make sense otherwise. +**Adriel:** Observability, to me, means that you are able to find the things that you didn't even know that you wanted to know because you have a lot of information. But just having information doesn't mean anything if you don't know how to interpret. So is a, as like they say that it actually comes from like mechanic engineering. That was just trying to understand like the system where you can just extrapolate this for everything. So finding a way to understand everything from your system can even bring to your life observability, understand what is going on. -**Adriel:** I actually associated that many years back... I was working with the telecom networks and yeah, observability was really, really crucial. So consider you make a phone call from here or for example for US. And the call goes through the ends of nodes and even multiple operators. So without observability, you couldn't pinpoint the problems. So the customer calls, hey, why are my calls dropped? So definitely you have to have a really strong observability in order to find out what's happening and pinpoint the issues. +**Marylia:** What does observability mean to me personally? It enables me to find things out that I didn't know and improve. And I've always been someone who loves continuously learning and continuous improvement and observability is the thing that helps me do that because there's a lot of things that I don't know. I think the more that I find out that I do now, the more I realize there's more things I don't know. So observability has really helped me to discover that. Both the technical level of various different applications and services. But it's also helped me do that in the socio-technical aspect. Right. So, the telemetry that I've been able to find and discover as part of just like the software development lifecycle has enabled me to be a better engineer. And so it's just helped for discovery in general for myself. -[00:10:30] **Jacob:** Before working on observability, I worked for many years on, I guess, SRE roles or equivalents. I've also done like incident management and, like, yeah, I've been an incidents commander as well. So I've been in many incidents trying to figure out its root causes. Very hard to figure out because it's on a Sunday morning and things like that. And yeah, I don't want to have to go through that again. And, working on observability ensures that, if I go back to being an SRE and like, operations role, things should be better. +**Hanson:** Observability. I mean, to boil Hazel Weekly's definition, it's about asking questions and getting answers, specifically ones that you didn't really think needed to ask initially, then doing something with that. Being able to act and learn from your data. It's not just telemetry. It's about understanding your system through data. -**Mikko:** And if they should be better for others. For me personally, it is understanding what is going on in your system at any given time. I think of observability generally as, you know, knowing the health of your running servers. The analogy that I use is when you're driving a car, you have a dashboard in front of you that has, you know, lots of instruments, lots of measurements that are telling you if you're operating the car effectively. Observability is the same thing. But for servers at a much, much larger scale than a single car. +### [00:09:06] Discussion on incident management -**Adriel:** Observability means a lot because I think that, you know, observability as a discipline, especially for cloud native infrastructure and applications, is a very essential part of guaranteeing that your applications and your infrastructure observable they work. And in the, you know, as a software engineer, especially as a distributed systems engineer, if you are looking in building applications, and using cloud native infrastructure, whether that is public cloud or, you know, non-public cloud on prem, Kubernetes based infrastructure, you are, inevitably, dealing with and working with a lot of complex microservices, which is absolutely essential for you to have observability baked into your application as well as your infrastructure day one. +**Hanson:** Observability means having insights into your system. It's about understanding how your applications work, how your systems are working, and having visibility into things you don't even know necessarily are important until something comes up. So being able to find, you know, those unknown unknowns in your services and be able to make sense of things that maybe don't make sense otherwise. -**Jamie:** And, in the case of observability, as it stands today, it’s not only collection of telemetry, which we have, you know, literally trillions and trillions of petabytes of data being generated by not only applications but models now as well as infra and but also, looking at how the whole solution works, in terms of storage, performance, analysis, as well as visualization. +**Adriel:** I actually associated that many years back. I was working with the telecom networks and yeah, observability was really, really crucial. So consider you make a phone call from here or for example for the US. And the call goes through the ends of nodes and even multiple operators. So without observability you couldn't pinpoint the problems. So the customer calls, hey, why are my calls dropped? So definitely you have to have a really strong observability in order to find out what's happening and pinpoint the issues. -[00:12:39] **Damien:** To me, OpenTelemetry means that you don't have to have the dependency on anyone and is also about the community. So it's a way for everybody to work together that you normally find your competitor, but actually you work together. You go to a meeting or do pair programming, and you just want to see that community to grow. +**Jamie:** Before working on observability, I worked for many years on, I guess, SRE roles or equivalents. I've also done like incident management and, like, yeah, I've been an incidents commander as well. So I've been in many incidents trying to figure out its root causes. Very hard to figure out because it's on a Sunday morning and things like that. And yeah, I don't want to have to go through that again. And, working on observability ensures that if I go back to being an SRE in like, operations role, things should be better. And if they should be better for others. -**Jacob:** What does OpenTelemetry mean? So many things, because it does so much. And having been able to touch all the different parts has really given this huge meaning to me, but it's really like it to me, it means that it's the central thing in any observability stack. If I have OpenTelemetry, then I can go figure out exactly the things I need to do. No matter what vendor I'm using as a backend. +### [00:10:50] Observability in cloud-native infrastructure -**Mikko:** I think OpenTelemetry is an opportunity for everybody to kind of work together, in, you know, maybe slightly different goals, but working on the same thing that will achieve it for everyone. I believe it's important to have an open standard so we speak the same language. We need a lingua franca, for observability. Instead of using proprietary things that really don't add a ton of value. +**Jacob:** For me personally, it is understanding what is going on in your system at any given time. I think of observability generally as, you know, knowing the health of your running servers. The analogy that I use is when you're driving a car, you have a dashboard in front of you that has, you know, lots of instruments, lots of measurements that are telling you if you're operating the car effectively. Observability is the same thing. But for servers at a much, much larger scale than a single car. -**Adriel:** Let's agree on the vocabulary. Let's agree on the letters. And I believe OpenTelemetry gives us an opportunity so we can all understand each other without having to literally be behind the same, you know, platform wall. +**Adriel:** Observability means a lot because I think that, you know, Observability as a discipline, especially for cloud native infrastructure and applications, is a very essential part of guaranteeing that your applications and your infrastructure work. And in the, you know, as a software engineer, especially as a distributed systems engineer, if you are looking at building applications and using cloud native infrastructure, whether that is public cloud or, you know, non-public cloud on-prem, Kubernetes-based infrastructure, you are inevitably dealing with and working with a lot of complex microservices, which is absolutely essential for you to have observability baked into your application as well as your infrastructure day one. -**Jamie:** OpenTelemetry is sort of special to me. Obviously I've been involved in OpenTelemetry for a few years, but I love this idea of this vendor neutral standard that people can use, that everyone sort of benefits from the standard being there and everyone is contributing from all over the place. From vendors, from end users to different people in the community who are passionate about it. And it's sort of like being around a lot of friends and people who work really hard and hold each other accountable and just try to make the best of this project. +**Adriel:** And in the case of observability, as it stands today, it’s not only the collection of telemetry, which we have, you know, literally trillions and trillions of petabytes of data being generated by not only applications but models now as well as infra, but also looking at how the whole solution works in terms of storage, performance, analysis, as well as visualization. + +### [00:13:00] OpenTelemetry community and collaboration + +**Mikko:** To me, OpenTelemetry means that you don't have to have the dependency on anyone and is also about the community. So it's a way for everybody to work together that you normally find your competitor, but actually you work together. You go to a meeting or do pair programming, and you just want to see the community grow. You want to see people be able to solve their problems just by working all together. + +**Adriel:** What does OpenTelemetry mean? So many things, because it does so much. And having been able to touch all the different parts has really given this huge meaning to me, but it's really like it to me, it means that it's the central thing in any observability stack. If I have OpenTelemetry, then I can go figure out exactly the things I need to do. No matter what vendor I'm using as a backend. + +**Hanson:** I think OpenTelemetry is an opportunity for everybody to kind of work together, in, you know, maybe slightly different goals, but working on the same thing that will achieve it for everyone. I believe it's important to have an open standard so we speak the same language. We need a lingua franca for observability. Instead of using proprietary things that really don't add a ton of value. Let's talk the same language. Maybe you will use the different ways to put things together. Let's agree on the vocabulary. Let's agree on the letters. And I believe OpenTelemetry gives us an opportunity so we can all understand each other without having to literally be behind the same platform wall. + +**Adriel:** OpenTelemetry is sort of special to me. Obviously, I've been involved in OpenTelemetry for a few years, but I love this idea of this vendor-neutral standard that people can use, that everyone sort of benefits from the standard being there and everyone is contributing from all over the place. From vendors, from end users to different people in the community who are passionate about it. And it's sort of like being around a lot of friends and people who work really hard and hold each other accountable and just try to make the best of this project. **Damien:** And I love the idea of vendors competing on their baguettes and competing on the features that they have in their products, and it's less difficult for end users to just instrument their applications just to get visibility into their system without having to deal with a different agent if they decide they want to switch to another vendor. It's just very open and a lot easier for people to use. -[00:15:30] **Mikko:** OpenTelemetry, personally, I feel it's such a great example of an open source project, but we are in a competitive industry and, and having so many companies, like +100 companies contributing to OTel and solving common problems and collaborating every day. So it feels really amazing. I feel it's about community and collaboration. +### [00:15:36] OpenTelemetry as a vendor-neutral standard + +**Jacob:** OpenTelemetry, personally, I feel it's such a great example of an open source project, but we are in a competitive industry and having so many companies, like 100 companies contributing to OTel and solving common problems and collaborating every day. So it feels really amazing. I feel it's about community and collaboration. -**Jacob:** To me, I think it's not just about like, solving engineering problems, but, I think really, like the global community is extremely nice and welcoming. I think it's extremely impressive what we have been able to succeed doing. Like having a common and shared understanding of how things should work across over 15 languages with very different needs and problems and ways of solving problems depending on the languages and the fact that as human beings, we have been able to solve that is extremely encouraging, I think. +**Adriel:** To me, I think it's not just about solving engineering problems, but I think really, like the global community is extremely nice and welcoming. I think it's extremely impressive what we have been able to succeed in doing, like having a common and shared understanding of how things should work across over 15 languages with very different needs and problems and ways of solving problems depending on the languages. And the fact that as human beings, we have been able to solve that is extremely encouraging, I think. -**Adriel:** And I would also add that it's extremely rare in the current environments to see like multiple companies, competitors aligned together to build something so that they can all be more, like, provide better value together rather than just, like, stick to their own little corner. And so that's, that's kind of the huge things are great things I find about OpenTelemetry. +**Adriel:** And I would also add that it's, I think it's extremely rare in the current environment to see multiple companies, competitors aligned together to build something so that they can all provide better value together rather than just stick to their own little corner. And so that's kind of the huge things or great things I find about OpenTelemetry. -**Jamie:** OpenTelemetry to me is the way that we get all of that data. Right? It's this really vast ecosystem of people who have agreed that there should be one way to do something, you know, that is new, generic to the field. Back to that car analogy. You know, you don't learn how to drive a Nissan or a Volvo. You learn how to drive a car. Right. And so in the same way that when you're learning how to engineer, it's important for there to be standards so that you don't have to relearn everything every time you go to a different company. And to me that is what OpenTelemetry means. +**Jamie:** OpenTelemetry to me is the way that we get all of that data. Right? It's this really vast ecosystem of people who have agreed that there should be one way to do something, you know, that is new, generic to the field. Back to that car analogy. You know, you don't learn how to drive a Nissan or a Volvo. You learn how to drive a car. Right. And so in the same way that when you're learning how to engineer, it's important for there to be standards so that you don't have to relearn everything every time you go to a different company. And to me, that is what OpenTelemetry means. -**Damien:** OpenTelemetry has almost 80 repos today on the project. And, as many of you may know, OpenTelemetry is a very large project. It is not only a collection of components, but also an amazing community in terms of the partnerships and the collaboration that vendors and end users work together on, in terms of solving technical challenges and building the best components in OpenTelemetry. +**Adriel:** OpenTelemetry has almost 80 repos today on the project. And, as many of you may know, OpenTelemetry is a very large project. It is not only a collection of components, but also an amazing community in terms of the partnerships and the collaboration that vendors and end users work together on, in terms of solving technical challenges and building the best components in OpenTelemetry. -[00:18:36] **Adriel:** To me, OpenTelemetry is not only an integral part of building collection architectures for cloud native applications, but it is also an amazing community where you actually see the use of interoperability across the different components in the project, as well as open standards such as the OpenTelemetry protocol being, implemented end to end, which really is a game changer for the industry. +**Jamie:** So to me, OpenTelemetry is not only an integral part of building collection architectures for cloud native applications, but it is also an amazing community where you actually see the use of interoperability across the different components in the project, as well as open standards such as the OpenTelemetry protocol being implemented end to end, which really is a game changer for the industry. -**Hanson:** OpenTelemetry protocol. The reason I call that out is because it really enables end users to be able to build solutions for observability, end to end and use observability out of the box without having to think about, oh, what's my protocol going to be for metrics or logs or traces or profiles at all of the data that we collect? Right. But open. +**Adriel:** The OpenTelemetry protocol. The reason I call that out is because it really enables end users to be able to build solutions for observability, end to end, and use observability out of the box without having to think about, oh, what's my protocol going to be for metrics or logs or traces or profiles at all of the data that we collect? -[00:19:20] **Jacob:** I love OpenTelemetry as a project and as a community. And, I love working on all the different pieces I worked on. I focused on working on the Collector. Collector contrib components, where we have added integrations, improving the operators for OpenTelemetry Collector, adding more metrics, performance features such as the targets allocator improvements in the operator, and also working on improving tracing, and logging. +**Mikko:** I love OpenTelemetry as a project and as a community. And I love working on all the different pieces I worked on. I focused on working on the Collector, Collector contrib components, where we have added integrations, improving the operators for OpenTelemetry Collector, adding more metrics, performance features such as the targets allocator improvements in the operator, and also working on improving tracing and logging. -**Mikko:** My favorite telemetry signal... I still like metrics. I know that you will start to find out a little bit about traces, but I find that metrics is still almost like the gateway for the signals, because it's very simple to explain to people. It’s a number. Then on top of the number you can add, for example, attributes and get more data on top of it. So this way people don't get scared with like a big trace or big span as soon as you start, but it's a way to at least get more people into this area. +**Jamie:** My favorite telemetry signal? I still like metrics. I know that you will start to find out a little bit about traces, but I find that metrics is still almost like the gateway for the signals because it's very simple to explain to people. It’s a number. Then on top of the number you can add, for example, attributes and get more data on top of it. So this way people don't get scared with like a big trace or big span as soon as you start, but it's a way to at least get more people into this area. And then from that, you can kind of grow on top of that. -**Jamie:** Favorite telemetry signal. That's such a hard thing to pick one because you can combine them so well. I think I started off as having metrics as my favorite because they were the thing that I looked at first when it came to the SDLC. But as I've gotten more into traces and tracing pipelines and all that stuff, I just realized how powerful it was. And then I can just derive, like any of the signals from those traces, and I can embed them directly in there. +### [00:20:48] Discussion on telemetry signals: metrics vs. traces -**Damien:** So I think I have to say right now, tracing is my favourite. +**Hanson:** Favorite telemetry signal. That's such a hard thing to pick one because you can combine them so well. I think I started off as having metrics as my favorite because they were the thing that I looked at first when it came to the SDLC. But as I've gotten more into traces and tracing pipelines and all that stuff, I just realized how powerful it was. And then I can just derive, like any of the signals from those traces, and I can embed them directly in there. So I think I have to say right now, tracing is my favourite. -**Adriel:** Well, traces really is the most powerful. So I would now pick that because you can. Yeah. Let's go spans. Yeah let’s go spans. Love spans. +**Hanson:** Well, traces really are the most powerful. So I would now pick that because you can... yeah. Let's go spans. Yeah let’s go spans. Love spans. -[00:21:40] **Mikko:** My favorite OpenTelemetry signal is probably going to be traces. I really like traces. I like the idea of starting in one place in your application, starting in one service, and seeing a request go from end to end service to service, and get an understanding of how that is flowing that you might not otherwise have visibility into because it's a full connected trace. So, yeah, traces. Traces would be my favorite signal I think. +**Jamie:** My favorite OpenTelemetry signal is probably going to be traces. I really like traces. I like the idea of starting in one place in your application, starting in one service, and seeing a request go from end to end service to service, and get an understanding of how that is flowing, that you might not otherwise have visibility into because it's a full connected trace. So, yeah, traces. Traces would be my favorite signal, I think. -**Jamie:** My favorite telemetry signal. They're all important, but I mostly I would select distributed traces. I find it special because with the single view, single waterfall view, you can easily get the overview. You can pinpoint problems. So if your request gets rejected, you can quite often see it's already promising. The view of what was the service causing the reject. Or if your request is returned really slowly or the system is running really slowly. You can see how each service is adding up, just from the distributed trace. So that's lots of insight. +**Damien:** My favorite telemetry signal? They are all important, but I mostly I would select distributed traces. I find it special because with the single view, single waterfall view, you can easily get the overview. You can pinpoint problems. So if your request gets rejected, you can quite often see it's already promising the view that what was the service causing the reject. Or if your request is returned really slowly or the system is running really slowly, you can see how each service is adding up, just from the distributed trace. So that's lots of insight. -**Adriel:** Oh, my colleagues are going to hate me for this, but it's tracing. And I'm saying that because, I work on profiling a lot, and the people that I work with really live and think by profiling. I think both are very important. But profiling is more like for, whatever you don't, everything works fine. And you want to improve things. And tracing is for, things are broken, and you need to figure out why. And that's kind of how I came into OpenTelemetry. And that's kind of my own work experience. So that's why my favourite signal is tracing. +**Adriel:** Oh, my colleagues are going to hate me for this, but it's tracing. And I'm saying that because I work on profiling a lot, and the people that I work with really live and think by profiling. I think both are very important. But profiling is more like for whatever you don't, everything works fine. And you want to improve things. And tracing is for things that are broken, and you need to figure out why. And that's kind of how I came into OpenTelemetry. And that's kind of my own work experience. So that's why my favourite signal is tracing. -**Jamie:** My favourite telemetry signal by far is tracing. Tracing is, you know, the best of every world, in my opinion. You can derive metrics, you can derive logs, you can do lots of really important visualizations that help you understand both, you know, the high level observability goals that you might have, as well as the really low level debugging that you might have to do. It is the most important one, and I think the most misunderstood as well. +**Jamie:** My favourite telemetry signal by far is tracing. Tracing is, you know, the best of every world, in my opinion. You can derive metrics, you can derive logs, you can do lots of really important visualizations that help you understand both, you know, the high-level observability goals that you might have, as well as the really low-level debugging that you might have to do. It is the most important one, and I think the most misunderstood as well. -[00:24:00] **Damien:** My favorite telemetry signal, I would say top of my list today. And that's metrics and traces. And the reason I say that is because when you're looking at observability real time, especially for AI applications, you know, in the new generation of applications coming in, tracing is very valuable, along with profiling, to be able to understand, you know, model behavior as well as software application behavior. And combined with metrics, you know, which actually are usually the standard way of getting telemetry and understanding of your infrastructure systems, it provides a very nice way of actually providing an end to end view of observability and observable components, all the way up the stack. +**Mikko:** My favorite telemetry signal, I would say top of my list today, and that's metrics and traces. And the reason I say that is because when you're looking at observability real-time, especially for AI applications, you know, in the new generation of applications coming in, tracing is very valuable, along with profiling, to be able to understand, you know, model behavior as well as software application behavior. And combined with metrics, which actually are usually the standard way of getting telemetry and understanding of your infrastructure systems, it provides a very nice way of actually providing an end-to-end view of observability and observable components, all the way up the stack. ## Raw YouTube Transcript diff --git a/video-transcripts/transcripts/2025-05-07T18:47:21Z-otel-night-berlin-v2025-05-spring-starter-for-opentelemetry.md b/video-transcripts/transcripts/2025-05-07T18:47:21Z-otel-night-berlin-v2025-05-spring-starter-for-opentelemetry.md index f734595..3311b6c 100644 --- a/video-transcripts/transcripts/2025-05-07T18:47:21Z-otel-night-berlin-v2025-05-spring-starter-for-opentelemetry.md +++ b/video-transcripts/transcripts/2025-05-07T18:47:21Z-otel-night-berlin-v2025-05-spring-starter-for-opentelemetry.md @@ -10,239 +10,356 @@ URL: https://www.youtube.com/watch?v=R0n6Ny588dk ## Summary -In this video, the speaker discusses the Spring Starter for OpenTelemetry, sharing insights from their extensive experience with Java and Spring Boot. They explain the motivations behind creating a Spring Boot starter for OpenTelemetry, highlighting its ease of use compared to existing solutions like Micrometer. The speaker also provides an overview of the current state of OpenTelemetry in Java, including the stability of various libraries and the importance of semantic conventions. A significant portion of the presentation is dedicated to a live demo, showcasing how the starter integrates seamlessly with Spring Boot applications for observability. Throughout the session, they engage with the audience, addressing questions and clarifying technical concepts related to the implementation and functionality of OpenTelemetry in Java applications. +In this video, the speaker discusses the Spring Boot Starter for OpenTelemetry, sharing their personal journey as a Java developer and their experiences with Spring Boot and OpenTelemetry. The presentation covers the current state of OpenTelemetry, particularly in Java, and outlines the motivations behind building a Spring Boot starter, comparing it to existing solutions like Micrometer. The speaker emphasizes the importance of a seamless integration experience for Spring Boot users and explains the technical aspects of OpenTelemetry, including bytecode manipulation and the significance of semantic conventions. A significant portion of the video is dedicated to a live demo showcasing how to implement OpenTelemetry in a Spring Boot application, followed by a Q&A session addressing various technical questions and challenges related to context propagation and service name configurations. Overall, the video aims to highlight the ease of integrating observability features into Spring Boot applications. ## Chapters -00:00:00 Introduction and background -00:01:30 Overview of OpenTelemetry -00:03:10 State of OpenTelemetry in Java -00:04:37 Spring Boot starter motivation -00:06:05 Spring Boot experience expectations -00:08:00 Protocol and semantic conventions -00:10:41 Comparison with Micrometer -00:12:00 Spring Boot starter vs existing technologies -00:15:40 Demo introduction -00:15:40 Starting the demo application -00:28:00 Demo results and Q&A +00:00:00 Introductions +00:01:34 Overview of OpenTelemetry state +00:02:36 Motivation for Spring Boot Starter +00:05:12 Comparison to Micrometer +00:10:24 Explanation of bytecode manipulation +00:15:36 Start of live demo +00:19:04 Context propagation discussion +00:25:08 Demo of OpenTelemetry integration +00:37:06 Q&A on service name configuration +00:47:40 Closing remarks -**Speaker 1:** All right. Yeah, I want to talk about the spring starter for open telemetry. And what I'll try to do is it just works, but except when it doesn't. So, that's for the demo gods and because it's spring boot. +## Transcript -**Speaker 1:** About myself. Actually, what I didn't put on here but 20 years ago I started my first job as a Java developer in this building. Just realized today I... Yeah. It was not the first location but the first company and we moved here from a different location in Berlin to this one and I worked here for about three years. +### [00:00:00] Introductions -**Speaker 1:** Yeah, 10 years later I started using Spring Boot. I still don't know everything about it. Five years ago I started working with hotel and I definitely know that I will never know everything about it. Yeah, and then I started working for Grafana two years ago and now I'm an approver for hotel Java instrumentation. So the stuff that sends all the data but you will see that later. And I work a bit on some other areas in hotel, operator, collector, and specification. +**Speaker 1:** All right. I want to talk about the spring starter for open telemetry. What I'll try to do is it just works, but except when it doesn't, so that's for the demo gods and because it's spring boot. -[00:01:30] **Speaker 1:** I want to give you like a quick overview of the state of hotel in 2025, what is new. Then the motivation why building a spring boot startup in the first place. I also want to compare it quickly to other solutions in spring boot which is called micrometer. I want to spend most of the time in the demo to actually see and show you that it is easy and leave time for Q&A, but you can also ask in between. I hope that we have enough time and if we don't then it's better if you ask before rather than I'm trying to finish the demo and it doesn't work. +**Speaker 1:** About myself. Actually, what I didn't put on here, but 20 years ago I started my first job as a Java developer in this building. Just realized today, yeah, it was not the first location, but the first company, and we moved here from a different location in Berlin to this one, and I worked here for about three years. -**Speaker 1:** As you can see, we have quite a lot of languages in hotel and a lot of them have actually moved to stable for traces, logs, metrics. I think that's quite a bit of change for last year but I haven't recorded it. So Java, which we are talking about, PHP, .NET, and C++. For some reason, profiling is not listed here but this is like a new topic that is an active development. I guess in a year or two this will also be listed here. +**Speaker 1:** Ten years later, I started using Spring Boot. I still don't know everything about it. Five years ago, I started working with hotel, and I definitely know that I will never know everything about it. Then I started working for Grafana two years ago, and now I'm an approver for hotel Java instrumentation. So, the stuff that sends all the data, but you will see that later. I work a bit on some other areas in hotel, operator collector, and specification. -[00:03:10] **Speaker 1:** And I will not show that. Okay, state of hotel in Java. Java is one of the more mature areas in hotel probably because a lot of e-commerce and business applications are written in Java and there's a lot of demand and there are many libraries. I just counted yesterday over 100 supported libraries are supported and it's also compatible with ancient versions of Java. So, Java 8 was released long more than 10 years ago but I did not count. +### [00:01:34] Overview of OpenTelemetry state -**Speaker 1:** And that is actually only possible because Java has a technology called byte code manipulation which means that you can write software that at runtime looks at all the methods and classes that are loaded and can basically change anything. It can change a return value and can look at parameters similar to how Python can do monkey patching. +**Speaker 1:** I want to give you a quick overview of the state of hotel in 2025, what is new. Then the motivation for building a spring boot startup in the first place. I also want to compare it quickly to other solutions in spring boot, which is called micrometer. I want to spend most of the time in the demo to actually see and show you that it is easy and leave time for Q&A, but you can also ask in between. I hope that we have enough time, and if we don't, then it's better if you ask before rather than I'm trying to finish the demo, and it doesn't work. -[00:04:37] **Speaker 1:** Now that I explained how the spring starter works, the question is, isn't that good enough? So the question is why did we or I and another engineer from Microsoft spend a huge amount of time last year working on making a spring boot startup? +### [00:02:36] Motivation for Spring Boot Starter -**Speaker 1:** Let's start with what we cannot do or cannot yet do. We cannot match the huge number of libraries, more than 100, simply because we cannot change old code. Also, because there's a lot of work being done and it would take a lot of time to have the same level. +**Speaker 1:** As you can see, we have quite a lot of languages in hotel, and a lot of them have actually moved to stable for traces, locks, metrics. I think that's quite a bit of change for last year, but I haven't recorded it. Java, which we are talking about, PHP, .NET, and C++. For some reason, profiling is not listed here, but this is like a new topic that is in active development. I guess in a year or two, this will also be listed here. -**Speaker 1:** Then the question is what we can do, and as a spring boot user, I like to have a native spring boot experience. That means I can just select a library, add it as a dependency, and it does not anything special and that is exactly what I wanted to have for observability. It should not be anything special. It should be built into the development like you have a database. Well, it's probably kind of hard because without a database, you could not do anything. You could not serve customer requests. But it should be as close as possible to just adding a database. +**Speaker 1:** Okay, the state of hotel in Java. Java is one of the more mature areas in hotel, probably because a lot of e-commerce and business applications are written in Java, and there's a lot of demand, and there are many libraries. I just counted yesterday over 100 supported libraries, and it's also compatible with ancient versions of Java. Java 8 was released long more than 10 years ago, but I did not count. -[00:06:05] **Speaker 1:** And that is not possible. If you are adding a Java agent, then it's kind of this magic. As a spring boot user, you also are used to have a very integrated experience. That means you can take your configuration file called application YAML and you can just add properties and the values and names of those properties are automatically suggested. +**Speaker 1:** And that is actually only possible because Java has a technology called byte code manipulation, which means that you can write software that at runtime looks at all the methods and classes that are loaded and can basically change anything. It can change a return value and can look at parameters, similar to how Python can do monkey patching. Now that I explained how the spring starter works, the question is, isn't that good enough? -**Speaker 1:** You can basically just select service name because we talked about service name a bit before and hopefully I can show you that this is very easy. And then you can also use another Java agent to do your other magic work because adding two Java agents doesn’t usually work well. And you can also use a GraalVM native image which compiles your entire application into an executable which starts faster but it takes ages to compile. I'm too impatient to show that to you. But there is a full working example that where you can try that out. +**Speaker 1:** The question is why did we, or I and another engineer from Microsoft, spend a huge amount of time last year working on making a spring boot startup? So, let's start with what we cannot do or cannot yet do. We cannot match the huge number of libraries, more than 100, simply because we cannot change old code, and also because there's a lot of work being done, and it would take a lot of time to have the same level. -**Speaker 1:** As I said, I spent a lot of time last year and it also became stable last year. Now I can talk about it. Before going into exactly how the spring starter works, this is a recap of how it works. This is not Java specific and not spring specific but in the next slide you will see how it relates to what I'm trying to show you. +### [00:05:12] Comparison to Micrometer -[00:08:00] **Speaker 1:** The order is my personal preference and from most important to least important for me. The most important one is the protocol because once you have that protocol and you have your application, you know data is sent out. I can switch it somewhere else I don't have to recompile the application even if I don't have any developers. +**Speaker 1:** Then the question is what we can do. As a spring boot user, I like to have a native spring boot experience. That means I can just select a library, add it as a dependency, and it does not do anything special, and that is exactly what I wanted to have for observability. It should not be anything special. It should be built into the development like you have a database. -**Speaker 1:** As we heard in the previous post, even if the data is kind of crappy you can use AI to make it better but you need to have the data. The next one is semantic conventions because semantic conventions define the names and attributes. So for a metric, it would be the metric name of the duration of a request for example, and also the units. +**Speaker 1:** Well, it's probably kind of hard because without a database, you could not do anything. You could not serve customer requests, but it should be as close as possible to just adding a database. And that is not possible. If you are adding a Java agent, then it's kind of this magic. As a spring boot user, you also are used to having a very integrated experience. -**Speaker 1:** So metrics and traces for logs. There is a bit of semantic conventions but not much. The third one is an API. The API is more for library authors who want to be compatible with hotel and they don't want to spend time in five years rebuilding their observability stack and for that, the APIs are stable. +**Speaker 1:** That means you can take your configuration file called application YAML, and you can just add properties, and the values and names of those properties are automatically suggested. You can basically just select service name because we talked about service name a bit before, and hopefully, I can show you that this is very easy. -**Speaker 1:** In hotel, they are stable as you have seen in like here what it means stable is means the API among others but one is that the API is stable. One special thing is that for tracing the API is also important so that different libraries in the same process know which parent span belongs to which child span. +**Speaker 1:** You can also use another Java agent to do your other magic work because adding two Java agents doesn't usually work well. And you can also use a GraalVM native image, which compiles your entire application into an executable, which starts faster, but it takes ages to compile. I'm too impatient to show that to you, but there is a full working example where you can try that out. -**Speaker 1:** Because there are two cases. The easy case is you have a thread in Java and the thread is for a request and the thing that is first in the thread is done first. This is a parent and the second one is the child. But sometimes you have something that is asynchronous because it has an executor framework or something like that and then you need to have some kind of object that says this is a parent, that is a child, and that is what the API is for. +**Speaker 1:** As I said, I spent a lot of time last year, and it also became stable last year. Now I can talk about it. Before going into exactly how the spring starter works, this is a recap of how it itself works. This is not Java specific and not spring specific, but in the next slide, you will see how it relates to what I'm trying to show you. -**Speaker 1:** Lastly, the SDK. The SDK is an implementation of the API and it also gives you configuration options for example with environment variables. The last one is not the least important one. It's just more like at the side, the tooling, the collector, and operator is actually very important and very helpful. +**Speaker 1:** The order is my personal preference, and from most important to least important for me. The most important one is the protocol because once you have that protocol and you have your application, you know data is sent out. I can switch it somewhere else; I don't have to recompile the application, even if I don't have any developers. -[00:10:41] **Speaker 1:** Now I want to compare the spring boot starter that I've worked on to the existing technologies in spring boot to explain a little bit why we built something where kind of something similar is already there. In micrometer, you have two things. Metrics is quite old and used in many applications as far as I know. More recently, a module has been added where you can export the metrics using OTLP, that's why OTLP the first one. +**Speaker 1:** As we heard in the previous post, even if the data is kind of crappy, you can use AI to make it better, but you need to have the data. The next one is semantic conventions because semantic conventions define the names and attributes. For a metric, it would be the metric name of the duration of a request, for example, and also the units. -**Speaker 1:** But as spring boot is older than hotel, they cannot just change the names of their metrics. I mean they could, but for spring, they hold it very dear to not change anything for old applications because there are many old spring applications. Changes are only in major versions and it takes a very long time and that's why they did not add yet support for semantic conventions. +**Speaker 1:** So metrics and traces for locks, there is a bit of semantic conventions, but not much. The third one is the API. The API is more for library authors who want to be compatible with hotel, and they don't want to spend time in five years rebuilding their observability stack, and for that, the APIs are stable. As you have seen in like here, what it means stable means the API, among others, but one is that the API is stable. -**Speaker 1:** In our spring starter, when I say our, it's not Grafana where I work for but it’s hotel as because this is something that I worked on just as a hotel contributor, you can import those metrics because there are many applications and libraries that emit those metrics and in some cases can be useful to use those micrometer tracing. +**Speaker 1:** One special thing is that for tracing, the API is also important so that different libraries in the same process know which parent span belongs to which child span because there are two cases. The easy case is you have a thread in Java, and the thread is for a request, and the thing that is first in the thread is done first. This is a parent, and the second one is the child. -**Speaker 1:** The second part that is a bit newer was called something different, Sleuth. I think before it was called micrometer tracing but it's still newer than metrics. In tracing, you can use the hotel SDK to emit traces but it also does not have the semantic conventions even though it's newer. +### [00:10:24] Explanation of bytecode manipulation -**Speaker 1:** Here you have the problem that it is not using the same API. That's why you have this problem here on the right side. That has actually created some questions and probably even more exploding heads that I'm not aware of for users who are trying to combine some library with some other library. They usually don't know that one is using micrometer tracing and the other one is using hotel and sometimes works, sometimes doesn't, and sometimes works kind of. +**Speaker 1:** But sometimes you have something that is asynchronous because it has an executor framework or something like that, and then you need to have some kind of object that says this is a parent, that is a child, and that is what the API is for. Lastly, the SDK is an implementation of the API, and it also gives you configuration options, for example, with environment variables. -**Speaker 1:** That's basically it. I would love to have a common approach where there would be only either the micrometer APIs or the hotel APIs but this has not happened because the spring folks have a rich tradition and they know when they create an API they also want to keep it. On the other hand, the hotel folks don’t want to say we just take another API because it is for many languages and picking a different API makes it very difficult for hotel users who actually love to have the same experience across all languages. +**Speaker 1:** The last one is not the least important one; it's just more like at the side, the tooling, the collector and operator are actually very important and very helpful. -**Speaker 1:** This has been a question that has been asked many times but so far nobody has really given an answer to this. This is basically what I said just more in a data flow kind of way. In hotel, we have instrumentation libraries and in spring, there are instrumentation libraries in spring. They are called native because they are not maintained by hotel and they are not repositories, but they are done by the authors of different libraries themselves. +**Speaker 1:** Now I want to compare the spring boot starter that I've worked on to the existing technologies in spring boot to explain a little bit why we built something where kind of something similar is already there. In micrometer, you have two things. Metrics is quite old and used in many applications, as far as I know, and more recently, a module has been added where you can export the metrics using OTLP. -**Speaker 1:** In spring, this is sometimes also not, they are also sometimes doing it for other libraries. Semantic conventions, I mentioned semantic conventions and later we will hopefully see why this is important. +**Speaker 1:** That's why OTLP is the first one, but as spring boot is older than hotel, they cannot just change the names of their metrics. I mean, they could, but for spring, they hold it very dear to not change anything for old applications because there are many old spring applications. Changes are only in major versions, and it takes a very long time, and that's why they did not add support for semantic conventions yet. -[00:15:40] **Speaker 1:** That's actually it. Now I want to start the demo, but if you have any questions, it probably makes sense to ask them now before we go to the demo. +**Speaker 1:** In our spring starter, when I say our, it's not Grafana where I work for, but it's hotel, as this is something that I worked on just as a hotel contributor. You can import those metrics because there are many applications and libraries that emit those metrics, and in some cases, it can be useful to use those micrometer tracing. -**Speaker 2:** All right. So, I used to work with Zipkin, right? +**Speaker 1:** The second part that is a bit newer was called something different, I think before it was called micrometer tracing, but it's still newer than metrics. In tracing, you can use the hotel SDK to emit traces, but it also does not have the semantic conventions, even though it's newer. Here you have the problem that it is not using the same API. -**Speaker 2:** Yes. And we have a Zipkin receiver in the collector. So, would it work to just send Zipkin to the collector and like does it work if I send Zipkin spans to the collector and hotel from my Python application to the collector and do they show up as one trace at the back end? I mean, I guess the question is on the context propagation side of things, right? +**Speaker 1:** That's why you have this problem here on the right side. That has actually created some questions and probably even more exploding heads that I'm not aware of for users who are trying to combine some library with some other library. They usually don't know that one is using micrometer tracing and the other one is using hotel, and sometimes it works, sometimes it doesn't, and sometimes it works kind of. -**Speaker 2:** You could send both and they would both show up, but they would not have the correct parent-child relationship because internally Spring does not use Zipkin. It uses something else which is like a common denominator of both Zipkin and hotel because those are the two targets for micrometer tracing. +**Speaker 1:** That's basically it. I would love to have a common approach where there would be only either the micrometer APIs or the hotel APIs, but this has not happened because the spring folks have a rich tradition, and they know when they create an API, they also want to keep it. On the other hand, the hotel folks can also or don't want to say, "Oh, we just take another API" because it is for many languages, and picking a different API makes it very difficult for hotel users who actually love to have the same experience across all languages. -**Speaker 2:** If they would say okay, we just want to optimize for hotel then it would work. Doesn't that work with the B3 headers? +**Speaker 1:** This has been a long question that has been asked many times, but so far nobody has really given an answer to this. This is basically what I said, just more in a data flow kind of way. In hotel, we have instrumentation libraries, and in spring, there are instrumentation libraries in spring. They are called native because they are not maintained by hotel, and they are not repositories, but they are done by the authors of different libraries themselves. -**Speaker 1:** The headers are basically where it’s between processes. +**Speaker 1:** In spring, this is sometimes also not. They are also sometimes doing it for other libraries. Semantic conventions, I mentioned semantic conventions, and later we will hopefully see why this is important. -**Speaker 2:** Yeah. I mean, I have a Python application that is sending data to my collector and hopefully that one generated a span that originates the Java spans. +### [00:15:36] Start of live demo -**Speaker 1:** Yeah, then it would work. So if here between parent and child you have a process boundary, then it would work. If it's in the same process, then it only works in some cases. So if it's in the same Java thread, then I think it will work. But if it's in a different Java thread because you have an executor then it will not work because the context is actually stored somewhere else as a matter object and Java does not have a context object like Go for example. +**Speaker 1:** That's actually it. Now I want to start the demo, but if you have any questions, it probably makes sense to ask them now before we go to the demo. -**Speaker 2:** Is that a property of Java or spring like kind of is that could this also happen just with general Java applications because I'm just thinking like we sometimes see that there are incorrect parent-child relationships in spam and we're mostly using Java but I'm not sure if they only use spring. +**Speaker 2:** All right. I used to work with Zipkin, right? Yes. And we have a Zipkin receiver in the collector. So, would it work to just send Zipkin to the collector? Does it work if I send Zipkin spans to the collector and hotel from my Python application to the collector, and do they show up as one trace at the backend? I mean, I guess the question is on the context propagation side of things, right? -**Speaker 1:** There are many scenarios where this can happen. Even if you just use the Java agent which is like one distribution, it usually is because there is some other context propagation mechanism that is not supported or a library version upgrade changes the version how context is propagated. That sometimes happens. But if it's between threads but in the same process, then it couldn't be, you said like if it's in the same process, it could still lead to this mismatch. +**Speaker 1:** You could send both, and they would both show up, but they would not have the correct parent-child relationship because internally, Spring does not use Zipkin. It uses something else, which is like a common denominator of both Zipkin and hotel because those are the two targets for micrometer tracing. If they would say, "Okay, we just want to optimize for hotel," then it would work. -**Speaker 1:** Yes. But if it's in the same process, it cannot be like different methods of propagation at least I understand propagation is in between processes. +**Speaker 2:** Doesn't that work with the B3 headers? -**Speaker 1:** Oh, there propagation is the term that is both used between processes and also within the same process. Between processes you have headers like B3 headers for example that are added to HTTP and in the same process it is some language-specific passing. +**Speaker 1:** The headers are basically where it's between processes. -**Speaker 1:** In Java, by default, the context is stored in a thread local because you can store a thread local without modifying the application. You don't have to pass a parameter. But that only works so long as the entire call is in one thread and new applications typically don't do that because this is not very efficient. +**Speaker 2:** Yeah. I mean, I have a Python application that is sending data to my collector, and hopefully, that one generated a span that originates the Java spans. -**Speaker 1:** In general, you have to expect that there are linkage breaking between parents and child with Java. +**Speaker 1:** Yeah, then it would work. So, if here between parent and child you have a process boundary, then it would work. If it's in the same process, then it only works in some cases. If it's in the same Java thread, then I think it will work, but if it's in a different Java thread because you have an executor, then it will not work because the context is actually stored somewhere else as a matter object, and Java does not have a context object like Go, for example. -**Speaker 2:** No, no, no, no. The Java agent authors have support for those technologies or libraries where it's not easy. So Java executor for example is a framework where you can pass a runnable to a different pool and then it will be picked up by a number of different threads. They do round robins so that you don't need a thread for every request. +**Speaker 2:** Is that a property of Java or Spring? -**Speaker 2:** This byte code manipulation thing of the Java agent can see when you put an object into this pool and then it will attach some metadata into that object. So maybe there's a map where you can put things into and then you can add a context ID, a trace ID, and a parent span ID, and then you can form the correct parent-child relationship. +**Speaker 1:** Kind of. Could this also happen just with general Java applications? Because I'm just thinking, like, we sometimes see that there are incorrect parent-child relationships in spans, and we're mostly using Java, but I'm not sure if they only use Spring. -**Speaker 1:** But if that support is missing in the Java agent and here in the spring boot starter it would be the same. If the spring boot starter does not support this then it would not work. What is even more difficult is if there are two different instrumentation technologies that don't use the same API then they also don't know what they should expect as a context object and that is what is making it so difficult to combine micrometer and hotel instrumentation. +**Speaker 1:** There are many scenarios where this can happen. Even if you just use the Java agent, which is like one distribution, it usually is because there is some other context propagation mechanism that is not supported or a library version upgrade changes the version of how context is propagated. That sometimes happens. -**Speaker 1:** This might be the beard talking, but I miss EJB context. +**Speaker 2:** But if it's between threads, but in the same process, then it couldn't be. You said if it's in the same process, it could still lead to this mismatch. -**Speaker 2:** EJB. I mean, EJB context was perfect for this kind of situation. Does EJB even still exist? +**Speaker 1:** Yes. But if it's in the same process, it cannot be like different methods of propagation. At least I understand propagation is in between processes. -**Speaker 1:** I don't. Maybe it would have been the solution. How old I am. The last time I used EJB was when I was in university. And as I just said, it's more than 20 years ago. +### [00:19:04] Context propagation discussion -**Speaker 1:** Let me try with the demo before we run out of time. How much time do we have left? +**Speaker 1:** Oh, there propagation is the term that is both used between processes and also within the same process. Between processes, you have headers, like B3 headers, for example, that are added to HTTP. In the same process, it is some language-specific passing. + +**Speaker 1:** In Java, by default, the context is stored in a thread local because you can store a thread local without modifying the application. You don't have to pass a parameter, but that only works so long as the entire call is in one thread, and new applications typically don't do that because this is not very efficient. + +**Speaker 1:** So in general, you have to expect that there are linkage breaking between parents and child with Java. + +**Speaker 2:** No, no, no. The Java agent authors have support for those technologies or libraries where it's not easy. Java executor, for example, is a framework where you can pass a runnable to a different thread in a pool, and then it will be picked up by a number of different threads, and they do round robins so that you don't need a thread for every request. + +**Speaker 1:** This byte code manipulation thing of the Java agent can see when you put an object into this pool, and then it will attach some metadata into that object. Maybe there's a map where you can put things into, and then you can add a context ID, a trace ID, and a parent span ID, and then you can form the correct parent-child relationship. + +**Speaker 1:** But if that support is missing in the Java agent, and here in the spring boot starter, it would be the same. If the spring boot starter does not support this, then it would not work. What is even more difficult is if there are two different instrumentation technologies that don't use the same API, then they also don't know what they should expect as a context object, and that is what makes it so difficult to combine micrometer and hotel instrumentation. + +**Speaker 2:** This might be the beard talking, but I miss EJB context. + +**Speaker 1:** EJB. I mean, EJB context was perfect for this kind of situation. Does EJB even still exist? + +**Speaker 2:** I don't. Maybe it would have been the solution. + +**Speaker 1:** How old I am. The last time I used EJB was when I was in university. As I just said, it's more than 20 years ago. + +**Speaker 1:** Let me try with the demo before we run out of time. How much time do we have left? **Speaker 2:** You have... -**Speaker 1:** Okay, good. Good. As long as you still have some energy left. So, yeah, this is my IntelliJ. Weirdly, it does not look black even though I have the black theme. Oh, no. Okay. Let's see if it does. Yep. +**Speaker 1:** Okay, good. Good. As long as you still have some energy left. This is my IntelliJ. Weirdly, it does not look black even though I have the black theme. + +**Speaker 1:** Oh, no. Okay, let's see if it does. Yep. + +**Speaker 1:** Oh, it's... oh, wow. Should have checked that table. + +**Speaker 2:** You just turned it off and turned it on. + +**Speaker 1:** Yeah, it's all you have to know as an S. Here I'm not just in any project, but I'm in the project called docker hotel lgtm, which both has examples of instrumenting and also has a complete observability stack based on Grafana technology, of course, because I'm working there. + +**Speaker 1:** I have created a new example just so that it to get us started a bit faster, and I'm going to explain to you what I have already done as the first iteration, and then we will add stuff later on. + +**Speaker 1:** Here I have two bill of materials. We are using Maven. This is a dependency management section, and first, we have open telemetry. This is the latest version. + +**Speaker 1:** Then we have spring boot dependencies. We have the open telemetry first so that if there are libraries with the same name, the one from the first one wins. Since spring boot has micrometer, which has the hotel exporter but in an older version, I want to make sure that we have the newer version and that they are compatible. + +**Speaker 1:** Then I just have a starter web, which is from spring, and then I have the open telemetry spring boot starter, and there's really nothing here. I don't know why we have repackaged; we have a spring boot app, but there's nothing interesting here, and then we have a small controller that is just doing rolls and it has a player name. + +### [00:25:08] Demo of OpenTelemetry integration + +**Speaker 1:** First, I want to start this. While this is starting, I'm going to switch here where I have two windows. The first one is actually starting the image where we have the complete observability stack. This one is actually just a docker script, but just to show you what this is really doing. + +**Speaker 1:** So, this is just a task runner, like make file but a bit nicer. This one is just a shell script, and the shell script is just Docker. The only thing why I don't call Docker is that I probably forget some of the ports. + +**Speaker 1:** By the way, I did not mention the ports. Yeah, I did not mention the whole slide because it said demo. This is explaining the project much better than I could do. + +**Speaker 1:** On the left side is the application that we're doing that is sending all the data to the hotel collector, which is sending the data to all the databases. The profiling one is not used here, and then we can view it with Grafana, and that's why we have two ports on the left and one port on the right side. + +**Speaker 1:** This is just passing the ports, and some people like to actually persist the data, but the goal of the project is more to have an easy tryout experience. So, it's not optimized for scalability, but it's optimized for startup time. + +**Speaker 1:** Did it work out? Yeah, it started in 3 seconds. That was the latest iteration that I did because it was taking half a minute before. + +**Speaker 1:** Then it also has an environment file, which is not really needed, but I set it here to an external endpoint so I can also view the data in Grafana cloud, but this is actually not needed. + +**Speaker 1:** Okay, so back to our application as it started now. Yeah, I think in the beginning, it sent some errors because the observability stack was not started, and now I will also put some traffic. + +**Speaker 1:** This is just a curl script that pulls the endpoint. Sometimes it gives an error because the error is simulated. Let's see if we actually get some data now. + +**Speaker 1:** Yeah, we have dice rolls. Now let's actually see if we can see some data. Yeah, we have some data. We have some traces. Are the traces? Oh, cannot scroll down. + +**Speaker 1:** Okay, so nothing really interesting. Why is it not interesting? Well, because the service is not calling any other service and it is not calling a database; it sometimes has errors. Do we see errors? Let's see if that at least works. -**Speaker 1:** Oh, wow. Should have checked that table. You just turned it off and turned it on. Yeah, it's all you have to know as an S. +**Speaker 1:** This is, by the way, the Grafana 12 that was just released today. I hope I'm using that correctly. Error. Yeah, it's an error. So, here we have an error. Let me see if that works. -**Speaker 1:** Here I'm not just in any project but I'm in the project called docker hotel lgtm which both has examples of instrumenting and also has a complete observability stack based on Grafana technology of course because I'm working there. And I have created a new example just so that it to get us started a bit faster. +**Speaker 1:** Copy that. Does it work? Oh, I did not copy the entire thing here. -**Speaker 1:** Here I have two bill of materials. In we are using Maven, this is a dependency management section and first we have open telemetry, this is the latest version and then we have spring boot dependencies. We have the open telemetry first so that if there are libraries with the same name the one from the first one win and since spring boot has a micrometer which has the hotel exporter but in an older version I want to make sure that we have the newer version and that they are compatible. +**Speaker 1:** There's copy. Yep, that's where the error is thrown. Okay, so that part works. -**Speaker 1:** Then I just have a starter web which is from spring and then I have the open telemetry spring boot starter and then there's really nothing here. I don't know why we have repackage we have a spring boot app but there's nothing interesting here and then we have a small controller that is just doing rolls and it has a player name. +**Speaker 1:** Oh, what I forgot to ask. How many of you have used Spring Boot? -**Speaker 1:** So, first I want to start this. And while this is starting, I'm going to switch here where I have two windows. The first one is actually starting the image where we have the complete observability stack. So this one is actually just a docker script but just to show you what this is really doing. +**Speaker 2:** Okay, at least a couple of people. -**Speaker 1:** MIS is just a task runner. So like make file but a bit nicer. And this one is just a shell script. The shell script is just Docker. The only thing why I don't call Docker is that I probably forget some of the ports. By the way, I did not mention the ports. +**Speaker 1:** How many have instrumented Open Telemetry in any language? -**Speaker 1:** I did not mention the whole slide because it said demo. This is explaining the project much better than I could do. On the left side is the application that we're doing that is sending all the data to the hotel collector which is sending the data to all the databases. The profiling one is not used here and then we can view it with Grafana and that's why we have two ports on the left and one port on the right side. +**Speaker 2:** Even more. Good. -**Speaker 1:** This is just passing the ports and some people like to actually persist the data but the goal of the project is more to have an easy try-out experience. So it's not optimized for scalability but it's optimized for startup time. Did it work out? Yeah, it started in 3 seconds. That was the latest iteration that I did because it was taking half a minute before. +**Speaker 1:** How many have used the two together? -**Speaker 1:** And then it also has an environment file which it's not really needed but I set it here to an external endpoint. So I can also view the data in Grafana cloud but this is actually not needed. +**Speaker 2:** Oh, one. Okay, good. -[00:28:00] **Speaker 1:** Okay, so back to our application as it started now. Yeah, I think in the beginning it sent some errors because the observability stack was not started and now I will also put some traffic. This is just a curl script that pulls the endpoint. Sometimes it gives an error because the error is simulated. See if we actually get some data now. +**Speaker 1:** Okay, but the next question is not about instrumenting. Do you see anything that could be improved, that we should make better, or that you want to have added? -**Speaker 1:** Yeah, we have dice rolls. Now let's actually see if we can see some data. Yeah, we have some data. We have some traces. Are the traces? Oh, cannot scroll down. Okay, so nothing really interesting. Why is it not interesting? Well, because the service is not calling any other service and it is not calling a database just sometimes has errors. +**Speaker 2:** More spans. -**Speaker 1:** Do we see errors? Let's see if that at least works. So this is by the way the Grafana 12 that was just released today. I hope I'm using that correctly. Error. Yeah, it's error. So, here we have an error. Let me see if that works. Copy that. +**Speaker 1:** More spans? -**Speaker 1:** Does it work? Oh, I did not copy the entire thing here. There's copy. Yep, that's where the error is thrown. Okay, so that part works. +**Speaker 2:** An unknown service. -**Speaker 1:** Oh, what I forgot to ask. How many of you have used Spring Boot? Okay, at least a couple of people. How many have instrumented Open Telemetry in any language? Even more. Good. How many have used the two together? Oh, one. Okay, good. +**Speaker 1:** Exactly. Unknown service. And I promise that this should be easy to do. -**Speaker 1:** But the next question is not about instrumenting. Do you see anything that could be improved that we should make better or that you want to have added? More spans? More spans. +**Speaker 1:** Let's see if we can do that. Where's my directory here? Very slow today. -**Speaker 2:** Yeah. An unknown service. Exactly. Unknown service. And I promise that this should be easy to do. So let's see if we can do that. Where's my directory here? Very slow today. +**Speaker 1:** Let me see. Service service hotel service name. No, but it's a different one. Okay. -**Speaker 1:** Let me see. Service service hotel service name. No, but it's a different one. Okay. Let me see if I have hotel. Oh, is it not recognizing? Maybe I not I have to reload. Maven did work today. But the goat... +**Speaker 1:** Let me see if I have hotel. Oh, is it not recognizing? Maybe I have to reload. Maven did work today, but the goat... -**Speaker 2:** Yeah, I did not sacrifice a goat. +**Speaker 2:** The goat. -**Speaker 1:** So, I guess I have a question. Why do you look it up? The new way of doing things is using hotel config files that can be reused across services. That is right. +**Speaker 1:** Yeah, I did not sacrifice a goat, so I guess I have a question. -**Speaker 1:** Would that come to the spring boot starter as well? Like can I could I just add my hotel.yaml file there instead of configuring to the partition? +**Speaker 1:** Why do you look it up? The new way of doing things is using hotel config files that can be reused across services. -**Speaker 1:** Before answering, just a bit on the background. Right now in hotel, the standard way of doing things is environment variables and that is quite limited. So the most fancy thing is if I write hotel resource at... Oh, it's even completing that but this is not spring, this is the AI. So I can say service name, service version, something like that. +**Speaker 2:** That is right. Would that come to the spring boot starter as well? Like, could I just add my hotel.yaml file there instead of configuring to the partition? -**Speaker 1:** Anything more complicated is specific to any language and is currently blocked. What Drusi is saying is that the plan currently is to have the same experience, well not in spring syntax but in YAML or in any structured data, to have the same experience. +**Speaker 1:** Before answering, just a bit on the background. Right now, in hotel, the standard way of doing things is environment variables, and that is quite limited. The most fancy thing is if I write hotel resource, oh, it's even completing that, but this is not spring; this is the AI, so I can say service name, service version, something like that. -**Speaker 1:** For spring, we would probably have a major version where we would tell you to switch from the syntax that I'm trying to show here to the one that is compatible. I have spent some time trying to figure out if this can be integrated with spring because here you can also use a spring expression. +**Speaker 1:** Anything more complicated is specific to any language and is currently blocked. What I'm unsuccessfully trying to show you is the only way where you can currently use YAML, as far as I know. -**Speaker 1:** In spring, you have a language where you can reference environment variables. I don't know if I can get it right if that is actually correct or if this is just an AI hallucination but there's a spring feature where you can reference other parts in spring and you can even reference Java beans and so on. +**Speaker 1:** What Drussi is saying is that the plan currently is to have the same experience, well, not in spring syntax, but in YAML or in any structured data, to have the same experience. For spring, we would probably have a major version where we would tell you to switch from the syntax that I'm trying to show here to the one that is compatible. -**Speaker 1:** What I want for the future is that you can also have this because you're used to it as a spring user but with the common YAML format. I'm not sure if that is possible because this is already stretching the limits of spring and it will be even more so with the new format. So I hope it's possible but no promises. +**Speaker 1:** I have spent some time trying to figure out if this can be integrated with spring because here you can also use a spring expression. In spring, you have a language where you can reference environment variables. I don't know if I can get it right if that is actually correct or if this is just an AI hallucination, but there is a spring feature where you can reference other parts in spring, and you can even reference Java beans and so on. -**Speaker 1:** Since I'm trying to do that here, I'll just look in my stash to see if I'm just too stupid to find the right format. Resource attributes, hotel service name. Yeah, the autocomplete just didn't work for some reason. It's not suggesting it. +**Speaker 1:** What I want for the future is that you can also have this because you're used to it as a spring user, but with the common YAML format. I'm not sure if that is possible because this is already stretching the limits of spring, and it will be even more so with the new format. -**Speaker 1:** So show up as a plugin on the right-hand side. There's no plugin, right? Actually, this is a feature of the dependency here. So it basically just has a JSON file where it lists all the properties. +**Speaker 1:** I hope it's possible, but no promises. -**Speaker 1:** Yeah, it should work. Let's see if it's working. If we restart the service. Tracing is always the tracing locks is the fastest to show because it's processed right away. Metrics is ingested every minute. You can change it to more often but for this demo it's not necessary. +**Speaker 1:** Since I'm trying to do that here, I'll just look in my stash to see if I'm just too stupid to find the right format. Resource attributes, hotel service name. -**Speaker 1:** See the last one. No, it still has unknown service. Well, what I had before is that when I started in spring, it's not working. But when I use Maven to start it then it is working. So that can be tricky. +**Speaker 1:** Yeah, the auto-completion just didn't work for some reason. It's not suggesting it. -**Speaker 1:** Let me see if that is the case. Oh, it's still unknown service. They look for unknown service. I think that's too old by now. That's still a node price that you were looking at. +**Speaker 1:** So, show up as a plugin on the right-hand side. There's no plugin, right? + +**Speaker 1:** Actually, this is a feature of the dependency here, so it basically just has a JSON file where it lists all the properties, and it should work. + +**Speaker 1:** Let's see if it's working. If we restart the service, tracing is always the fastest to show because it's processed right away. Metrics are ingested every minute. You can change it to more often, but for this demo, it's not necessary. + +### [00:37:06] Q&A on service name configuration + +**Speaker 1:** So, see the last one? No, it still has an unknown service. What I had before is that when I started in spring, it's not working, but when I use Maven to start it, then it is working. + +**Speaker 1:** That can be tricky. Let me see if that is the case. Oh, it's still unknown service. + +**Speaker 1:** They look for unknown service. I think that's too old by now. That's still a node price that you were looking at. **Speaker 2:** Can you filter by service? -**Speaker 1:** It should. It actually says service name here. So, it's automatically grouping by service name. That is not the problem. I'm probably doing something wrong because I put it in a wrong directory or something. +**Speaker 1:** It should; it actually says service name here, so it's automatically grouping by service name. That is not the problem. I'm probably doing something wrong because I put it in a wrong directory or something. + +**Speaker 1:** Something that is a typical spring problem that happens every time when I try to do something new. + +**Speaker 1:** I'm just going to delete that file and apply the stash as I had it before. + +**Speaker 1:** Oh yeah, it's in a different directory. That's why my error. + +**Speaker 1:** Good that I had the stash. Let me see if I can also type service. Yep. Now it works. + +**Speaker 1:** Okay, at least that worked. But now I made the mistake that I started it here and also in the other one. It's probably saying that the port is already used. + +**Speaker 1:** Started it here and no, seems that it is loaded. + +**Speaker 1:** Okay, see if it works better here. Here it is. + +**Speaker 1:** Okay, here's rolled dice. Now let's just look at that instead. + +**Speaker 1:** Here now I can show you where the service name is. Here is service name, and then you also have a bunch of stuff that we did not specify, and that is because there are resource detectors so that you don't have to do all the work yourself. + +**Speaker 1:** For example, you have the process ID that was taken from the operating system, and you can also see that it was created by the spring boot starter. So if you're reporting a bug, then it's easier for the authors to find where this is coming from. + +**Speaker 1:** You also have this correlation that you have between some metric types. You can actually see the trace that started before the log message was emitted. + +**Speaker 1:** Now that we have that, let's look at some metrics. There is this duration metric that I talked about before, and that has the same name across all languages. + +**Speaker 1:** That is really useful for having an APM system where you can see what the request rates for all of your services are. This is the open-source version of Grafana, so you don't have the APM tool here. + +**Speaker 1:** I would have some things that we could do. We could add a manual metric or an additional metric where you can count the number of requests or players. We can add more spans, I heard that before, or we can add a database, or we do nothing of it, and we just leave more time for questions. + +**Speaker 1:** So if you don't have anything that you want, then I think we can just cut it here, but if you want something, then I will add it. + +**Speaker 2:** I just have a quick question. Why is service name not on the resource attribute? + +**Speaker 1:** Oh yeah, that's a good one. Let's... I think we saw it here. + +**Speaker 1:** That is the visualization of Loki. Loki shows everything flat, and actually, you're not the first one to notice that it would be more useful if that was structured in traces. + +**Speaker 1:** It is structured. + +**Speaker 2:** Yeah, it's a good point. I think it would be better. + +**Speaker 1:** Now it's also showing up here for some reason it didn't before. + +**Speaker 1:** So here it is in resource attributes, and it's down there. It's there. Service name, right? + +**Speaker 2:** Yeah, here it is. -**Speaker 1:** Something that is a typical spring problem that happens every time when I try to do something new. Okay. I'm just going to delete that file and apply the stash as I had it before. +**Speaker 1:** But in the configuration, you have to add it to a different place, or is that kind of free? -**Speaker 1:** Oh yeah, it's in a different directory. That's why my error. Yeah. Good that I had the stash. Let me see if I can also type service. Yep, now it works. Okay, at least that worked. +**Speaker 1:** You can... Oh, service name is special. In open telemetry, service name is special. -**Speaker 1:** But now I made the mistake that I started it here and also in the other one. It's probably saying that the port is already used. Started it here and no seems that it is loaded. +**Speaker 1:** It internally translates into resource attribute, but it is like the single one that is required by every telemetry data. I could also add it here, by the way. -**Speaker 1:** Okay, see if it works better here. Here it is. Okay, here's rolled dice. Okay, then let's just look at that instead. +**Speaker 2:** Yeah, I also find that confusing that there are two ways, and I just had a customer call today where we found out that in some cases it does matter where you put it, even though it should not. -**Speaker 1:** Here now I can show you where the service name is. Here is service name and then you also have a bunch of stuff that we did not specify and that is because there are resource detectors so that you don't have to do all the work yourself. +**Speaker 1:** Yeah, I mean, I think it's an implementation detail. Basically, service name is the only required attribute for open telemetry, and it is top-level. -**Speaker 1:** For example, you have the process ID that was taken from the operating system and you can also see that it was created by the spring boot starter. So if you're reporting a bug, then it's easier for the authors to find where this is coming from. +**Speaker 1:** An implementation detail is it is a resource attribute, and so that's why sometimes it works and sometimes it doesn't. -**Speaker 1:** You have also this correlation that you have between some metric types. You can actually see the trace that was the trace that started before the log message was emitted. +**Speaker 1:** I mean, in theory, it should only work if it is at top level, right? Because if you add it as a resource attribute, then it's probably not recognized at the top level. -**Speaker 1:** Now that we have that, let's look at some metrics. There is this duration metric that I talked about before and that has the same name across all languages. That is really useful for having an APM system where you can see what the request rates for all of your services are. +**Speaker 2:** So, when I see in Grafana on the resource attributes, that's just the dual thing, basically. -**Speaker 1:** Here this is the open-source version of Grafana. So you don't have the APM tool here. I would have some things that we could do. We could add a manual metric or an additional metric where you can count the number of requests or players. +**Speaker 1:** Yeah, is that really true? It is how it is internally stored. -**Speaker 1:** We can add more spans, I heard that before. Or we can add a database or we do none of it and we just leave more time for questions. So if you don't have anything that you want then I think we can just cut it here. But if you want something then I will add it. +**Speaker 1:** In Grafana, I know if you go like to the trace query, instead of using traceQL itself, it has this kind of building query by the same, then there's like the top two or so. -**Speaker 2:** I just have a quick question. Why is the service name not on the resource attribute? +**Speaker 2:** Yeah, exactly. -**Speaker 1:** Oh yeah, that's a good one. Let's I think we saw it here. That is the visualization of Loki. Loki shows everything flat and it's actually you're not the first one to notice that it would be more useful if that was structured in traces. +**Speaker 1:** So, if you go to search now, not the trace. -**Speaker 1:** It is structured. Yeah. It's a good point. I think it would be better. And now it's also showing up here for some reason didn't before. So here it is in resource attributes and it's down there. It's there. Service name, right? +**Speaker 1:** So, service name, span name, and status are like the special one. -**Speaker 1:** Yeah, here it is. But in the configuration, you have to add it to a different place or is that kind of free? You can... Oh, service name is special. In open telemetry, service name is special. It internally translates into resource attribute but it is like the single one that is required by every telemetry data that I could also add it here by the way. +**Speaker 1:** For open telemetry, only the service name is special. -**Speaker 1:** I also find that confusing that there are two ways and I just had a customer call today where we found out that in some cases it does matter where you put it even though it should not. +**Speaker 1:** The folks at Tempo, they said, you know span name is interesting to have it as part of the trace syntax, and so it's there. -**Speaker 1:** Yeah, I mean I think it's an implementation detail. Basically, service name is the only required attribute for open telemetry and it is top level. An implementation detail is it is a resource attribute and so that's why sometimes it works sometimes it doesn't. +**Speaker 1:** But from the hotel perspective, the only thing that we require is a service name. Even then, we don't throw an error. -**Speaker 1:** I mean in theory it should only work if it is at top level right because if you add it as resource attribute then it's probably not recognized at the top level. +**Speaker 1:** The specification says that if a service name is not specified, the unknown service should be used, which is what we've seen before, right? -**Speaker 1:** Confusion in OTLP, it is only resource attributes so it's only within the resource attributes that is a special one called service name and that's it, so that's why some if you specify an environment variable with the service name it might work but then depending on the library if you don't have the proper service name environment variable it overrides the one that you would override with the resource attributes with a default one so that's why sometimes it doesn't work sometimes it does. +**Speaker 1:** So, the spring boot application was working without the service name, and it was showing unknown service, but it is against this spec. -**Speaker 1:** So if as a user you should only use it at the top level not at the resource attributes. +**Speaker 1:** It should resources then in the application YAML; you should not put it under resources. -**Speaker 2:** So when I see in Grafana on the resource attributes that's just the dual thing basically. +**Speaker 1:** No, it's perfectly fine; it works, but as a user, you should not. -**Speaker 1:** Yeah. Is that really true? It is how it is internally stored. In Grafana, I know if you go like to the traceQL query instead of using traceQL itself it has this kind of building query by the same then there's like the top two or so. +**Speaker 1:** As a user, you should probably not do that. -**Speaker 1:** Yeah, exactly. So if you go to search now, not the trace. So service name, span name, and status are like the special ones. +**Speaker 1:** Got it. Okay. I mean, it might work for spring, but it might not work for only lambdas. -**Speaker 1:** For open telemetry, only the service name is special. The folks at Tempo, they said you know span name is interesting to have it as part of the trace syntax and so it's there. But from the hotel perspective, the only thing that we require is a service name. Even then we don't throw an error. +**Speaker 1:** I think, per spec, it should work, but the case I had today was where it actually did not work. They were putting it in an environment variable, but it's basically the same. -**Speaker 1:** The specification says that if a service name is not specified the unknown service should be used which is what we've seen before, right? So the spring boot application was working without the service name and it was showing unknown service but it is against this spec it should resources then in the application YAML you should not put it under resources. +**Speaker 1:** Most people, I think most authors expected it in service name; that's why it's safer to put it in there. -**Speaker 1:** No, it's perfectly fine. It works. But as a user you should not, as a user you should probably not do that. +**Speaker 1:** This is also just my last obligatory slide; that's why I put it on here. -**Speaker 2:** Got it. Okay. I mean, it might work for spring but it might not work for only lambdas. I mean I think per spec it should work but the case I had today was where it actually did not work. They were putting it in the environment variable but it's basically the same. Most people I think most authors expected in service name that's why it's safer to put it in there. +### [00:47:40] Closing remarks -**Speaker 1:** This is also just my last obligatory slide. That's why I put it on here. Cool. All right. Thanks for your time. +**Speaker 1:** Cool. All right. Thanks for your time. ## Raw YouTube Transcript diff --git a/video-transcripts/transcripts/2025-05-07T18:48:36Z-otel-night-berlin-v2025-05-leveraging-ai-for-opentelemetry-data.md b/video-transcripts/transcripts/2025-05-07T18:48:36Z-otel-night-berlin-v2025-05-leveraging-ai-for-opentelemetry-data.md index 450bb02..db36940 100644 --- a/video-transcripts/transcripts/2025-05-07T18:48:36Z-otel-night-berlin-v2025-05-leveraging-ai-for-opentelemetry-data.md +++ b/video-transcripts/transcripts/2025-05-07T18:48:36Z-otel-night-berlin-v2025-05-leveraging-ai-for-opentelemetry-data.md @@ -10,186 +10,292 @@ URL: https://www.youtube.com/watch?v=8JlFuGTDCXQ ## Summary -In this video, Lariel, a principal AI engineer at Dash Zero, discusses his experience applying AI to open telemetry data, specifically focusing on a case study involving log parsing. He outlines the challenges of working with unstructured logs that lack the attributes needed for effective analysis and explains the motivation behind developing their log AI solution. Lariel explains their approach, which involves combining various methods for log pattern recognition without relying on custom regular expressions or large language models (LLMs) for every log. He details their process of clustering logs by resource attributes and using a combination of heuristics and prompt engineering to extract valuable information. The results show a high success rate in accurately parsing log severity and patterns, and he emphasizes the importance of context and evaluation in AI applications. The video concludes with key takeaways and an invitation for questions. +In this video, Lariel, a principal AI engineer at Dash Zero, shares insights from her experience applying AI to open telemetry data, specifically focusing on a case study about their log AI system. The discussion delves into the challenges of parsing unstructured logs from various applications, highlighting the inadequacies of existing models and traditional methods. Lariel explains their innovative approach that combines resource clustering, log parsing, and the use of language models to identify patterns without human intervention. She describes their evaluation process, which shows a high success rate in log parsing and the efficacy of their system in categorizing log severities. Additionally, Lariel touches on the importance of context and semantic conventions in improving AI output and discusses ongoing experiments with AI agents for query writing and root cause analysis. The video concludes with key takeaways about the application of AI in production environments, emphasizing the balance between cost and functionality. ## Chapters -00:00:00 Welcome and intro -00:00:40 Overview of D-Zero -00:02:40 Log AI motivation -00:04:10 Limitations of existing models -00:06:40 Challenges with LLMs -00:09:46 Combining different approaches -00:10:40 Predicting log formats -00:12:00 Evaluation methodology -00:13:01 Results and success metrics -00:19:20 Key takeaways +00:00:00 Introductions +00:01:04 Overview of Dash Zero +00:02:28 Challenges with unstructured logs +00:03:42 Motivation behind log AI +00:05:33 Limitations of existing models +00:07:24 Exploring alternatives for log parsing +00:09:52 Resource clustering for log patterns +00:11:40 Evaluation process and success rates +00:19:14 Key takeaways from the case study +00:32:14 Audience Q&A and discussions -**Lariel:** All right. So, hello everyone. I'm Lariel. Today I'm going to share a little bit of my experience in applying AI to open telemetry data. So, short intro about myself. I'm from Brazil, moved to Berlin a couple of years ago. Nowadays, I work as principal AI engineer at Dash Zero with these guys here. Before that, I had been around playing different roles in data and AI, ranging from data engineering to data scientist, ML ops. My CV kind of looks like a mess nowadays. +## Transcript -[00:00:40] Also, for those of you who have not heard of D-Zero, just a quick introduction: we are an open telemetry native SAS solution for observability. We work with metrics, traces, logs, we do dashboarding, alerting. We're built on open standards like Prometheus and Persis. Obviously, we are building lots of cool AI capabilities to help our users get more value from the data that they send to us. And that brings me to the subject of today's talk. Actually, the main subject, which is a case study on our log AI. So how to make sense of unstructured logs. +### [00:00:00] Introductions -Many of you probably had a similar disappointment at some point. You plug in the open telemetry demo data set and then the logs look beautiful. The logs are shipped via OTLP. They have plenty of useful attributes. They are formatted as JSON, so they're very easy to work with, right? But in reality, most people's logs look like this. You have something running on Kubernetes, and you have those file listeners that get the logs from every node, and in the end, the result is something like this. Your log has some raw string body and some attributes saying what file it was collected from, and that's not very useful. But you know that there is information there. If you look into the log, there is something that looks like a module name, something that looks like a duration, a timestamp, a severity at debug level. How can we harness that information? That's the motivation behind the log AI that we developed. +**Lariel:** All right. So, hello everyone. I'm Lariel. Today I'm going to share a little bit of my experience in applying AI to open telemetry data. -[00:02:40] We want to do log parsing and log abstraction at scale for any logs of any application, without a human in the loop, without having to write custom regular expressions. For example, if I have an application writing logs like this, I want to parse them by separating this prefix from the message. I have in the prefix timestamp, host, severity text, and we can see that the messages follow different patterns. I have some "ad lookup failed," "display this ad to this user." There are basically two different patterns of log messages that my application is writing, and there are some structured fields that I would like to be able to query. There is some ad ID in the middle, there is some user ID, and we want to abstract that in the form of a pattern without writing a regular expression manually. We want to do that for every log of every application. That's the challenge and the motivation behind our log AI. +So, short intro about myself. I'm from Brazil, moved to Berlin a couple years ago. Nowadays, I work as principal AI engineer at Dash Zero with these guys here. Before that, I had been around playing different roles in data and AI, ranging from data engineering to data scientist, ML ops. So my CV kind of looks like a mess nowadays. -[00:04:10] We started looking into the available options to do that. The most famous one is the drain model that some of you might have seen already. Drain comes from this family of natural language processing models that work with word frequency. They basically compare the parts of the text that change frequently with the parts that are often the same. They can identify where the variables are and what the patterns should be. But those algorithms have plenty of limitations. First, they assume that the log is already prepared, so you have already separated that prefix from the message that can change from one pattern to another. For that, you need custom regular expressions written by hand for each log source. That is already a deal breaker. Secondly, those algorithms are very dependent on pre-processing. +### [00:01:04] Overview of Dash Zero -There is this paper called "pre-processing is all you need," where they describe this manual iterative process of coming up with tokenization rules, variable identification rules, text cleaning rules for each of your applications. Unless you do that, you cannot expect these models to produce any decent results for your custom logs. Believe me or not, the screenshot here on the left side is real code from the repository where they have the benchmarks for these models. You can see that for each log data set that they are benchmarking on, they have hardcoded rules; otherwise, the model doesn't work. That's a real deal breaker for us. We cannot just apply this drain model to every log and expect it to work. +Also, for those of you who have not heard of D-Zero, just a quick introduction: we are an open telemetry native SAS solution for observability. We work with matrix, traces, logs; we do dashboarding, alerting, and we're built on open standards like Prometheus and Persis. And obviously, we are building lots of cool AI capabilities to help our users get more value from the data that they send to us. -We started looking into other alternatives. What's very popular nowadays is just throwing things at a large language model. If you ask GPT to parse a log, it's going to get everything right on the first try. You get the fields really nice. The problem is, with hundreds of customers instrumenting thousands of applications and sending billions of logs every day, using an LLM for everything would be very cost ineffective and slow. So what do we do? +And that brings me to the subject of today's talk, actually the main subject, which is a case study on our log AI. So how to make sense of unstructured logs. -[00:06:40] Yet another option is, like this paper suggests, using a deep learning model that is trained to predict the pattern on every log. But in order to train a model in the first place, you need to have lots of labeled data, logs where you already know what the correct pattern should be, so you can teach the model how to infer that. We cannot possibly have such a dataset for all of our customers. Yet another problem with these models is that although they have nice benchmarks, those can be interpreted as a form of data contamination because the logs used to benchmark the model were produced from the same applications with the same templates as the logs that were used to train the model in the first place. This indicates that the model can actually only identify patterns that it is already familiar with or parse logs of applications that it is already familiar with. It doesn't generalize for any logs of any applications. +Many of you probably had a similar disappointment at some point. You plug in the open telemetry demo data set and then the logs look beautiful. The logs are shipped via OTLP. They have plenty of useful attributes. They are formatted as JSON, so they're very easy to work with, right? But in reality, most people's logs look like this. -This was the status quo: plenty of challenges, different options, each one with their limitations. How we solved this in the end was no magic at all. We actually just combined different approaches, trying to balance out the limitations of some with the advantages of others. The first thing we did was figure out the right level of granularity for predicting log formats and patterns. I don't know if anyone here has already worked with predictive AI in production, but you usually have the concept of an instance. For example, if you are doing customer churn prediction, then every customer is an instance. If you're doing product sales forecast, then every product is an instance. It's quite straightforward. +You have something running on Kubernetes, and you have those file listeners that get the logs from every node. In the end, the result is something like this. Your log has some raw string body and some attributes saying what file it was collected from, and that's not very useful. But you know that there is information there. If you look into the log, there is something that looks like a module name, something that looks like a duration, a timestamp, a severity at debug level. -When it comes to open telemetry data, it's quite challenging because there are different levels of granularity, and the schema is very nested. What is an instance? We started with the concept of the open telemetry resource. We tried to predict the log format and patterns for every resource, but it wasn't efficient at all. If you think for example of a Kubernetes deployment with autoscale, it is creating new pods and killing old pods all the time, and every new pod has different resource attributes. So it's treated as a new resource. If we would predict log formats and patterns for every resource, our model would be working all the time, which would not be efficient at all. +### [00:02:28] Challenges with unstructured logs + +How can we harness that information? That's the motivation behind the log AI, the thing we developed. We want to do log parsing and log abstraction at scale for any logs of any application without a human in the loop, without having to write custom regular expressions. + +For example, if I have an application writing logs like this, I want to parse them by separating this prefix from the message. I have in the prefix a timestamp, host, a severity text, and we can see that the messages follow different patterns. I have some "ad lookup failed" displayed this ad to this user. There are basically two different patterns of log messages that my application is writing, and there are some structured fields that I would like to be able to query. There is some ad ID in the middle, there is some user ID, and we want to abstract that in the form of a pattern without writing a regular expression manually. + +We want to do that for every log of every application. That's the challenge and the motivation behind our log AI. + +### [00:03:42] Motivation behind log AI + +First, we started looking into the available options to do that. Now, the most famous one is the drain model that some of you might have seen already. Drain comes from this family of natural language processing models that work with word frequency. They basically compare the parts of the text that change frequently with the parts that are often the same. They can identify where the variables are and what the patterns should be. + +But those algorithms have plenty of limitations. First, they assume that the log is already prepared. You have already separated that prefix from the message that can change from one pattern to another. For that, you need custom regular expressions written by hand for each log source. That is already a deal breaker. + +Secondly, those algorithms are very dependent on pre-processing. There is this paper called "pre-processing is all you need" where they describe this manual iterative process of coming up with tokenization rules, variable identification rules, text cleaning rules for each of your applications. Unless you do that, you cannot expect these models to produce any decent results for your custom logs. + +Believe me or not, the screenshot here on the left side is real code from the repository where they have the benchmarks for these models. You can see that for each log data set that they are benchmarking on, they have hardcoded rules; otherwise, the model doesn't work. Yeah, that's a real deal breaker for us. We cannot just apply this drain model to every log and expect it to work. + +### [00:05:33] Limitations of existing models + +Okay, so we started looking into other alternatives. What's very popular nowadays is just throwing things at a large language model. If you ask GPT to parse a log, it's going to get everything right on the first try. You get the fields really nice. The problem is, with hundreds of customers instrumenting thousands of applications and sending billions of logs every day, using an LLM for everything would be very cost ineffective and slow. + +So what do we do? Another option is, like this paper suggests, using a deep learning model which is trained to predict the pattern on every log. But in order to train a model in the first place, you need to have lots of labeled data, logs where you already know what the correct pattern should be. You can teach the model how to infer that, and we cannot possibly have such a data set for all of our customers. + +Yet another problem with these models is that, although they have nice benchmarks, those can be interpreted as a form of data contamination because the logs used to benchmark the model were produced from the same applications with the same templates as the logs that were used to train the model in the first place. This indicates that the model can actually only identify patterns that it is already familiar with or parse logs of applications that it is already familiar with. It doesn't generalize for any logs of any applications. + +### [00:07:24] Exploring alternatives for log parsing + +So yeah, this was the status quo. Plenty of challenges, different options, each one with their limitations. How we solved this in the end was no magic at all. We actually just combined different approaches, trying to balance out the limitations of some with the advantages of others. + +First thing we did was figure out the right level of granularity for predicting log formats and patterns. I don't know if anyone here has already worked with predictive AI in production, but you usually have the concept of an instance. For example, if you are doing customer churn prediction, then every customer is an instance. If you're doing product sales forecast, then every product is an instance. It's quite straightforward. + +When it comes to open telemetry data, it's quite challenging because there are different levels of granularity. The schema is very nested. What is an instance? We started with the concept of the open telemetry resource. We tried to predict the log format and patterns for every resource, but it wasn't efficient at all. + +If you think, for example, of a Kubernetes deployment with autoscale, it is creating new pods and killing old pods all the time, and every new pod has different resource attributes. If we would predict log formats and patterns for every resource, our model would be working all the time, which would not be efficient at all. Instead of doing this, we came up with resource clustering rules that we apply to resource attributes of different resource types. This allows us to scope and cache and reuse the predicted log formats and log patterns between different resources that belong in the same group. To make this work in production, we also came up with a recommended configuration for the open telemetry collector that guarantees that the logs that we get from our customers always contain all the resource attributes that are relevant for our resource clustering rules. -[00:09:46] Now we have a concept of an instance that we want to make predictions for. Next step, how do we predict the patterns? We came up with this pipeline where we start with all the logs, and then we first cluster by resource attributes, like I said before. Then we focus on parsing. We use a combination of heuristics and prompt engineering to identify what is the prefix, what is the message, what parts of the prefix are worth extracting, like the log severity, and we come up with log formats for each resource cluster. +### [00:09:52] Resource clustering for log patterns + +All right, we have now a concept of an instance that we want to make predictions for. Next step: how do we predict the patterns? We came up with this pipeline where we start with all the logs, and then we first clusterize by resource attributes, like I said before. + +Then we focus on parsing. We use a combination of heuristics and prompt engineering to identify what is the prefix, what is the message, what parts of the prefix are worth extracting, like the log severity, and we come up with log formats for each resource cluster. After the logs are parsed, we apply those traditional word frequency models like drain in order to clusterize the log messages by structure. + +Every log message that has kind of the same structure goes into the same cluster. Then, looking at each log cluster, we take the output of the drain model and inject it into the prompt for a language model in order to identify the final variables and also give names and types to the variables. + +At this stage, we observe that including semantic convention information and resource attributes in the prompt helps a lot in getting variable names and types that make sense for each application. Then in the end, we just apply some heuristics to optimize the patterns that we get so they match as best as possible the respective log cluster. + +At ingestion time, it's actually really straightforward in our open telemetry collector. The processor is going to match the logs against patterns from the respective cluster and attach the attributes with the variables that it finds. + +### [00:11:40] Evaluation process and success rates + +This is an overview of our solution. But before I show you what the results look like in our beautiful UI, I'm just going to have a quick word about evaluation. This is really important. Before we applied this to every log of every customer all the time in production, we needed to build some confidence to know that it would produce the expected results. + +We came up with an evaluation data set that combines logs from community data sets with logs from our own proprietary data. The focus was not to have too many logs or quantity but rather to have diversity of logs so we could really stress the model and confirm that it was working as expected for many different edge cases. + +Our main goal with this evaluation was to confirm that our approach had this kind of built-in graceful degradation. What does that mean? It means that the model doesn't need to work all the time. It doesn't need to work for every log from the billions of logs that we are ingesting. But whenever it works, it needs to produce correct information. When it doesn't work, then it cannot produce any unwanted side effects. + +Graceful degradation, and that's basically what we observed in the evaluation. We had a 98% success rate at the log parsing step with 100% log severity accuracy among the cases that are successfully parsed. The model almost always understands the log format, and when it does, the log severity that it extracts is always correct. When it does not, then it doesn't extract anything, and the log stays as it was before. -[00:10:40] After the logs are parsed, we apply those traditional word frequency models like drain in order to cluster the log messages by structure. Every log message that has kind of the same structure goes into the same cluster. Looking at each log cluster, we take the output of the drain model and inject it into the prompt for a language model in order to identify the final variables and also give names and types to the variables. At this stage, we observe that including semantic convention information and resource attributes in the prompt helps a lot in getting variable names and types that make sense for each application. +Besides that, the patterns that we extracted for each application’s logs cover an average of 85% of that application's logs. That means we successfully abstract almost all the information that is available to be abstracted. -Then, in the end, we just apply some heuristics to optimize the patterns that we get so they match as best as possible the respective log cluster. At ingestion time, it's actually really straightforward. In our open telemetry collector, the processor is going to match the logs against patterns from the respective cluster and attach the attributes with the variables that it finds. +Sorry, ask a question right at the end. Let's save the questions for the end if you don't mind. Don't forget it. -[00:12:00] This is an overview of our solution. But before I show you what the results look like in our beautiful UI, I'm just going to have a quick word about evaluation. This is really important. Before we applied this to every log of every customer all the time in production, we needed to build some confidence to know that it would produce the expected results. We came up with an evaluation dataset that combines logs from community datasets with logs from our own proprietary data. The focus was not to have too many logs or quantity, but rather to have diversity of logs so we could really stress the model and confirm that it was working as expected for many different edge cases. +Okay, I'm just quickly going to show what the results look like in our UI. This is a histogram of log severities over time. The gray bars are logs with unknown severity. As soon as we plug in the new log processor that applies the log formats and patterns, we start having colorful logs in the histogram indicating the information and warning and error logs. -[00:13:01] Our main goal with this evaluation was to confirm that our approach had this kind of built-in graceful degradation. What does that mean? It means that the model doesn't need to work all the time. It doesn't need to work for every log from the billions of logs that we are ingesting. Whenever it works, it needs to produce correct information. When it doesn't work, then it cannot produce any unwanted side effects. That’s graceful degradation, and that's basically what we observed in the evaluation. We had a 98% success rate at the log parsing step with 100% log severity accuracy among the cases that are successfully parsed. The model almost always understands the log format, and when it does, the log severity that it extracts is always correct. When it does not, then it doesn't extract anything, and the log stays as it was before. +Also, we observe a couple examples here of errors and warnings that would have passed by unnoticed if it wasn't for the correct log parsing. What else? We come up with this dedicated visualization for the patterns. This is data from the open telemetry demo, and then we have different patterns where the variable is in the middle. -Besides that, the patterns that we extracted for each application's logs cover an average of 85% of that application's logs. That means we successfully abstracted almost all the information that is available to be abstracted. +Things that say, for example, "targeted ad request received for ad category," and then "ad category" is a variable or a method name called with user ID, and then the method name and the user ID are variables. If we open one log record like this one with product ID name, then we know which pattern it matches, and we have the product ID and product name as dedicated variables. This is structured data that can be referenced in queries, in filters, in group by expressions, also in Prometheus query language. -Oh, sorry. Ask a question right at the end. Let's save the questions for the end if you don't mind. Don't forget it. I'm just quickly going to show what the results look like in our UI. This is a histogram of log severities over time. The gray bars are logs with unknown severity. As soon as we plug in the new log processor that applies the log formats and patterns, we start having colorful logs in the histogram indicating the information and warning and error logs. +For example, if I want to create an alert based on log frequency using the patterns, I have some logs that say "product ID lookup failed," and then it has some ID in the end. This one matches a pattern, and the product ID is a variable. -We also observe a couple of examples here of errors and warnings that would have passed by unnoticed if it wasn't for the correct log parsing. We come up with this dedicated visualization for the patterns. This is data from the open telemetry demo, and we have different patterns where the variable is in the middle, things that say, for example, "targeted ad request received for ad category," and then ad category is a variable or a method name called with user ID. The method name and the user ID are variables. If we open one log record like this one with product ID name, then we know which pattern it matches, and we have the product ID and product name as dedicated variables. This is structured data that can be referenced in queries, in filters, in group by expressions, also in Prometheus query language. +So I can create a log frequency alert with the filter saying that the pattern should be that one and grouping by the product ID field that comes from the semi-structured text. This is the Prometheus query language expression to alert on that log frequency, and then the result is when I'm having too many lookup errors for that product, I get a nice failed check with the custom message up here that says "lookup failures increasing for product with the ID of the product." -For example, if I want to create an alert based on log frequency using the patterns, I have some logs that say "product ID lookup failed," and then it has some ID in the end. This one matches a pattern, and the product ID is a variable. So I can create a log frequency alert with the filter saying that the pattern should be that one and grouping by the product ID field that comes from the semi-structured text. +Basically, alerting based on semi-structured information that was abstracted from logs. This happens without any metrics, without needing an extra metric, without needing to write any regular expression, without depending on traces and anything else, just based on the logs. -This is the Prometheus query language expression to alert on that log frequency, and the result is when I'm having too many lookup errors for that product, I get a nice failed check with the custom message up here that says "lookup failures increasing for product with the ID of the product." So basically, alerting based on semi-structured information that was abstracted from logs. +Yeah, this is really cool. This was the case study on the log AI that we have been developing. I'm just going to quickly comment on a couple other experiments that we have been running. -This happens without any metrics, without needing an extra metric, without needing to write any regular expression, without depending on traces and anything else, just based on the logs. This is really cool. So yeah, this was the case study on the log AI that we have been developing. +We have been experimenting with AI agents to write and debug Prometheus query language expressions and also to diagnose and find the root cause of failed checks and alerts. The agent that we work with has access to a model context protocol, MCP server that allows it to browse through every metric and log and trace, kind of like the same way that a human would do in our UI. -I'm just going to quickly comment on a couple of other experiments that we have been running. We have been experimenting with AI agents to write and debug Prometheus query language expressions and also to diagnose and find the root cause of failed checks and alerts. The agent that we work with has access to a model context protocol, MCP server that allows it to browse through every metric and log and trace, kind of like the same way that a human would do in our UI. +In this context, we also observe that if we feed the agent with information about semantic conventions and resource attributes, then it gets way better at writing correct queries in the first try. So it doesn't take so many iterations to figure out the correct metric names, the relevant attributes, and so on because it builds on top of the semantic conventions. -In this context, we also observe that if we feed the agent with information about semantic conventions and resource attributes, then it gets way better at writing correct queries in the first try. It doesn't take so many iterations to figure out the correct metric names, the relevant attributes, and so on because it builds on top of the semantic conventions. Yet another use case is our trace AI, where we group duplicated spans in our trace explorer by their attribute similarity, and we also generate trace names and trace descriptions to clusters of similar traces. +Yet another use case is our trace AI, where we group duplicated spans in our trace explorer by their attribute similarity, and we also generate trace names and trace descriptions to clusters of similar traces. In this context, we also observe that giving semantic convention information to the prompt increases the quality of the names and the descriptions that we get. -In this context, we also observe that giving semantic convention information to the prompt increases the quality of the names and the descriptions that we get. Also, when clusterizing the traces by their semantic embeddings, if we include the resource attributes in the embedding, we get way more meaningful embeddings, therefore a better clustering of similar traces. +Also, when clusterizing the traces by their semantic embeddings, if we include the resource attributes in the embedding, we get way more meaningful embeddings, therefore a better clustering of similar traces. -[00:19:20] Those were just a couple of other examples of use cases that we have been working on. Before we go on to the questions, I just wanted to leave you with four key takeaways. +### [00:19:14] Key takeaways from the case study -First, applying AI in production at scale can be expensive. It can get expensive very quickly. But remember that in the open telemetry schema, we have plenty of attributes that you can use to cluster similar resources. This way, you can scope, cache, and reuse AI predictions between different resources. +Yeah, those were just a couple other examples of use cases that we have been working on. Before we go on to the questions, I just wanted to leave you with four key takeaways. + +First, applying AI at production at scale can be expensive. It can get expensive very quickly. But remember that in the open telemetry schema, we have plenty of attributes that you can use to clusterize similar resources. This way, you can scope, cache, and reuse AI predictions between different resources. Second, LLMs are cool, but they are expensive. We always try to see what we can do with traditional machine learning techniques or how we can combine them with LLMs to get the best of both worlds. -Third, evaluation is very important. Even if you're building a simple LLM application and you're not really training or fine-tuning any model, you can always try to model the expected behavior of your application and measure how often and how close it gets to the expected results. You kind of quantify the limitations and the risks before shipping it to production. +Third, evaluation is very important. Even if you're building a simple LLM application and you're not really training or fine-tuning any model, you can always try to model the expected behavior of your application and measure how often and how close it gets to the expected results. You can quantify the limitations and the risks before shipping it to production. Lastly, context is everything when it comes to LLMs. Always benefit from resource attributes from the rich open telemetry schema and the semantic conventions so you contextualize your prompts and get better results from language models. -That was it. If you want to connect with us, please follow us on LinkedIn. We also have a blog on our website. We're always posting some cool stuff. We have the Code Red podcast on Spotify and Apple. We have a couple of open positions on our careers website too, including one for another AI engineer and for a product manager with a background in observability. +Yeah, that was it. If you want to connect with us, please follow us on LinkedIn. We also have a blog on our website. We're always posting some cool stuff. We have the Code Red podcast on Spotify and Apple. We have a couple open positions on our careers website too, including one for another AI engineer and for a product manager with a background in observability. That was it. Thanks for your attention. -**Audience Member:** I'm sorry. Do you remember your question? +I'm sorry, do you remember your question? + +Yes. I think it was about the log levels. You had 98% with log levels, but there are some logs that do not have a log level. How is that working out? + +We only measure the accuracy where there is a ground truth. For a log that doesn't make sense to assign a level to, it doesn't have the level, even in an unstructured way or in the form of a status code or anything, then we do not assign it a log level. + +Actually, in that case, if we would assign a log level, let's say error, warning, or anything to a log that doesn't contain any textual information that references that, it would count as a false positive, and we have a separate metric for measuring how often that occurs. It's the specificity metric, so it's how often we avoid false positives, and that one is maximized as well. + +But it wasn't on the slide. + +Okay. I think there's even an unspecified level for open telemetry, right? + +There is, yes, but it is the default. At least when we ingest, if that field is not set, we set it to unspecified, and then after the AI runs, if we don't identify an explicit level, it remains at unidentified. + +Any other questions? + +I didn't get this specifically about the semantic conventions and resource. Did you use them to insert resource attributes and semantic conventions into logs that don't have it, or what was the... + +No. The thing is, whenever we get any signal that has attributes or a span name, a metric name that matches the latest semantic conventions, we have access to the documentation of that metric or attribute, and we include that documentation in the prompt. + +The model has context of what that metric means and which system it belongs to, so it is able to, for example, give an appropriate name to a variable or give a better description to a trace. + +That's how we work with semantic conventions for prompt contextualization. + +That's very interesting. I was thinking it would be also nice to have these kinds of resource attributes and conventions. Sometimes, the values are not correct, especially if you have your own semantic conventions on top of OpenTelemetry, like business-relevant semantic conventions. If you could use AI to rectify basically the discrepancies in what the users are putting in because usually, they put in a lot of stuff and a lot of different stuff. + +Even if you have written it down, "please use these values," they make up their own values and their own writing systems. Instead of going there and telling them every time, "no, please lower case, not uppercase," or something along the lines, if we could use AI to understand the... + +I'm not sure if AI is really necessary. Sometimes regular expressions do the trick, but sometimes they don't. + +That can actually be a challenge. Every case where I mentioned that we employed so many conventions, it was the official ones, like from the open source repositories, theoretically. + +Theoretically, yes. + +Sorry, I have a follow-up question on this. A question I asked myself: I don't know how D-Zero works as an observability platform, but my assumption is that the log AI feature is the same for every customer. + +So you found an approach to leverage different kinds of AI technologies to have log AI enabled, and it's the same for every customer. + +Do I train this follow-up question on your... + +With my own semantics, everyone has something like a slightly different implemented AI approach, so that's a good question. + +The approach right now is the same for everyone, but we work in a multi-tenant way. Even if two customers have, let's say, Oracle databases, we don't mix the log formats and patterns of one customer with the other. + +So all of your data will be processed with formats and patterns that we inferred from your data only. + +It is, but it's always pre-trained. + +Yes. We are not customizing by customer, although not right now. Maybe in the future. + +Yeah. -**Lariel:** Yes. I think it was about the log levels. You had 98% with log levels, but there are some logs that do not have a log level. How is that working out? +Thank you very much. Are there plans on open sourcing any of that? -**Lariel:** We only measure the accuracy where there is a ground truth. For a log that really doesn't make sense to assign a level to, it doesn't have the level, even in an unstructured way or in the form of a status code or anything, then we do not assign it a log level. Actually, in that case, if we would assign a log level, let's say error, warning, or anything to a log that doesn't contain any textual information that references that, it would count as a false positive. We have a separate metric for measuring how often that occurs. It's the specificity metric. It's how often we avoid false positives, and that one is maximized as well, but it wasn't on the slide. +I suppose that, I mean I love the log AI part, but I suppose the Prometheus part would also have that kind of feature, and I think I heard other companies doing the same, right? -**Audience Member:** Okay. I think there's even an unspecified level for open telemetry, right? +I think I heard some other companies doing something very similar. I see a lot of potential collaboration opportunities there. -**Lariel:** Yes, but it is the default. At least when we ingest, if that field is not set, we set it to unspecified, and then after the AI runs, if we don't identify an explicit level, it remains unidentified. +Yeah, there could be. Regarding any kind of open source, you know, need to talk to our CTO. What we are planning to do is to release an MCP, so anyone can connect their agents to D-Zero data if they want to over there. -**Audience Member:** Any other questions? +So here to me, do you apply a model to every single log? -**Audience Member:** I didn't get this specifically about the semantic conventions and resource. Did you use them to insert resource attributes and semantic conventions into logs that don't have it, or what was the... +No. -**Lariel:** No. The thing is, whenever we get any signal that has attributes or a span name, a metric name that matches the latest semantic conventions, then we have access to the documentation of that metric or attribute, and we include that documentation in the prompt. The model has context on what that metric means in which system it belongs. It is able to, for example, give an appropriate name to a variable or give a better description to a trace. So that's how we work with semantic conventions for prompt contextualization. +Okay. So how do we apply? -**Audience Member:** That's very interesting. I was thinking it would also be nice to have these kinds of resource attributes and conventions. Sometimes the values are not correct, especially if you have your own semantic conventions on top of OpenTelemetry, like business-relevant semantic conventions. If you could use AI to rectify basically the discrepancies in what the users are putting in, because usually they put in a lot of stuff, a lot of different stuff. Even if you have it written down, please use these values, they make up their own values and their own writing systems. Instead of going there and telling them every time, "No, please lower case, not uppercase," or something along the lines, if we could use AI to understand the... +We do it in a batch where the logs are already clustered by resource, and then they are clustered by structure. Within each log cluster, we identify the pattern with the LLM, and then we store that pattern. -**Lariel:** I'm not sure if AI is really necessary. Sometimes regular expressions do the trick, but sometimes they don't. That can actually be a challenge. Every case where I mentioned that we employed so many conventions, it was the official ones, like from the open source repositories. +During ingestion time, we don't invoke the LLM at all; we just apply the patterns that we have already cached. -**Audience Member:** I have a follow-up question on this. A question I asked myself, I don't know how D-Zero works as an observability platform, but my assumption is that the log AI feature is the same for every customer because you used something like... +So by pattern, you mean regular expression? -**Audience Member:** Yeah, now someone is turning around if you're not... +Yes. -**Lariel:** You found an approach to leverage different kinds of AI technologies to have log AI enabled, and it's the same for every customer. +No. -**Audience Member:** With my own semantics, does my own and everyone have something like a slightly different implemented AI approach? +You can try that. Maybe the newer ones will succeed for some cases, but LLMs are usually bad at writing regular expressions on their first try. We actually use them to infer parts of the log, extract variables, get some insights on the log structure, and then we apply heuristics to these results to compose the regular expression ourselves. -**Lariel:** That's a good question. The approach right now is the same for everyone, but we work in a multi-tenant way. Even if two customers have, let's say, Oracle databases, we don't mix the log formats and patterns of one customer with the other. All of your data will be processed with formats and patterns that we inferred from your data only. It is pre-trained, yes. We are not customizing by customer, although not right now, maybe in the future. +Yeah, I hope that rests on the data at rest. -**Audience Member:** Thank you very much. Are there plans on open sourcing any of that? +Yes, it was also part of the processor, which was done on the fly. -**Lariel:** I suppose that—I mean, I love the log AI part, but I suppose the Prometheus part would also have that kind of feature, and I think I heard other companies doing the same, right? +On the fly, during ingestion, we apply log formats and patterns that have already been cached. So there is like this pattern identification phase, and there is the production time, which is exploiting the patterns that have already been cached. -**Lariel:** I think I heard some other companies doing something very similar. I see a lot of potential collaboration opportunities there. +Question. So at what frequency do you refresh the cache? -**Audience Member:** Regarding any kind of open source, you know, need to talk to our CTO, but what we are planning to do is to release an MCP, so anyone can connect their agents to D-Zero data if they want to over there. +That's in our documentation. It should be every two hours. -**Audience Member:** So you apply a model to every single log. Do you use an LLM? +Why should... -**Lariel:** No. +It's in beta phase; I'm working on it. But yeah, it's meant to be every two hours. There is obviously a quota per rate limits per customer. So if you're sending logs of different formats and different patterns all the time, like generating random formats just to make us spend money, it's not going to work. -**Audience Member:** Okay. So how do we apply? +So refreshes for when patterns change, but actually it's only needed when it changes. -**Lariel:** We do it in a batch where the logs are already clustered by resource, and then they are clustered by structure. Within each log cluster, we identify the pattern with the LLM, and then we store that pattern. During ingestion time, we don't invoke the LLM at all; we just apply the patterns that we have already cached. +Sorry, you said the cache refreshes every two hours? -**Audience Member:** So by pattern, you mean regular expression? +Yes. -**Lariel:** Yes. No. You can try that. Maybe the newer ones will succeed for some cases, but LLMs are usually bad at writing regular expressions on their first try. We actually use them to infer parts of the log, extract variables, get some insights on the log structure, and then we apply heuristics to these results to compose the regular expression ourselves. +And so that cache is for... you're caching the log patterns? -**Audience Member:** I hope that rests on the data at rest. +Yes. -**Lariel:** Yes, it was also part of the processor, which was done on the fly. +So when the log pattern changes, the cache needs to be refreshed too? -**Audience Member:** On the fly during ingestion, we apply log formats and patterns that have already been cached. +Yes. -**Audience Member:** So at what frequency do you refresh the cache? +So then, yeah, it would be good if you could do this on demand. -**Lariel:** That's in our documentation. It should be every two hours. +It is on demand every two hours. -**Audience Member:** Every two hours? +Okay. Seriously, that wasn't a joke. I mean, if they change, we realize that they changed at latest two hours after, and then update them. -**Lariel:** Yes. It's in beta phase. I'm working on it, but yeah, it's meant to be every two hours. There is obviously a quota per customer. If you're sending logs of different formats and different patterns all the time, like generating random formats just to make us spend money, it's not going to work. Refreshes are for when patterns change, but actually, it's only needed when it changes. +I mean, one very important aspect of Dash Zero is that we do not sell AI capabilities as separate modules. We give them kind of for free. You only pay for the data that you send, and every capability, all the cool stuff in the UI, you get out of the box. -**Audience Member:** Sorry, you said the cache refreshes every two hours? +So whenever we implement something like this, we have to establish rate limits and optimize it as best as we can to make it cost-effective. -**Lariel:** Yes. +But if I, as a customer, know that my logs won't change, let's say, or I know when it will change, is there a possibility to change that, to tell D-Zero that now there's a new pattern coming? -**Audience Member:** So that cache is for, you're caching the log patterns? +Right now, the way to do that would be you message our customer success people, and then they would ping me. But we wouldn't need to do anything if they change, and at latest two hours later, the pipeline would realize. -**Lariel:** Yes. When the log pattern changes, the cache needs to be refreshed too. +Exactly. -**Audience Member:** Yes. So then, yeah, it would be good if you could do this on demand. +What it means is, within the first two hours, I might not recognize all of the new fields or all of the new patterns that are flowing through the pipeline. -**Lariel:** It is on demand, every two hours. +After two hours, it's going to be recognized. -**Audience Member:** Okay. Seriously, that wasn't a joke. I mean, if they change, we realize that they changed at latest two hours after, and then update them. One very important aspect of Dash Zero is that we do not sell AI capabilities as separate modules. We provide them kind of for free. You only pay for the data that you send, and all the capabilities, all the cool stuff in the UI, you get out of the box. Whenever we implement something like this, we have to establish rate limits and optimize it as best as we can to make it cost-effective. +### [00:32:14] Audience Q&A and discussions -**Audience Member:** But if I, as a customer, know that mine won't change, let's say, or I know when it will change, is there a possibility to tell Dash Zero that now there's a new pattern coming? +That graph I've shown with the gray bars, we could see a difference there for the first two hours, and then it goes back to the same level? -**Lariel:** Right now, the way to do that would be you message our customer success people, and then they would ping me. But we wouldn't need to do anything if they change, and at latest two hours later, the pipeline would realize exactly. +Exactly. -**Audience Member:** So what it means is, within the first two hours, I might not recognize all of the new fields or all of the new patterns that are flowing through the pipeline? +That's very cool. -**Lariel:** Yes. After two hours, it's going to be recognized. +Wow. -**Audience Member:** That graph you've shown with the gray bars, we could see a difference there for the first two hours, and then it goes back to the same level. +Go on. -**Lariel:** Exactly. That's very cool. +So one question is, you mentioned that you have some kind of prompt engineering in the pipeline detecting the patterns. Do customers somehow, can they involve, are they involved in this process? Can they change something like, for example, adding their own smart? -**Audience Member:** Go on. +We use the same workflow for every customer, but the more information we have for each customer, the better we could contextualize the prompt. Maybe having custom semantic conventions for each customer's domain would be something to help improve the prompt. -**Audience Member:** One question is, you mentioned that you have some kind of prompt engineering in the pipeline detecting the patterns. Do customers somehow, can they involve, are they involved in this process? Can they change something like, for example, adding their own smart? +Yeah, that's something we could build. -**Lariel:** We use the same workflow for every customer, but the more information we have for each customer, the better we could contextualize the prompt. Maybe having custom semantic conventions for each customer's domain would be something to help improve the prompt. That's something we could build. +And a second question: how do you monitor, like end to end, for example, how much money you spent on how many tokens and so on? -**Audience Member:** And a second question, how do you monitor end to end, for example, how much money you spent on how many tokens and so on? +Well, we use D-Zero to monitor D-Zero, right? We also use the community libraries for instrumenting LLM applications, the ones from the open Python OpenTelemetry contrib. From those, we get LLM traces and a cost for every LLM request that we make. -**Lariel:** We use D-Zero to monitor. We also use the community libraries for instrumenting LLM applications, the ones from the OpenTelemetry Python contrib. From those, we get LLM traces and a cost for every LLM request that we make. The costs are also contextualized with the price per token of each vendor and model variant. We have a nice LLM trace view in our UI where we can debug LLM interactions, like with what tools were called and so on. +The costs are also contextualized with the price per token of each vendor and model variant, and we have a nice LLM trace view in our UI where we can debug LLM interactions, like with what tools were called and so on. -**Lariel:** Folks, if anyone has other questions, we can chat later, here around in the kitchen. I'm going to give it to the next, and thanks for having me. +Folks, if anyone has other questions, we can chat later here around in the kitchen. I'm going to hand it over to the next speaker. Thanks for having me. ## Raw YouTube Transcript diff --git a/video-transcripts/transcripts/2025-06-11T05:54:28Z-otel-me-with-oluwatomisin-taiwo-and-andrei-morozov.md b/video-transcripts/transcripts/2025-06-11T05:54:28Z-otel-me-with-oluwatomisin-taiwo-and-andrei-morozov.md index 938b3ca..b3cdf2f 100644 --- a/video-transcripts/transcripts/2025-06-11T05:54:28Z-otel-me-with-oluwatomisin-taiwo-and-andrei-morozov.md +++ b/video-transcripts/transcripts/2025-06-11T05:54:28Z-otel-me-with-oluwatomisin-taiwo-and-andrei-morozov.md @@ -10,42 +10,46 @@ URL: https://www.youtube.com/watch?v=tTCuTAPE5aQ ## Summary -In this episode of "Hotel Me," hosts Adriana Vila and Andre Kapolski are joined by guests Tommy and Andre M. from Compass Digital, who discuss their experiences with observability in distributed systems using OpenTelemetry. Tommy focuses on the reliability and scalability of their web platform, while Andre M. specializes in developer experience and platform engineering. The conversation delves into the architecture of their systems, which primarily utilize AWS services like Lambdas and ECS, as well as the challenges they face with distributed tracing, performance monitoring, and error tracking. They share insights on their instrumentation practices, including the use of OpenTelemetry for logs, metrics, and traces, and discuss their approach to managing large amounts of telemetry data. Additionally, they highlight their journey towards adopting OpenTelemetry, the importance of community engagement, and the potential for future contributions to the project. The session concludes with a Q&A from the audience, covering topics such as instrumentation of legacy systems and reporting business value metrics. The hosts encourage viewers to participate in future sessions and community discussions. +In this episode of "Hotel Me," hosts Adriana Vila and Andre Kapolski engage with guests Tommy and Andre M. from Compass Digital, discussing their roles in ensuring system reliability and observability. They delve into the architecture of their systems, which primarily leverage AWS services like Lambda and ECS with a focus on using OpenTelemetry for observability. Key challenges addressed include the need for distributed tracing in microservices, performance monitoring, and effective error tracking. The conversation highlights the importance of context propagation and manual instrumentation for critical business logic, as well as the evolving landscape of observability practices. The session concludes with insights on community engagement and future directions for OpenTelemetry, emphasizing the need for practical examples and improved tooling. ## Chapters -00:00:00 Welcome and intro -00:01:30 Guest introductions -00:03:35 System architecture overview -00:06:32 Observability challenges -00:10:01 Instrumentation methods -00:13:12 Context propagation issues -00:19:00 OpenTelemetry setup discussion -00:22:50 Data storage optimization -00:30:00 Collector setup and management -00:36:10 Future of OpenTelemetry +00:00:00 Introductions +00:01:36 Guest introduction: Tommy +00:03:08 Guest introduction: Andre M +00:05:20 Discussion about system architecture +00:10:24 Need for observability in microservices +00:12:48 Introduction of OpenTelemetry +00:19:50 Discussion about instrumentation challenges +00:25:36 Audience Q&A session +00:36:48 Discussion on data management and optimization +00:43:12 Future directions for OpenTelemetry -**Adriana:** Heat. Heat. Hello everyone and welcome to our latest hotel me. Super excited for those who are able to join and for those who are not, we do have this recording available after the fact both on LinkedIn and YouTube. My name is Adriana Vila and I am one of the maintainers of the hotel enduser SIG, and I am happy to introduce my co-presenter today, Andre. +## Transcript -**Andre:** Hi everyone and yeah, awesome to be here. My name is Andre Kapolski and I'm a fairly recent contributor to the end user SIG. So yeah, I'm super excited to hear more from our guests today. +### [00:00:00] Introductions -[00:01:30] **Adriana:** Yeah, and I think this is a perfect segue as we bring on our guests. And as we do, folks on the chat, please feel free to say where you're watching from. +**Adriana:** Heat. Heat. Hello everyone and welcome to our latest hotel me. I'm super excited for those who are able to join and for those who are not, we do have this recording available after the fact both on LinkedIn and YouTube. My name is Adriana Vila and I am one of the maintainers of the hotel end-user SIG, and I am happy to introduce my co-presenter today, Andre. -**Andre:** Yeah, I can perhaps start. I'm watching from the Czech Republic, in the EU, specifically in Brno. Adriana, what about you? Where are you joining from? +**Andre:** Hi everyone, and yeah, awesome to be here. My name is Andre Kapolski and I'm a fairly recent contributor to the end-user SIG. So yeah, I'm super excited to hear more from our guests today. -**Adriana:** Oh, yeah, that's right. I'm in Toronto, Canada, so it's, I guess, 1:00 for me and I guess it's evening for you, right? You're probably 7 o'clock in the... +### [00:01:36] Guest introduction: Tommy -**Andre:** That's correct. That's correct. +**Adriana:** Yeah, and I think this is a perfect segue as we bring on our guests. And as we do, folks on the chat, please feel free to say where you're watching from. -**Adriana:** Yeah. Awesome. Awesome. All right. I guess let's bring our guests on. Hello everyone. +**Andre:** Yeah, I can perhaps start. I'm watching from or I'm joining from the Czech Republic and in the EU, specifically in Brno. Adriana, what about you? Where I don't think you mentioned it, have you? + +**Adriana:** Oh, yeah. That's right. I'm in Toronto, Canada, so it's I guess 1:00 for me and I guess it's evening for you, right? You're probably 7 o'clock in the... -**Tommy:** Hello. +**Andre:** That's correct. That's correct. + +**Adriana:** Yeah. Awesome. Awesome. All right. I guess let's bring our guests on. Hello everyone. -**Andre M:** Hey. +**Tommy:** Hello. -**Adriana:** So, why don't you both introduce yourselves? Let's start with Tommy. +**Andre:** Hey. So, why don't you both introduce yourselves? Let's start with Tommy. -**Tommy:** My name is Tommy. I'm an SRE here at Compass Digital and my primary responsibility essentially is ensuring the reliability, scalability, and observability of our web platform. I work closely with the engineering team, product teams, and test engineers to ensure that our systems are robust and we have actionable insights into what's going on within our services. So yeah, that's me. +**Tommy:** My name is Tommy. I'm an SRE here at Compass Digital and my primary responsibility essentially is ensuring the reliability, scalability, and observability of our web platform. I work closely with the engineering team, product teams, and test engineers to ensure that our systems are robust and we have actionable insights into what's going on within our services. So yeah, that's me. **Adriana:** And where are you calling from? @@ -53,217 +57,167 @@ In this episode of "Hotel Me," hosts Adriana Vila and Andre Kapolski are joined **Adriana:** Awesome. We're practically neighbors, a few hours away from each other. -**Tommy:** Oh yeah. Awesome. +**Tommy:** Oh yeah. Awesome. -**Adriana:** Okay. And then we have another Andre today. +**Andre:** Okay. And then we have another Andre today. -**Andre M:** Hi. I'm out from the Bonneville Durham region. So, same type of area. And for me, I've been in this industry for 11 years, but always been into the tech stuff since I was a kid. I work with Tommy on the same team, SRE at Compass Digital here, and very similar responsibilities across observability, on-call, PI, all the type of stuff for our systems. +### [00:03:08] Guest introduction: Andre M -[00:03:35] **Adriana:** Awesome. Well, we're super excited to have you both on, and just a reminder for folks watching, if you do have questions along the way, please feel free to post them in the chat. And also, we will be taking questions at the end if anyone has questions. So I guess let us get started. So I know you guys both talked a little bit about your intros. Maybe you could talk a little bit about your respective roles at Compass Digital in a little bit more detail. +**Andre M:** Hi. I'm out from the Bonneville Durham region. So, same type of area. And for me, I've been in this industry for 11 years, but always been since I was a kid into the tech stuff. So I work with Tommy and the same team SRE at Compass Digital here and very similar responsibilities across observability to on-call PI, all the type of stuff for our systems. -**Tommy:** Sounds good. I'll start. - -**Andre M:** Okay. Yeah, sorry. I'll go first then. As I said, most of my role is primarily focused on ensuring the reliability and scalability of our systems. Very recently, or for the most part that I've been here, I've been working on one of our products. It's a company I actually bought over, it's e-club, to help get observability into what they're doing there. And largely, I work with things like AWS SDK, the CDK, TF rather, and PA duty. We have observability backend data trace where we send all our OpenTelemetry stuff to. So pretty much just SRE, there's nothing too crazy happening on there, but I believe as we go along on this podcast, I'll get more nuanced into what I do and how your architecture is structured and the stack there. +**Adriana:** Awesome. Well, we're super excited to have you both on and just a reminder for folks watching, if you do have questions along the way, please feel free to post them in the chat. And also we will be taking questions at the end if anyone has questions. So I guess let us get started. So I know you guys both talked a little bit about your intros. Maybe you could talk a little bit about your respective roles at Compass Digital in a little bit more detail. -**Adriana:** Sounds good. And Andre M, if you could elaborate a little bit on your... +**Tommy:** Sounds good. I'll start. -**Andre M:** Yeah, certainly. So when I started with Compass about five years now, I started off as a senior software engineer. So that's my background in software engineering, less so on the ops and infra side, but that's kind of been a new love with Terraform and services. But primarily about the developer experience, the platform engineering aspect of it. And more recently, it's more about understanding the system that you can worry about and they'll tell us that linkability, how potentially across that, and give us the insights with our platform and a vendor that we utilize that with. +### [00:05:20] Discussion about system architecture -**Tommy:** So just to give you some pretty cool things is like identifying any services that are looping into each other, for instance, via API rather than just calling it internally with a function. So it's more about just setting up these guard rails and looking into the optimization of our system. And the other aspect is, because we are a microservice shop, it's been a challenge for us to get that context propagation and seeing all the services all together. And now we're able to kind of have that full aspect all together and actually run real, I guess, business impacts and analysis onto that. +**Andre M:** Okay. Yeah, sorry. I'll go first then. As I said, most of my role is primarily focused on ensuring the reliability and scalability of our systems. Very recently or for the most part that I've been here, I've been working on one of our products. It's a company I actually bought over is e-club to help get observability into what they're doing there. And largely, I work with things like AWS SDK, the CDK, TF rather, and PA duty. We have observability back-end data trace where we send all our open telemetry stuff to. So pretty much just SRE. There's nothing too crazy happening on there, but I believe as we go along on this podcast, I'll get more nuanced into what I do and how your architecture is structured and the stack there. -**Adriana:** Yeah, that's primarily an area I've been looking at more or less. +**Adriana:** Sounds good. And Andre M, if you could elaborate a little bit on your role? -**Tommy:** Cool. Folks, can you start with, can you tell us a bit more about the system that you are operating? What is its architecture? What programming languages are used and how it is deployed and so on? +**Andre M:** Yeah, certainly. So when I started with Compass about 5 years now, I started off as a senior software engineer. So that's my background in software engineering, less so on the ops and infra side, but that's kind of been a new love with Terraform and services. But primarily about the developer experience, the platform engineering aspect of it. And more recently, it's more about understanding the system that you can worry about and they'll tell us that linkability how potentially across that and give us the insights with our platform and a vendor that we utilize that with. So just to give you some pretty cool things is like identifying any services that are looping into each other, for instance, via API rather than just like calling it internally with a function. So it's more about just setting up these guard rails and looking into the optimization of our system. And the other aspect is because we are a microservices shop, it's been a challenge for us to get that context propagation and seeing all the services all together. And now we're able to kind of have that full aspect all together and actually run real, I guess, business impacts and analysis onto that. Yeah, that's primarily an area I've been looking at more or less. -**Tommy:** Okay. So we have two products. So I'll talk about the one that I primarily focus on. So for us, our main infrastructure and architecture all runs, 90, 95% is all Lambdas. We have about 300 plus Lambdas in our ecosystem. We run a hybrid of DynamoDB with Aurora RDS and that's the exact thing progress, and we're slowly getting into Argate services. So that's a little bit different model, but that's the primary architecture of our ecosystem. You know the standard services like S3, API Gateway, just standard service stuff that the ecosystem provides. +**Tommy:** Cool. Folks, can you start with can you tell us a bit more about the system that you are operating? What is it? What is its architecture? What programming languages are used and how it is deployed and so on? -**Andre M:** Oh yeah. On the E-Club side, what we have is a blend of Python and Django on the backend. And then JavaScript, TypeScript on the frontend, and we run everything on that side on AWS ECS, which gives us that flexibility and consistency across all our environments. +**Tommy:** Okay. So we have two products. So I'll talk about the one that I primarily focus on. For us, our main infrastructure and architecture all runs, 90 to 95% is all lambdas. We have about 300 plus lambdas in our ecosystem. We run a hybrid of DynamoDB with Aurora RDS and that's the exact thing progress and we're slowly getting into Fargate services. So that's a little bit different model but that's the primary architecture of our ecosystem. You know, the standard services like S3, HTTP gateway, just standard service stuff that the ecosystem provides. -**Tommy:** Okay, thank you. Our CI/CD pipelines automate the process of building those Docker images and then deploying them onto ECS so we can roll out those changes quickly and safely. But to do a quick walkthrough across this little diagram that I have here, so imagine a user interacting with the application. What they hit first is the load balancers that we have here, and then it distributes traffic to the web service that we have on the backend, which runs ECS and is instrumented with the OpenTelemetry SDK. +**Andre M:** Oh yeah. On the E-Club side, what we have is a blend of Python and Django on the back end and then JavaScript, TypeScript on the front end. And we run everything on that side on AWS ECS, which gives us that flexibility and consistency across all our environments. -[00:10:01] As this web service or web services begin to process this request, what it does is offload some heavy tasks or async tasks to this Celery worker down here, which also runs on ECS and is also instrumented with the OpenTelemetry SDK. Thanks to the context—oops, typo—context propagation here, the trace context can flow seamlessly between the web service and the worker. With the introduction of the OpenTelemetry SDK, now we can follow a request end to end from inception to the end, right? Both these services, that's the web service and Celery worker, emit traces, logs, and metrics, and we sample them, make sure they have high cardinality and they are batched also for efficiency. All this data that I'm talking about is sent to OpenTelemetry collectors that run as a sidecar on ECS to send this data to our observability backend. So this is pretty much in a nutshell what the architecture on the E-Club side of what we are doing looks like. +**Tommy:** Okay, thank you. Our CI/CD pipelines automate the process of building those Docker images and then deploying them onto ECS so we can roll out those changes quickly and safely. But to do a quick walkthrough across this little diagram that I have here, imagine a user interacting with the application. What they hit first is the load balancers that we have here and then it distributes traffic to the web service that we have on the back end which runs ECS and is instrumented with the open telemetry SDK. Now as the web service or web services begin to process this request, what it does is offload some heavy tasks or async tasks to this Celery worker down here which also runs on ECS and is also instrumented with the open telemetry SDK. Thanks to the context, oops typo, context propagation here, the trace context can flow seamlessly between the web service and the worker. With the introduction of the open telemetry SDK, we can follow a request end to end from inception to the end, right? Both these services, that's the web service and Celery worker emit traces, logs, and metrics. We sample them, make sure they have high cardinality and they are batched also for efficiency. All this data that I'm talking about is sent to open telemetry collectors that run as a sidecar on ECS to send this data to our observability back end. So this is pretty much in a nutshell what the architecture on the E-Club side of what we are doing looks like. -**Adriana:** Awesome. Thanks for that. As a follow-up question, you know, given your system architecture, what were the top three problems that you were facing that kind of compelled you to say, "Hey, I need observability"? +### [00:10:24] Need for observability in microservices -[00:06:32] **Tommy:** Oh yeah, sure. I'm sure you know, and probably everyone listening knows that observability in a distributed system is never a solved problem. Some of the nuanced challenges that we faced was, again, as I said, distributed tracing. With microservices, a single user request can touch so many services and background jobs, and without distributed tracing, it's almost impossible to reconstruct that full journey of a request from the user's end to the response back to the user, especially when something goes wrong. For instance, if a user says that they had a slow order placement, we need to trace our request from the web frontend, the backend, the Celery workers, and even to third-party APIs. The introduction of OpenTelemetry helps us stitch all these things together, right? And also have consistent context propagation across all these boundaries. +**Adriana:** Awesome. Thanks for that. As a follow-up question, given your system architecture, what were the top three problems that you were facing that compelled you to say, "Hey, I need observability"? -One of the things that we were also looking at was performance monitoring because we rely on Celery heavily on E-Club for background processing. However, these things can be black boxes if you don't instrument them properly. So, we need to know not just if a task has succeeded, but how long it took, what resources consumed, and whether it's causing bottlenecks along the way. Introducing Celery with, sorry, instrumenting Celery with OpenTelemetry required us to go with out-of-the-box solutions, again thanks for that. To be able to add custom spans and metrics to these things. Also, error tracking, you know, errors can manifest in logs, traces, or metrics, wherever. The most important thing for us at the time was to be able to correlate these things together, so that's having your trace ID and span ID in logs and being able to look at that trace ID in the log and correlate it with an actual trace. A spike in error log sometimes might correspond with a specific trace or a drop in a metric that probably we're tracking. What we've done is we've worked to ensure that our telemetry includes enough context, like I said, the trace ID, request ID, so we can pivot between the logs, traces, and metrics seamlessly. So yeah, that's what got us to where we are now. +### [00:12:48] Introduction of OpenTelemetry -**Andre M:** Awesome. +**Tommy:** Oh yeah. Sure. I'm sure you know and probably everyone listening knows that observability in a distributed system is never a solved problem. Some of the nuanced challenges that we faced was again, as I said, distributed tracing. With microservices, a single user request can touch so many services and background jobs and without distributed tracing, it's almost impossible to reconstruct that full journey of a request from the user's end to the response back to the user, especially when something goes wrong. For instance, if a user says that they had a slow order placement, we need to trace our request from the web front end, the back end, the Celery workers, and even to third-party APIs. The introduction of open telemetry helps us stitch all these things together and also have consistent context propagation across all these boundaries. One of the things that we were also looking at was performance monitoring because we rely on Celery heavily on E-Club for background processing. However, these things can be black boxes if you don't instrument them properly. So we need to know not just if a task has succeeded, but how long it took, what resources were consumed, and whether it's causing bottlenecks along the way. Introducing Celery with open telemetry required us to go with out-of-the-box solutions. Again, thanks for that. To be able to add custom spans and metrics to these things and error tracking. You know, errors can manifest in logs, traces, or metrics, wherever. The most important thing for us at the time was to be able to correlate these things together, so that's having your trace ID and span ID in logs and being able to look at that trace ID in the log and correlate it with an actual trace. A spike in error logs sometimes might correspond rather with a specific trace or a drop in a metric that we're tracking. What we've done is we've worked to ensure that our telemetry includes enough context, like as I said, the trace ID, request ID so we can pivot between the logs, traces, and metrics seamlessly. So yeah, that's what got us to where we are now. -**Adriana:** Andre, do you have anything else to add? +**Andre M:** Awesome. Andre, do you have anything else to add? -[00:13:12] **Andre M:** So for our end, I think the primary problem again really was that context propagation because again, the microservices and distributed nature of that. I think the other part that we weren't even really aware about is just how the system really behaves. So a big part of it is tying back into alerting. So depending on what vendor you use, it might have automatic alerting, but being able to provide that to our developers, in our case via Terraform. So developers can open up PR, create their own alerts, and now we have that across our ecosystems has really changed a lot of how we're able to empower the different developer teams to just take control of their own alerting and systems rather than just one team kind of owning that. +**Andre M:** So for our end, I think the primary problem again really was that context propagation because again the microservice and distributed nature of that. I think the other part that we weren't even really aware about is just how the system really behaves. So a big part of it is tying back into alerting. So depending on what vendor you use, it might have automatic alerting, but being able to provide that to our developers in our case via Terraform. So developers can open up PR, create their own alerts, and now we have that across our ecosystems has really changed a lot of how we're able to kind of empower the different developer teams to just take control of their own alerting and systems rather than just one team owning that. **Adriana:** Can you tell us a bit more about how your systems are instrumented? What types of instrumentation are you using? -**Andre M:** Certainly. So on our platform, currently, for us, we do use our vendor agent. So in our end, the agent's really nice for the most part. It works seamlessly with both of our infrastructure pieces for Lambda. Lambda has the concept of a Lambda layer where you can basically put on something. So their agent just goes onto that, kind of bolts right on and works essentially right out of the box. The second part to that that we had to play around with a little bit was our ECS Fargate. There is a solution from AWS called AWS FireLens, which allows you to kind of have your task logs, standard error, and send it out to basically be sent to a specific sidecar where then that sidecar will run something like Fluent Bit or they could be it, and essentially take your logs and ship them to where you need them. - -**Adriana:** Yeah, that makes sense. I would ask a follow-up right away. So are you using auto instrumentation or are you manually instrumenting your services? +**Andre M:** Certainly. On our platform, currently for us, we do use our vendor agent. So in our end, the agent's really nice for the most part. It works seamlessly with both of our infrastructure pieces for Lambda. Lambda has the concept of a Lambda layer where you can basically put on something. So their agent just goes onto that, kind of bolts right on and works essentially right out of the box. The second part to that that we had to play around with a little bit was our ECS Fargate. There is a solution from AWS called AWS FireLens which allows you to kind of have your task logs, standard error, and send them out to basically be sent to a specific sidecar where then that sidecar will run something like Fluent Bit or the Kube and essentially take your logs and ship them to where you need them. -**Tommy:** So for us, because we do have the 300 Lambdas, each one need, we basically have like a core library that all these Lambdas will utilize, a very thin layer, but in there we do have OpenTelemetry instrumentation for Node.js. I believe it's just the auto instrument or I think we might have changed that for the fine-tuner because it needs to be small because Lambdas, you want to keep them really tiny. So I think we do have the custom one where it's not the auto instrument. +**Adriana:** Yeah, that makes sense. I would ask a follow-up right away. So, are you using auto-instrumentation or are you manually instrumenting your services? -**Adriana:** And how was your experience with doing that so far? +**Andre M:** So for us, because we do have the 300 lambdas, each one needs, we basically have like a core library that all these lambdas will all utilize, very thin layer. But in there we do have hotel instrumentation for the Node.js. I believe it's just the auto-instrument or I think we might have changed that for the finetuner because it needs to be small because lambdas, you want to keep them really tiny. So I think we do have the custom one where it's not the auto-instrument. -**Tommy:** I haven't heard any issues. So I didn't do that one. It was actually our coworker Matt who did that one, but I didn't see any too many issues with that. PR went through pretty smoothly and the primary purpose was we use our own custom logger, one that isn't supported by OpenTelemetry or our vendor. So we had to use that to kind of pull in the trace ID from the headers and everything else and be able to actually populate the spans that we require. +**Tommy:** And yeah, how was your experience with doing that so far? -**Andre M:** Cool. So to clarify then, in a nutshell, you basically have like a wrapper around OpenTelemetry from what it sounds like. +**Andre M:** I haven't heard any issues. I didn't do that one. It was actually our coworker Matt who did that one, but I didn't see any too many issues with that. The PR went through pretty smoothly and the primary purpose was we use our own custom logger, one that isn't supported by the hotel or vendor. So we had to use that to kind of pull in the trace ID from the headers and everything else and be able to actually populate the spans that we require. -**Tommy:** Kind of, I guess so. +**Adriana:** Cool. So, to clarify then in a nutshell, you basically have like a wrapper around hotel from what it sounds like. -**Andre M:** Yeah. +**Andre M:** Kind of, it's, I guess so. -**Adriana:** Cool. And a question for you around just to keep on the instrumentation thread. Who is responsible for instrumenting? +**Adriana:** Yeah. And question for you around just to keep on the instrumentation thread, who is responsible for instrumenting? -**Andre M:** Well, that's an interesting journey. So we actually do have a new project upcoming. So the way our system has kind of evolved with Terraform in particular is, you know, we're IAC first for no matter what we do, infrastructure as code. A big part of that is we use the Cloud Development Kit extension on Terraform. So it gives us the ability to kind of have this inheritance model and allows us to build these L1, L2 constructs. So if anyone's familiar with that, it's a very useful way of building abstractions for your infrastructure and enabling teams to kind of handle that as well. +**Andre M:** Well, that's an interesting journey. So we actually do have a new project upcoming. So the way our system has kind of evolved with Terraform in particular is, you know, we're IA first for no matter what we do, infrastructure as code. A big part of that is we use the cloud development kit extension on Terraform. So it gives us the ability to kind of have this inheritance model and allows us to build these L1, L2 constructs. If anyone's familiar with that, it's a very useful way of building abstractions for your infrastructure and enabling teams to handle that as well. In our case, what we're trying to build right now is kind of an L3 construct which is like an application service level where we can give it to developers and they can essentially have all the nuts and bolts already instrumented out of the box with the right permissions and the right access and the right agents and everything else. We kind of support this golden path for them. -In our case, what we're trying to build right now is kind of an L3 construct, which is like an application service level where we can give it to developers and they can essentially have all the nuts and bolts kind of already instrumented out of the box with the right permissions and the right access and the right agents and everything else. We kind of support this golden path for them. +**Tommy:** Awesome. Do you have anything else that you want to add? -**Adriana:** Awesome. Tommy, do you have anything else that you want to add? +**Tommy:** No, not really. As I said, what I think we're trying to get to a point where all of these things are, would I say, prepackaged for the devs to be able to use them, to leverage them, to do their own instrumentation on their own. I would say for the most part we are automating many of the collector setup or the open telemetry setup. But I think at the beginning what we did, especially on the E-Club side, was to auto-instrument so we can have quick wins and move quickly. But when we're getting down to things like business logic, salary tasks, and things that were unique to the E-Club side, we manually created spans along some critical code paths and added some custom attributes here and there so we can get more than just what open telemetry offers out of the box. For E-Club, it's just a little slightly different from what we have on the concentric side. -**Tommy:** No, not really. As I said, what I think we're trying to get to a point where all of these things are, would I say, prepackaged for the devs to be able to use them, to leverage them, to do their own instrumentation on their own. I would say for the most part, we are automating many of our collector setup or the OpenTelemetry setup. But I think at the beginning, especially on the E-Club side, was to auto instrument so we can have quick wins and move quickly. But when we're getting down to things like business logic, salary tasks, and things that were unique to the E-Club side, we manually created spans along some critical code paths and added some custom attributes here and there so we can get more than just what OpenTelemetry offers out of the box. So I think for E-Club, it's just a little slightly different from what we have on the Centric side. +**Adriana:** Nice. That's really cool because I think that's a path that a lot of organizations start with anyway. Like the auto-instrumentation is that quick win and then it's like, "Oh yeah, we're lacking a little bit more. Let's go with the manual instrumentation." That's awesome. -**Adriana:** Nice. That's really cool because I think that's a path that a lot of organizations start with anyway. Like the auto instrumentation is that quick win and then it's like, oh yeah, we're lacking a little bit more. Let's go with the manual instrumentation. That's awesome. +**Adriana:** We do have a question from someone in the audience from Buddha. Hi Buddha, nice to have you join us. Buddha asks, "Are you currently using all hotel signals, logs, metrics, traces, profiles or a combination of them?" -[00:19:00] We do have a question from someone in the audience, from Buddha. Hi Buddha, nice to have you join us. So, Buddha asks, are you currently using all OpenTelemetry signals: logs, metrics, traces, profiles, or a combination of them? +### [00:19:50] Discussion about instrumentation challenges -**Tommy:** So we are using logs, metrics, and traces. Logs are coming through depending on which infrastructure piece. So if it's Fargate, it'll come through the AWS FireLens and Fluent Bit solution and then be sent to our vendor. Sorry, to our vendor. Otherwise, if it's Lambda, it comes straight from the Lambda layer agent. +**Tommy:** So we are using logs, metrics, and traces. Logs are coming through depending on which infrastructure piece. So if it's Fargate, it'll come through the AWS FireLens and Fluent Bit solution and then be sent to our agent, sorry, to our vendor. Otherwise, if it's Lambda, it comes straight from the Lambda layer agent. Then for metrics, that comes in through a different connector with our vendor. They have a solution we install in our cloud environment and then it kind of pulls all the metrics over to that. Now profiles is interesting. We don't use profiles, but the use case I think for profiles is enabling maybe common attributes or standardization, and we handle that through again our vendor has a solution out of the box that allows us to kind of set the schema and richer data as it gets ingested. -For metrics, that comes in through a different connector with our vendor. They have a solution we install in our cloud environment, and then it kind of pulls all the metrics over to that. Now profiles is interesting. We don't use profiles but the use case I think for profiles is enabling maybe common attributes or standardization, and we handle that through, again, our vendor has a solution out of the box that allows us to kind of set the schema and richer data as it gets ingested. +**Adriana:** Great. And then another follow-up question from Buddha. And by the way, actually before I ask that, there was a question from Manish, who's asking if we are recording the session and yes, we are recording the session. It should be available on LinkedIn and YouTube on demand after the fact. So, great question. The follow-up question from Buddha was, "Was this how you started, or slowly built up to it?" Who would like to take that one on? -**Adriana:** Great. And then another follow-up question from Buddha. And by the way, actually before I ask that, there was a question from Manish, who's asking if we are recording the session, and yes, we are recording the session. So it should be available on LinkedIn and YouTube on demand after the fact. So, great question. So the follow-up question from Buddha was, was this how you started or slowly built up to it? Who would like to take that one on? - -**Tommy:** Was that one for like the Terraform aspect or just in general? +**Tommy:** Was that one for like the Terraform aspect or just in general? **Adriana:** I'm assuming it was for the instrumentation in general, I would say. -**Tommy:** Yeah. At least one for Tommy. Tommy had a fun time doing that. - -**Tommy:** Okay. Well, this was not how we started. And for context, I joined about 10 months ago. And yeah, as time I joined, we did not have OpenTelemetry setup anywhere, at least best of my knowledge. I know that some of, yeah, a couple of the motivations for adopting OpenTelemetry on our side was standardization because we had like a patchwork of monitoring tools, like logs, metrics, everything scattered all over the place, but OpenTelemetry gave us a way to now unify all these and to reduce cognitive load and then the interaction overhead that comes with that. - -Additionally, we wanted to be vendor neutral, you know, using OpenTelemetry allows us not to be locked to a single vendor. I say, I know I'm getting into the motivations for why we adopted it, but it also cuts back into the fact that we did not start out that way, slowly built into it, and these were some of the decisions that drove us to using OpenTelemetry. - -**Adriana:** Yeah, I hope that answered the question. - -**Tommy:** I hope so as well. - -**Adriana:** How long did it take you to get to the point where you are at currently? - -**Tommy:** I'd say we're still getting there, right? If I remember correctly, we started somewhere around September, Andre, keep me honest here. I think by March, we closed the chapter on that one. From, I would say, from conceptualization, but I say from implementation to when we thought that we were in a good place, so roughly seven months trying out stuff. Yeah, I think we're in a good place now. - -**Adriana:** Alrighty. +**Tommy:** Yeah, at least one for Tommy. Tommy had a fun time doing that. -[00:22:50] **Tommy:** Yeah, we have a question in chat from Manish. When we talk of OpenTelemetry data, there is always a huge amount of data. How are you optimizing your storage and managing doing it? +**Tommy:** Okay. Well, this was not how we started. And for context, I joined about 10 months ago. As I joined, we did not have open telemetry setup anywhere, at least best of my knowledge. I know that some of the motivations for adopting open telemetry on our side was standardization because we had like a patchwork of monitoring tools, like logs, metrics, everything scattered all over the place. But open telemetry gave us a way to now unify all these and reduce cognitive load and the interaction overhead that comes with that. Additionally, we wanted to be vendor neutral. You know, using open telemetry allows us not to be locked to a single vendor. I know I'm getting into the motivations for why we adopted it, but it also cuts back into the fact that we did not start out that way slowly built into it and these were some of the decisions that drove us to using open telemetry. -**Tommy:** Okay. I'll take this one a bit and then I'll let Andre finish off because he's the geek with that. One of the things we do is sampling, of course. I know when we query for stuff, we do a sampling ratio of I think 1 to 1,000 for traces at least on the E-Club side. This is tuned in this way to balance cost, performance, and visibility. Of course, we monitor the effectiveness of this sampling setting and adjust it dynamically during incidents or specific services that require deeper analysis. +**Adriana:** Yeah, I hope that answers the question. -For logs, for querying and most of the other things we do with logs, we enforce a scan limit of 500 GB per query to keep the backend performant and cost-effective. But I know again, Andre geeks out on things like this, so I think you'll be able to give a better response. +**Tommy:** I hope so as well. How long did it take you to get into the point where you are at currently? -**Andre M:** Technically, we've got a really big budget for our vendors, so we actually have zero sampling at the moment. So we just grab it all. We're just trying to understand from that point and slim it down. You know, the first exercise that we kind of ran through was grab all the logs, run a pivot on it like the error logs, do I count on, you know, just general logs, and are there any useless logs that are being constantly created by the system or just noise? Can we remove that? And the answer was yes. So I think we had something like two billion, three billion logs coming in that are just like success that we just didn't need to have in place for it. +**Tommy:** I'd say we're still getting there, right? If I remember correctly, we started somewhere around September. Andre, keep me honest here. I think by March, we closed the chapter on that one. From, I would say from conceptualization but I say from implementation to when we thought that we were in a good place, so roughly seven months, right? Trying out stuff and yeah, I think we're in a good place now. -Yeah, we could have been cleaning up the system a lot. We go through all our pods and kind of hand out tickets to them as we find through the investigation and be like, "Yeah, let's get rid of this," or "You make it a debug log potentially if you guys actually need it, enable it during your debug sessions." Otherwise, let's not try to have too much noise in our ecosystem. I will say though that one caveat is because we are running Lambda, that one agent and CloudWatch requires certain logs to come out. So we can't get away from certain noise. Right now, I think we're ingesting 54 million logs a day that we just can't get away from and they're just noise at the moment. So another motivation to move to Fargate where we have a bit more control over the system as we adapt a bit more of the observability ecosystem. +**Adriana:** All righty. We have a question in chat from Manish. When we talk of open telemetry data, there is always a huge amount of data. How are you optimizing your storage and managing doing it? -**Adriana:** Cool. +**Tommy:** Okay. I'll take this one a bit and then I'll let Andre finish off because he's the geek with that. One of the things we do is sampling, of course. I know when we query for stuff, we do a sampling ratio of I think 1 to 1,000 for traces at least on the E- Club side. This is tuned in this way to balance cost, performance, and visibility. Of course, we monitor the effectiveness of this sampling setting and adjust it dynamically during incidents or specific services that require deeper analysis. And for logs, for querying and most of the other things we do with logs, we enforce a scan limit of 500 GB per query to keep the back end performant and cost-effective. But I know again, Andre geeks out on things like this. I think you'll be able to give a better response. -**Tommy:** Yeah, Manish, if you have any follow-ups, feel free to post them in chat. I would continue with the next question about collectors. Can you folks tell us a bit about the way how you set up your collector or collectors? What is the distribution and what is the architecture overall? Super curious about that. +**Andre M:** Technically, we got a really big budget for our vendors. So we actually have zero sampling at the moment. So we just grab it all. We're just trying to understand from that point and slim it down. The first exercise that we kind of ran through was grab all the logs, kind of run like a pivot on it like the error logs. Do I count on general logs and are there any useless logs that are being constantly created by the system or just noise, and can we remove that? The answer was yes. I think we had something like two billion, three billion logs coming in that are just like success that we just didn't need to have in place for it. So yeah, we could clean up the system a lot. We go through all our pods and kind of hand out tickets to them as we find through the investigation and be like, "Yeah, let's get rid of this," or "You make it a debug log potentially if you guys actually need it, enable it during your debug sessions." Otherwise, let's not try to have too much noise in our ecosystem. I will say though that one caveat is because we are running Lambda, that one agent and CloudWatch requires certain logs to come out. So we can't get away from certain noise. Right now I think we're ingesting 54 million logs a day that we just can't get away from and they're just noise at the moment. So another motivation to move to Fargate where we have a bit more control over the system as we adapt a bit more of the observability ecosystem. -**Tommy:** Okay, well, I'll begin again with the E-Club side. And again, for context, as I said, I handle most of the E-Club side of things. For E-Club, what we do is try to make our collector environment aware. So in Sandbox, because what we have is Sandbox, staging, and production on E-Club. So on Sandbox, we sample a little more aggressively to capture as much data as possible for debugging, especially for me, right? To really know what's going on there and also help the devs really debug stuff that they have going on there. But in production, what we do is a little more probabilistic, I hope I got that right, to balance visibility with cost. +### [00:25:36] Audience Q&A session -What we do is we dynamically also adjust that sampling ratio if there's an incident, and then we temporarily increase that to capture more data. The collector definitely does all the batching, filtering, and exporting telemetry to the backend of our vendor. Of course, this is done in a YAML config to define our pipeline for traces, for metrics, and logs, and also set the environment-specific parameters like service name, the host type, and then all the OpenTelemetry options we want to set on that. So that's what we have for our collector on E-Club. As I said, it runs as a sidecar through the services that Celery, the Django application also. +**Adriana:** Cool. Cool. Yeah, Manish, if you have any follow-ups, feel free to post them in chat. I would continue with the next question about collectors. Can you folks tell us a bit about the way how you set up your collector or collectors? What is the distribution and what is the architecture overall? Super curious about that. -**Andre M:** So as a follow-up on your collector setup, do you have a way to manage a fleet of your... because it sounds like you have a number of collectors. Do you have a way to manage them effectively? Like do you use OpenTelemetry for that or do you use a homegrown thing? +**Tommy:** Okay, well I'll begin again with the E-Club side. For context, as I said, I handle most of the E-Club side, the concentric. For E-Club, what we do is try to make our collector environment aware. So in Sandbox, because what we have is Sandbox, staging, and production on E-Club. In Sandbox, we sample a little more aggressively to capture as much data as possible for debugging, especially for me, right? To really know what's going on there and also help the devs really debug stuff that they have going on there. But in production, what we do is a little more probabilistic, I hope I got that right, to balance visibility with cost. What we do is we dynamically also adjust that sampling ratio if there's an incident and then we temporarily increase that to capture more data. The collector definitely does all the batching, filtering, and exporting telemetry to the back end, our vendor. Of course, this is done in a YAML config to define our pipeline for traces, metrics, and logs and to also set the environment-specific parameters like service name, the host type, and then all the open telemetry options we want to set on that. So that's what we have for our collector on E-Club. As I said at the beginning, it runs as a sidecar through the services that Celery, the Django application also. -**Tommy:** And also, do you use a collector gateway to funnel all of your collectors into a central gateway before pushing that off to your observability? +**Andre M:** So as a follow-up on your collector setup, do you have a way to manage a fleet of your collectors? It sounds like you have a number of collectors. Do you have a way to manage them effectively? Like do you use OpAmp for that or do you use like kind of a homegrown thing? And also, do you use like a collector gateway to funnel all of your collectors into like a central gateway before pushing that off to your observability? -**Andre M:** To be honest, not really. We just manage the fleet using ECS, as I said, and then we run them as a sidecar to our standalone services. The environment variables that I said help us customize those things, I said earlier, which are the sampling rates, the exporter, and the service names. So we don't really manage the fleets independently or independent of our services. We just run them as a sidecar and let them do the rest for us. +**Tommy:** To be honest, not really. We just manage the fleet using ECS as I said and then we run them outside car to our standalone services and the environment variables that I said help us customize those things I said earlier which are the sampling rates, the exporter, and the service names. So we don't really manage the fleets independently or independent of our services. We just run them as a sidecar and let us do the rest for us. We also monitor the performance, right? Definitely. Things like the Q length and then we have some drop spans along the way. So we can scale them horizontally if we do need to. For high throughput services, like I can't remember the name, one of our Celery, I think it's Celery Flour, right? We have of course a higher instance for that to manage the size and scale of how things can be and to avoid bottlenecks. -We also monitor the performance, right? Definitely. Things like the queue length and then we have some drop spans along the way, so we can scale them horizontally if we do need to. But for high throughput services, like I can't remember the name, one of our Celery, I think it's Celery Flower, right? We have, of course, a higher instance for that to manage the size and scale of how things can be and to avoid bottlenecks. +**Adriana:** So just to clarify, do all your sidecar collectors then, do each of them go directly to your back end? -**Adriana:** So just to clarify, do all your sidecar collectors then, do each of them go directly to your backend? - -**Andre M:** Oh, okay, got it. +**Tommy:** Oh, okay, got it. **Adriana:** And it sounds like Andre M had a screen share for us. -**Andre M:** Yeah. So just, it's a visual essentially the same idea that Tommy was walking through there. Before we get started though, shout out to Skydraw for anyone using that, the best platform for brainstorming ideas and a little whiteboard place. But essentially the ECS Fargate, again I was mentioning the AWS FireLens configuration for us. The management all of it comes back to Terraform. So that allows us to easily manage our collectors because even the configurations are all in code. So it makes that part a little bit easier. - -[00:30:00] If that's what you're trying to guess, prelude to our vendor does have that instance within our account, but we don't utilize that as we've found that sometimes the logs are being dropped. Either we under-provisioned it, and we found that if we just directly send them straight to the vendor, there weren't any issues that we've noticed aside from losing some of the benefits of that additional collecting if we have to do some data masking or dropping those logs prior to being sent and ingested by the vendor there. So definitely an area we're trying to balance. - -**Adriana:** But the other part is, obviously, the Lambda, which again is super straightforward. It's just straight from the Lambda and straight to the vendor, again, without any additional pieces there. - -**Tommy:** So, that may be an area now that we're talking here, an area for improvement because if we have those 54 million logs that are just noise, we might benefit by going to that filtering agent first. So very useful. +**Andre M:** Yeah. And just it's a visual, essentially the same idea that Tommy was walking through there. Before we get started though, shout out to SkyDraw for anyone using that. The best platform for brainstorming ideas and a little whiteboard place. Essentially the ECS Fargate, again I was mentioning the AWS FireLens configuration for us. The management, all of it comes back to Terraform. That allows us to easily manage our collectors because even the configurations, everything in code. So it makes that part a little bit easier. If that's what you're trying to guess, prelude to our vendor does have that instance within our account, but we don't utilize that as we found that sometimes the logs are being dropped. Either we under-provisioned it and we found that if we just directly send them straight to the vendor, there weren't any issues that we've noticed aside from losing some of the benefits of that additional collecting if we have to do some data masking or dropping those logs prior to being sent and ingested by the vendor there. So definitely an area we're trying to balance. The other part is obviously the Lambda, which again is super straightforward. It's just straight from the Lambda and straight to the vendor again without any additional pieces there. -**Adriana:** Cool. Thanks for sharing. You already mentioned areas for improvement, and I'm wondering, do you have any challenges with OpenTelemetry currently and anything that perhaps the project could do better to serve you as its user? +**Adriana:** So, it may be an area now that we're talking here, an area for improvement because if we have those 54 million logs that are just noise, we might benefit by going to that filtering agent first. So very useful. -**Tommy:** Well, I'll say for me, one of the biggest challenges was just initial setup, right? Configuring the connector and ensuring context propagation across all the services, which required me to do manual instrumentation. Of course, there's a learning curve. Telemetry is powerful, but again, the API is just so large. Of course, in terms of best practices, we are still evolving. +**Tommy:** Cool. Thanks for sharing. You already mentioned areas for improvement and I'm wondering, do you have any challenges with open telemetry currently and anything that perhaps the project could do better to serve you as its user? -One of the things that I'll probably like to see, and I think I'm speaking to the community when I say this, is maybe more complex example projects, speaking to myself too, of course, putting it out there. But I would say the community has been really helpful, right? There were so many things that I benefited from, from GitHub mainly, and then a bunch of articles that people have written on Medium, right? For me, it's just more real-world examples, more complex setups like probably what we have. Maybe, you know, somebody just doing something that big and putting it out there for anybody to be able to use, but say for me that's just been the challenge. Nothing too crazy. +**Tommy:** Well, I'll say for me, one of the biggest challenges was just initial setup, right? Configuring the connector and ensuring context propagation across all the services which required me to do manual instrumentation and of course, there's a learning curve. Telemetry is powerful, but again, the API offers is just so large. Of course, with in terms of best practices, we are still evolving. One of the things that I'll probably like to see, and I think I'm speaking to the community when I say this, is maybe more complex example projects, speaking to myself too, of course, putting it out there. For me, it's just more real-world examples, more complex setups like probably what we have. Maybe, you know, somebody just doing something that big and putting it out there for anybody to be able to use, but I say for me that's just been the challenge. Nothing too crazy. **Andre M:** What about you? Do you have anything in mind? -**Andre M:** Not so much I can give feedback to really improve. I mean, the whole time has just been learning, learning, learning, right? There's just a lot. I mean, it's my first exposure to it really for me. It's not even that I had experience with other observability tooling; it's just straight to our vendor and then also OpenTelemetry. So just learning the best of the best right off the bat basically, right? And then learn the history. For me, it's just been a constant uphill battle to learn all the different intricacies. Probably just want to echo Tommy's message: just more examples, right? Being able to pull things down, play with it, getting it into your hands really quickly. That's usually the best place to learn. - -**Adriana:** Yeah, I totally agree. I think we're at where OpenTelemetry has been around long enough now that I think many organizations are kind of past that 101 phase and are eager to get into the more gnarly use cases. And that's why, you know, we really appreciate folks like you jumping onto these Hotel Me sessions talking about your mature OpenTelemetry setups because it really helps others in the community. We definitely appreciate that. - -Switching gears a little bit, we were wondering if you could, if you have had any interactions with the OpenTelemetry community, things around, you know, like if you got stuck on a particular issue, like how did you go about it? Did you go on Slack or did you Google stuff? How did that work for you? - -**Tommy:** I remember the first thing, it was for E-Club. When we were trying to instrument it originally, there’s an extension we use for, is it parallel processing, Tommy? Do you remember that one? It's not AWS Wake, there's another one. Well, it's a Django app that we run. The problem with that is that there's an extension we use, I think it optimizes performance. The problem with that is that it somehow was fighting OpenTelemetry. - -**Andre M:** Yeah, I really can't remember the name, but I know they were fighting for the threading running in the background. I think that was one of the challenges that we had in the beginning. I really can't remember what specifically it was, but I know that was a big uphill battle back then. - -**Tommy:** Yeah, so we had an open issue on GitHub with that. Luckily our vendor was able to get their agent working, so it worked for that environment. But I think the issue is still open, but wasn't sure if it was like upstream with Django or was it so much with OpenTelemetry itself. - -**Adriana:** Cool. Cool. - -**Adriana:** As a follow-up, have you gotten to the point where you have made any contributions to OpenTelemetry or planning to make any contributions? - -**Tommy:** 100%. Nothing yet in terms of contribution or commit, but 100% we'll probably be looking at something within the ecosystem of probably Node or Go, one of those two. - -[00:36:10] **Adriana:** Awesome. Final thing. I think we've covered most of the base questions. We do have a cool audience question that came in from Esther. It says, "How do you see OpenTelemetry evolving in your workflow over the next year?" Really great question. - -**Tommy:** That is a tough question. Well, because I mean OpenTelemetry, I mean it's more about is it so much like the framework itself or is it more about the data we're getting out of it? Because for me, I think the value, I mean both are great, right? Because one gives us a standardized tooling and framework to be able to switch vendors or anything else we have the capability now. - -For me, I think the real value comes into what type of data do we really pull out of that OpenTelemetry and what can we really get from it. I think that's where real value is. So being able to now take that data and apply user journeys and business events, like those things, I think are where our organization can really, I guess, expand or benefit from for the most part. +**Andre M:** Not so much I can give feedback to really improve. I mean, the whole time has just been learning, learning, learning, right? There's just a lot. I mean, it's first exposure to it really for me. It's not even that I had experience with other observability tooling. It's just like straight to our vendor and then also hotel, so just learning the best of the best right off the bat basically and then learn the history. For me, it's just been a constant uphill battle to learn all the different intricacies. Probably just want to echo Tommy's message, just more examples, right? Being able to pull things down, play with it. Getting into your hands really quickly. That's usually the best place to learn. -**Andre M:** Tom, do you have anything to add? +**Adriana:** I totally agree. I think we're at OpenTelemetry has been around long enough now that I think many organizations are kind of past that 101 phase and are eager to get into like the more gnarly use cases. That's why, you know, we really appreciate folks like you jumping onto these Hotel Me sessions talking about your mature hotel setups because it really helps others in the community. We definitely appreciate that. -**Tommy:** Not much, but I think just like OpenTelemetry also keeps maturing in our observability journey, especially around logs and for complex setups like Celery or serverless with Lambda. Because I know there were also some battles fought there. +**Adriana:** Switching gears a little bit, we were wondering if you could, if you have had any interactions with the hotel community, things around, you know, if you got stuck on a particular issue, how did you go about it? Did you go on Slack or did you like Google stuff? How did that work for you? -I also think we are probably going to have more out-of-the-box integration and tools to simplify the configuration for our collector, right? I mean the ecosystem is growing so fast. So I would not be surprised to see us adopting it a little bit more. As Andre said, we are growing every day. We are learning every day. But I think we'll be leaning more towards probably much better or more standard industry practices when it comes to adopting OpenTelemetry and using it to really observe our system. +**Andre M:** I remember the first thing it was for E-Club. When we were trying to instrument it originally, there was an extension we use for, is it parallel processing Tommy? Do you remember that one? It's not AWS Wake, there's another one. -**Adriana:** So that's what I see happening over the next year, but who knows? +**Tommy:** Well, it's a Django app that we run. The problem with that is that there's an extension we use. I think it optimizes performance and the problem with that is that it somehow was fighting hotel. -**Adriana:** Sounds good. Sounds good. +**Andre M:** Yeah, I really can't remember the name but then I know they were fighting for the threading running in the background. I think that was one of the challenges that we had in the beginning. I really can't remember what specifically it was, but I know that was a big uphill battle back then. -**Adriana:** Yeah, we have more audience questions. Today we just talked over at the end with Adrian that we have a really, really engaged audience. So that’s amazing. So Manish is asking, how are you doing the instrumentation of legacy systems? Is anything like that happening? Do you have a use case for that? +**Tommy:** Yeah. So we had an open issue on GitHub with that. Luckily, our vendor was able to get their agent working, so it worked for that environment but I think the issue is still open but wasn't sure if it was like upstream with Django or was it so much with hotel itself. -**Tommy:** I think we have some older .NET projects, but it seems as if our vendor supports out of the box, and anything I think we even with that solution, the vendor allows us to still have it in OpenTelemetry format as it comes through. So not really a concern for on our side from what I've seen so far. +**Adriana:** Cool. Cool. As a follow-up, have you gotten to the point where you have made any contributions to hotel or planning to make any contributions? -**Adriana:** Alrighty. Alrighty. +**Tommy:** 100%. Nothing yet in terms of contribution, commit, but 100% we'll probably be looking at something within the ecosystem of probably node or go, one of those two. -**Adriana:** And one more from Buddha. How do you report on business value metrics for decision makers? What metrics do you report on? +**Adriana:** Awesome. Final thing. I think we've covered most of the base questions. We do have a cool audience question that came in from Esther. It says, "How do you see open telemetry evolving in your workflow over the next year?" Really great question. -**Tommy:** Yeah, I know I think Tommy you mentioned it that for that business logic you had to do some manual instrumentation. Can you tell us a bit more about that? +### [00:36:48] Discussion on data management and optimization -**Tommy:** Yeah. So, as I said earlier, there were certain business logic that we needed to especially on the Celery side because that's where we upload stuff to really see how data was flowing through that and then report on, I think specifically it was a reporting service or I think an order service at the time. +**Andre M:** That is a tough question. Well because I mean open telemetry, I mean, it's more about is it so much like the framework itself or is it more about the data we're getting out of it? Because for me, I think the value, I mean both are great, right? Because one gives us a standardized tooling and framework to be able to switch vendors or anything else we have the capability now, but for me, I think the real value comes into what type of data do we really pull out of that open telemetry and what can we really get from it? I think that's where real value is. So being able to now take that data and apply user journeys and business events, like those things I think are where our organization can really expand or benefit from for the most part. -What we got out of that really was just the rise and fall or the spike of the, would I say the order process within that within a certain period or within a certain trace. But you know reporting back to business people, I think that one comes more out of the box with our vendor. And we, as Andre said, build stuff that aggregates the amount of logs that we use that translate into how much we pay for these things. Even though we have a big budget, we still don't want to be overshooting that. But I think again, Andre would have more context into this because he builds most of the stuff that I report back to business leaders and that's something he's so in love with. +**Tommy:** Do you have anything to add? -**Andre M:** So I think for me, the biggest one at least it's been months in development is the most important key metric I've come to love recently is users. So what is a single user doing across a system and aggregate that over a certain period of time? That alone allowed us to find a particular leak where we had a single user, was a bug in our client and within our service that wasn't like a batch request where this one user within a short period of time was invoking over 200,000 requests into our system. +**Tommy:** Not much, but I think just like OpenTelemetry also keeps maturing in our observability journey, especially around logs and for complex setups like Celery or serverless that's with Lambda because I know there were also some battles fought there. I also think we are probably going to have more out-of-the-box integration and tools to simplify the configuration for our collector, right? I mean the ecosystem is growing so fast. I would not be surprised to see us adopting it a little bit more. As Andre said, we are growing every day. We are learning every day. But I think we'll be leaning more towards probably much better or more standard industry practices when it comes to adopting hotel and using it to really observe our system. So, that's what I see happening over the next year, but who knows? -With Lambda, pay as you go, it's not a good model there. So without this observability in place, we wouldn't even know that there is, you know, one single user somehow invoking 200,000 requests within like a six-hour period. So these tips, it's small little things where now we can focus on a single user or aggregate it across a grouping and be able to find behaviors. And it's really good when you throw into like a time series or flame graph and you can see these kind of fat spots where you can just visually identify something doesn't look good here. Investigate. +**Adriana:** Sounds good. Sounds good. Yeah, we have more audience questions. Today, we just talked over at the end with Adrian that we have a really, really engaged audience. So, that's amazing. Manish is asking how are you doing the instrumentation of legacy systems? Is anything like that happening? Do you have a use case for that? -**Adriana:** That's great. Thank you so much, both of you, Andre and Tommy, for joining us. And thank you to my co-host, the other Andre, for joining. And also this is Andre K's first time on a stream, so big kudos. Awesome job as co-hosts. +**Andre M:** I think we have some older .NET projects, but it seems as if our vendor supports out of the box and anything I think even with that solution, the vendor allows us to still have it in hotel format as it comes through. So, not really a concern for on our side from what I've seen so far. -I want to give a shout-out, and you know, as always, we appreciate all the stories that we hear from our community members showing how they use OpenTelemetry out in the wild. It really helps us. And it helps show folks in the community, like, you know, OpenTelemetry is here to stay. We've got some gnarly use cases of it being used out in the wild and it's great to hear those stories. +**Adriana:** All righty. All righty. And one more from Buddha. How do you report on business value metrics for decision-makers? What metrics do you report on? Yeah, I know I think Tommy you mentioned that for that business logic you had to do some manual instrumentation. Can you tell us a bit more about that? -Coming up next, stay tuned. We should have another Hotel Me planned, I think, in July. So, stay tuned on the hotel socials for that. In the meantime, coming up at the end of this month, we have Hotel Community Day. It's part of Open Observability Con. It's combined with Hotel Community Day. So that's happening on the heels of Open Source Summit in Denver. Open Source Summit North America in Denver. So if you're around for Open Source Summit, check out Hotel Community Day and Open Observability Con that are happening together. +**Tommy:** Yeah. So, as I said earlier, there were certain business logic that we needed to especially on the salary side because that's where we upload stuff to really see how data was flowing through that and then report on, I think specifically it was a reporting service or I think an order service at the time. What we got out of that really was just the rise and fall or the spike of the, would I say, the others process within a certain period or within a certain trace. But you know, reporting back to business people, I think that one comes more out of the box with our vendor and we, as Andre said, build stuff that aggregates the amount of logs that we use that translate into how much we pay for these things. Even though we have a big budget, we still don't want to overshoot that. But I think again Andre would have more context into this because he builds most of the stuff that I report back to business leaders and that's something he's so in love with. -I think we've got a couple of KubeCons also coming up. We've got, I think, I want to say this week is KubeCon China. And then next week, I'm actually jetting off to Japan at the end of this week for KubeCon Japan. And it's the first KubeCon Japan, so very exciting stuff. So if anyone's around for KubeCon Japan, come say hello. We'd love to meet you and talk. +**Andre M:** So I think for me, the biggest one at least it's been months in development is the most important key metric I've come to love recently is users. So what is a single user doing across a system and aggregate that over a certain period of time. That alone allowed us to find a particular leak where we had a single user was a bug in our client and within our service that wasn't like a batch request where this one user within a short period of time was invoking over 200,000 requests into our system. With Lambda pay as you go, it's not a good model there. Without this observability in place, we wouldn't even know that there is, you know, one single user somehow invoking 200,000 requests within like a six-hour period. These tips, it's small little things where now we can focus on a single user or aggregate it across a grouping and be able to find behaviors and it's really good when you throw into like a time series or flame graph and you can see these kind of like fat spots where you can just visually identify something doesn't look good here. Investigate. -I think that is a wrap. And again for those who attended, tell your friends if they missed it, the recording will be available both on LinkedIn and YouTube. Also, check out the hotel end user SIG. We also have a group on CNCF Slack we are called, I think we're called SIG-User, so come share your stories on the hotel end user SIG chat. Also, if you'd love to join us for one of our meetings, we meet every two weeks, so I think our next one is next week. +### [00:43:12] Future directions for OpenTelemetry -Thank you everyone once again for joining and we'll see you next time. +**Adriana:** That's great. Thank you so much, both of you, Andre and Tommy for joining us and thank you to my co-host, the other Andre for joining. And also this is Andre K's first time on a stream. So, big kudos. Awesome job as co-hosts. I want to give a shout out. As always, we appreciate all the stories that we hear from our community members showing how they use open telemetry out in the wild. It really helps us and helps show folks in the community like, you know, open telemetry is here to stay. We've got some gnarly use cases of it being used out in the wild and it's great to hear those stories. Coming up next, stay tuned. We should have another Hotel Me planned, I think, in July. So stay tuned on the hotel socials for that. In the meantime, coming up at the end of this month, we have Hotel Community Day. It's part of Open Observability Con. It's combined with Hotel Community Day. That's happening on the heels of Open Source Summit in Denver. Open Source Summit North America in Denver. If you're around for Open Source Summit, check out Hotel Community Day and Open Observability Con that are happening together. I think we've got a couple of KubeCons also coming up. I want to say this week is KubeCon China. Next week, I'm actually jetting off to Japan at the end of this week for KubeCon Japan. It's the first KubeCon Japan, so very exciting stuff. If anyone's around for KubeCon Japan, come say hello. We'd love to meet you and talk. I think that is a wrap. Again, for those who attended, tell your friends if they missed it. The recording will be available both on LinkedIn and YouTube. Also, check out the hotel end-user SIG. We also have a group on CNCF Slack. We are called end-user. So come share your stories on the hotel end-user SIG chat. Also, if you'd love to join us for one of our meetings, we meet every two weeks. I think our next one is next week. Thank you everyone once again for joining and we'll see you next time. **Tommy:** Thanks so much for having us. diff --git a/video-transcripts/transcripts/2025-07-16T20:50:15Z-otel-in-practice-alibabas-opentelemetry-journey.md b/video-transcripts/transcripts/2025-07-16T20:50:15Z-otel-in-practice-alibabas-opentelemetry-journey.md index ac034e1..5f16567 100644 --- a/video-transcripts/transcripts/2025-07-16T20:50:15Z-otel-in-practice-alibabas-opentelemetry-journey.md +++ b/video-transcripts/transcripts/2025-07-16T20:50:15Z-otel-in-practice-alibabas-opentelemetry-journey.md @@ -10,98 +10,148 @@ URL: https://www.youtube.com/watch?v=fgbB0HhVBq8 ## Summary -In this episode of "Hotel in Practice," Dan, a member of the End User Special Interest Group, hosts Hushing Jang and Steve Ra from Alibaba to discuss the adoption of OpenTelemetry within the company. Hushing, a staff engineer, and Steve, a senior software engineer, share their experiences transitioning Alibaba's observability tools from a legacy system to OpenTelemetry. They detail the migration process, emphasizing the challenges faced, the enhancements made to the OpenTelemetry Java agent, and their approach to compile-time instrumentation for Go applications. They highlight the benefits of the OpenTelemetry ecosystem, such as easier context propagation and better scalability, and discuss their contributions to the OpenTelemetry community. The session also addresses internal user adoption strategies and the importance of auto instrumentation in ensuring proper context propagation. The discussion concludes with an invitation for viewers to continue the conversation through community resources. +In this edition of "Hotel in Practice," host Dan discusses the adoption of OpenTelemetry at Alibaba with experts Hushing Jang and Steve Ra. Hushing, a staff engineer, and Steve, a senior software engineer, share insights into their journey of migrating Alibaba's Java agent from an older framework to OpenTelemetry. They highlight the challenges faced during the transition, including the need for enhancements in instrumentation, profiling capabilities, and the importance of maintaining compatibility with existing frameworks. The duo emphasizes the benefits of OpenTelemetry's semantic conventions, improved user autonomy, and the scalability of the architecture, which has facilitated better observability across Alibaba's services. They also touch on the implementation of compile-time instrumentation for Go applications and their ongoing contributions to the OpenTelemetry community, including organizing events and collaborating on new projects. The discussion underscores the complexities of context propagation in asynchronous environments and the team's commitment to ensuring effective observability for both internal and external users. ## Chapters -00:00:00 Welcome and intro -00:01:36 Guest introductions -00:03:00 Presentation begins -00:05:00 Background on Alibaba's observability -00:07:40 Migration to OpenTelemetry -00:10:00 Enhancements on Java agent -00:12:40 Context propagation challenges -00:18:00 Go applications instrumentation -00:24:00 Adoption cases overview -00:30:24 Future work and contributions +00:00:00 Introductions +00:01:38 Discussion on OpenTelemetry adoption +00:04:34 Guest introduction: Hushing +00:05:30 Guest introduction: Steve +00:06:32 Background on Alibaba's observability +00:10:37 Migration challenges to OpenTelemetry +00:13:53 Enhancements in Java agent +00:18:47 Compile-time instrumentation for Go +00:27:36 Adoption cases in Alibaba +00:43:07 Context propagation challenges -[00:01:36] **Dan:** Good morning, good afternoon and good evening whenever you're watching this or wherever you're watching this. Welcome to a new edition of hotel in practice where industry experts, end users and contributors tell us about a specific area of open telemetry and how it can benefit or it has already benefited end users. My name is Dan. I'm a member of the end user special interest group and I'm also a member of the governance committee and I'm very excited today to have Hushing and Steve to talk about adoption of open telemetry within Alibaba. I'm connecting from Edinburgh in Scotland. You can see the hills here. And if you're watching live on YouTube or LinkedIn, please say hi in the chat and tell us where you're watching from or where you're connecting from. Before we get started, I just wanted to cover a couple of housekeeping items. During the presentation, please drop your questions in the chat and although we'll make our best to get to all of your questions, I can't promise that we'll get to all of them. But fear not, if that's the case, if we can get to all of them and your question is not answered or you want to continue the conversation, you can go to the end user s and then we've got resources in this link and you can continue the conversation there. You find your way to Slack where we have a big community of end users including Hushing and Steve and we will be happy to provide some of their insights there. Also, please remember to keep the questions to date and topic of the presentation. And yeah, so with that out of the way, let's get started and welcome Hushing Jang, staff engineer at Alibaba and hotel compile instrumentation maintainer. Hi Hushing, can you tell us a bit more about what you do? +## Transcript -**Hushing:** Hello Dan. Very nice to meet you and hello. I'm Hushing. I'm from Alibaba. Yeah, I'm working as an observability engineer in Alibaba cloud and we are also working on the client instrumentations. So we work very closely with the open telemetry community. We're working on different kinds of language instrumentations, collectors, and something like that. We also provide service both for internal and external clients, customers. Thanks. +### [00:00:00] Introductions -**Dan:** That's awesome. And also welcome Steve, Steve Ra, senior software engineer at Alibaba and hotel Java instrumentation approver. Hi Steve. +**Dan:** Good morning, good afternoon, and good evening whenever you're watching this or wherever you're watching this. Welcome to a new edition of hotel in practice where industry experts, end users, and contributors tell us about a specific area of open telemetry and how it can benefit or it has already benefited end users. My name is Dan. I'm a member of the end user special interest group and I'm also a member of the governance committee. I'm very excited today to have Hushing and Steve to talk about the adoption of open telemetry within Alibaba. I'm connecting from Edinburgh in Scotland. You can see the hills here. If you're watching live on YouTube or LinkedIn, please say hi in the chat and tell us where you're watching from or where you're connecting from. -**Steve:** Yeah. Hi Dan. Hi. I am. I'm Steve. I'm also from Alibaba cloud. I work in the observability team in Alibaba and I participate in Java agent and I also responsible for profiling in our team and provide out of box profiling ability for our users. Thank you. +Before we get started, I just wanted to cover a couple of housekeeping items. During the presentation, please drop your questions in the chat and although we'll make our best to get to all of your questions, I can't promise that we'll get to all of them. But fear not, if that's the case and your question is not answered or you want to continue the conversation, you can go to the end user resources in this link and you can continue the conversation there. You can find your way to Slack where we have a big community of end users including Hushing and Steve, and we will be happy to provide some of their insights there. Also, please remember to keep the questions to the date and topic of the presentation. -**Dan:** Nice. Well, welcome both. I'm really excited to hear more about how you're approaching open telemetry in Alibaba and how you ended up contributing as well, being major contributors to open telemetry. So, now that we're all set, you've got a presentation and you'll guide us through it. I'm going to move out and then please take it away. +With that out of the way, let's get started and welcome Hushing Jang, staff engineer at Alibaba and hotel compile instrumentation maintainer. Hi Hushing, can you tell us a bit more about what you do? -[00:05:00] **Hushing:** Okay. Hello everyone. We are Steve and Hushing from Alibaba and we are very glad to share our talk Alibaba practice experience. Today we will explain our talk from the following five parts and I will introduce the first two parts and the remaining are given to my colleague. The reason for choosing hotel. Let me briefly introduce our background. In 2013, we provided observability capability for our internal users by manual instrumentation and we provided this ability into our internal software frameworks and we also designed a set of trace propagation protocol called Eagle I. In 2017, we developed the Alibaba Java agent based on the pinpoint Java agent. After several years of use, we found some problems such as pinpoint cannot support verifying the supporting version of a framework. For example, if a framework released a new version, the instrumentation may not take effect and we can perceive this immediately. Another thing sounds like pinpoint cannot support very well in some scenarios such as trace propagation synchronously. It doesn't support it automatically and there are also some other reasons due to the limited time I cannot list them one by one. +**Hushing:** Hello Dan. Very nice to meet you and hello. I'm Hushing. I'm from Alibaba. Yeah, I'm working as an observability engineer in Alibaba Cloud and we are also working on the client instrumentations. So we work very closely with the open telemetry community. We're working on different kinds of language instrumentations, collectors, and something like that, also providing service both for internal and external clients and customers. Thanks. -From 2020 to 2023, we researched some other solutions from open source. At the time, we found open telemetry is a very strong ecosystem. It has rich semantic conventions and it represents the standard in observability. After several internal discussions, we made up our mind to migrate our Alibaba Java agent from pinpoint to open telemetry. After several months of development, we launched our Alibaba Java agent based on open telemetry in 2024. We also launched our Golang and Python instrumentation based on the semantic convention or SDK. Currently we embraced open telemetry completely. +**Dan:** That's awesome. And also welcome Steve. Steve Ra, senior software engineer at Alibaba and hotel Java instrumentation approver. Hi Steve. -[00:07:40] After introducing our background, during our migration, we really did a lot of enhancements on the hotel Java agent. The first one is instrumentation. At the time we found open telemetry not widely used in China because there are some frameworks that are not supported by the hotel Java agent but they are widely used in China. We had supported them before. So we needed to implement them based on the hotel Java agent architecture. The second point is profiles. As we know profile is a very powerful tool. It can help us to inspect the behavioral performance of our application at runtime. It can help us to know which code is responsible for the spikes in CPU memory usage. We have supported profiling for several years. So the migration of this feature is also very important for us. Other abilities such as abundant sampling or more metrics and we also need to debug a very popular debugging tool in China. Last but not least, we not only support observability by Java agent, we also support microservices governance such as traffic control or security management by this Java agent. +**Steve:** Yeah. Hi Dan. Hi, I am. I'm Steve. I'm also from Alibaba Cloud. I work in the observability team in Alibaba and I participate in Java agent and I'm also responsible for profiling in our team and provide out-of-box profiling ability for our users. Thank you. -[00:10:00] So how to combine them in the hotel Java agent architecture? It's a toss-up we need to solve. Due to the limited time at the time and how to migrate them, it's challenging for us. In fact, at the time we weren't very familiar with telemetry and in order to achieve this goal, we came up with a two-step solution to solve this question. First one, we forked a copy of the Java agent repository and we promptly implemented all our commercial enhancements on the Java agent. In this process, it's not very easy because there are a lot of commercial features. If we migrate to the open telemetry, there are a lot of issues or problems and due to the rich business scenarios and during migration, we also found some problems and we needed to fix them and some of them are strongly related to the upstream and we also contributed them to the upstream. After about six months from research to launch the first release of our Alibaba Java agent based on open telemetry. After finishing this phase, we knew that it's not enough because we know the biggest charm of open telemetry is its stronger ecosystem. If we compare our code with open source, it's challenging for us to merge some upstream updates to our Alibaba Java agent. +**Dan:** Nice. Well, welcome both. I'm really excited to hear more about how you're approaching open telemetry in Alibaba and how you ended up contributing as well, being major contributors to open telemetry. Now that we're all set, you've got a presentation and you'll guide us through it. I'm going to move out and then please take it away. -So on the one hand, we promoted our users to use our first version and then we also invited some developers to design some extensions or use some extensions from open source to migrate our commercial feature. This process is time-consuming because we need to design some extensions based on the Java agent and we also need to contribute them to the upstream. The other things we are currently doing, after this step we can see the open source hotel Java agent just like a dependency for us and we can decouple our commercial code and the open source code and we can merge upstream easily. +### [00:04:34] Guest introduction: Hushing -[00:12:40] This is our whole approach to achieve migration. After migration to open telemetry, we gained a lot from open source and we also got something we didn't anticipate at the beginning. The first one I want to share is the hotel semantic convention. I guess that is the soul of open telemetry and when we migrated to open telemetry and we told our users we achieved this Java agent based on open telemetry and they can do a lot of things by themselves such as they can search some materials, some documentation by themselves and they can learn how to use some extensions to write something by themselves and they can debug some problems by themselves. I think that is a very good point for us to embrace open telemetry and our users can do a lot of things by themselves and it rarely reduces users' learning burden. Because they just need to learn the standards, the semantic convention or implementation of open telemetry and they can do a lot of things by themselves. +**Hushing:** Okay. Hello everyone. We are Steve and Hushing from Alibaba and we are very glad to share our talk about Alibaba's practice experience. Today we will explain our talk from the following five parts, and I will introduce the first two parts and the remaining are given to my colleague. -This is the first point. The second point is good architectural design and better scalability of hotel instrumentation. It's very useful for us to promote our users to use our Java agent. How to understand this? For example, before Java hotel Java agent, if a user used a framework that is not widely used and we don't support that framework in our Java agent, they will create an issue for us and ask us to schedule resources to support this framework. But sometimes, we take the cost and energy into consideration, we are reluctant to support that. So how to communicate with our user that is a question at that time. Nowadays, this is not a big question. Because we can tell our users there are a lot of extensions; they can use this extension to support their own framework and they can do it by themselves and a lot of users are very interested in this because they can extend a lot of things by themselves and don't rely on us. +### [00:05:30] Guest introduction: Steve -This is all the things I want to share today. The remaining time, I will introduce something from other aspects. Thank you. +The reason for choosing hotel. Let me briefly introduce our background. In 2013, we provided observability capability for our internal users by manual instrumentation and we provided this ability into our internal software frameworks. We also designed a set of trace propagation protocol called Eagle. In 2017, we developed the Alibaba Java agent based on the pinpoint Java agent. After several years of use, we found some problems such as pinpoint cannot support verifying the supporting version of a framework. For example, if a framework released a new version, the instrumentation may not take effect and we can perceive this immediately. -**Steve:** Thank you, Hushing. I will next introduce. Hello. Can you hear me very well? +### [00:06:32] Background on Alibaba's observability -Yes. Okay. Sorry. There's some network issues. Maybe I just lost the connections. I will go into for the remaining part. +Another issue was that pinpoint cannot support very well in some scenarios such as trace propagation synchronously. It doesn't support it automatically. There are also some other reasons due to the limited time I cannot list them one by one. So from 2020 to 2023, we researched some other solutions from open source. At the time, we found open telemetry is a very strong ecosystem. It has rich semantic conventions and it represents the standard in observability. After several internal discussions, we made up our mind to migrate our Alibaba Java agent from pinpoint to open telemetry. After several months of development, we launched our Alibaba Java agent based on open telemetry in 2024. We also launched our Golang and Python instrumentation based on the semantic conventions and SDK. Currently, we embraced open telemetry completely. -[00:18:00] Yes. Actually, this solution has been started in 2023 and at that time we wanted to look for a solution that can be easily applied for Go applications for the to-do instrumentations. Actually, when we were trying the eBPF based approach that is already been provided by the open telemetry community, but it turns out that there are some limitations. We finally figured out a way to do the instrumentations to satisfy our needs that is in a compile time instrumentation. If you're looking at the figure on the right, there's an approach that the injection has happened in the middle of the compilation process. Actually, we hooked the Golang compile command to make sure that can be done for users without any modifications to users' code. That's a key feature that we want to achieve. +After introducing our background, during our migration, we really did a lot of enhancements on the hotel Java agent. The first one is instrumentation. At the time, we found open telemetry was not widely used in China because there are some instrument frameworks that are not supported by the hotel Java agent but they are widely used in China. We had supported them before, so we needed to implement them based on hotel Java agent architecture. -Next, I want to address several highlights of this approach. First is the async context propagations. Actually, we encourage our users to follow the guidelines, but actually not all the users follow the rules. Sometimes they do not pass the context object correctly and the traces may break and become incomplete. So this feature can actually do the context propagation automatically for us even when we are not passing the object correctly. The second one is that it is actually compatible with the old SDK. That means that users who have used manual instrumentations can work well with our approach. So the traces can be merged. The last one is the most powerful feature that we have done is kind of like customer extension. That means that you can instrument any code into any point, any line of that application. This allows us to solve some special requirements from users. Then they can do some customizations like traffic management and security. +The second point is profiles. As we know, profiling is a very powerful tool. It can help us inspect the behavioral performance of our application at runtime. It can help us know which code is responsible for the spikes in CPU and memory usage. We have supported profiling for several years, so the migration of this feature is also very important for us. Other abilities such as abundant sampling or more metrics, we also needed to debug a very popular debugging tool in China. Last but not least, we not only support observability by Java agent, we also support microservices governance such as traffic control or security management by this Java agent. -Next, we will introduce some practices that we have in the LLM of observability. As we know, the LLM has been popular recently, especially this year a lot of AI agents have become active and there's a typical use case that we found. In the use cases, a user sends a request to the API gateway and the API gateway sends to the LLM applications, which are written in maybe Python, Java, or Go and the application will call different kinds of models, for example, DeepSea Queen or OpenAI. +So how to compile them in the hotel Java agent architecture? It’s a task we need to solve. Due to the limited time and how to migrate them, it’s a challenging task for us. In fact, at the time, we were not very familiar with telemetry and in order to achieve this goal, we came up with a two-step solution to solve this question. The first one, we forked a copy of the Java agent repository and promptly implemented all our commercial enhancements on the Java agent. -We have adopted an AI gateway in the middle that can do a lot of things for us. For example, it can do unified access and token limitations, token cache, and other stuff like that. We have adopted the open source project called Highress that can do it very well for us. +### [00:10:37] Migration challenges to OpenTelemetry -When we are building observability across the different components, we make sure we're following this latest JNI semantic convention. Because the convention is still in development, we're trying to keep pace with them as much as possible. We have done some instrumentations based on this semantic convention. This includes major AI agent frameworks like high code frameworks such as AgentScope, Agono, Lang, Spring, Alibaba, etc. +In this process, it was not very easy because there were a lot of commercial features. If we migrated to open telemetry, there would be a lot of issues or problems. Due to the rich business scenarios, during migration we also found some problems and we needed to fix them. Some of them are strongly related to the upstream, and we also contributed them to the upstream. After about six months from research to launch, we had our first release of our Alibaba Java agent based on open telemetry. -For low code platforms, DIY is a very popular open source project to build AI agents with low code. We also do some instrumentation for the official MCP clients. For example, we want to capture the MCP2 courses; we want to capture the M32 responses and to make sure our observability is end-to-end. We want to put our instrumentations into the internal models that come up with serving frameworks like VM or SGAN, which are actually Python processes running there. So we can put our Python instrumentations into that Python process to capture traces and key metrics like TTFT (time to first token) and TBOT (time per output token), which are very important metrics for us to identify bottlenecks of LLM serving issues. +After finishing this phase, we knew that it was not enough because we knew the biggest charm of open telemetry is its stronger ecosystem. If we compared our code with open source, it was challenging for us to merge some upstream updates to our Alibaba Java agent. So on the one hand, we promoted our users to use our first version, and then we also invited some developers to design some extensions or use some extensions from open source to migrate our commercial features. This process is time-consuming because we need to design some extensions based on the Java agent and we also need to contribute them to the upstream. -[00:24:00] Next, talking about the adoption cases. Actually, we have many adoption cases. I want to introduce several of them today. I divided them into two parts. The first part is the cloud service data flow. When users use cloud service, we make sure all the services that are in key positions of the application embrace the official W3 tracing goal by default. That means that balancer way or they support the W3 protocol by default. They will have the async messaging, like how a consumer is consuming the messages, to measure the latency or identify bottlenecks. They are using open telemetry to do that. +After this step, we can see the open source hotel Java agent just like a dependency for us and we can decouple our commercial code and the open source code and we can merge upstream easily. This is our whole approach to achieve migration. After migrating to open telemetry, we gained a lot of things from open source and we also got something we didn't anticipate at the beginning. -For the AI studio, they are using Java and Python hotel instrumentation for end-to-end observability. The next part is the cloud service management console. A lot of management console services have embraced open telemetry as well. For example, the elastic cloud computing service is using hotel Java instrumentation quite a lot and they extend and make some extensions to that. They want to do request-based traffic routing. They want to route the service to different machine groups. +### [00:13:53] Enhancements in Java agent -The video on demand service is also running a large amount of clusters written in Go and they have adopted the Golang compile time instrumentation mostly for runtime monitoring. They want to monitor the runtime metrics and also provide data. Another case is the institution detection service. There’s an agent running on the host written in Go. Before open telemetry, they had nothing to do with observability because they lacked techniques and tools to do that. Now they can use hotel Go compile time instrumentation for their runtime monitoring and profiling. +The first one I want to share is the hotel semantic convention. I guess that is the soul of open telemetry. When we migrated to open telemetry and we told our users we achieved this Java agent based on open telemetry, they could do a lot of things by themselves. They can search for materials, documentation by themselves, learn how to use extensions, and debug some problems by themselves. I think that is a very good point for us to embrace open telemetry. Our users can do a lot of things by themselves and it really reduces users' learning burden because they just need to learn the standards, the semantic convention, or implementation of open telemetry, and they can do a lot of things by themselves. -As our own service, the cloud monitoring service also implements self-observability with Java instrumentation. We do a lot of things with Java instrumentation to do traces and profiling. This is a good example of dogfooding, right? +This is the first point. The second point is good architectural design and better scalability of hotel instrumentation. It’s very useful for us to promote our users to use our Java agent. How to understand this? For example, before the hotel Java agent, if a user used a framework that is not widely used and we didn’t support that framework from our Java agent, they would create an issue for us and ask us to schedule resources to support this framework. But sometimes, taking the cost and energy into consideration, we were reluctant to support that. -Yes. Besides our internal adoption, we are also trying to contribute these enhancements or extensions back to the open telemetry community. We have mentioned before that a lot of instrumentations, some of them have been contributed and some are also on the way to contributing back. We also helped promote open telemetry in the APAC area. We helped organize some meetings that are friendly to the Asia-Pacific areas like Java, Go, and Jingi meetings. We also engaged with conferences like KubeCon and the hotel community and we are also organizing events like KCD to help promote open telemetry in Asia-Pacific areas. +So how to communicate with our users about this was a question at that time. Nowadays, this is not a big question because we can tell our users there are a lot of extensions. They can use these extensions to support their own framework and they can do it by themselves. A lot of users are very interested in this because they can extend a lot of things by themselves and don’t rely on us. -In 2024, when we decided to donate this Golang compile time instrumentation to the open telemetry community and after a long discussion with the community, we also found that DataDog showed a very strong interest in this project. They actually submitted a proposal to donate their own framework, which is also a compile time instrumentation project, to open telemetry. So we decided that we do not want to depend on a single company's effort. We formed a special interest group (SIG) to build this project from scratch. This SIG is co-founded by Alibaba, DataDog, and Quesma. We have a common effort to implement this compile time instrumentation for Golang. This project is still in active development and we are working on the first release and the first MVP version of that this year. +And yeah, that’s all the things I want to share today. The remaining time is to introduce something from another aspect. -[00:30:24] If you are interested, please definitely check it out. Talking about future work, for Java instrumentations, we are planning to implement the proxy agent mode. That means that we want to use the Java agent as a dependency rather than booking upstream projects because we do not want to maintain such kind of project. We want to make sure all the extension points can be merged into the upstream so that we can extend this project very well. +**Steve:** Thank you, Hushing. I will next introduce. Hello. Can you hear me very well? -For the Golang compile time instrumentation, we are working well with the MVP version and we're making sure we are working towards the first release. We will also support more signals including metrics and profiles. For the GI, we also want to contribute some of the instrumentations back to the community because the GI is still in active development. We put some of them in our repo and we want to contribute them back as well. +**Dan:** Yes. -Actually, we also made our own open telemetry distribution called Long Sweet. Long is the Chinese version of Dragon. It's kind of a distribution built on various language instrumentation as well as our collector project called the Long Collector. We want to make sure that all the enhancements we want to contribute back to the hotel project as much as possible. But for some reasons, the community may not accept our contributions; they will be in our own distributions. +**Steve:** Okay. Sorry. There are some network issues. Maybe I just lost the connection. I will go into the remaining part. + +Actually, this solution has started in 2023 and at that time we were looking for a solution that can be easily applied to Go applications for the auto instrumentations. When we were trying the eBPF-based approach that is already been provided by the open telemetry community, it turned out that there were some limitations. Finally, we figured out a way to do the instrumentations to satisfy our needs, which is in a compile-time instrumentation. + +### [00:18:47] Compile-time instrumentation for Go + +If you’re looking at the figure on the right, the injection happens in the middle of the compilation process. Actually, we hooked the Golang compile command to make sure that it can be done for users without modifications to users' code. That’s a key feature that we want to achieve. + +Next, I want to address several highlights of this approach. First is the async context propagations. Actually, we encourage users to follow the guidelines, but not all users follow the rules because they do not pass the context object correctly. The traces may break and become incomplete. This feature can actually handle context propagation automatically for us, even when we are not passing the object correctly. + +The second highlight is that it is compatible with the old SDK. This means that users who have used manual instrumentations can work well with our approach, so the traces can be merged. The last and I think the most powerful feature that we have done is kind of customer extension. This means that you can instrument any code at any point in any line of the applications. This allows us to solve some special requirements from users. They can do some customizations like traffic management and security. + +Next, we will introduce some practices that we have in the LM of observability. As we know, the LM has become popular recently, especially this year. A lot of AI agents have become active and there are typical use cases that we found. In these use cases, a user sends a request to the API gateway, and the API gateway sends it to the LM applications, which are written in maybe Python, Java, or Golang. The application will call different kinds of models, for example, DeepSpeed, GPT-3, or OpenAI. + +We have adopted an AI gateway in the middle, which can do a lot of things for us, such as unified access, token limitations, token cache, and other stuff. We have adopted the open-source project called Highress that can do it very well for us. When we are building observability across different components, we ensure we follow the latest JNI semantic conventions. Because the convention is still in development, we are trying to keep pace with them as much as possible. We have done some instrumentations based on this semantic convention. + +This includes major AI agent frameworks like Highcode frameworks, Agono, Lang, Spring, Alibaba, etc. For low-code platforms, DIY is a very popular open-source project to build AI agents with low code. We also do some instrumentation for the official MCP clients. For example, we want to capture the MCP2 calls and the M32 responses, and to ensure our observability is end-to-end. We want to put our instrumentations into the internal models that work with serving frameworks like VM or SGAN, which are actually Python processes running there. + +So we can put our Python instrumentations into that Python process to capture traces and key metrics like TTFT (time to first token) and TBOT (time per output token), which are very important metrics for us to identify bottlenecks of LM serving issues. + +### [00:27:36] Adoption cases in Alibaba + +Next, talking about the adoption cases, we have many adoption cases. Today I want to introduce several of them. I divided them into two parts. The first part is the cloud service data flow. When users are using cloud service, we make sure all the services in the key positions of the application embrace the official W3 tracing goal by default. This means that balancers or other key components that support the W3 protocol by default will have users using open telemetry for async messaging. + +For the AI studio, as I mentioned in the previous slides, they are using Java and Python hotel instrumentation for end-to-end observability. The second part is the cloud service management console. A lot of management console services have embraced open telemetry as well. For example, the elastic cloud computing service is using hotel Java instrumentation quite a lot and they extend and make some extensions to that. They want to do request-based traffic routing and want to route the service to different machine groups. + +The video-on-demand service is also running large clusters written in Golang and they have adopted the Golang compile-time instrumentation mostly for runtime monitoring. They want to monitor the runtime metrics and also provide data. Another case is the institution detection service. There is an agent running on the host written in Go. Before open telemetry, they had nothing to do with observability because they lacked the techniques and tools. Now they can use hotel Go compile-time instrumentation for their runtime monitoring and profiling. + +As for our own service, the cloud monitoring service, we also implement self-observability with Java instrumentation. We do a lot of things with Java instrumentation to do traces and profiling. This is a good example of dogfooding, right? + +Besides our internal adoption, we are also trying to contribute these enhancements or extensions back to the open telemetry community. As I mentioned before, a lot of instrumentations have been contributed and some are also on the way to contributing back. We also helped to promote open telemetry in the APAC area. We help to organize some meetings that are friendly to the Asia-Pacific area like Java, Go, and Jingi meetings. We also engaged with conferences like KubeCon and the hotel community. + +We are also organizing events like KCD to help promote open telemetry in the Asia-Pacific areas. In 2024, we decided to donate the Golang compile-time instrumentations to the open telemetry community. After a long discussion with the community, we found that DataDog showed a very strong interest in this project and they submitted a proposal to donate their own framework, which is also a compile-time instrumentation project, to open telemetry. + +We decided that we do not want to depend on the effort from a single company, so we formed a SIG to build this project from scratch. This SIG is co-founded by Alibaba, DataDog, and Quesma. We have a common effort to implement this compile-time instrumentation for Golang. This project is still in active development and we are working on the first release and the MVP version this year. + +If you are interested, please definitely check it out. + +Talking about future work for Java instrumentations, we are planning to implement the proxy agent mode. This means that we want to use the Java agent as a dependency rather than hooking upstream projects because we do not want to maintain that kind of project. We want to ensure all the extension points can be merged into the upstream so that we can extend this project very well. + +For Golang compile-time instrumentation, we are working on the MVP version and we are making sure we are very close to the first release. We will also support more signals, including metrics and profiles. As for GI, we also want to contribute some of the instrumentations back to the community because this GI is still in active development. We put some of them in our repo and we want to contribute them back as well. + +Actually, we also have our own open telemetry distribution called Long Sweet. Long is the Chinese version of Dragon. It’s a distribution built on various language instrumentations as well as our collector project called the Long Collector. We want to make sure that all the enhancements we try to contribute back to the open telemetry project as much as possible, but for some reasons the community may not accept our contributions, and they will be in our own distributions. That is all for the presentation and thanks for everyone for listening. -**Dan:** Thank you, Hushing and Steve. That was great. I think feel free to drop any questions in the chat and then, if you're watching live on YouTube or LinkedIn, I have one which is you mentioned the adoption of open telemetry within Alibaba. I wanted to know how easy was it for teams within Alibaba to migrate to open telemetry? Is there anything you can share in terms of facilitating a migration, maybe they use something like a common set of libraries or migration script or something else? +**Dan:** Thank you, Hushing and Steve. That was great. I think feel free to drop any questions in the chat. If you're watching live on YouTube or LinkedIn, have one which is, you mentioned the adoption of open telemetry within Alibaba and I wanted to know how easy was it for teams within Alibaba to migrate to open telemetry? Is there anything you can share in terms of facilitating a migration, maybe they use something like a common set of libraries or migration script with code generation or something else? + +**Steve:** Yeah, I think it’s not quite easy actually. It’s not quite easy for them to migrate. We have done a lot of things to advocate for that. First, we wrote some internal articles to share some of our practices, how like a kind of team they adopt this technology and how they benefit from that. We put that into the internal article to help promote this open telemetry. -**Steve:** Yeah, I think it's not quite easy actually. It's not quite easy for them to migrate. We have done a lot of things to advocate for that. First, we wrote some internal articles to share some of our practices, like how a team adopted this technology and how they benefited from that. Then we put that into the internal article to help promote open telemetry. Secondly, we actually wanted to try to minimize the effort for the user to migrate because we have a legacy project called Ecoi and they already have some instrumentations there. If we are switching to this architecture to the auto instrumentations, they might encounter some conflicts or something like that. We actually have done a lot of things to prevent such kind of things from happening and we built that within our agent with our instrumentations as much as possible to make sure the users have minimized their effort to start up and to migrate. The idea is to try to put things to our team rather than put the efforts to the users to make sure that a seamless migration from the old architecture to the open telemetry based one. +Secondly, we wanted to minimize the effort for users to migrate because we have a legacy project called Ecoi and they already have some instrumentation there. If we are switching to this architecture, they might encounter some conflicts or something like that. We have done a lot of things to prevent such kind of things from happening and we built that within our agent with our instrumentations as much as possible to make sure the users have minimized their effort to start up and to migrate. + +I think the idea is to try to put things on our team rather than putting the efforts on the users to make sure that seamless migration from the old architecture to the open telemetry-based one. **Dan:** Nice. So you dogfooded that to make sure that you adopted it internally as well. -**Steve:** Yeah. +**Steve:** Yeah, yeah, yeah. + +**Dan:** Makes sense. Okay. I think we've got some comments in the chat but no questions so far as I can tell. I have another one as well from myself. You mentioned that you run a fork or a distribution of the hotel Java agent. In terms of API usage, do internal users in Alibaba use the open telemetry API directly when instrumenting applications or do you have some kind of abstraction on top or a helper library that they use instead of the hotel API? -**Dan:** Makes sense. Okay. I think we've got some comments in the chat but no questions so far as I can tell. I have another one as well from myself. You mentioned that you run on the hotel Java agent stuff. You run a fork or a distribution of that agent but in terms of API usage, do internal users in Alibaba use the open telemetry API directly when instrumenting applications or do you have some kind of abstraction on top or a helper library that they use instead of the open telemetry API? +**Steve:** I think for most of the cases, we would urge them to use the auto instrumentations as the first choice because that is less effort for them to start their adoption. But for some special cases, actually we can’t satisfy them, so we will encourage them to use the SDK one to add some custom instrumentations. For example, they want to add some custom spans or metrics. -**Hushing:** I think for most cases, we would have them use the auto instrumentations as the first choice because that is less effort for them to start their adoption. But for some special cases, actually, we can't satisfy. We will encourage them to use the SDK to add some custom instrumentations. For example, they want to add some custom spans or metrics. When that happens, we encourage them to use the official open telemetry SDK as much as possible because that is well documented and we actually have battle-tested it with the latest version of that SDK. So we do not want them to build another abstraction layer on top of that. But there are special cases that we introduced, like LLM application observability recently. Our AI studio is using the open telemetry SDK; they found that they have to write a lot of code in order to add their instrumentations because there are so many things they want to capture in a span. So they have to write a lot of instrumentations. So they asked us if we can provide an abstraction layer for them to simplify their work to do their instrumentation. So in that case, we actually worked with them to provide a higher level of that SDK. Their instrumentation effort has been reduced and they are very happy to do that. +Normally, we use the official hotel SDK as much as possible because that is well documented and we actually battle tested with the latest version of that SDK. We do not want them to build another abstraction layer on top of that. But there are special cases, like the LLM application observability recently, where our AI studio is using the OT SDK. They found that they have to write a lot of code in order to add their instrumentations because there’s so much they want to capture in a span, so they have to write a lot of instrumentations. -**Dan:** Yes. So that's one case I can share, and for most cases, I think it's using the official OT SDK. +They asked us if we could provide an abstraction layer for them to simplify their work for instrumentation. So in that case, we actually worked with them to provide a higher level of that SDK. Their instrumentation effort has been reduced and they are very happy with that. Yes. -**Hushing:** Yes. And in terms of the API as well because it's got that decoupling of the API and the implementation, like you can always swap the implementation underneath with the SDK, right? +So that is one case I think I can share, and for most cases, I think it’s using the official OT SDK. + +**Dan:** And in terms of the API as well, because it's got that decoupling of the API and the implementation, you can always swap the implementation underneath with the SDK, right? **Steve:** Yes. @@ -109,19 +159,27 @@ That is all for the presentation and thanks for everyone for listening. **Hushing:** Sure. -**Steve:** For our internal users, some of them, if we don't provide auto instrumentation such as C++ applications, we encourage them to use the API just like Hushing mentioned. Another point for Java applications, some users, as we know, if they use trace or span, sometimes that is a monitoring spot. If we just instrument some frameworks, some key methods and if the latency belongs to users' code, maybe they can't get the information why the latency is so long. During the traces, at this time, we provide out of box profiling ability. We correlate the traces with the profile and we can provide diagnosis for users to understand why the latency is so long and which code is responsible for the latency. So they don't need to use the API to instrument their own methods manually. +**Dan:** For our internal users, some of them, if we don’t provide auto instrumentation such as C++ applications, we encourage them to use the API just like HI mentioned. Another point for Java applications, some users, as we know, if they use trace or span, sometimes that is a monitoring spot. If we just instrument some frameworks or some key methods and if the latency belongs to users' code, they may not get the information why the latency is so long during the traces. At this time, we provide out-of-box profiling ability. We correlate the traces with profiles and we can provide diagnosis for users to understand why the latency is so long, which code is responsible for the latency, so they don’t need to use the API to instrument their own methods manually. + +**Dan:** Okay, that makes sense. + +**Dan:** Last question from me. Again, the audience, if you're watching and want to drop a question, feel free. Both of you talked about context propagation as something that has definitely improved a lot thanks to open telemetry instrumentation. I know this from experience, especially when we’re talking about async tasks. However, sometimes it’s still challenging in certain scenarios to make sure that context is propagated correctly within a service. How do you encourage teams to ensure that context is propagated correctly, not just across services, but also within their own services? And do you measure if context is broken somewhere or leaking? + +**Steve:** Yeah, that’s a good question. Actually, I think for our internal users, it’s very difficult for them to follow this rule. Even if you provide guidelines or encourage them to do so, they may not follow it. There’s a lot of opportunity for them to miss it or fail to follow that. + +To solve this problem, it’s our team’s duty to make sure all possible ways of context propagation are handled correctly by our auto instrumentations. That is how we want to solve this problem. We have popular frameworks, for example, in Java, the thread pool, or some similar scenarios, and we do tests to ensure that asynchronous context can be propagated properly. -**Dan:** Okay. Yeah, that makes sense. Last question from me. Again, if you're watching and want to drop a question, feel free. Both of you talked about context propagation as something that has definitely improved a lot thanks to open telemetry instrumentation. I know this from experience, especially when we're talking about async tasks. However, sometimes it's still challenging in certain scenarios to make sure that internally in a service, the context is propagated correctly. How do you encourage teams to ensure that context is propagated correctly within not just across services but also within their own services? And also, do you know if they measure that in some way if context is broken somewhere or leaking? +### [00:43:07] Context propagation challenges -**Steve:** Yeah, that's a good question. Actually, I think for our internal users, it's very difficult for them. Even if we provide guidelines or encourage them to do so, they may not follow this rule. There is very much opportunity for them to miss that or fail to follow that. To solve this problem, it's our team's duty to make sure all the possible ways of asynchronous context propagations have been propagated correctly by our auto instrumentations. That is the way we solve this problem. We have popular frameworks in Java, for example, the thread pool or some kind of similar scenarios, and we do the test to make sure that the asynchronous context can be propagated properly. +These frameworks are very challenging for us to handle. For example, reactor netty or such frameworks are very high frequency. For them, switching context and asynchronous tasks is common. For this kind of scenario, we might have something working not very well or as expected. For this case, we will try to work with the community or figure it out ourselves. At the same time, we seek help from the community to identify the issue and try to fix it. -There are other frameworks that are very challenging for us to do. For example, Reactor Netty or such kinds of frameworks are very high frequency for them to switch context and asynchronous is common for them. For this kind of scenario, we might have something working not very well or as expected. For this case, we would like to try and work with the community to identify the issue and try to fix it. +This is the common case I think. I used to remember the days where you had to pass a context object around in function calls, and it wasn't pretty and it wasn't good either. I think auto instrumentation is the way to go. Anything that can be done transparently will always cover more and result in better context propagation. -**Dan:** Yeah, I used to remember the days where you had to pass a context object around in function calls and it wasn't pretty and it wasn't good either. I think auto instrumentation is the way to go and anything that can get done transparently is also always going to be covering more and resulting in better context propagation, right? +**Dan:** Right. As long as the users stick to that automatic instrumentation approach, they will ask us about any issues they encounter. -**Steve:** Yes. As long as the users stick to that automatic instrumentation approach, they will ask us if we satisfied this or make why did that happen? So they will come to us looking for help rather than doing it by themselves. +**Steve:** Yes, they will come to us looking for help rather than doing it themselves. -**Dan:** Indeed. Okay. I think that's all we have time for today. Thank you so much for joining us both Hushing and Steve. It was great knowing more about Alibaba's approach to open telemetry. Thanks to all that joined live on LinkedIn and YouTube. Remember that if you want to continue the conversation, we've got the end user SEG resources as part of the opentelemetry.io website. Feel free to continue the conversation there and hope to see you there as well. Thank you again, both, and see you in the next one. +**Dan:** Okay. I think that’s all we have time for today. Thank you so much for joining us both, Hushing and Steve. It was great knowing more about Alibaba's approach to open telemetry. Thanks to all who joined live on LinkedIn and YouTube. Remember that if you want to continue the conversation, we have the end user SEG resources as part of the open telemetry.io website. Feel free to continue the conversation there and hope to see you there as well. Thank you again both, and see you in the next one. **Hushing and Steve:** Thank you. Bye-bye. diff --git a/video-transcripts/transcripts/2025-09-25T05:32:25Z-otel-in-practice-how-we-scaled-kafkalog-ingestion-for-otel-by-150.md b/video-transcripts/transcripts/2025-09-25T05:32:25Z-otel-in-practice-how-we-scaled-kafkalog-ingestion-for-otel-by-150.md index 06b7dc1..a24153c 100644 --- a/video-transcripts/transcripts/2025-09-25T05:32:25Z-otel-in-practice-how-we-scaled-kafkalog-ingestion-for-otel-by-150.md +++ b/video-transcripts/transcripts/2025-09-25T05:32:25Z-otel-in-practice-how-we-scaled-kafkalog-ingestion-for-otel-by-150.md @@ -10,100 +10,116 @@ URL: https://www.youtube.com/watch?v=GrQyUPeCljo ## Summary -In this episode of "Hotel in Practice," host Dakota Passman from Bindplane discusses scaling open telemetry logs using Kafka. Dakota, a software engineer with over two years at Bindplane, presents a case study involving a customer who faced performance bottlenecks pulling events from Kafka, initially processing only 12,000 events per second per partition, while needing to reach 30,000. Key strategies to resolve this included switching to a more efficient Kafka client, optimizing configuration settings, adjusting log encoding from OTLP JSON to raw JSON, and repositioning the batching processor to improve throughput. The presentation also features a live demo showcasing the improvements in event processing capabilities after implementing these changes. Audience questions addressed the impacts of different exporters on performance, the configuration of batching, and resource usage considerations. The session concludes with reminders about upcoming events and opportunities for community engagement. +In this episode of "Hotel in Practice," host Dakota Passman from Bindplane discusses scaling open telemetry logs using Kafka. He introduces himself as a software engineer with a focus on the hotel project and shares insights from a recent customer experience where they faced performance issues while processing Kafka events. The session highlights the critical need to avoid default settings, optimize configurations, and understand observability goals. Key improvements included switching to a more efficient Kafka client, adjusting log encoding, repositioning batch processing in the pipeline, and changing the export protocol from gRPC to HTTP. The demonstration showcased a significant increase in throughput, allowing the customer to meet their data processing needs effectively. The video concludes with a Q&A segment addressing audience queries related to performance and configuration adjustments. Viewers are encouraged to check the recording on their YouTube channel and LinkedIn page. ## Chapters -00:00:00 Welcome and intro -00:01:40 Guest introduction: Dakota Passman -00:02:50 Overview of scaling issues -00:05:01 Key takeaways for performance -00:06:30 Customer performance metrics -00:09:00 Changes made to improve performance -00:12:00 Demo setup explanation -00:15:02 Live demo of performance changes -00:19:00 Q&A session begins -00:25:26 Closing remarks and housekeeping +00:00:00 Introductions +00:01:36 Guest introduction: Dakota +00:03:12 Discussion about performance issues +00:05:20 Key takeaways for optimization +00:08:32 Changes made to Kafka receiver +00:10:40 Demo of Kafka performance improvements +00:12:48 Configuration comparison +00:15:30 Discussion on batching and throughput +00:19:12 Q&A session begins +00:25:04 Closing remarks and announcements -**Host:** Hello everyone and welcome to our latest edition of Hotel in Practice. We are so excited to have you join here today. And remember, tell your friends for anyone who wasn't able to make it today that we will have our recordings available after the fact. You can go to our YouTube channel at hotel-official, and you can also check out the hotel LinkedIn page. We should have the recording available on there too. +## Transcript + +### [00:00:00] Introductions + +**Host:** Hello everyone and welcome to our latest edition of Hotel in Practice. We are so excited to have you join here today. And remember, tell your friends for anyone who wasn't able to make it today that we will have our recordings available after the fact. You can go to our YouTube channel at hotel-official, and you can also check out the hotel LinkedIn page. We should have the recording available on there too. + +### [00:01:36] Guest introduction: Dakota We have a very special hotel in practice today. We have Dakota Passman from Bindplane talking about doing some gnarly scaling of open telemetry logs with Kafka. So without further ado, let's bring Dakota on. **Dakota:** Hello. -**Host:** Welcome Dakota. +**Host:** Welcome, Dakota. **Dakota:** Hi. Thanks for having me. **Host:** Super excited to have you here. Would you like to do a brief intro of yourself before starting? -[00:01:40] **Dakota:** Yeah. Yeah. I'm Dakota. I've been a software engineer at Bindplane for a little over two and a half years now. Working on our platform, focusing a lot on the hotel project and making some different contributions. Focusing a lot these days on the supervisor. It's a pretty cool project going on there. But yeah, happy to be here. +**Dakota:** Yeah. I'm Dakota. I've been a software engineer at Bindplane for a little over two and a half years now, working on our platform. Focusing a lot on the hotel project, making some different contributions. Focusing a lot these days on the supervisor. It's a pretty cool project going on there. But yeah, happy to be here. + +**Host:** Excellent. So if you're ready to go, then let's get started. By the way, for anyone who is interested, we will have some time after Dakota's presentation for questions. So please feel free to post them in the chat and we will address them after the presentation. -**Host:** Excellent. If you're ready to go, then let's get started. By the way, for anyone who is interested, we will have some time after Dakota's presentation for questions. So please feel free to post them in the chat, and we will address them after the presentation. +### [00:03:12] Discussion about performance issues -[00:02:50] **Dakota:** Yeah. Cool. All right. So yeah, today I want to talk about a situation we handled with a customer of ours using the Kafka receiver at scale. We had to scale their environment. They were having performance issues. They were running into a bottleneck trying to pull events from Kafka. When we started talking to them, they were pulling only 12,000 events per second per partition in their topic, which was just not enough to keep up. +**Dakota:** Yeah. Cool. All right. Today I want to talk about a situation we handled with a customer of ours using the Kafka receiver at scale. So yeah, we had to scale their environment. They were having performance issues. They were running into a bottleneck trying to pull events from Kafka. When we started talking to them, they were pulling only 12,000 events per second per partition in their topic, which was just not enough to keep up. -So we're going to talk about that bottleneck, how we got them to get up to a target EPS of 30,000. And then we're going to demo those changes live in a replica environment. To start, we're going to go over some of the takeaways just so that we can keep these in mind as we're going through this and see where they're coming up. +So we're going to talk about that bottleneck, how we got them to get up to a target EPS of 30,000. And then we're going to demo those changes live in a replica environment. To start, we're going to go over some of the takeaways just so that we can keep these in mind as we're going through this and see where they're coming up. For starters, don't trust defaults. The Kafka receiver by default uses a certain Kafka client, and these clients behave differently. They perform differently. It's a default. However, shortly before we had this issue with the customer, there was a new client implementation added that we were able to use, and using that client over the default client was a big improvement. -Secondly, configuration of the collector makes a huge difference. That might seem kind of obvious, but it's sometimes easy to overlook that, especially if you're not entirely sure of what configuration options you have and what they might exactly do. Something else that I think gets overlooked sometimes is configuration of the pipeline matters a lot too, especially in this situation. +Secondly, configuration of the collector makes a huge difference. That might seem kind of obvious, but it's sometimes easy to overlook that, especially if you're not entirely sure of what configuration options you have and what they might exactly do. Something else that I think gets overlooked sometimes is configuration of the pipeline matters a lot too, especially in this situation. Finally, it's important to understand your observability goals so you can tune the environment to get it right and make sure that your Kafka receiver or really any receiver that you might be using is performing well in your environment and hitting the observability goals that you're looking for. -To kind of set the stage a bit, this customer of ours was pulling 192,000 events per second across all of their partitions. They had just a single topic with 16 partitions, again doing roughly 12,000 events per second per partition, which sums up to 192,000 events per second. However, they needed to really be at 480,000 events per second in order to keep up with the amount of data going into Kafka. At that point, they would be able to start cutting down the backlog that they had going. +### [00:05:20] Key takeaways for optimization -They were using the open telemetry collector using the Kafka receiver with a lot of default options set. One topic, 16 partitions, each one only doing 12,000 events per second. The backlog was growing rapidly. Kafka was ingesting 30,000 events per second per partition roughly, and so their backlog was growing by 288,000 events per second across all the partitions. They were quickly falling behind. +To kind of set the stage a bit, this customer of ours, they were pulling 192,000 events per second across all of their partitions. They had just a single topic with 16 partitions, again doing roughly 12,000 events per second per partition. As a sum, that's 192,000 events per second. However, they needed to be at 480,000 events per second in order to keep up with the amount of data going into Kafka. At that point, they would be able to start cutting down this backlog that they had going. -[00:06:30] In this situation, it was security telemetry that they were collecting from Kafka and trying to send to a SIEM backend. Already, they were dealing with delayed telemetry, and at this poor performance, they were at risk of starting to drop some of this telemetry too. +They were using the open telemetry collector using the Kafka receiver, a lot of default options set. One topic, 16 partitions, each one only doing 12,000 events per second. The backlog was growing rapidly. Kafka was ingesting 30,000 events per second per partition roughly, and so their backlog was growing by 288,000 events per second across all the partitions. They were quickly falling behind. -Some of the changes we made, this is kind of like an illustration of the configuration and the pipeline that they had going. You can see some of these changes as we moved through the pipeline. So first off, the Kafka receiver using the new FronGo client, which was the Kafka client I mentioned before, using that implementation, we saw a big jump. We started, we jumped up to about 30% just using that client instead of the default. +In this situation, it was security telemetry that they were collecting from Kafka and trying to send to a seam backend. They were already dealing with delayed telemetry, and at this poor performance, they were at risk of starting to drop some of this telemetry too. + +Some of the changes we made: this is an illustration of the configuration and the pipeline that they had going. You can see some of these changes as we moved through the pipeline. First off, the Kafka receiver using the new FronGo client, which was the Kafka client I mentioned before. Using that implementation, we saw a big jump; we jumped up to about 30% just using that client instead of the default. Another thing, the log encoding that we were pulling from Kafka made a big difference. The receiver was configured to use OTLP JSON, which didn't really make sense in their situation. There was a lot of unnecessary work being done to handle OTLP JSON. So instead, we switched it to just raw JSON or normal JSON, not OTLP specifically, and that was another big performance boost there. -One other thing, batching. We moved our batch processor to the start of the processor pipeline, right after the receiver. The reason we did this, it allowed the receiver to pump out large quantities of events rather than being restricted by the data flow of the processor in front of it. It was reliably able to push out thousands of events in a batch at once rather than maybe a couple hundred and then jump up to a couple thousand back to a couple hundred. It was much more consistent performance batching right after the receiver. That's what we saw. As a result, we were able to boost our throughput doing that. +One other thing: batching. We moved our batch processor to the start of the processor pipeline, right after the receiver. The reason we did this was that it allowed the receiver to pump out large quantities of events rather than being restricted by the data flow of the processor in front of it. It was reliably able to push out thousands of events in a batch at once rather than maybe a couple hundred and then jump up to a couple thousand back to a couple hundred. It was much more consistent performance batching right after the receiver. That's what we saw. As a result, we were able to boost our throughput doing that. + +One final change we made, this is again specific to their environment and their pipeline. They were sending the SecOps seam backend, and we saw that changing the protocol used from gRPC to HTTP actually increased the performance. In their specific case, this made a big impact. We were able to get increased throughput doing this as well. The idea here is similar to the batch processor we discussed here. The exporter is just able to handle throughput better at HTTP or using HTTP instead of gRPC. This affected the receiver upstream from the exporter, allowing it to push data out a lot faster. + +In this demo, the customer had one topic, 16 partitions. They had a single collector per partition. In this demo, we're just going to be using a single collector, a single partition. Demoing with 16 partitions and managing collectors at that scale is difficult to do manually. We're just going to keep it simple here, just a single collector. + +### [00:10:40] Demo of Kafka performance improvements -[00:09:00] One final change we made, this is again specific to their environment and their pipeline. They were sending the SecOps SIEM backend, and we saw that changing the protocol used from gRPC to HTTP actually increased performance. In their specific case, this made a big impact. We were able to get increased throughput doing this as well. The idea here is similar to the batch processor we discussed. The exporter is just able to handle throughput better at HTTP than using gRPC. This affected the receiver upstream from the exporter, allowing it to push data out a lot faster. +I'm going to start the environment; it should be about 12,000 events per second that we're pulling. Then we're going to apply the changes to the environment, and then we'll see how it's performing afterwards. If I come over here and check this out, I started this up shortly before we started here. This green line up here, this is going to be the events per second that the Kafka topic in partition is pulling in. We're stable at around about 26,000-27,000 events per second. -In this demo, the customer had one topic, 16 partitions. They had a single collector per partition. In this demo, we're just going to be using a single collector, a single partition. Demoing with 16 partitions and managing collectors at that scale is difficult to do manually. We're just going to keep it simple here, just a single collector. +Down here is our single receiver, single copy receiver. This is using the default client. It's pulling in these events as OTLP JSON, and batching is being done afterwards. Here, you can see that we're stable at 12,000 events. Now, I'm going to just quickly stop this collector, and then I'm going to start it up using the new config. Also crucially here, I'm going to start it up using this feature gate to enable the FronGo client. -We're going to start the environment, should be about 12,000 events per second that we're pulling. Then we're going to apply the changes to the environment, and then we'll see, you know, we'll discuss those changes more in depth and see how it's performing afterwards. +I'm going to start that. I'm going to let that run in the background for a little bit, let these changes trickle through, and I'm going to talk about the changes we made in more depth. On the left here, this is the first configuration that the collector was using, and this is the configuration for the Kafka receiver. This is a pretty default config. The key difference here again is this OTLP JSON encoding versus on the right here in the new config, we're going to be using text. -So if I come over here and check this out. I started this up shortly before we started here. This green line up here, this is going to be the events per second that the Kafka topic in the partition is pulling in. We're stable at around 26,000 to 27,000 events per second. And then down here is our single receiver, a single copy receiver. This is using the default client. It's pulling in these events as OTLP JSON, and batching is being done afterwards. Here you can see that we're stable at 12,000 events. +Using text is just going to be able to pull the data faster. It's not going to have to worry about transforming or handling that data as much. It can just push the events through faster. It's not going to get bogged down trying to transform it. I've got some metrics collecting from this receiver. Here, I've got some processors. These define the same across the two here. This is to generate or to simulate how moving the batch processor around in the pipeline affects the actual throughput. -Now, I'm going to just quickly stop this collector, and then I'm going to start it up using the new config. Crucially here, I'm going to start it up using this feature gate to enable the FronGo client. I'm going to start that. I'm going to let that run in the background for a little bit, let these changes trickle through. I'm going to talk about the changes we made more in depth. +### [00:12:48] Configuration comparison -[00:12:00] On the left here, this is the first configuration that the collector was using. This is the configuration for the Kafka receiver. This is a pretty default config. The key difference here again is this OTLP JSON encoding versus on the right here in the new config. We're going to be using text. Using text is just going to be able to pull the data faster. It's not going to have to worry about transforming or handling that data as much. It can just push the events through faster. It's not going to get bogged down trying to transform it. +These processors are all the same, just pretty basic adding some fields. Next, exporters. We've got no exporter. We're not going to show the effects of the SecOps exporter in this demo just because that's a little much. We're going to just keep it simple here using the OP exporter to eliminate that lever from the situation. We're sending our internal telemetry. -I've got some collecting the metrics from this receiver. Here I've got some processors. These are defined the same across the two here. This is to generate or to simulate how moving the batch processor around in the pipeline affects the actual throughput. These processors are all the same, just pretty basic, adding some fields. +The other important part here, this is the initial configuration. We can see here the order we define our processors. We've got our two transform processors and then the batch. Again, the improvement we saw was adding the batch processor at the start of this processors definition list. This is so that the receiver is sending directly to the batch processor. It can get consistent performance consistently pushing out a high number of events rather than being bottlenecked by the other processors in front of it. -Next exporters, we've got no exporter. We're not going to show the effects of the SecOps exporter in this demo just because it's a little much. We're going to just keep it simple here using the OP exporter, you know, eliminate that lever from the situation. We're sending our internal telemetry. +The other crucial thing, again I pointed out when I restarted the collector, I used the new feature gate to use the FronGo client instead of the default. We'll come back here. We'll give this a refresh. This is going to keep going. But at the moment, we can see this new collector here. This is the old one turning off, and this is the new one coming back on. We briefly spike up, and I expect if we let this go for a little while, we'll see this stabilize and mirror the Kafka line. -The other important part here, this is the initial configuration. We can see here the order we define our processors. We've got our two transform processors and then the batch. Again, the improvement we saw was adding the batch processor at the start of this processors definition list. This is so that the receiver is sending directly to the batch processor. It can get consistent performance, consistently pushing out a high number of events rather than being bottlenecked by the other processors in front of it. +### [00:15:30] Discussion on batching and throughput -The other crucial thing, again I pointed out when I restarted the collector, I used the new feature gate to use the FronGo client instead of the default. +Let this go for a bit. Just to kind of go back here and reiterate some of these points again. Use the FronGo feature flag to use a FronGo client; it just performs a lot better than the default client in the Kafka receiver. Your log encoding, again, if you're handling the data unnecessarily—in this case, we're pulling data from Kafka that's not necessarily OTLP JSON—but because we have a receiver set up for that encoding, there's unnecessary work being done to transform it into that. We can just pull it in as normal JSON and handle it afterwards. That was a big source of improvement. -We'll come back here. We'll give this a refresh. This is going to keep going. But at the moment, we can see this new collector here. This is the old one turning off, and this is the new one coming back on. We briefly spike up, and I expect if we let this go for a little while, we'll see this stabilize and mirror the Kafka line. +Again, batching early, the receiver is able to consistently push out a consistent number of events rather than having seen that fluctuate based on what's happening upstream of it. Again, in a similar vein—not shown in the demo—but for SecOps or for this exporter, tuning and configuring that affects the receiver upstream of it. -[00:15:02] Let this go for a little bit. But I guess, just to kind of go back here and reiterate some of these points again, use the FronGo feature flag to use a FronGo client; it just performs a lot better than the default client in the Kafka receiver. Your log encoding, again, if you're handling the data unnecessarily, in this case, we're pulling data from Kafka that's not necessarily OTLP JSON, but because we have a receiver set up for that encoding, there's unnecessary work being done to transform it into that. We can just pull it in as normal JSON and handle it afterwards. That was a big source of improvement. +### [00:08:32] Changes made to Kafka receiver -Again, batching early, the receiver is able to consistently push out a consistent number of events rather than having seen that fluctuate based on what's happening upstream of it. Again, in a similar vein, not shown in the demo but for SecOps or for this exporter, tuning and configuring that affects the receiver upstream of it. +Those were again some of the changes we made. We'll see if we've got some better data here. It's still early, but you can see how this collector is starting to stabilize kind of right around the throughput that the Kafka topic is getting. I expect this to get closer as it comes online and starts to stabilize. You can see we jumped from doing 12,000 events per second, and then with all those changes we made, now we're closing in around 25,000-26,000 events per second. Drastically better improvement. This is how we were able to help this customer scale up their throughput so that they're able to keep up with Kafka. -Those were again some of the changes we made. We'll see if we've got some better data here. It's still early, but you can see how this collector is starting to stabilize, kind of right around the throughput that the Kafka topic is getting. I expect this to get closer as it comes online and starts to stabilize. But again, you can see we jumped from doing 12,000 events per second, and then with all those changes we made, now we're closing in around 25,000 to 26,000 events per second. Drastically better improvement. This is how we were able to help this customer scale up their throughput so that they're able to keep up with Kafka. In their case, we were able to get past what Kafka was ingesting so that we could also start cutting down the backlog and catch up. +In their case, we were able to get past what Kafka was ingesting so that we were able to also start cutting down the backlog and catch up. Got to refresh this a bit. You can see this is stabilizing. So again, why it worked? Back pressure. Back pressure is the term for this idea I've been talking about where what's happening upstream or downstream of the receiver is affecting the receiver. It's just not able to push events through as fast because it's got to wait on the components ahead of it to process their events. -Got to refresh this a bit. Cool. You can see this is stabilizing. Why it worked? Back pressure. Back pressure is the term for this idea I've been talking about where what's happening upstream or downstream of the receiver is affecting the receiver. It's just not able to push events through as fast because it's got to wait on the components ahead of it to process their events. You don't notice it until, in this case, the receiver starts falling behind and is affected by it. +You don't notice it until in this case the receiver starts falling behind and is affected by it. Again, very similar to the early batching. The FronGo client just performs a lot better than the default client in terms of pulling events from Kafka. Again, use that feature gate to enable it. Finally, avoid doing unnecessary encoding conversions. This is understanding your telemetry goals in your environment, understanding what the data you're sending looks like so that you can make sure that your receivers and other components are configured properly to handle it. -Again, very similar to the early batching. The FronGo client just performs a lot better than the default client in terms of pulling events from Kafka. Again, use that feature gate to enable it. Finally, avoid doing unnecessary encoding conversions. Again, this is understanding your telemetry goals in your environment, understanding what the data you're sending looks like so that you can make sure that your receivers and other components are configured properly to handle it. +I think that's all I have for a demo. We can keep checking out that graph to see how it's looking as it's coming online. I've got a couple links here. One to the blog post that we made about this, and then also the link out to the CNCF Slack. I think that is it for my slides. -I think that's all I have for a demo. We can keep checking out that graph to see how it's looking as it's coming online. Got a couple links here, one to the blog post that we made about this, and then also the link out to the CNCF Slack. I think that is it for my slides. +**Host:** Cool. Sorry, muted. Looks like we have a question coming in from LinkedIn. How do different exporters impact performance? -[00:19:00] **Host:** Cool. Sorry, muted mic. Looks like we have a question coming in from LinkedIn. How do different exporters impact performance? +### [00:19:12] Q&A session begins -**Dakota:** Yeah. I think that's definitely going to be very dependent on the exporters being used. It's going to be something you have to investigate per case and, again, fine-tune the configuration of them. In this case for the customer, the SecOps exporter using gRPC just wasn't able to keep up with the throughput we had, and in this situation, HTTP was able to perform a lot better. It's just a matter of trial and error to see what works best for your environment based on your telemetry and your goals. +**Dakota:** I think that's definitely going to be very dependent on the exporters being used. It's going to be something you have to investigate per case and fine-tune the configuration of them. Again, in this case for the customer, a SecOps exporter using gRPC just wasn't able to keep up with the throughput we had, and in this situation, HTTP was able to perform a lot better. It's just a matter of trial and error to see what works best for your environment based on your telemetry and what your goals are. **Host:** Awesome. Looks like we have another question. Batch has been recommended to be the first processor of the pipeline. What was the initial reason to put it at the end of the first pipeline? -**Dakota:** Yeah, so the reason for putting it at the end, you know, I'm not entirely sure of the decision to do that or what went into that decision. I think it's a perfectly valid approach. I know maybe in other environments that works better. I think it depends on the processors in your pipeline. I think some processors, it would be better to batch afterwards rather than before. Again, it's very dependent on what your pipeline looks like, what your goals are, and what you're trying to achieve. +**Dakota:** The reason for putting it at the end, you know, I'm not entirely sure of the decision to do that or what went into that decision. I think it's a perfectly valid approach. I know maybe in other environments that works better. I think it depends on the processors in your pipeline. I think some processors it would be better to batch afterwards rather than before. Again, it's very dependent on what your pipeline looks like, what your goals are, and what you're trying to achieve. **Host:** Awesome. Another follow-up question or another question. You did not use the memory limiter. Would that bring extra performance from using it if you used the memory limiter? @@ -111,15 +127,15 @@ I think that's all I have for a demo. We can keep checking out that graph to see **Host:** Fair enough. Another one that we have is, the FronGo feature flag, is it only useful for the Kafka receiver? -**Dakota:** It's a good question too. Yeah. No, the FronGo client implementation is just a general implementation not specific to the receiver. Some of the other Kafka components in the project, it's a matter of whether or not they utilize this client as well, and looking into seeing if there's a way to enable them to use this client or not. If not, I imagine that's something that will happen shortly, the ability to use the other Kafka components with that FronGo client. +**Dakota:** It's a good question too. The FronGo client implementation is just a general implementation not specific to the receiver. I think some of the other Kafka components in the project, it's a matter of whether or not they utilize this client as well and looking into seeing if there's a way to enable them to use this client or not. If not, I imagine that's something that'll happen shortly, the ability to use the other Kafka components with that FronGo client. -**Host:** All right. Awesome. Another question we have. Did you size the batching to get improvements, and did you adjust the exporter batch as well? +**Host:** All right. Awesome. Another question we have. Did you size the batching to get improvements and did you adjust the exporter batch as well? -**Dakota:** Yeah, that's another good question. We definitely did play around with the size of the batching. In this demo, we've got it configured like this. That's definitely something that you can play around with, toggle that, fine-tune it, see what works best for your environment and your throughput. +**Dakota:** That's another good question. We definitely did play around with the size of the batching. I think in this demo, we've got it configured like this. That's definitely something that you can play around with, toggle that, fine-tune it, see what works best for your environment and your throughput. **Host:** And then the final question that we have here, what about the resource usage? Does it have any consequences? -**Dakota:** Yeah, that's a great question too. Definitely something you want to be aware of. With our customer, making these configuration changes, they didn't affect the resource consumption and usage like that. The amount of resources the collector was using was still pretty, not negligible, but it didn't affect the system. It wasn't like all of a sudden we jumped from using 20% memory to 80% memory or CPU. It was very reasonable, so it wasn't necessarily a concern. +**Dakota:** That's a great question too. Yeah, I mean, definitely something you want to be aware of. With our customer, making these configuration changes, they didn't affect the resource consumption and usage like that. The amount of resources the collector was using was still pretty—not negligible—but it didn't affect the system. It wasn't like all of a sudden we jumped from using, you know, 20% memory to 80% memory or CPU. It was very reasonable, so it wasn't necessarily a concern. **Host:** Excellent. Do we have any other questions from the audience? Let me share these links in the chat as well. @@ -133,27 +149,27 @@ I think that's all I have for a demo. We can keep checking out that graph to see **Dakota:** Perfect. -**Host:** Amazing. Well, I suppose then if we don't have any other questions, I guess that is a wrap. Short and sweet informative. That's a lot to cover in a short time and also lots of great questions, lots of great considerations from a performance side as well. So definitely appreciate the questions that we've gotten. Again, tell your friends for those who are on the stream that this recording will be available after the fact, both on our LinkedIn page, the open telemetry LinkedIn page, and on the open telemetry YouTube channel, which is hotel-official. +### [00:25:04] Closing remarks and announcements -[00:25:26] A couple of housekeeping notes for anyone who is interested. The CFPs are still open for CubeCon, and I think in the last week, the CFPs have also opened for the CubeCon collocated events. That's the CubeCon in Europe in Amsterdam. Not the upcoming one; the upcoming one is done. Those CFPs are closed. But if you are interested in applying to CubeCon EU and Amsterdam and/or to the collocated events, these are the links. Remember folks that the submission limit for CubeCon is now three proposals per person, and that includes whether you're the main submitter or a co-speaker. The submission limit for the collocated events is 10 across all collocated events, which is very cool. +**Host:** Amazing. Well, I suppose then if we don't have any other questions, I guess that is a wrap. Short and sweet, informative. That's a lot to cover in a short time and also lots of great questions, lots of great considerations from a performance side as well. So definitely appreciate the questions that we've gotten. Again, tell your friends for those who are on the stream that this recording will be available after the fact both on our LinkedIn page, the OpenTelemetry LinkedIn page, and on the OpenTelemetry YouTube channel which is hotel-official. -For anyone who is interested in sharing their stories with the hotel end user SIG, we love hearing from the community. Please reach out to us. You can find us in CNCF Slack. We have a lovely Slack channel, which is, I believe, SIG hotel end user. Henrik has been nice enough to put the link to our Slack channel on there as well. +A couple of housekeeping notes for anyone who is interested: the CFPs are still open for CubeCon, and I think in the last week the CFPs have also opened for the CubeCon co-located events. That's the CubeCon in Europe in Amsterdam. Not the upcoming one; the upcoming one, that's done. Those CFPs are closed. But if you are interested in applying to CubeCon EU in Amsterdam or to the co-located events, these are the links. Remember folks that the submission limit for CubeCon is now three proposals per person, and that includes whether you're the main submitter or a co-speaker. The submission limit for the co-located events is 10 across all co-located events, which is very cool. -Oh, yeah, sorry, I had it backwards; it's hotel-end user. We would love to hear your stories, whether it is through hotel in practice, this type of presentation. If you have a presentation that you want to test out, this is a great proving ground. Or if it's a talk you've given somewhere, and you just want to share it with more folks, this is also another great way to get the topic out there. +For anyone who is interested in sharing their stories with the hotel end user SIG, we love hearing from the community, so please reach out to us. You can find us in CNCF Slack. We have a lovely Slack channel which is, I believe, SIG hotel end user. Henrik has been nice enough to put the link to our Slack channel on there as well. Oh, sorry, I had it backwards; it's hotel-end user. We would love to hear your stories, whether it is through Hotel in Practice, this type of presentation, or if you have a presentation that you want to test out, this is a great proving ground. -Dakota, you presented this at open source summit, right? The collo observability day or the, I forget what the hotel day was, is that correct? +If it's a talk you've given somewhere and you just want to share it with more folks, this is also another great way to get the topic out there. Dakota, you presented this at Open Source Summit, right? The colloquium observability day or the—I forget what the hotel day was. Is that correct? -**Dakota:** I didn't present anything, but one of our coworkers at Bindplane did present a topic about the Kafka receiver. +**Dakota:** I didn't present anything, but one of our co-workers at Bindplane did present a topic about the Kafka receiver. **Host:** Ah, okay. **Dakota:** His presentation was a deep dive into his understanding of it and some of the unique aspects of it he figured out. -**Host:** Oh nice. Very nice. That's awesome. We would love to hear topics like that. We also love to hear anything where you've learned cool stuff about open telemetry. We would love to hear from you, so hit us up on our Slack channel. We also have hotel Q&A, so for anyone who wants to talk about their hotel journey, this one's an interview style format. Both of these are live streams. We would love to hear from you. +**Host:** Oh, nice. Very nice. That's awesome. Yeah. We would love to hear topics like that. We also love to hear anything you've learned, cool stuff about OpenTelemetry. We would love to hear from you. Hit us up on our Slack channel. We also have Hotel Q&A for anyone who wants to talk about their hotel journey. This one's an interview style format. Both of these are live streams. We would love to hear from you. -We have recently had, I think our previous hotel in practice was with folks from Alibaba in the APAC region. We are looking for more folks in the APAC region as well who would love to share their story because, you know, open telemetry and CNCF, it's a global undertaking. We have lots of love from folks all across the globe. If you're in the APAC region and you have a story to share, we would schedule an APAC-friendly hotel in practice or hotel Q&A to cater to the time zone. +We recently had, I think our previous Hotel in Practice was with folks from Alibaba in the APAC region. We are looking for more folks in the APAC region as well who would love to share their story because, you know, OpenTelemetry CNCF, it's a global undertaking. We have lots of love from folks all across the globe. If you're in the APAC region and you have a story to share, we would schedule an APAC-friendly Hotel in Practice, Hotel Q&A to cater to the time zone. -That is it for us. If you're going to CubeCon North America, you'll probably see some of our crew at CubeCon North America. Excited to have folks connect in person on open telemetry. Thank you so much everyone for attending. +That is it for us. Hope to see you at CubeCon North America; you'll probably see some of our crew there. Excited to have folks connect in person on OpenTelemetry. Thank you so much everyone for attending. **Dakota:** Thanks. diff --git a/video-transcripts/transcripts/2025-10-22T04:08:27Z-whats-new-in-otel.md b/video-transcripts/transcripts/2025-10-22T04:08:27Z-whats-new-in-otel.md index 58c26b7..52af2f4 100644 --- a/video-transcripts/transcripts/2025-10-22T04:08:27Z-whats-new-in-otel.md +++ b/video-transcripts/transcripts/2025-10-22T04:08:27Z-whats-new-in-otel.md @@ -10,22 +10,26 @@ URL: https://www.youtube.com/watch?v=NFtcp0LPtFA ## Summary -In this special edition of "Open Telemetry and User SIG Live," hosts Bree Lee and Adriana Vila introduced a new segment called "What's New in Hotel," aimed at sharing updates within the OpenTelemetry community. They engaged with guests Severin Newman and Marilia Gutierrez, who discussed recent developments in OpenTelemetry including community growth, updates on the configuration system, and upcoming events like KubeCon. Severin highlighted impressive contributor statistics, while Marilia detailed the transition from environment variables to a more structured declarative configuration system for SDKs, emphasizing its advantages for complex setups. The session concluded with a Q&A segment addressing various queries about OpenTelemetry functionalities and contributions, encouraging community engagement and participation. +In this special edition of "What's New in OpenTelemetry," hosted by Bree Lee from New Relic and Adriana Villa from Dynatrace, the video features discussions with guests Severin Newman and Marilia Gutierrez on the latest updates and community developments within the OpenTelemetry project. The session highlights the growing community involvement, with over 20,000 contributors and numerous organizations participating. Key topics include the introduction of a new declarative configuration format, which allows for more structured and robust setup options compared to traditional environment variables, as well as updates on sampling methodologies and upcoming events like KubeCon. The hosts encourage viewer interaction and emphasize the importance of community contributions, inviting newcomers to engage with existing projects and discussions. The video concludes with audience questions and a promise for future sessions to keep the community updated on OpenTelemetry developments. ## Chapters -00:00:00 Welcome and intro -00:01:10 Introduction of guests -00:03:56 Severin Newman introduction -00:06:00 Community growth statistics -00:12:00 Community updates -00:15:30 Governance committee election details -00:19:50 OpenTelemetry Unplugged announcement -00:23:20 Technical highlights overview -00:29:00 Marilia Gutierrez introduction -00:31:30 Declarative configuration discussion +00:00:00 Introductions +00:05:00 Guest introduction: Severin +00:10:12 Discussion about OpenTelemetry community growth +00:22:40 Introduction of declarative configuration format +00:30:56 Guest introduction: Marilia +00:32:52 Discussion on declarative configuration implementation +00:41:56 Audience Q&A session begins +00:52:08 Discussion on OpenTelemetry governance committee elections +00:58:08 Audience question on long-running background processes +01:00:04 Closing remarks and future events -**Bree:** Heat. Heat. Hello. +## Transcript + +### [00:00:00] Introductions + +**Bree:** Heat. Heat. Hello. **Adriana:** Hello everyone. Welcome. @@ -33,289 +37,331 @@ In this special edition of "Open Telemetry and User SIG Live," hosts Bree Lee an **Adriana:** You're welcome. -[00:01:10] **Bree:** Well, hi. Welcome to this special edition of Open Telemetry and User Sig Live. We're very excited to bring to you a brand new segment called What's New in Hotel. I am Bree Lee. I'm a senior developer relations engineer at New Relic, and I get to work with the lovely Adriana Villa. Would you like to introduce yourself? +**Bree:** Well, hi. Welcome to this special edition of Open Telemetry and User Sig Live. We're very excited to bring to you a brand new segment called What's New in Hotel. I am Bree Lee. I'm a senior developer relations engineer at New Relic. I get to work with the lovely Adriana Villa. Would you like to introduce yourself? -**Adriana:** Yes. Yeah. My name is Adriana Vila. I work with Bree in the hotel and user SIG, and I am a principal developer advocate at Dynatrace. And like Bree said, this is our very first What's New in Hotel. And I think this is really exciting because I don't know about you, but for me, I can never keep up with all the stuff that's going on in hotel. So I kind of need someone to tell me. So this is a little self-serving, and we're basically bringing folks in from the hotel community to tell us some of the cool stuff that is going on. +**Adriana:** Yes. My name is Adriana Vila. I work with Bree in the hotel and user SIG and I am a principal developer advocate at Dynatrace. Like Bree said, this is our very first What's New in Hotel. I think this is really exciting because I don't know about you, but for me, I can never keep up with all the stuff that's going on in hotel. So I kind of need someone to tell me. This is a little self-serving and we're basically bringing folks in from the hotel community to tell us some of the cool stuff that is going on. **Bree:** Yes. And hello, Michael. **Michael:** Hi. Yay. Thanks for joining us. I know it's lateish for you. -**Bree:** It's way 5:00 PM for you. So thank you for joining, Michael. +**Bree:** It's way 5:00 PM for you. So, thank you for joining, Michael. -**Michael:** Yes. And by the way, I am coming in from Portland, Oregon. And if you would like to share where you're tuning in from in the chat of whichever app you're using, we would love to see where you're from because there's a pretty good variety of time zones on this call as well. For example, Adriana is coming from Toronto. +**Michael:** Yes. And by the way, I am coming in from Portland, Oregon. If you would like to share where you're tuning in from in the chat of whichever app you're using, we would love to see where you're from because there's a pretty good variety of time zones on this call as well. -**Adriana:** That's right. Yeah. And you said it right. You said Toronto, not Toronto. +**Adriana:** For example, I am coming from Toronto. -**Bree:** Amazing. Yes, I'm from Canada, so representing. +**Bree:** That's right. Yeah. And you said it right. You said Toronto, not Toronto. -**Adriana:** Yeah. +**Adriana:** Amazing. Yes, I'm from Canada, so representing. **Bree:** So I guess this is a good time to bring in our first guest. -**Adriana:** Oh yeah, sorry. Sorry. Yes. +**Adriana:** Oh yeah, sorry. Sorry. Yes. -[00:03:56] **Bree:** Yes. So just to give you all a sneak peek into what our new segment is going to look like, we are going to speak with Severin Newman. And then we're going to speak with Marilia Gutierrez. And finally, we are going to bring on Lisa Jung to host our audience Q&A at the very end. So if you have questions, put them in the chat, and we will get to them at the end of both of the first two interviews during the audience Q&A. Hopefully, that'll make sense. And yes, like Adriana said, let's bring on Severin. Oh, Egypt is also watching. +**Bree:** Just to give you all a sneak peek into what our new segment is going to look like, we are going to speak with Severin Newman. Then we're going to speak with Marilia Gutierrez. Finally, we are going to bring on Lisa Jung to host our audience Q&A at the very end. If you have questions, put them in the chat and we will get to them at the end of both of the first two interviews during the audience Q&A. Hopefully that'll make sense. Yes, like Adriana said, let's bring on Severin. **Severin:** Hey, happy to be here. **Bree:** Thanks for joining us. -**Severin:** Yeah, sure. Where are you? Where are you coming to us live from? +**Severin:** Yeah, sure. Where are you coming to us live from? -**Severin:** Yeah, I'm based in Germany, so I'm close to Nuremberg. So yeah, Central European time. +**Bree:** I'm based in Germany, so I'm close to Nuremberg. -**Bree:** So it's evening for you. +**Adriana:** So it's evening for you. -**Severin:** Yeah, it's 5:00 PM. So it's getting dark again. So summer is officially over, unfortunately. +**Severin:** Yeah, it's 5:00 PM. It's getting dark again. Summer is officially over unfortunately. -**Bree:** And then next week you guys get the time change of 1 hour back, and then North America gets the time change of 1 hour back this week because it's like screwing up my whole calendar. So it's like, oh, everything is now at 5:00 PM, which should be at 6:00 PM and everything. So yeah, let's get over it. +**Bree:** And then next week you guys get the time change of 1 hour back and then North America gets the time change of 1 hour back this week because it's screwing up my whole calendar. Everything is now at 5:00 PM which should be at 6:00 PM. -**Severin:** Yeah. Well, at least we have like one more week of not having to worry about that. +**Severin:** Yeah, let's get over it. -**Bree:** Yeah. +**Bree:** Well, at least we have one more week of not having to worry about that. -**Severin:** So do you want to introduce yourself briefly? +**Severin:** Yeah. -**Severin:** Yeah, sure. So, sorry. Tell us all the things you do for hotel. +**Bree:** Do you want to introduce yourself briefly? -**Severin:** Right. Thank you so much. I try my best. Yeah. So I'm Severin, however you pronounce it. I always also pronounce it differently when I speak English or German. I'm head of community and developer relations at Cosley. I'm part of the Open Telemetry governance committee, and I'm also one of the co-maintainers of what we call sigcoms. So what we do is the website, the blog, social media channels and like everything around that. So that's a very short version of what I do for the Open Telemetry community. Yeah, and I'm super excited to be here today to give you a first start into what's new in Open Telemetry. So yeah, that's me. +**Severin:** Yeah, sure. I'm Severin, however you pronounce it; I also pronounce it differently when I speak English or German. I'm head of community and developer relations at Cosley. I'm part of the Open Telemetry governance committee and I'm also one of the co-maintainers of what we call sigcoms. What we do is the website, the blog, the social media channels, and everything around that. That's a very short version of what I do for the Open Telemetry community. I'm super excited to be here today to give you a first start into what's new in Open Telemetry. So yeah, that's me. **Bree:** Amazing. So I guess let's get started then. Do you have some goodies that you can share with us? -[00:06:00] **Severin:** Yeah, absolutely. So I have prepared a little bit of slides, not to like bore everybody out with like a lot of text and more like to make a few of these things I wanted to share with you a little bit more visual. So I have three topics to talk about, right? A little bit hotel and numbers because I'm always excited about seeing how big this project has grown, and then a little bit about the community and then a little bit about the tech stuff. I think that's the stuff, the things that most people are excited about. So yeah, let me share my screen and get started with a little bit on the numbers. +**Severin:** Yeah, absolutely. I have prepared a little bit of slides not to bore everybody out with a lot of text, but more to make a few of these things I wanted to share with you a little more visual. I have three topics to talk about: a little bit about hotel and numbers because I'm always excited about seeing how big this project has grown, a little bit about the community, and then a little bit about the tech stuff. I think that's the stuff that most people are excited about. So yeah, let me share my screen and get started with a little bit on the numbers. + +**Severin:** So those numbers are like for all the time, right? Since Open Telemetry was created, what is it now, 6 years ago? We had almost 20,000 contributors from 120 countries and almost 2,000 organizations. I also looked up the numbers for the last 12 months which is still like 6,000 contributors over 70 countries and over 600 organizations. You see there's a lot of velocity in our project, right? This is growing every year and we see it also with the PRs and the work that's happening on GitHub. We have to add here right something like this session here is not recorded by anything that you see on CNCF Devstat; there's a lot of work going on in our community that's not tracked on those boards unfortunately. + +**Severin:** Also, a big shout out to everybody doing the work outside of the repositories, but I still think those numbers are impressive. Since the foundation of the project, we had like those 60,000 PRs reviewed; as you can see, most of them three to four times and then over half a million comments. That means there were over half a million times someone wrote something on a below our issue or pull request or something like that, which I think is really great. + +**Severin:** What's really exciting is like in the last year, I think 20% of those PRs have been just done in the last year, which is huge growth and especially like from the reviews, 30% of them have been done in the last year meaning we're leveling up our reviews. We have more and more repositories where we say, "Hey, you're not done with one review; you maybe need two reviews." For example, in our Open Telemetry IO repository, we need a review from docs maintainers and from the co-owners sig. I think this really helps with the quality of the project. + +**Severin:** And who's doing that, right? This is a number I'm super happy to be able to share because we built a page for that specifically on the Open Telemetry website where we list all the triagers, all the approvers, and all the maintainers. As you can see, those are people that stepped up and took over a role where they said, "Hey, I help with triaging. I help with managing the repositories. I help with running the SIG meetings." If you do the math, there's almost 300 people just having those kinds of roles, which I think is really, really great to have. A big shout out to everybody who's helping with that and also a big shout out to all the people I'm not listing here because there are so many people in our community that are helping or that just land in our community and then wanting to step up and do much more things. + +### [00:10:12] Discussion about OpenTelemetry community growth -**Severin:** Yeah. So those numbers are like for all the time, right? So since Open Telemetry was created, what is it now, 6 years ago, we had almost 20,000 contributors from 120 countries and like I call it organizations, but think about it's like companies and foundations, whatever, like almost 2,000 organizations. I also looked up the numbers for the last 12 months, which is still like 6,000 contributors, over 70 countries, over 600 organizations. So you see there's a lot of velocity in our project, right? So, and this is growing and growing every year, and we see it also with the PRs and the work that's happening on GitHub. I mean, we have to add here, right? Something like this session here is not recorded by anything that you see on CNCF Devstat, right? There's a lot of work going on in our community that's not tracked on those boards unfortunately. So also big shout out to everybody doing, let's say, the work outside of the repositories, but I still think those numbers are impressive, right? Since the foundation of the project, we had like those 60,000 PRs reviewed, as you can see, like most of them three to four times and then like over half a million comments, right? So that means like there were over half a million times someone wrote something on a below our issue or pull request or something like that, which I think is really great. And what's really exciting is like in the last year, I think 20% of those PRs have been just done in the last year, which is like huge growth, and especially like from the reviews, 30% of them have been done in the last year, meaning like we're leveling up our reviews, right? So we have more and more repositories where we say like, hey, you're not done with one review, you maybe need two reviews or for example, in our Open Telemetry IO repository, we need a review from docs maintainers and from like the co-owners sig, and I think this really helps with the quality of the project. +**Severin:** Then of course, who is using Open Telemetry, right? There's always like three kinds of groups that we mention: first of all, of course, the vendors or the backends. This is of course companies that consume Open Telemetry data and do something with it, but also open source projects. This also includes Jaeger or OpenSearch. Then there's also what we call integrations. That number is probably much bigger. This is just what we have on our website. Those are projects that said, "Hey, we add Open Telemetry to our project natively." Things like Kubernetes has their API server supporting Open Telemetry or there's the PHP server called Roadrunner that has Open Telemetry. I always love mentioning that one because it's written in Go and then doing PHP, so you can even do tracing from Go into the PHP in this web server. I think that's also very fun. -**Severin:** And who's doing that, right? And this is some number I'm super happy to be able to share because we built a page for that specifically on the Open Telemetry website where we list all the triagers, all the approvers, and all the maintainers. And as you can see, like those are like people that stepped up and took over a role where they said like, hey, I help us triaging, I help us like managing the repositories, I help with running the sig meetings. And if you do the math, there's like almost 300 people just having those kinds of roles, which I think is really, really great to have. So yeah, a big shout out to everybody who's helping with that, and also a big shout out to all the people I'm not listing here because there's so many people in our community that are helping or that are just landing in our community and then wanting to step up and do much more things. +**Severin:** And then of course thousands of adopters. I mean every time I'm in a presentation or in a room and I ask people, "Hey, who of you knows about Open Telemetry?" most of their hands go up. Then if I say, "Hey, and who of you is using Open Telemetry?" also like 80% of the people are raising their hand. Unfortunately, this is something we don't have that many listed on our website. Here’s the first call to action: if you are an Open Telemetry adopter, if you have maybe even written a blog from your company where you say, "Hey, here's how we use Open Telemetry. Here's how it's helping us," just use that link, and you can reach out, and we can have you add it there. -**Severin:** Yeah, and then of course, who is using Open Telemetry, right? And there's always like three kinds of groups that we mention. There's first of all, of course, like what we call the vendors or like the backends. So this is of course companies that consume Open Telemetry data and do something with it. But also open source projects, right? This also includes Jaeger or OpenSearch. But then there's also what we call integrations, right? And that number is probably much, much bigger. So this is just what we have on our website. Those are projects that said like, hey, we add Open Telemetry to our project natively. So things like Kubernetes has their API server supporting Open Telemetry or there's the PHP server called Roadrunner that has Open Telemetry. I always love mentioning that one because like it's written in Go and then doing PHP. So you can even do tracing from Go into the PHP in this web server. This is really something I always love seeing. Oh, and also Docker, right? So if you, I think it's Docker Buildkit and another project of Docker, like you can see your containers being built with Open Telemetry. I think that's also very fun. +**Severin:** That's a little bit Open Telemetry numbers. I think that's just a good starting point and let's talk about the community or is there anything you'd like to deep dive on that slide? -**Severin:** And then of course thousands of adopters, right? I mean every time I'm in a presentation or in a room and I ask people like, hey, who of you knows about Open Telemetry, most of their hands go up, and then if I say like, hey, and who of you is using Open Telemetry, also like 80% of the people are raising their hand. Unfortunately, this is something we don't have that many listed on our website. So here's the first call to action. If you are an Open Telemetry adopter, if you have maybe even written a blog from your company where you say like, hey, here's how we use Open Telemetry, here's how it's helping us, just use that link and you can reach out and we can have you add it there, right? So yeah, that's a little bit Open Telemetry numbers. I think that's just a good starting point, and let's talk about the community or is there anything you'd like to deep dive on that slide? I think it's just good to get started. +**Adriana:** I think let's dive into the next topic. -**Adriana:** Yeah, I think let's dive into the next topic. +**Severin:** Awesome. I think that's now more exciting. I always love to start with those numbers to just emphasize how big this community has grown. That also means there's a lot of community updates and especially like we all know KubeCon is coming. For whatever reasons, September, October, November is like all those community changes that are going on. One I'd like to highlight, because I'm one of the maintainers of this project, is if you go to the OpenTelemetry.io website, there's now nine languages that we have started to translate the OpenTelemetry website to. This includes English just like the base language, but then we have Bengali, Spanish, French, Japanese, Portuguese, Romanian, Ukrainian, and Chinese. I have written it down so that I don't forget anybody. I think that's amazing. We started with that a little bit more than a year ago with four languages and it has grown to nine languages. This is really amazing community because I think that's a really great way to get started with Open Telemetry, right? You can translate the things into your native language, learn about them, and then maybe move into different projects of the OpenTelemetry project with this additional understanding. -[00:12:00] **Severin:** Awesome. I think that's now more exciting. Right. As I said, I always love to start with those numbers to just emphasize how big this community has grown. And that also means like there's a lot of community updates, and especially like we all know KubeCon is coming, and for whatever reasons September, October, November is like all those community changes that are going on. One I'd like to highlight because like I'm one of the maintainers of this project is like if you go to the Open Telemetry.io website, there's now nine languages that we have started to translate the Open Telemetry website to. So that includes English just like the base language. But then we have Bengali, Spanish, French, Japanese, Portuguese, Romanian, Ukrainian, and Chinese. I have written it down so that I don't forget anybody. But I think that's amazing, right? I mean we have started with that a little bit more than a year ago with four languages, and it has grown to nine languages. And this is really amazing community because I think that's a really great way to get started with Open Telemetry, right? So you can translate the things into your native language, learn about them, and then maybe move into different projects of the Open Telemetry project with this additional understanding. +**Severin:** A big call out: the technical committee has added two members. David Ashpole and Josh McDonald have joined the TC. David is a subject matter expert on metrics Prometheus. He's a Kubernetes contributor as well. Josh has stepped down from the TC a while ago and is now coming back in. He's in the sampling SIG. He is one of the people behind the OpenTelemetry Arrow and so many other things. I'm probably not doing him justice only calling out those two things, but yeah, big congratulations to the two for joining the TC and we will see the impact they will have on the project very soon. -**Severin:** Yeah, then maybe a big call out, the technical committee has added two members. So David Ashpole and Josh McDonald have joined the TC. David is a subject matter expert on metrics Prometheus. He's a Kubernetes contributor as well. And Josh has stepped down from the TC a while ago, is now coming back in, and he's like in the sampling sig. He is one of the people behind the Open Telemetry Arrow and so many other things. So I'm probably not doing him justice only calling out those two things. But yeah, big congratulations to the two for joining the TC, and we will see the impact they will have on the project very soon. +**Severin:** Another one: the governance committee election is coming up next week. We just announced the candidates. If you are a member of standing, as we call it in the OpenTelemetry community, you should have a notification about being allowed to elect. If you think that you're allowed to be in that election as a voter and have not gotten your access to that, reach out. There's a link for that also. You can find it from here to be added to that. -**Severin:** Then another one, the governance committee election is coming up next week. So we just announced the candidates. If you are a member of standing, how we call it in the Open Telemetry community, you should have a notification about you being allowed to elect. If you think that you're allowed to be in that election as a voter and have not gotten your access to that, reach out. There's a link for that also. You can find it from here to be added to that. For people that are new to the Open Telemetry project, a new Slack channel that we called Open Telemetry New Contributors. So if you want to start contributing, go over there, and you can get started or can ask your questions and say like, hey, I picked up that issue and I need some help with that or hey, I want to help with documentation, I want to help as the collector, can somebody guide me with that? So I think that's a really great opportunity to get started. +**Severin:** For people that are new to the OpenTelemetry project, a new Slack channel that we called OpenTelemetry New Contributors. If you want to start contributing, go over there and you can get started or can ask your questions and say, "Hey, I picked up that issue and I need some help with that," or "Hey, I want to help with documentation. I want to help as the collector. Can somebody guide me with that?" I think that's a really great opportunity to get started. -[00:15:30] **Severin:** Yeah, another call to action, I think I have a few more coming. So take down your paper and take some notes. We have community awards, right? So, we started with this last year, and we wanted to redo this once again. So at KubeCon, we will call out a few members of the community where the community thinks like they deserve an award, right? So everybody is allowed to nominate someone, and this also does not only include like people contributing to the project. This also includes people that are promoting the project in any way, right? So let's say you have a person in your company that's running around and convincing everybody to use Open Telemetry. You can nominate them, right? If you find four or five friends in your company and say like, "Hey, let's nominate that person. Let's add them to that list." Perfectly fine with us, right? We want to hear those stories. We want to hear about people that are promoting Open Telemetry. +**Severin:** Another call to action: we started the community awards last year and we wanted to redo this once again. At KubeCon, we will call out a few members of the community where the community thinks they deserve an award. Everybody is allowed to nominate someone and this does not only include people contributing to the project. This also includes people that are promoting the project in any way. Let's say you have a person in your company that's running around and convincing everybody to use OpenTelemetry. You can nominate them. If you find four or five friends in your company and say, "Hey, let's nominate that person. Let's add them to that list." Perfectly fine with us. We want to hear those stories. -**Severin:** Speaking about KubeCon, right? I mean, the observatory is back to KubeCon. So if you want to speak with us at KubeCon, I will be first time for me at KubeCon North America this year, by the way. So if you want to catch up with us, you will find us there. I think we also have some schedule and some program there planned. You can probably tell more than I can do on that, but yeah, I'm very excited about that. +**Severin:** Speaking about KubeCon, right? The observatory is back to KubeCon. If you want to speak, I will be first time for me at KubeCon North America this year by the way. If you want to catch up with us, you will find us there. I think we also have some schedule and program planned. You can probably tell more than I can do on that, but I'm very excited about that. -**Adriana:** Yeah. And if I may interject really quickly on a couple of things, I wanted to just dig in quickly to the GC elections because that's pretty much open to anyone, right? I know you said it's happening next week. But anyone who's been involved in Open Telemetry can nominate themselves for the GC elections, right? +**Adriana:** If I may interject really quickly on a couple of things, I wanted to dig in quickly to the GC elections. That's pretty much open to anyone, right? I know you said it's happening next week, but anyone who's been involved in OpenTelemetry can nominate themselves for the GC elections. **Severin:** Sorry, can you now? I lost you for a minute. -**Adriana:** Oh, can anyone nominate themselves for the GC elections that's been working in Open Telemetry? Is that correct? +**Adriana:** Oh, anyone can nominate themselves for the GC elections that's been working in OpenTelemetry. Is that correct? -**Severin:** Yeah, but I think the nomination is closed since like— +**Severin:** Yeah, but I think the nomination is closed since... -**Adriana:** Okay. Okay. Yeah. Yeah. So that's why I did not call that the nomination. But yeah, no, fair enough. +**Adriana:** Okay. Okay. -**Severin:** I think even you don't have to be like— +**Severin:** Yeah. -**Adriana:** So yeah, everybody can be nominated. I think you need two supporters, and then you're in. I said it's closed for this year, but next year, we will do this once again. So maybe it would be good for anyone who's interested in running next year to just give them a heads up of what the process might be like. +**Adriana:** I think you even have to be like... -**Severin:** Yeah, sure. So what we normally do is we will send out a blog post, I think, end of September, early October, where we say like, hey, elections are coming up. If you want to run as a candidate, create a pull request against the community repository and have two people supporting you or saying like, yeah, I think that person is a good candidate. I can recommend that if you want to do that for the first time, wait for the first people to submit their nomination or self-nomination. Let’s be honest, most people self-nominate, and there’s nothing wrong with that. I do that. And just use that as a template and then think about like if you would be elected in the governance committee of Open Telemetry, which is like responsible for managing the whole project, for the health of the project, for the community as a whole, what are the things that you would do for the project, right? So the GC, just to call this out, this is less about like, oh, I want the collector to do x, y, and z, or I want to have, I don’t know, added some specific language to the Open Telemetry project. Of course, from a GC perspective, let's say you can engage in those conversations with those communities, but the GC is much more around like how can we grow and make our community stronger and better and how can we manage the project as a whole? So that means like how can we get projects added to what we are doing? How can we make sure that they're sustainable? How can we make sure that our project is thriving, right? So yeah, next year around the same time window, if you want to run, that’s where you should take a look. +**Severin:** Yeah, everybody can be nominated. I think you need two supporters and then you're in. I said it's closed for this year, but next year we will do this once again. Maybe it would be good for anyone who's interested in running next year to just give them a heads up of what the process might be like. -[00:19:50] **Adriana:** There you go. And for anyone who's planning for next year, I guess keep an eye on the nominees this year. See what, like you said, see what they've written for their nominations to get an idea of the types of things that get you elected. You're on the GC, right? +### [00:05:00] Guest introduction: Severin -**Severin:** Yeah. +**Severin:** Sure. What we normally do is we will send out a blog post, I think end of September, early October, where we say, "Hey, elections are coming up. If you want to run as a candidate, create a pull request against the community repository and have two people supporting you or saying, 'Yeah, I think that person is a good candidate.'" I can recommend that if you want to do that for the first time, wait for the first people to submit their nominations or self-nomination. Let's be honest, most people self-nominate, and there's nothing wrong with that. I do that. Just use that as a template and think about if you would be elected in the governance committee of OpenTelemetry, which is responsible for managing the whole project, for the health of the project, for the community as a whole, what are the things that you would do for the project? + +**Adriana:** The GC is less about, "Oh, I want the collector to do x, y, and z," or "I want to have added some specific language to the OpenTelemetry project." Of course, from a GC perspective, you can engage in those conversations with those communities, but the GC is much more about how can we grow and make our community stronger and better, and how can we manage the project as a whole so that means how can we get projects added to what we are doing, how can we make sure that they're sustainable, how can we make sure that our project is thriving. + +**Severin:** Next year, around the same time window, if you want to run, that's where you should take a look. + +**Adriana:** There you go. For anyone who's planning for next year, keep an eye on the nominees this year. See what, like you said, see what they've written for their nominations to get an idea of what gets you elected. + +**Severin:** You're on the GC, right? + +**Adriana:** Yeah. + +**Severin:** What gets you elected? I think it's that you show that interest in the community. At the end of the day, it's also like... -**Adriana:** So yeah, I think what gets you elected is like that you show that interest in the community, right? I mean at the end of the day, it’s also like— +**Adriana:** It's very hard to say that since I'm a member of the GC myself, so what I look for, but before I was a member of the GC, I always looked at what have those people done before that. Sure, putting on your nomination is one thing, like writing down, "Here's the things I want to do for the community," is one thing, but it's more about what have you done in the last few months to help people with that. -**Severin:** It’s very hard to say that since I'm a member of the GC myself. So what I look for, but before I was a member of the GC and before I was voting with this insight and perspective, I always looked at like what have those people done before that, right? Sure, putting on your nomination is one thing, like writing down, hey, here’s the things I want to do for the community is one thing. But yeah, it’s more about like what have you done in the last, I don’t know, few months to help people with that, right? +**Adriana:** From a voter perspective, it definitely helps if you've been visible in the community. It could be in a number of ways, right? Code contributions, talks, blog posts, being active in the channels, answering questions. -**Adriana:** Yeah, and I think from a voter perspective, it definitely helps if you've been visible in the community. And it could be in a number of ways, right? Code contributions, talks, blog posts, being active in the channels, answering questions. +**Severin:** That's maybe a very important point, right? You don't have to be part of the GC to do that. We need a lot of people to do a lot of work outside of writing all the code, outside of writing all the documentation. There are so many great things you can contribute to this project. I say it very often to people that want to break into the OpenTelemetry community or any open-source community. You have skills already. You could be a good marketer or designer or technical writer or a good developer, and think about the things that you bring to the table. That's helping you to be appreciated in the community. Don't think, "Oh, I have to be a good Go programmer to be valuable to the OpenTelemetry community." That's just not true. -**Severin:** Yeah. And that’s maybe a very important point, right? You don’t have to be part of the GC to do that, right? I mean, we need a lot of people to do a lot of work outside of writing all the code, outside of writing all the documentation. There are so many great things you can contribute to this project, and I say it very often to people that want to either break into the Open Telemetry community or in any open-source community. I mean, you have skills already, right? You could be a good marketer person, or you could be a designer, or you could be a good technical writer, or you could be a good developer, right? And think about the things that you bring to the table, and that’s helping you to be appreciated in the community. Don’t think like, oh, I have to be a good Go programmer to be valuable to the Open Telemetry community. That’s just not true. +**Bree:** That's a great point. Before we transition over to Marilia, you still had one more thing that you wanted to show us really quickly. I think you're going to show OpenTelemetry Unplugged. -**Adriana:** That’s a great point. And I think before we transition over to Marilia, you still had one more thing that you wanted to show us really quickly. I think you're going to show hotel unplugged. +### [00:22:40] Introduction of declarative configuration format -[00:23:20] **Severin:** Yeah, I have like the slide and another one. I'm not sure how we’re doing on timing. I'm talking a little bit long now. So I still have like one more. But let's see that we can get through that. I think people are more excited about the declarative configuration part, but people should be very excited about OpenTelemetry Unplugged. This is like an unconference we're planning for happening after FOSDEM next year. So Monday after FOSDEM, if you're still around, register for this event and then you can meet with the OpenTelemetry community. There's a blog post on that. I'm very excited about that one. And then last but not least, because people are asking about that, graduation is in progress. I can share a little bit more in the tech overview. So yeah, that’s on the community updates. Let me hopefully go into the technical highlights. +**Severin:** Yeah, I have this slide and another one. I'm not sure how we are doing on timing. I'm talking a little bit long now. I still have one more. Let's see that we can get through that. I think people are more excited about the declarative configuration part, but people should be very excited about OpenTelemetry Unplugged. This is like an unconference we're planning for happening after FOSDEM next year. So, Monday after FOSDEM, if you're still around, register for this event and you can meet with the OpenTelemetry community. There's a blog post on that. I'm very excited about that one. -**Severin:** This was the text I wanted to show first, right? There are so many updates. I will not touch into anything and everything that the hotel project is doing right now, right? So, this is almost impossible. But I picked like the things that showed up recently, and since I saw this question coming up very early on when we started this session today, it’s like how can I stay updated? The OpenTelemetry blog is a really good place, and we try to motivate also SIGs to put things there. For example, the profiling SIG will come out very soon with like, hey, we are moving towards alpha. I think that's a really great thing that we're seeing in the community. And equally, we had a blog post just recently about sampling that they're going to adopt the W3C trace context level two. That’s probably worth another What's New in Hotel. I try to do like a two-sentence summary of that. It’s very difficult, but in a long story to make a long story short, it's about mainly about randomness and trace ID to ensure that there's enough randomness in the trace ID, and that allows you to do certain things and sampling. I can, like if we have time in the Q&A, I can try to dive into that for a little bit. +**Severin:** Last but not least, because people are asking about that, graduation is in progress. I can share a little bit more in the tech overview. This was the text I wanted to show first, right? There are so many updates. I will not touch into anything and everything that the hotel project is doing right now. This is almost impossible. But I picked things that showed up recently. Since I saw this question coming up very early on when we started this session today is like how can I stay updated? The OpenTelemetry blog is a really good place, and we try to motivate also SIGs to put things there. For example, the profiling SIG will come out very soon with like, "Hey, we are moving towards alpha." I think that's a really great thing that we're seeing in the community. -**Severin:** The mainframe community has been doing a survey just recently, and I don't know if people know that we have a mainframe community with OpenTelemetry, and they just asked a bunch of questions to people on the mainframe like do you know about OpenTelemetry, how do you want to see it used? And they were talking about like, hey, we need good Python and Java support. We need to see that the OpenTelemetry collector is able to take metrics out of a mainframe, and of course the question about like what to do in COBOL. I have no more details into that, but it's a very exciting community that like, hey, even the mainframe, which is like a technology for how many years around, probably longer than I'm living, and they're looking into OpenTelemetry just to close this gap for end-to-end visibility. +**Severin:** We had a blog post just recently about sampling that they're going to adopt the W3C trace context level two. That's probably worth another What's New in Hotel. I try to do like a two-sentence summary of that. It's very difficult, but in a long story to make a long story short, it's about randomness and trace ID to ensure that there's enough randomness and the trace ID, and that allows you to do certain things in sampling. I can, if we have time in the Q&A, I can try to dive into that for a little bit. -**Severin:** Declarative configuration, I just have it here for completeness, looking forward to hear about this more myself. And then we have next Android road to stable. So they're working towards a version 1.0. They have done a ton of work over the last few months. So also very excited about that one. There's even a demo app now in the hotel demo, so you can even play around with that. I think that's a really good starting point into that. And they're looking for feedback, right? So go to the OpenTelemetry website, go to the OpenTelemetry blog. In that blog, there's a link to like, hey, what do you want the hotel Android community doing for their next release? +**Severin:** The mainframe community has been doing a survey just recently. I don't know if people know that we have a mainframe community with OpenTelemetry, and they just asked a bunch of questions to people on the mainframe like, "Do you know about OpenTelemetry? How do you want to see it used?" They were talking about like we need good Python and Java support, and we need to see that the OpenTelemetry collector is able to take metrics out of a mainframe. Of course, the question about what to do in COBOL. I have no more details into that, but it's a very exciting community. Even the mainframe, which is a technology for how many years around, probably longer than I'm living, and they're looking into OpenTelemetry just to close this gap for end-to-end visibility. -**Severin:** And then what I said about like we are working towards graduation. We met with the TOC of CNCF. There were a few adopter interviews we were discussing what are the things to go next. So there’s an issue on the CNCF TOC repository, it's number, I wrote it down 1739. You can look this up, and there's a summary of like what are the next steps, right? And one big thing, and this is maybe exciting because this is not new feedback for us, that people are looking for like how can we improve stability? How can we communicate stability better? And how can we look into ease of use? How can we make it easier for people to get started with OpenTelemetry, right? +**Severin:** Declarative configuration, I just have it here for completeness. Looking forward to hearing about this more myself. Then we have next Android road to stable. They're working towards a version 1.0. They have done a ton of work over the last few months. Also very excited about that one. There's even a demo app now in the hotel demo, so you can even play around with that. I think that's a really good starting point into that, and they're looking for feedback, right? Go to the OpenTelemetry website, go to the OpenTelemetry blog. In that blog, there's a link to like, "Hey, what do you want the OpenTelemetry Android community doing for their next release?" -**Severin:** And I said probably this slide alone would have covered like another 20 to 25 minutes if I would have talked about browser phase one, the bail donation, which is now called OBI, the injector weaver, hotel arrow, they're doing a new phase, we have a call for contributors for Kotlin, a lot of good changes in sigcom. So yeah, I tried to start with that, and I really hope we do those kinds of sessions more regularly so people can stay on top of things. So yeah, this is how you say like scratching the surface, and then let's dig deeper next time. So yeah, hope it's helping. +**Severin:** What I said about like we are working towards graduation, we met with the TOC of CNCF. There were a few adopter interviews. We were discussing what are the things to go next. There’s an issue on the CNCF TOC repository, it's number 1739, you can look this up. There's a summary of what are the next steps. One big thing, and this is maybe exciting because this is not new feedback for us, is that people are looking for how we can improve stability, how we can communicate stability better, and how we can look into ease of use, how we can make it easier for people to get started with OpenTelemetry. This slide alone would have covered another 20 to 25 minutes if I would have talked about browser phase one, the bail donation, which is now called OBI, the injector weaver OpenTelemetry Arrow, and there are a lot of good changes in SIGCOM. + +**Severin:** I tried to start with that, and I really hope we do those kinds of sessions more regularly so people can stay on top of things. This is how you say like scratching the surface, and then let's dig deeper next time. I hope it's helping. **Adriana:** This was amazing. -**Severin:** Oh, sorry. Go ahead. +**Severin:** Sorry, go ahead. -**Adriana:** No, I was going to say thank you so much. Like that was really helpful for me. +**Adriana:** I know. I was going to say thank you so much. That was really helpful for me. **Severin:** Yeah, definitely. I was going to say this proves that we need more of these. -**Adriana:** Yeah. Yeah. No, I mean I was learning a bunch of things while preparing. I was like, "Hey, let me ask around and let's talk with people like what should what's going on in your SIG? What’s happening?" Right? But I mean, our project is growing, and we are adding so many cool things to it. And yeah, I'm very excited about a lot of those things. Again, I only can recommend two things. Go to the OpenTelemetry blog from time to time. We have those kinds of announcements there. And if you're a maintainer or contributor, come to SIGCOMs and let us know about the latest and greatest of your SIG so we can share this with the wider world, right? +**Adriana:** Yeah. No, I mean I was learning a bunch of things while preparing. I was like, "Hey, let me ask around and talk to people like what should what's going on in your SIG? What's happening?" Our project is growing and we are adding so many cool things to it. I'm very excited about a lot of those things. Again, I can only recommend two things: go to the OpenTelemetry blog from time to time. We have those kinds of announcements there. If you're a maintainer or contributor, come to SIGCOMs and let us know about the latest and greatest of your SIG so we can share this with the wider world. -**Bree:** And again, if Severin covered a lot, if you have any questions, please put them in the chat of whichever platform you're tuning in from, and we will get to them after we speak with our next guest. Thank you so much, Severin. +**Bree:** If Severin covered a lot, if you have any questions, please put them in the chat of whichever platform you're tuning in from, and we will get to them after we speak with our next guest. Thank you so much, Severin. -**Severin:** Yeah, thank you. Speak to you in a minute. Thank you. +**Severin:** Yeah, thank you. Speak to you in a minute. -[00:29:00] **Bree:** And we are getting ready now to bring on Marilia Gutierrez to talk about what is she talking about? +**Bree:** Thank you. We are getting ready now to bring on Marilia Gutierrez to talk about... What is she talking about? Bree, remind me. -**Adriana:** Amazing. Hi, Marilia. +**Adriana:** Amazing. Hi Marilia. -**Marilia:** Hello everyone. Well, continuing on the theme of where you're tuning from, I'm also tuning from Toronto. So yeah, me and Adriana. +**Marilia:** Hello everyone. Continuing on the theme of where you're tuning from, I'm also tuning from Toronto. So yeah, me and Adriana. -**Adriana:** Yeah, and we're both Brazilians tuning in from Toronto, which is a coincidence. +**Adriana:** We're both Brazilians tuning in from Toronto, which is a coincidence. **Marilia:** Small subset. That's awesome. So Marilia does a lot in the community. Would you like to share some of the work that you do before you get into declarative configuration? -**Marilia:** Yes, of course. So yeah, that is one thing that I was even talking to colleagues yesterday that I keep finding more things in hotel, and I just trying to get involved, and I just like keep spreading the things that I work on. So for example, I am a maintainer for the contributor experience. So if you want to be a contributor and you're having some challenges, that would be the group that will help you with. I am an approver for the JavaScript SDK for Portuguese localization, database semantic conventions, and as of a couple of days ago, also for the communications SIG as well. +**Marilia:** Yes, of course. That is one thing that I was even talking to colleagues yesterday that I keep finding more things in hotel and I just try to get involved and I just keep spreading the things that I work on. For example, I am a maintainer for the contributor experience. If you want to be a contributor and you're having some challenges, that would be the group that will help you with. I am an approver for the JavaScript SDK, for Portuguese localization, database semantic conventions, and as of a couple of days ago, also for the communications SIG as well. **Adriana:** Yay. Amazing. -**Marilia:** Dang, that is so productive. - -**Adriana:** No, you're an inspiration. +**Bree:** Dang, that is so productive. **Marilia:** Thank you. **Adriana:** Seriously. -**Marilia:** Yeah. So, even like for this one that we were like, okay, we need to pick one topic to go into deep, like so many things like, okay, I was working on this, I'm working on that. Let's pick one. But I feel like the declarative config is something that is really cool that is getting to a very almost stable place, and we're going to see this implementation in several SDKs now. So I think it's a really cool one to talk about. +**Marilia:** Even for this one, we were like, okay, we need to pick one topic to go into deep. There are so many things. Okay, I was working on this. I'm working on that. Let's pick one. I feel like the declarative config is something that is really cool that is getting to a very almost stable place, and we're going to see this implementation in several SDKs now. I think it's a really cool one to talk about. -[00:31:30] **Adriana:** Cool. So yeah, let's start. So before I start with the config, I feel like I should give a little context like what is this? Why do we need this? What are we talking about? So how you would set up your SDK, you’re like you're using hotel and you want to set up some specific things for your case, things like even like my server's name or like what is going to be my tracer exporter. So how it was done up until now is basically using environment variables, and it was very easy to do this because environment variables are just universally available on all languages. So it doesn't matter the SDK, there was like some way to do this. But then more complex things came up and like oh I need this, and we just published a blog post yesterday talking about the story of the declarative config. There was an issue created five years ago on the Java SIG like I just want to be able to filter out health checks traces. I don't want to keep sending this. There is no easy way to just like filter that specific one. Like I have my sampler, but how do I pass what is the thing that I want to filter? So that started to get very complex to do just with an environment variable, so we needed something more robust. So that is the idea that came for, let's work on with the declarative config. +### [00:30:56] Guest introduction: Marilia -**Marilia:** So at this point, we now disallowed environment variables just so we don't keep like adding more when we have to transition, and then we started with now declaring config. So this is pretty much a YAML file, and you can just put all the things that you care about on that one. It is a little more robust when just compared to environment variables. It's easier to expand because you can have like all the objects and types that you need there. And because this was a project, of course, I'm the one talking about it today, but there were several people that worked on this for several months, and now we're just on the phase of really implementing on several SDKs. But the process of actually creating the convention, there is this own repo specific for this one; it takes a lot of time to get feedback from people and see what's going to work with all the languages. +**Bree:** Cool. So yeah, let's start. Before I start with the config, I feel like I should give a little context: what is this? Why do we need this? What are you talking about? How would you set up your SDK? You're using hotel and you want to set up some specific things for your case. Things like even my server's name or what is going to be my tracer exporter. How it was done up until now is basically using environment variables. It was very easy to do this because environment variables are universally available in all languages. It doesn't matter the SDK; there was some way to do this. -**Marilia:** And also, people have concerns like I'm not going to use environment variables anymore, but I'm used to them. So you can still have environment variables on your config file. So I can put an example here like how do you use this? And like if I have this, use it; otherwise, use this other thing. And to just start using, which is available today in Java and also starting on JavaScript, I do recognize the irony that you have to set up the environment variable config file to use the config file. I do see the irony in that. Let's say like don't use my environment variables, use the file. But that's how you do it. And then from that means like the SDKs will be able to just use your config file instead of having to set up all the environments. So if you're curious, this is how a basic one looks like. +**Marilia:** But then more complex things came up, and like, "Oh, I need this." We just published a blog post yesterday talking about the story of the declarative config. There was an issue created five years ago on the Java SIG like, "I just want to be able to filter out health checks traces. I don't want to keep sending this." There was no easy way to just filter that specific one. I have my sampler, but how do I pass what is the thing that I want to filter? That started to get very complex to do just with an environment variable, so we needed something more robust. That is the idea that came for, "Let's work on the declarative config." -**Marilia:** So here you see like the YAML. So we have like some resources. Here we have like an example that still points to environment variable if you care. And then I have like some examples like tracer provider. I only care about the endpoints. So this is all anything that I put in here. And then here, the example that I mentioned for like the issue that was created, like I just want to filter my health checks. This is an example how you do this. In Java, it's already working, and you can see here just like a sampler, and I say like my rule, I want to drop if I have these attributes with this pattern. So it's a lot easier than having to actually go into the code or your instrumentation, trying to like create your sampler or like trying to have this environment somehow. Now you just create this object, and that's it. You don't have to care about like how the SDK would do this. We are the ones that instrumented, like doing that for you, so you don't have to do it. +### [00:32:52] Discussion on declarative configuration implementation -**Marilia:** And of course, there's a lot of things like I'm just trying to like put a few screenshots. This can be quite big. So you can see here like, oh, I have a processor, I have a batch with this schedule with this timeout, my exporter is this type, and it's a good one that we have a file called kitchen sync on that repo that every single spec that is created is added to this file, which is the one that I'm showing here. So if you want to see examples of everything you have here, and it has the comments of what it means, what are the values, what are like the default values if you don't put them. So it's a great one, and we also have one for migration. If you’re like I just want the basic because I'm using my environment variables and I don't want to have to like copy. So you just copy that file, and it's still going to use your environment variables for some basic stuff. +**Marilia:** At this point, we now disallowed environment variables just so we don't keep adding more when we have to transition, and then we started with now declaring config. This is pretty much a YAML file, and you can just put all the things that you care about in that one. It is a little more robust compared to environment variables. It's easier to expand because you can have all the objects and types that you need there. This was a project, of course, I'm the one talking about it today, but there were several people working on this for several months, and now we're just in the phase of really implementing it in several SDKs. -**Marilia:** If you want to know what is available today, we have this matrix of compliance, which Java is fully compliant, PHP, JavaScript is partially, and we do have a lot of other languages that are also working on it. I think we just need to update this for a few of them, like for example Go and I believe Python are also have some implementation. And when I say like it's fully compliant, it does not mean that it's ended; it's like complete. That is not what it means. There are still features that still need to be added, but at least the basic of what you're seeing here is working for those languages. +**Marilia:** The process of creating the convention is its own repo specific for this one. It takes a lot of time to get feedback from people and see what's going to work with all the languages. People have concerns like, "I'm not going to use environment variables anymore, but I'm used to them." You can still have environment variables in your config file. I can put an example here: how do you use this? If I have this, use it; otherwise, use this other thing. To just start using it, which is available today in Java and also starting in JavaScript, I do recognize the irony that you have to set up the environment variable config file to use the config file. I do see the irony in that. -**Marilia:** And here I can probably copy this on the chat, and we can share, but here I put a few links for resources. So the blog post that I mentioned, which is in English and in Portuguese, that is like, see one of the cool things about localization that I already created like both of them. I have the file like the repo for the configuration and the examples that I mentioned. So the kitchen sync, the migration itself, and if you have questions about the config file in general, you can use the Slack channel. So the hotel config file, if you want to have specific for the SDK. So for example, how is this working on the Java one or the JavaScript one. So then you go to those specific channels, so like hotel Java, JavaScript, Go, and so on. And yeah, that's what I have for an overview for today. +**Marilia:** Let's say, "Don't use my environment variables, use the file." But that's how you do it. The SDKs will be able to just use your config file instead of you having to set up all the environments. If you're curious, this is how a basic one looks like. Here you see the YAML. We have some resources. Here we have an example that still points to an environment variable if you care. I have some examples like tracer provider. I only care about the endpoints. This is all anything that I put in here. -**Adriana:** This is great. Now I had a follow-up question for you, Marilia. So these configurations, it sounds like they are language dependent. So you do them at the language level, not at the collector level. Is that correct? +**Marilia:** The example that I mentioned for the issue that was created: "I just want to filter my health checks." This is an example of how you do this. Java is already working, and you can see here just like a sampler, and I say, "My rule, I want to drop if I have these attributes with this pattern." It's a lot easier than having to go into the code or your instrumentation trying to create your sampler or trying to have this environment somehow. Now you just create this object, and that's it. You don't have to care about how the SDK would do this. -**Marilia:** So the, so one thing is supposed to be like completely agnostic. So you can use the same file; it doesn't matter which SDK you're using. But there are parts that are also for the collector. So for example, the collector is already using the declarative config for a couple of things. +**Marilia:** Of course, it has a lot of things. I'm just trying to put a few screenshots. This can be quite big. You can see here like, "Oh, I have a processor. I have a batch with this schedule with this timeout. My exporter is this type." We have a file called kitchen sink on that repo that every single spec that is created is added to this file, which is the one that I'm showing here. If you want to see examples of everything you have here, it has comments of what it means, what are the values, what are like the default values if you don't put them. It's a great one, and we also have one for migration. If you're like, "I just want the basic because I'm using my environment variables and I don't want to have to copy," you just copy that file, and it's still going to use your environment variables for some basic stuff. -**Adriana:** Right. +**Marilia:** If you want to know what is available today, we have this matrix of compliance: Java is fully compliant; PHP, JavaScript is partially compliant. We do have a lot of other languages that are also working on it. I think we just need to update this for a few of them, like for example Go and Python are also having some implementation. When I say it's fully compliant, it does not mean that it's ended; that is not what it means. There are still features that still need to be added, but at least the basic of what you're seeing here is working for those languages. -**Marilia:** So it's pretty much like already added there. I don't have a list with me easily to just show what are the things that are working, but the idea is to have all components will be able to have the declarative config; it's just a matter of now we need to go and implement on all those places. +**Marilia:** Here I can probably copy this in the chat and we can share, but I put it a few links for resources: the blog post that I mentioned, which is in English and in Portuguese. That is one of the cool things about localization that I already created, like both of them. I have the file, the repo for the configuration, and the examples that I mentioned: the kitchen sink, the migration itself, and if you have questions about the config file in general, you can use the Slack channel, so the hotel config file if you want to have specifics for the SDK. For example, how this is working on the Java one or the JavaScript one. -**Adriana:** But does like how does the config file get loaded up? Does it get loaded like as part of, yeah, I guess I'm having trouble like understanding where in the flow it fits in. +**Bree:** That's awesome. -**Marilia:** Are you talking specifically about the collector or in general? +**Marilia:** Yeah, that's what I have for an overview for today. + +**Bree:** I had a follow-up question for you, Marilia. These configurations, it sounds like they are language-dependent. So you do them at the language level, not at the collector level. Is that correct? + +**Marilia:** So one thing is supposed to be completely agnostic. So you can use the same file; it doesn't matter which SDK you're using. There are parts that are also for the collector. For example, the collector is already using the declarative config for a couple of things. + +**Bree:** Right. + +**Marilia:** It's pretty much already added there. I don't have a list with me easily to just show what are the things that are working, but the idea is to have all components able to have the declarative config. It’s just a matter of now we need to go and implement it in all those places. -**Adriana:** Oh no, just in general, like the declarative config, like for the environment variables. +**Bree:** How does the config file get loaded up? Does it get loaded as part of...? -**Marilia:** So for example, I can talk about like the JavaScript, which is the one that I'm working on. So when you initialize the SDK, usually you do like for example new SDK, and sometimes you have to like pass parameters. This time, you’re not going to need to do this. So your file is going to exist on the same place that you're adding your instrumentation. +**Marilia:** Are you talking specifically about the collector or in general? + +**Bree:** Oh, no, just the declarative config, like for the environment variables. -**Adriana:** So it picks it up from, does it pick it up like from as long as you have that file in a particular directory in your code, it'll pick it up? Is that the idea? +**Marilia:** For example, I can talk about the JavaScript, which is the one that I'm working on. When you initialize the SDK, usually you do like, for example, new SDK, and sometimes you have to pass parameters. This time, you're not going to need to do this. Your file is going to exist in the same place that you're adding your instrumentation. -**Marilia:** Yeah, because the environment variable points to the file. So as long as you put the like this is the path to the file. So when you initialize the SDK, it's going to check like do you have set a config file? If you don't, it's gonna ignore and use whatever you were doing before. But if it has, it's gonna say, "Okay, let me check what you have on that file." And for some things, it's pretty much saying like if you have this value, I'm going to use it; otherwise, it has default values for a lot of things. So it's basically that is the moment when you initialize the SDK that it's going to read and do whatever it needs to do. +**Bree:** Does it pick it up from... as long as you have that file in a particular directory in your code, it'll pick it up. Is that the idea? -**Adriana:** Okay. Okay. That makes a lot of sense. This is cool because I guess it also lets you, I guess if you're using like just the old way of like just the environment variables, that was very flat, right? And this kind of gives it a little bit more structure too. So it kind of lets you like lump configurations together, which is very cool. +**Marilia:** Yeah, because the environment variable points to the file. As long as you put this is the path to the file. When you initialize the SDK, it's going to check like, "Do you have a config file?" If you don't, it's going to ignore and use whatever you were doing before. But if it has, it's going to say, "Okay, let me check what you have on that file." For some things, it's pretty much saying, "If you have this value, I'm going to use it, otherwise it has default values for a lot of things." It's basically that. The moment when you initialize the SDK, that's when it's going to read and do whatever it needs to do. -**Marilia:** Yeah, because we have a section for example this works for everybody. So it's supposed to be like very agnostic. So now my application is in Python, and I want to use the same. But of course, there are things that are very specific for languages, like this like agents only exist in Java. So, but oh, but I still want to configure this. So the config file has sections for languages as well. So if you have like this is specific for Java, do this. We have those types of things there as well. +**Bree:** Okay. That makes a lot of sense. This is cool because I guess it also lets you... I guess if you're using just the old way of just the environment variables, that was very flat, right? This kind of gives it a little bit more structure too. It lets you lump configurations together, which is very cool. -**Adriana:** That's awesome. Bree, do you have anything that you wanted to chime in on? +**Marilia:** Yeah, because we have a section for everybody. It's supposed to be very agnostic. Now my application is in Python; I can use the same. But of course, there are things that are very specific to languages, like agents only exist in Java. But, oh, I still want to configure this. The config file has sections for languages as well. If you have, like, this is specific for Java, do this. We have those types of things there as well. -**Bree:** I'm still processing. +**Bree:** That's awesome. Ree, do you have anything that you wanted to chime in on? I'm still processing. -**Marilia:** It's so cool, right? Like my mind is blown. I'm like where were you like five years ago? +**Ree:** It's so cool, right? My mind is blown. I'm like, where were you like five years ago? -**Marilia:** Yeah. And the idea is even like for some of the things you don't have to like restart some of the components and can just like read as is. So that is also an advantage as well. But yeah. +**Marilia:** Yeah. The idea is even that for some of the things, you don't have to restart some of the components, and can just read as is. That is also an advantage as well. -**Bree:** Well, I think we can get right into some Q&A because I do have questions, and I think our audience has some questions. So Lisa, we would love to bring you on. +**Bree:** I think we can get right into some Q&A because I do have questions, and I think our audience has some questions. So Lisa, we would love to bring you on. -**Lisa:** Hi, Lisa. +**Lisa:** Hi, Lisa. -**Lisa:** All right, thank you so much. Hello everybody. I'm Lisa Jung, a staff developer advocate at Grafana Labs. I'm part of the communications and the end-user SIG. So yeah, now we are gathering questions from the audience and asking our lovely feature speakers here. So the first question is from Cecil. So he said if you have both options, environment variables or the configuration in a project, which one takes precedence? +### [00:41:56] Audience Q&A session begins -**Marilia:** So if you do have the environment variable saying like this is my config file, that is the one that takes precedence, and it's going to use this, and it's not going to use the environment variable as a backup at all. It's going to ignore that unless you have your config file had the environment variable on that file. But that takes precedence. If you don't have that environment variable set up, then it uses, even if you have the file existing somewhere, it doesn't know. So it's going to ignore and use environment variables. +**Lisa:** All right, thank you so much. Hello everybody. I'm Lisa Jung, a staff developer advocate at Grafana Labs. I'm part of the communications and the end-user SIG. Now we are gathering questions from the audience and asking our lovely feature speakers here. The first question is from Cecil. He said if you have both options, environment variables or the configuration in a project, which one takes precedence? -**Lisa:** Gotcha. Thank you. And then we have a question from Kieran. Russ is still a four-letter word. Long way to go. Question mark. +**Marilia:** If you do have the environment variable saying, "This is my config file," that is the one that takes precedence, and it's going to use this. It's not going to use the environment variable as a backup at all. It's going to ignore that unless you have your config file that has the environment variable on that file. But that takes precedence. -**Marilia:** For specific for the declarative config, I believe this question came up during your— +**Lisa:** Gotcha. Thank you. Then we have a question from Kieran. "Rust is still a four-letter word. Long way to go?" -**Lisa:** Okay. +**Marilia:** For the declarative config, I believe this question came up during your... -**Marilia:** So yeah, it is. So that is the thing because when people are creating the semantic convention and stuff, they try to get people from different languages so you know what's going on. And then you go to your own SIG and start implementing, but a few of the SIGs have very small groups, so it's hard sometimes for them to handle because they have to review PRs that come in and do releases, so it's hard sometimes to do and be able to keep up with everything going on. The same thing like database and conventions came out; a lot of SDKs still don't have implementation. So if you, for example, really care about REST, that is your chance to join the SIG and actually be a contributor. So people are really, all the materials are always happy to help and have people joining in. You can say like this is something that I'm interested in, and I really care about how do I start contributing? And it's a way for you to be able to have things faster that you want, and if your goal is to be like an approver or maintainer for small things, that is going to happen faster because your contribution is going to have a lot of weight if there are not a lot of people helping out. So that is something also to keep in mind. +**Marilia:** It is hard sometimes for the SIGs to handle because they have to review PRs that come in and do releases. It's hard sometimes to keep up with everything that's going on. The same thing like database conventions came out. A lot of SDKs still don't have implementation. If you really care about REST, that is your chance to join the SIG and actually be a contributor. People are always happy to help and have people joining in. You can say, "This is something that I'm interested in, and I really care about how do I start contributing?" It's a way for you to be able to have things happen faster because your contribution is going to have a lot of weight if there are not a lot of people helping out. -**Lisa:** Gotcha. +**Lisa:** Gotcha. All right. We have a question for Severin. What does W3C trace context L2 for tail sampling bring to the table? -**Lisa:** All right, we have a question for Severin. So what does W3C trace context L2 for tail sampling bring to the table? +**Severin:** Yeah, I said I had to take some notes on that myself because it's a very complex topic, and I'm also not a sampling expert, but I try my very best. We've been doing sampling in OpenTelemetry for I think over four years now, and it has been evolving. We've added a lot of good things. One of the things that we had is this trace ID ratio-based sampler. This sampler always had this to-do where we said, "Hey, all you can do safely with that is sampling on root spans." You had not a lot of good guarantees on probability downstream. -**Severin:** Yeah, I had to take some notes on that myself because it's a very, very complex topic, and I'm also not like a sampling expert, but I try my very best, right? So we have been doing sampling in OpenTelemetry for I think over four years now, and it has been evolving, and like we added a lot of good things. One of the things that we had is this trace ID ratio-based sampler, right? And this sampler had always like this is just me quoting straight from that blog post. It always had this to-do where we said like, hey, all you can do safely with that is sampling on root spans, right? So you had not a lot of good guarantees on probability downstream, right? Because what's happening is that like you use the trace ID as a random n-sized word, like you use some part of the trace ID, and then you compare it to like, let's say a boundary that’s like 2 to the ^ of n times the ratio. So you can think about it like if you take 50%, it’s like I don't know 128. If you have six, seven, six bits, something like that. And if the number is smaller or bigger, then it comes in or does not come in. But the big problem is that in the trace ID until level two, there were no guarantees on how many of those bits are truly random, right? Or are maybe encoding some additional information, right? And this is what has been changed in that new standard, and OpenTelemetry is adopting that, right? So you send an additional flag when you use that, and with that, you get 56 bits of sufficient randomness, and you can use that, right? So again, this is me just talking from a few notes that I took from the blog post. I can highly recommend reading into that. I think if we talk about sampling, we all know that like with the amount of data that people are consuming these days with OpenTelemetry, this is urgently needed, and they need a lot of good hands to try out those things. So give this a look, and yeah, try it out. +**Severin:** The big problem is that in the trace ID until level two, there were no guarantees on how many of those bits are truly random or are maybe encoding some additional information. This is what has been changed in that new standard and OpenTelemetry is adopting that. You send an additional flag when you use that, and with that, you get 56 bits of sufficient randomness, and you can use that. Again, this is me just talking from a few notes that I took from the blog post. I can highly recommend reading into that. If we talk about sampling, we all know that with the amount of data that people are consuming these days with OpenTelemetry, this is urgently needed, and they need a lot of good hands to also try out those things. -**Lisa:** Gotcha. And to piggyback on that, Henrik, what about consistent probabilistic sampling? +**Lisa:** Gotcha. To piggyback on that, Henrik, what about consistent probabilistic sampling? -**Severin:** I don't know. Very, very honest. I'm not exactly sure. So that's like maybe a good thing for the next round, right? Maybe invite someone from the sampling SIG and have them talk about that specifically, right? It looks like there's some interest in that. +**Severin:** I'm not exactly sure, so that's maybe a good thing for the next round, right? Maybe invite someone from the sampling SIG and have them talk about that specifically, right? It looks like there's some interest in that. **Lisa:** Yeah, absolutely. So Marilia, Fernando has a question for you. Can I have more than one config file, like a file with generated configs or another with only specific JavaScript configs? -**Marilia:** So the file itself is going to be one because otherwise, we don't know which one takes precedence. But for example, the part that I mentioned that are specifics to language is still part of the same file. So you have, for example, like oh, like tracer exporter, like resources, and then there is a section that we call like you should put at the bottom that like, okay, now it's specific, like some resources and stuff for like JavaScript for Java. So that would be the parts that you have that are specific for JavaScript. At the moment, there's nothing specific for JavaScript, so you wouldn't have that at the moment. But if you have things different for different applications, you need one file for each one. So this is something like, for example, that you can take advantage of some things environment-variable-related. Imagine that you have, oh, it’s exactly the same thing, but I want to have the service name different on those two. So what you do then, your config file points just to say like use the environment variable for this case, and then you have your two environments that use the different name. +**Marilia:** The file itself is going to be one because otherwise we don't know which one takes precedence. For example, if the part that I mentioned that are specific to language is still part of the same file. If you have something that you want to have different for different applications, you need one file for each one. This is something like, for example, that you can take advantage of some things related to environment variables. Imagine that you have the exact same thing, but I want to have the service name different on those two. What you do then, your config file points just to say like, "Use the environment variable for this case," and then you have your two environments that use the different name. **Lisa:** Gotcha. Another question from Henrik for Marilia. Would the OTL operator provide a config CRD that could be deployed across Kubernetes objects? -**Marilia:** Not sure because I don't see like for example a reason to add new config because the idea is to have all possible config that we want to exist on that file. It's just a matter of like does it exist already or something that needs to be added because it's a little more also about getting now feedback from the community just because we have something a little more stable and say like what is missing and what we can add there. So those are the points that we need feedback from people. I can check this one specifically, but from the top of my mind, I don't know the answer if we have something already for this one. +**Marilia:** Not sure because I don't see a reason to add new config. The idea is to have all possible config that we want to exist in that file. It's just a matter of getting feedback from the community because we have something a little more stable and say like, "What is missing and what we can add there?" Those are the points that we need feedback from people. I can check this one specifically, but from the top of my mind, I don't know the answer if we have something already for this one. + +**Lisa:** Gotcha. We have a question from Cecil, so please feel free to jump in, Severin or Marilia. Is there any guidance you can share around OpenTelemetry for long-running background processes? + +**Severin:** Yeah, I can pick that. This is actually a very complicated topic, right? I just looked it up. There's an issue on the OpenTelemetry specification from 2019 since the project was founded by Armen around that topic. The very first guidance I can give is go to that issue, to that repository, and call out that you're interested in that, and maybe even interested in helping with that. It's a very gnarly topic. + +**Severin:** What can you do until then? I mean, at the end, it depends a little bit on what does "long" mean to you. If we talk about spans and traces taking a few minutes, then we are totally fine. You should be just fine with the tracing that you're doing today. But if you say, "Hey, this is the classic thing," you have a background job, and maybe this is doing some video encoding, and maybe you want to do something around that. One thing you definitely can do today, right, is use logging and events for that and maybe announce those kinds of things and say, "Okay, now I'm starting, now I'm ending." It’s not perfect, and let's be super honest with that, and there are probably still things you need to do on your back end. You have to work around that and just let us know that we need to do more on that topic. + +**Lisa:** Thanks, everyone. As we wrap up our Q&A, I'm going to ask one question to both Severin and Marilia. You've been contributing to hotel for a long time. I'm going to take you all the way to the beginning. How did you get your start in contributing to hotel? Do you have any advice for new contributors? + +**Marilia:** Who goes first? You want to start, or should I start? -**Lisa:** Gotcha. +**Severin:** You can start. -**Lisa:** We have a question from Cecil. So please feel free to jump in, Severin or Marilia. Is there any guidance you can share around OpenTelemetry for long-running background processes? +**Marilia:** Yeah. My reason to start with OpenTelemetry was a very selfish reason. This is something I always tell people: if you want to get into open source, have your own good reason to start with open source. -**Severin:** Yeah, I can pick that. So this is actually like a very complicated topic, right? And I just looked it up. There's an issue on the OpenTelemetry specification from 2019. So since the project was founded by Armen around that topic, and the first guidance I can give is— +**Severin:** Have something where you say, "I don't know; I want sampling to be better in OpenTelemetry because I have way too much telemetry, so let me get started with sampling and then make this thing work." -**Marilia:** Yeah, no worries. +**Marilia:** My situation back then was like I was in a consulting position, and I was talking with a lot of people about what we back then called APM. We were still transitioning to call everything observability. People had questions about OpenTelemetry. They told me like, "Hey, what is this thing? Should I use that or should I stick with this thing I have already?" This was something where I recognized, "Hey, I have to level up on that." -**Severin:** Now, I lost track for a minute. Yeah, so the very, very first guidance I can give you is like go to that issue, go to that repository, and call out that you're interested in that and maybe even interested in helping with that. It's a very gnarly topic, right? So what can you do until then, right? I mean at the end it depends a little bit on what does long mean to you, right? I mean if we talk about if we talk about spans and traces taking a few minutes, then we are totally fine. You maybe should be just fine with tracing that you're doing today. But if you say like, hey, and this is the classic thing, right? You have a background job, and maybe this is doing some video encoding and maybe you want to do something around that. One thing you definitely can do today, right, is use logging and events for that and maybe announce like those kinds of things and say like, okay, now I'm starting, now I'm ending. It's not perfect, right? And let’s be super honest with that, and there’s probably still a lot of things you need to do on your back end of things. But yeah, you have to work around that and just let us know that we need to do more on that topic. So that's the guidance here. +**Marilia:** I went to the community and said, "Let me get started on that. Let me learn myself what this OpenTelemetry thing is so that I can speak better to my users, to our customers, to people that want to get into our product." I started to help with documentation, help us getting started. I started actually also in the JavaScript community, but then I stuck with documentation because again, this was the most helpful thing for myself to just explain OpenTelemetry better to other people. -**Lisa:** Thanks, everyone. All right, I need to stop just reading straight. I need to process a little bit more before reading the questions out loud. So sorry about that. +**Severin:** For me, it was basically I had been working on observability at a few companies before, and I was like, "Oh, I like this observability thing, and I want to focus on that." I was looking for a job that basically is my job. I only have to think about the observability, and this is why I went to Grafana, that the team was like, "You have to contribute to hotel." That was exactly what I wanted. -**Lisa:** Okay. So as we wrap up our Q&A, I'm going to ask one question to both Severin and Marilia. So, you've been contributing to hotel for a long time. So, I'm going to take you all the way to the beginning. How did you get your start in contributing to hotel? And do you have any advice for new contributors? Who goes first? You want to start or should I start? +**Severin:** I do actually have, I was going to say, we just created a post that I can share here. There are ideas on how to contribute because if you wanted again, you can have several reasons for it, or you're just working on something because your company needs or something your own personal project needs, and you want to develop something you just want to learn. You want to increase your network. Pick something that really interests you like, "I already have knowledge and I want to focus on that," or something like, "I'm just curious about this thing I don't know, but I want to learn." You can have both ways and really dig deep into those ones. OpenTelemetry is a huge project with so many components and languages. You will find it; it's just a matter of finding what is your place there, and you can try it out. There is no thing like, "I picked this. Now I have to go through this one until the end." No, join a couple of calls, just listen in. You don't have to say anything like, "Oh, this thing seems interesting." Continue a little more; if no, this is not for me, go to the next one until you find what is the thing that really excites you. -**Marilia:** You can start. +**Lisa:** Nice. Thanks, Marilia. Severin, Adriana, and Bree have been amazing guides and mentors when I was first getting started with OpenTelemetry. -**Severin:** Yeah. Yeah. Let me get started. So my reason to start with OpenTelemetry was a very selfish reason, right? And this is something I always tell people, right? If you want to get into open source, have your own good reason to start with open source, right? Have something where you say like I don't know, I want sampling to be better in OpenTelemetry because I have way too much telemetry, so let me get started with sampling and then make this thing work. So my situation back then was like I was in a consulting position, right? And I was talking with a lot of people about what we back then called APM, and we were still transitioning to call everything observability, and people had questions about OpenTelemetry, right? They told me like, hey, what is this thing? Should I use that or should I stick with this thing I have already? And this was like every time like something where it was like recognizing like, hey, I have to level up on that, right? So I went to the community and I said like okay let me get started on that. Let me learn myself what this OpenTelemetry thing is so that I can speak better to my users, to our customers, to people that want to get into our product. So I started to help with documentation, help us getting started. I started actually also in the JavaScript community, but then I stuck with documentation because again, this was the most helpful thing for myself to just explain OpenTelemetry better to other people. So yeah, for me, it was basically I have been working on observability in a few companies before, and I really liked this observability thing, and I really wanted to focus on that. So I was looking for a job that is basically my job. I only have to think about like the observability, and this is why I went to Grafana, that the team was like you have to contribute to hotel, so it was like oh exactly what I wanted. So this is kind of how I started contributing. So it was a mix of things that I found interesting, like oh I think this is important, I should contribute it, and a mix of what also makes sense for my own company. +**Lisa:** Now, there's one more burning question for both of you. The question is, what kind of interesting usage trends in terms of implementation, observability stack, docs page views, or anything have you seen over the past year? -**Marilia:** And I do actually have, I was going to say, we just created a post that I can share here. There are ideas on how to contribute because if you wanted again, you can have several reasons for it, or like you're just working on something because your company needs or something your own personal project needs, and you want to develop something that you just want to learn. You want to increase your network; people that you want to meet. So pick something that really interests you, like oh I already have knowledge, and I want to focus on that, or something like oh, I'm just curious about this. I don't know what this is, but I want to learn. You can have both ways and really dig deep into those ones. OpenTelemetry is a huge project, and there are so many components, so many languages, you will find. It's just a matter of finding what is your place there, and you can try it out. There is no thing like I picked this; now I have to go through this one until the end. Like no, join a couple of calls, just listen in. You don’t have to say anything. Like oh this thing seems interesting, like continue a little more; oh no, this is not for me, go to the next one until you find what is the thing that really excites you. +**Marilia:** I think this is more of like whatever people did not think was necessarily important before to have observability on. They're like, "Oh yeah, that would be good if I actually know what the hell was going on." You're going to find a little of everything. People are really now until mobile, like nobody was paying attention or the browser one that's been really active. It's just like what we forgot to add it, okay, let's work on all this thing because it's really important. So it's just like all over. You're going to find from every single place, and even things that people do for fun, like I want to monitor the quality of the air and humidity on my plants. There are people that do this to say, "Oh no, I'm actually working at NASA and want to know what's going on." -**Lisa:** Nice. Thanks, Marilia. +### [00:58:08] Audience question on long-running background processes -**Lisa:** So Severin, Marilia, Adriana, and Bree have been amazing guides and mentors when I was first getting started with OpenTelemetry. +**Severin:** I can try to make it quick because there are so many things I could now answer that I was just thinking about it. I had five or ten things in my head, but let me pick a few. One thing is, and this may be close to like, "Hey, let's measure the humidity in our room." I saw more and more databases being used. Observability is inside the database. I know that now people are thinking, "Why not do tracing inside the database?" It’s something I find very funny. Similar to what I mentioned before, right? Your Docker build. -**Severin:** Now we— I lied; there's one more burning question for both of you. So the question is, what kind of interesting usage trends in terms of implementation, observability stack, docs page views, or anything have you seen over the past year? +**Severin:** You can put OpenTelemetry into just everything and get amazing data. The other thing, and then I tried to make it quick so we can get to an end, is I think people start to recognize that with OTLP, they can send their data everywhere. I mean, Adriana, you wrote about that. Don't send your traces, logs, and metrics into different places. That's not what I'm talking about. It's more like, "Wait a minute, I can send a bunch of my data over into that back end, and I can send maybe another portion of that data into another specific tool." -**Marilia:** I think this is more of like whatever people did not think it was necessarily important before to have observability on, they're like, oh yeah, that would be good if I actually know what the hell was going on. So I think—and then just like time, people keep adding more and more. So it's a lot of old type of applications that people did not think about like oh yeah I should add it, but also new things. But I guess you're going to find a little of everything like people like really now until like mobile, like nobody was paying attention or like the browser one, that's being really active. So it's just like what we forgot to add it. Okay, let's work on all this thing because it's really important. So it's just like all over. So you're going to find from every single place and then even things that people do for fun like I want to monitor the quality of the air and humidity on my plants. There are people that do this to like oh no I'm actually working at NASA and want to know what's going on. So you're going to find so much spread. +**Severin:** People are starting to recognize they have a lot of freedom with the data, right? They have a lot of freedom with the observability data beyond what observability vendors offer today or what we do with trace explorers, alerting, and all the capabilities that we had before. This is actually a topic I'm very curious and excited about right now that we say, "Wait a minute, now we have all this data. What else can we do with that?" I think that's super, super interesting. -**Severin:** Yeah, I can try to make it quick because there's so many things I could now answer that I was just thinking about it like yeah I had like five or ten things in my head. But let me pick a few. Like the one thing is like and this may be close to like hey let’s measure like the humidity in our room. I saw more and more also like databases being used, observability inside the database, right? I mean we saw like hey tracing going up down to the database, but I know that now people are thinking about like why not do tracing inside of the database, right? I mean that's something I found very funny. Similar to what I mentioned before, right? Your Docker build. So this is just this thing where we need to recognize like every software can be traced, logged, and metrics. Is there even a verb? I don't know. But I think you get the gist, right? So you can put OpenTelemetry into just everything and get out amazing data. And the other thing, and then I tried to make it quick so that we can get to an end is like I think people start to recognize that with OTLP, they can send their data everywhere, right? I mean, Adriana, you wrote about that. Don’t send your traces, logs, and metrics into different places. That's not what I'm talking about. It's more like hey, wait a minute. I can send a bunch of my data over into that back end, and I can send maybe another portion of that data into another specific tool. And people start to recognize like they have a lot of freedom with the data, right? They have a lot of freedom with the observability data beyond, let's say, what observability vendors offer today or what we do with trace explorers, alerting, and all the capabilities that we had before. This is actually a topic I'm very, very curious and excited about right now that we say like wait a minute now we have all this data, what else can we do with that, right? I think that's super, super interesting. +### [01:00:04] Closing remarks and future events -**Lisa:** Thank you. I—I—yeah, it has been really interesting. And thank you all so much for following along. If you have additional questions, please find us on the CNCF Slack instance at hotel-sig-end-user channel. There is a QR code on the screen for you to use. +**Lisa:** Thank you. It has been really interesting, and thank you all so much for following along. If you have additional questions, please find us on CNCF Slack instance at hotel-sig-end-user channel. There is a QR code on the screen for you to use. Huge shout out to Henrik Rexet, who is doing all this live streaming stuff for us on the back end. Thank you so much to Lisa for hosting our Q&A, to Adriana for being my co-host, and Severin and Marilia for taking the time to come on and keep us updated. Thank you so much. -**Lisa:** Also, huge shout out to Henrik Rexet, who is doing all this live streaming stuff for us on the back end. Thank you so much to Lisa for hosting our Q&A, and Adriana for being my co-host, and Severin and Marilia for taking the time to come on and keep us updated. Thank you so much. +**Severin:** Thank you for having us. -**Severin:** Yeah, thank you for having us. +**Marilia:** Happy to be here. -**Marilia:** Yeah, happy to be here. +**Lisa:** We got a question asking when the next one is. This means that I guess we'll have to put on another one. We definitely plan on putting on another one, hoping for a monthly cadence. But we also have KubeCon coming up in November, so we'll see. Speaking of KubeCon, as Severin mentioned, the Hotel Observatory is happening. If you haven't been to the Hotel Observatory booth, it's loads of fun. This is where all the cool hotel peeps come to hang out. -**Lisa:** We got a question asking when the next one is. So, this means that I guess we'll have to put on another one. We definitely plan on putting on another one, hoping for a monthly cadence. But we also have KubeCon coming up in November, so we'll see. There will be—speaking of KubeCon, as Severin mentioned, the Hotel Observatory is happening. If you haven't been to the Hotel Observatory booth, it's loads of fun. This is where all the cool hotel peeps come to hang out. And we're also doing as part of a thing that we've been doing the last little while for KubeCon, we're doing a live stream from KubeCon. And I guess it will be—we call it the Humans of Hotel live stream. I guess it will be to a certain extent a What's New in Hotel. So keep an eye out for that. I believe it's happening on Wednesday, November 12th from starting at 2:30 PM Eastern time. So keep an eye out for that. +**Lisa:** We're also doing a live stream from KubeCon, and I guess we'll call it the Humans of Hotel live stream. I guess it will be, to a certain extent, a What's New in Hotel. Keep an eye out for that. I believe it's happening on the Wednesday, November 12th, starting at 2:30 PM Eastern time. -**Lisa:** And I think that's it. Do you have anything else, Bree, to add to that? +**Lisa:** I think that's it. Do you have anything else, Bree, to add to that? -**Bree:** No, I thank you all for being here. +**Bree:** No, thank you all for being here. -**Lisa:** Yeah, thank you. And also if you have a cool OpenTelemetry story to share, whether it's like how you use OpenTelemetry in your organization or you have like a presentation that you want to do about cool stuff you've done with OpenTelemetry, reach out to us in the hotel user SIG. Again, it’s hotel-sig-end-user on Slack. We would love to hear from you. +**Severin:** Yeah, thank you. -**Bree:** Yes. Thank you so much, everyone. We'll see you next time. +**Marilia:** If you have a cool OpenTelemetry story to share, whether it's like how you use OpenTelemetry in your organization or you have a presentation that you want to do about cool stuff you've done with OpenTelemetry, reach out to us in the hotel and user SIG again. It's hotel-sig-end-user on Slack. We would love to hear from you. -**All:** Thank you, everyone. +**Lisa:** Yes. Thank you so much, everyone. We'll see you next time. -**Marilia:** Bye. +**All:** Thank you, everyone. Bye. ## Raw YouTube Transcript diff --git a/video-transcripts/transcripts/2025-11-13T08:33:00Z-humans-of-otel-live-from-kubecon-na-2025.md b/video-transcripts/transcripts/2025-11-13T08:33:00Z-humans-of-otel-live-from-kubecon-na-2025.md index fb36108..56b2a34 100644 --- a/video-transcripts/transcripts/2025-11-13T08:33:00Z-humans-of-otel-live-from-kubecon-na-2025.md +++ b/video-transcripts/transcripts/2025-11-13T08:33:00Z-humans-of-otel-live-from-kubecon-na-2025.md @@ -10,33 +10,36 @@ URL: https://www.youtube.com/watch?v=NzbDui8hDdo ## Summary -In this live segment from KubeCon North America 2025, hosts Reese and Sophia discuss various topics related to open telemetry with their guest, Jacob Marino, a maintainer in the open telemetry community. Jacob shares insights from his recent talk on introductory concepts of open telemetry and Kubernetes, highlighting the surprising engagement from the audience, with a wide range of questions from beginners to experienced users. The conversation also touches on the collaborative efforts in the open telemetry community, including the introduction of a new auto-instrumentation project using the Zig programming language, and the importance of user feedback for improving documentation and functionality. The hosts also welcome Diana, a developer experience engineer and recent Open Source Community Star award winner, who discusses her contributions to localization efforts within the open telemetry community, emphasizing the challenges and motivations in translating technical documentation for various languages. The session concludes with a focus on the community's commitment to improving user experiences and engagement through better practices and collaborative efforts. +In this video, hosted by Reese and co-hosted by Sophia, live from CubeCon North America 2025 in Atlanta, they engage in a conversation with Jacob Marino, a maintainer of OpenTelemetry projects. Jacob discusses his recent talk on OpenTelemetry and Kubernetes, which garnered a surprisingly large audience and numerous questions, indicating a strong community interest in the topic. They explore the challenges of user engagement and feedback in open-source projects, highlighting the importance of documentation and user education on existing tools and features. The discussion also touches upon the OpenTelemetry Injector project, which aims to simplify auto-instrumentation for various environments using the Zig programming language. The second guest, Diana, a developer experience engineer, shares her experiences in observability and her involvement in localization efforts within the OpenTelemetry community, emphasizing the importance of making technical documentation accessible in multiple languages. Throughout the conversation, they highlight the need for collaboration, user feedback, and community support to enhance the OpenTelemetry ecosystem. ## Chapters -00:00:00 Welcome and intro -00:01:00 Guest introduction: Jacob -00:02:40 Jacob's talk experience -00:04:50 Audience Q&A insights -00:06:00 Community feedback challenges -00:08:40 Discussion on the injector project -00:10:50 Stability and project maturity -00:12:00 Graduation criteria explanation -00:14:50 Importance of documentation -00:27:20 Guest introduction: Diana -00:30:01 Localization efforts and challenges +00:00:00 Introductions +00:01:38 Guest introduction: Jacob +00:02:17 Discussion about Jacob's talk +00:04:54 Discussion about user feedback +00:08:10 Introduction to OpenTelemetry Injector project +00:10:37 Discussion on project stability +00:11:55 Explanation of project graduation criteria +00:25:19 Guest introduction: Diana +00:29:04 Diana's localization efforts +00:41:30 Discussion on instrumentation best practices + +## Transcript + +### [00:00:00] Introductions **Reese:** Just the light. Oh, action. **Sophia:** Oh, you did. You did. -**Reese:** Hello everybody. If you've been waiting around, we are so sorry for the delay, but we are very excited to be coming to you live from Atlanta, Georgia. But pretty much everybody was very surprised by a cold front that had us looking at 40°, which I think is 3°C, the past couple days. Luckily, it's warming back up, so thank goodness. But yeah, we are coming to you live from CubeCon North America 2025. I'm Reese and I am joined here by my co-host Sophia. +**Reese:** Hello everybody. If you've been waiting around, we are so sorry for the delay, but we are very excited to be coming to you live from Atlanta, Georgia. But pretty much everybody was very surprised by a cold front that had us looking at 40°, which I think is 3° C the past couple days. Luckily, it's warming back up, so thank goodness. But yeah, we are coming to you live from KubeCon North America 2025. I'm Reese and I am joined here by my co-host Sophia. **Sophia:** Hi. **Reese:** And our first guest, Jacob. -[00:01:00] **Jacob:** Hello, my name is Jacob Marino. I'm a maintainer for, let's see, OpenTelemetry operator, OpenTelemetry Helm charts, hotel injector, and I maintain a few random collection things and I contribute to OpAmp and, and I'm missing one. [laughter] I was wondering who had a hands for all 10 honestly. +**Jacob:** Hello, my name is Jacob Marino. I'm a maintainer for, let's see, OpenTelemetry Operator, OpenTelemetry Helm Charts, Hotel Injector, and I maintain a few random collection things, and I contribute to OpAmp. I'm missing one. I can't remember. [laughter] I was wondering who had hands for all ten, honestly. **Reese:** If Josh Sar were here, he's everywhere. @@ -44,11 +47,13 @@ In this live segment from KubeCon North America 2025, hosts Reese and Sophia dis **Sophia:** That's wild and awesome at the same time. +### [00:01:38] Guest introduction: Jacob + **Jacob:** No, it was very impressive talking with him. Learned so much. -**Reese:** So, how I feel talking to you all? Um, now Jacob, you did a talk or was it two talks? +**Reese:** So, how do you feel talking to you all? Now Jacob, you did a talk, or was it two talks? -**Jacob:** Uh, one talk. So I gave a talk about two hours ago, three hours ago. +**Jacob:** One talk. So I gave a talk about two hours ago, three hours ago. **Reese:** Cool. @@ -56,53 +61,67 @@ In this live segment from KubeCon North America 2025, hosts Reese and Sophia dis **Sophia:** It was packed. -**Jacob:** It was like I was worried that—I've given this is my second time giving a talk. First talk that I gave was in Chicago two years ago. +**Jacob:** I was worried that I've given—this is my second time giving a talk. The first talk that I gave was in Chicago two years ago. **Reese:** Oh, nice. -[00:02:40] **Jacob:** And the room we got a very low slot, like it was like towards the end of the day on Thursday, I think. So it was a tough tougher day to give a talk. We had a solid audience, and for this one I was worried that with the content we were doing, which is like introductory to hotel and Kubernetes, I always worry with these introduction ones, you know, it's like is this going to be enough, is it going to be too much? So my fear is that nobody shows up and I'm speaking to an empty room. This was the reverse where it was a full room and people were standing in the room and then people got turned away from that door. +### [00:02:17] Discussion about Jacob's talk + +**Jacob:** And the room, we got a very low slot. It was like towards the end of the day on Thursday, I think. So it was a tougher day to give a talk. We had a solid audience. For this one, I was worried that with the content we were doing, which is like introductory to Hotel and Kubernetes, I always worry with these introduction ones, you know, it's like, is this going to be enough? Is it going to be too much? So my fear is that nobody shows up and I'm speaking to an empty room. This was the reverse, where it was a full room and people were standing in the room, and then people got turned away from that door. **Sophia:** Oh my gosh. -**Jacob:** Which I was happily surprised by. Then [laughter] I also like we only reserved five minutes for questions at the end because I was like, you know, I don't know what questions people will have. Usually people aren't asking a ton of questions like after they'll come up to you later, but it's like usually like a Q&A session does not last very long. So oh five minutes for questions, like that's fine. We do questions; there's a line of like 15 people waiting to ask questions. Like they just kept growing and growing. We stayed after for another 30 minutes just answering questions. +**Jacob:** Which I was happily surprised by. [laughter] I also like—we only reserved five minutes for questions at the end because I was like, you know, I don't know what questions people will have. Usually, people aren't asking a ton of questions. They'll come up to you later, but it's like usually a Q&A session does not last very long. So, oh, five minutes for questions, like that's fine. We do questions, and there's a line of like 15 people waiting to ask questions. It just kept growing and growing. We stayed after for another 30 minutes just answering questions. -**Reese:** Interesting. What level of questions? Like was it more, um, you know, people who are newer to the topic or who are asking like more like stage two kind of questions? +**Reese:** Interesting. What level of questions? Was it more, um, you know, people who are newer to the topic, or who are asking more like stage two kind of questions? -**Jacob:** Huge range. So we had some people who were, you know, very introductory level, just like I'm getting my bearings here. How does this part of it work? You know, what's the deal with Prometheus and hotel and what do you use for a backend? You know, that's like a solid starter question. And then other people came in they were, "I've been using this chart for the past year and there's this one thing that I want to be able to do and when are you going to like do the work to like enable that and how can we like architect this?" Big range. +**Jacob:** Huge range. + +**Jacob:** So we had some people who were, you know, very introductory level, just like I'm getting my bearings here. How does this part of it work? You know, what's the deal with Prometheus and Hotel, and what do you use for a backend? That's like a solid starter question. And then other people came in, they were like, I've been using this chart for the past year, and there's this one thing that I want to be able to do. And when are you going to like do the work to enable that? And how can we like architect this big range? **Sophia:** Very big and each one was like a different part of the area. **Jacob:** Okay, that's so cool. -**Reese:** Yeah, very, very cool and very surprising. I'm always happy to because, you know, I think as a maintainer it's like we get people who come to us with [laughter] issues on GitHub and it's like you'll get someone complaining on a GitHub issue or like, "Hey, this thing broke. The most recent release broke me, um, you know, what are we doing about that?" +**Jacob:** Yeah, very, very cool and very surprising. I'm always happy to, 'cause you know, I think as a maintainer, it's like we get people who come to us with [laughter] issues on GitHub, and it's like you'll get someone complaining on a GitHub issue or like, hey, this thing broke. The most recent release broke me. You know, what are we doing about that? -**Sophia:** Broke! [laughter] +**Reese:** But no one's broke their spirit. -**Jacob:** But no one's broke their spirit. You know, I've had releases break my spirit. +**Jacob:** You know, I've had releases break my spirit. -**Sophia:** [laughter] +**Jacob:** But the real thing is we don't get positive feedback. We don't get a lot of questions 'cause realistically, most people are asking track or going on Stack Overflow. -[00:04:50] **Jacob:** But the real thing is we don't get positive feedback. We don't get a lot of questions because realistically most people are asking track or going on Stack Overflow, right? It's like they're not bringing the questions to us. So we don't really hear or understand what usage is. I mean, it's kind of the benefit [laughter] and the curse of open source is that like we don't necessarily get that user feedback directly. It's why like I love what you all do because getting the user feedback is one of the most valuable things to the community. Like in order for us to push forward, we need to know what isn't working right now where people just don't see the effort. +**Sophia:** Right. -**Reese:** Um, we actually—I hosted a meetup in New York last month, two months ago, something like that. +### [00:04:54] Discussion about user feedback -**Sophia:** Yeah. +**Jacob:** It's like they're not bringing the questions to us. So we don't really hear or understand what usage is. I mean, it's kind of the benefit [laughter] and the curse of open source is that like we don't necessarily get that user feedback directly. It's why like I love what you all do, 'cause getting the user feedback is one of the most valuable things to the community. Like in order for us to push forward, we need to know what isn't working right now where people just don't see the effort. -[00:06:00] **Jacob:** And that was eye-opening because it was like wow, people are really using these things and things that I feel like I've talked about so many times are still not known, you know? And it's like I'll mention like how many people know what like the target allocator is in a show of hands and nobody raised their hand in a room of like 40, you know, senior SREs. I'm like, okay, uh, would anybody like it if you could do distributed, sharded, Prometheus scraping? And everybody raises their hand and I'm like, well we have that, like it's done, it's there, you know? You can use it, it's been a thing for a few years. You know, so I guess all to say the community just is still—there are still things for people to learn about the vast ecosystem that we have here. +**Jacob:** We actually—I hosted a meetup in New York last month, two months ago, something like that. -**Sophia:** Yeah, I think, you know, I think a common theme just in general is people not being aware of, you know, different abilities and features and yeah on the end you see too, you know, it's a bit of a challenge because like we feel like we're talking about it, um, you know, at our work and like in the Ozone community and then, yeah, like you said, we have all these conscious people who don't know and we're trying to figure out like where are they going for like how can we make them aware of, you know, all these cool things that are happening. +**Reese:** Yeah. + +**Jacob:** And that was eye-opening because it was like wow, people are really using these things, and things that I feel like I've talked about so many times are still not known. You know, and it's like I'll mention like how many people know what the target allocator is in a show of hands and nobody raised their hand in a room of like 40, you know, senior SREs. I'm like, okay, would anybody like it if you could do distributed, sharded Prometheus scraping? And everybody raises their hand, and I'm like, well, we have that. Like it's done, it's there. You know, you can use it, it's been a thing for a few years. -**Jacob:** Um yeah, so, you know, one of the things we started doing is um the what's the hotel segment, which we just had our first one um a month ago or so. And so we're hoping to, you know, I don't know, have something we can point people to. +**Sophia:** You know, I think a common theme just in general is people not being aware of, you know, different abilities and features. + +**Jacob:** Yeah. -**Reese:** Of course. +**Sophia:** And yeah, on the end, you see too, you know, it is a bit of a challenge 'cause like we feel like we're talking about it, um, you know, at our work and like in the Ozone community, and then, yeah, like you said, we have all these conscious people who don't know, and we're trying to figure out like where are they going for, like how can we make them aware of, you know, all these cool things that are happening. + +**Jacob:** Um, yeah, so you know, one of the things we started doing is the—what's the Hotel segment, which we just had our first one um a month ago or so. And so we're hoping to, you know, I don't know, have something we can point people to. + +**Sophia:** Of course. -**Sophia:** But you mentioned the injector. +**Jacob:** But you mentioned the injector. -**Jacob:** I did. Tell us more about please do. So I am the, uh, I'm a maintainer for it but I'm mostly just doing reviewing because this is actually a joint effort between uh Splunk and D-Zero. Both of them have already written their own uh injectors for auto instrumentation. +**Sophia:** I did. Tell us more about it, please do. -**Reese:** Oh, right. +**Jacob:** So I am the—I'm a maintainer for it, but I'm mostly just doing reviewing because this is actually a joint effort between Splunk and D-Zero. Both of them have already written their own injectors for auto instrumentation. -**Jacob:** And they're essentially donating a merge of both of those projects to the tot alpha version of this for their customers. Um, so we're just testing out that everything is working the way we need to. [clears throat] There's a lot of really random edges here. Um, interestingly enough, this does introduce a new language to hotel, which is exciting, which is it's very rare for us to not use a language in a project that is meant to instrument languages. +**Sophia:** Oh, right. + +**Jacob:** And they're essentially donating a merge of both of those projects to the alpha version of this for their customers. Um, so we're just testing out that everything is working the way we need to. [clears throat] There's a lot of really random edges here. Um, interestingly enough, this does introduce a new language to Hotel, which is exciting, which is very rare for us to not use a language in a project that is meant to instrument languages. **Sophia:** Right. @@ -112,103 +131,119 @@ In this live segment from KubeCon North America 2025, hosts Reese and Sophia dis **Jacob:** Zig um is a fun language that is sort of a merge between like Go, Rust, C++, and C. -**Reese:** Oh, super fun. [laughter] +**Sophia:** Oh, super fun. [laughter] + +### [00:08:10] Introduction to OpenTelemetry Injector project -[00:08:40] **Jacob:** But so the real value of it is it doesn't require uh a lot of like core Linux dependencies. So like GCC is one that we really think about a lot. Uh, if you're on AWS, something that I've been bitten by is not all of the nodes have GCC. And so when you're trying to do injection, like say you're trying to run a load balancer like to or Envoy, um, is similarly injected like we inject auto instrumentation and that will just not work. It used to not work. I think they've changed this but it used to not work uh on certain types of AWS nodes. So we have that same issue because we're trying to inject to all these different environments. So, Zig is pre-built with a bunch of these C libraries, but from scratch, part of its standard library and not based on anything existing. So it has all of its own dependencies. So, the benefit of this injector project is that we're going to be able to really meet users where they're at without needing to know a ton about their environment. And that's the real problem today. +**Jacob:** But so the real value of it is it doesn't require a lot of like core Linux dependencies. So like GCC is one that we really think about a lot. Uh, if you're on AWS, something that I've been bitten by is not all of the nodes have GCC. And so when you're trying to do injection, like say you're trying to run a load balancer like Envoy, um, is similarly injected, like we inject auto instrumentation, and that will just not work. It used to not work. I think they've changed this, but it used to not work on certain types of AWS nodes. So we have that same issue because we're trying to inject to all these different environments. So, Zig is pre-built with a bunch of these C libraries, but from scratch, part of its standard library is not based on anything existing. So, it has all of its own dependencies. So, the benefit of this injector project is that we're going to be able to really meet users where they're at without needing to know a ton about their environment. And that's the real problem today. **Sophia:** Yeah. -**Jacob:** Um, so it's an exciting project. I think we're a few months away from it being alpha beta. Other people are on the release timeline. Ted Young is thinking a lot about the release timeline there. +**Jacob:** Um, so it's an exciting project. I think we're a few months away from it being alpha-beta. Other people are on the release timeline. Ted Young is thinking a lot about the release timeline there. -**Sophia:** Um, but all TC. +**Sophia:** Um, that's so cool. -**Jacob:** TD, that's so cool. +**Reese:** What are the parts of, you know, whether it's something you're working on or not working on directly that you are excited about? -**Reese:** Yeah. What are the parts of, you know, whether it's something working on or not working on directly that you are excited about? +### [00:10:37] Discussion on project stability -[00:10:50] **Jacob:** Good question. I think that the biggest thing for the project right now that everybody cares about is stability, right? I think that uh we're trying to get to the place where users can reliably get, you know, a new update without them needing to do any configuration changes, without needing to worry about uh components being deprecated, things like that, right? I talked to, at that meetup in New York, I talked to a developer at a major bank who was, you know, they've developed all of their own uh custom components for the collector and so every time that we've been doing these reworkings, the collector's architecture, they need to then refactor a bunch of their code as well. And you know, after a few months you lose track of all these updates. So it becomes like a weeks-long project to keep up to date. When you have to do that, their release cycle is every quarter, right? It gets a lot to manage and a lot to update. So for the users, I think what people really want to see is that level of maturity and stability. We can get to that graduated status with the CCF would be huge. That would be massive for the project to have. Uh, I think it would really—I mean our adoption already has been really high. I think that's when we would come to the next level. +**Jacob:** Good question. I think that the biggest thing for the project right now that everybody cares about is stability, right? I think that we're trying to get to the place where users can reliably get, you know, a new update without them needing to do any configuration changes, without needing to worry about components being deprecated, things like that, right? I talked to, at that meetup in New York, I talked to a developer at a major bank who was, you know, they've developed all of their own uh custom components for the collector, and so every time that we've been doing these reworkings of the collector's architecture, they need to then refactor a bunch of their code as well. And you know, after a few months you lose track of all these updates. So it becomes like a weeks-long project to keep up to date. When you have to do that, their release cycle is every quarter, right? It gets a lot to manage and a lot to update. So for the users, I think what people really want to see is that level of maturity and stability. -**Sophia:** Great. Can you explain a little bit for those of us like CCS stages like what going from incubated, incubating projects to graduated means? +**Jacob:** We can get to that graduated status with the CCF would be huge. That would be massive for the project to have. Uh, I think it would really—I mean, our adoption already has been really high. I think that's when we would come to the next level. -**Jacob:** Sure. +**Sophia:** Can you explain a little bit for those of us like CCF stages? Like what going from incubated, incubating projects to graduated means? -**Sophia:** Yeah. +**Jacob:** Sure. -**Jacob:** I'm not the best. I can tell you what I know. I'm definitely not the person in charge of this. There are people who are far more confident in the stability part of this governance than I. +**Jacob:** Yeah. I'm not the best. I can tell you what I know. I'm definitely not the person in charge of this. There are people who are far more confident in the stability part of this governance than I. **Sophia:** Yeah. -**Jacob:** So I will speak on only what I know. +**Jacob:** So I will speak on only— **Sophia:** That's all we're asking for. [laughter] -[00:12:00] **Jacob:** So right now we're in, as you said, the project is relatively lenient in terms of its standards. There are standards, but graduation is really strict, I would say for good reason, right? Like you want to be sure that when you are considered a graduated project you have a healthy community, which [laughter] we definitely do. You have stability guarantees, which is the main thing we're working towards, and lastly, adoption. So adoption has to be at a certain amount of companies; you have to have testimonials [laughter] from those companies about how the project's been helpful. That's another thing that we have plenty of. So now it really just comes down to stability and stability. You can think of Kubernetes as the example here. Kubernetes is, you know, the first graduated project. I imagine it's the first one. Maybe there maybe someone beat them to it, but you don't [laughter] know. But essentially, every time that Kubernetes improves their security and reliability, uh, you know, release stability, anything like that, it raises the bar for everybody else to continue to improve. So as we are getting towards stability, Kubernetes is chugging along, you know, really positively and improving their posture constantly. So it's up to us to not only meet where we started for our graduation criteria, but also to continue building towards that. So that means that you're going to see more security features, much more security guarantees, which is going to be very helpful for enterprises. M um, you're going to hear more about quarterly releases rather than uh the sort of disparate release calendar that we have today. Uh, you'll hear more about documentation which is also going to be huge. So uh later when that comes on, I assume that we'll be talking about some mobilization efforts and documentation. +### [00:11:55] Explanation of project graduation criteria + +**Jacob:** So right now we're in—as you said, the project is relatively lenient in terms of its standards. There are standards, but graduation is really strict, I would say for good reason, right? Like you want to be sure that when you are considered a graduated project, you have a healthy community, which [laughter] we definitely do. You have stability guarantees, which is the main thing we're working towards, and lastly, adoption. So adoption has to be at a certain amount of companies. You have to have testimonials [laughter] from those companies about how the project's been helpful. That's another thing that we have plenty of, so now it really just comes down to stability. + +**Jacob:** And stability, you can think of Kubernetes as the example here. Kubernetes is, you know, the first graduated project. I imagine it's the first one. Maybe there—maybe someone beat them to it, but you don't [laughter] know. But essentially, every time that Kubernetes improves their security and reliability, uh, you know, release stability, anything like that, it raises the bar for everybody else to continue to improve. So as we are getting towards stability, Kubernetes is chugging along, you know, really positively and improving their posture constantly. So it's up to us to not only meet where we started for our graduation criteria but also to continue building towards that. So that means that you're going to see more security features, much more security guarantees, which is going to be very helpful for enterprises. + +**Jacob:** Um, you're going to hear more about quarterly releases rather than the sort of disparate release calendar that we have today. Uh, you'll hear more about documentation, which is also going to be huge. So, uh, later when that comes on, I assume that we'll be talking about some mobilization efforts and documentation. **Reese:** Ah yes, uh that is super, super valuable, right? -**Jacob:** Um, all of that is going to get us to that graduation phase. Uh, it's just time, it's time and effort and uh we have a dedicated group of people who want to see this happen and are spending, you know, all of our time at all of these large companies trying to make that happen. Very exciting. +**Jacob:** All of that is going to get us to that graduation phase. Uh, it's just time, it's time and effort, and we have a dedicated group of people who want to see this happen and are spending, you know, all of our time at all of these large companies trying to make that happen. Very exciting. **Sophia:** Awesome. -**Reese:** That is very exciting. And if you are yourself an open end user and um your company is using in fashion, um, and you want to share your story, we would love to have your story as part of um our legacy [laughter] our catalog. +**Reese:** That is very exciting. And if you are yourself an open end user and um, your company is using it in fashion, um, and you want to share your story, we would love to have your story as part of um our legacy [laughter] our catalog. -**Jacob:** Yeah. +**Jacob:** Yeah, like to help, you know, help graduate the project and help take us out to us. Um, we'll have it in the show notes um after this. But yeah, it's pretty easy. As long as you remember, Slack, you can find us at hotel— + +**Sophia:** Dash— + +**Jacob:** —dash and dash. + +**Reese:** I think that's all the dashes. + +**Jacob:** I thought we were going to Dash Zero. [laughter] -**Reese:** Like to help, you know, help graduate the project and help take us out to us. Um, we'll have it in the show notes um after this. But yeah, it's pretty easy. As long as you remember me Slack, you can find us at hotel-dash-dash-and-dash. +**Sophia:** Oh, so we're like Dash— -**Jacob:** I think that's all the dashes. +**Jacob:** —and similarly, if anybody wants to help out with documentation, documentation is one of the best ways to contribute to the project. It is so valuable. We had somebody for the operator group contribute a whole uh bit of documentation for the target allocator, and it was so well done. Adriana did this as well. Super helpful, and it has been such a resource for me to be able to point users that come into our Slack, come into our S meetings, to just say, "Hey, you have a question about this? We have a full document to answer everything you might need to know about it." That has been such a way to offer. -**Reese:** I thought we were going to dash zero. [laughter] +**Sophia:** Um, it's lovely. So, great way to contribute and get started. Doesn't require writing code. Requires asking questions and writing a bunch of stuff down. -**Jacob:** Oh, so we're like dash. +**Jacob:** Yeah. And I can confirm that everyone that I met in the local community has been super helpful. Um, anything, you know, you can take on and be responsible for, like documentation, that's something that you're taking off their plate. So I know they're having to help you. And it's also cool to see your work on an official website. -[00:14:50] **Reese:** And similarly, if anybody wants to help out with documentation, documentation is one of the best ways to contribute to the project. It is so valuable. We had somebody for the operator group contribute a whole uh bit of documentation for the target allocator and it was so well done. Adriana did this as well, super helpful, and it has been such a resource for me to be able to point users that come into our Slack, come into our S meetings to just say, "Hey, you have a question about this? We have a full document to answer everything you might need to know about it." That has been such a way to offer. Um, it's lovely. So, great way to contribute and get started. Doesn't require writing code. Requires asking questions and writing a bunch of stuff down. +**Sophia:** Yeah, like definitely join the communication SIG for that. I'm currently in it, and we're working on a lot of refactoring of the documentation. Like just like the early stages since I'm more of a beginner with OpenTelemetry and everything. I've been working on kind of refactoring those beginner-level documentations. So like it's been really helpful getting people acquainted with that and localization efforts too. Like I know Lisa is working on a lot of localization efforts. -**Sophia:** Yeah. And I can confirm that everyone that I met in the local community has been super helpful. Um, anything you know you can take on and be responsible for, like documentation, that's something that you're taking off their plate. So I know they're having to help you. And it's also cool to see your work on an official website. +**Jacob:** So yeah, it'll help. -**Jacob:** Yeah. Like, uh, definitely join the communication SIG for that. I'm currently in it and we're working on a lot of refactoring of the documentation, like just like the early stages since I'm more of a beginner with the OpenTelemetry and everything. I've been working on kind of refactoring those beginner level documentations. So like it's been really helpful getting people like acquainted with that and localization efforts too. Like I know Lisa is working on a lot of localization efforts. +**Sophia:** Super helpful. And the beginner stuff especially. I mean, again from the talk today, I think it's just so—it was so useful to learn where people are at in their implementation journeys. -**Reese:** So yeah, it'll help. +**Jacob:** Right. -**Jacob:** Super helpful. And the beginner stuff especially. I mean, again from the talk today I think it's just so—it was so useful to learn where people are at in their implementation journeys, right? Uh we do have people that really need reference architectures, a ton of code examples, right? These things help project adoption that also make it so that users actually are delighted by the experience, right? Ultimately, that's what I think we all want to see and that's what users want. Like you want to be able to install hotel wherever you need it and not have to scour the internet to figure out how to make it work, right? Different companies have so many different needs and so getting started is always going to be the most important thing for us to really nail. +**Sophia:** We do have people that really need reference architectures, a ton of code examples, right? These things help project adoption, that also make it so that users actually are delighted by the experience, right? That and ultimately that's what I think we all want to see, and that's what users want. Like you want to be able to install Hotel wherever you need it and not have to scour the internet to figure out how to make it work, right? -**Sophia:** Uh, the next thing that I heard from, uh, when I did the event in New York is reference architectures and showing how we can do hotel in all of these various environments, so that people can have some code samples to copy from, uh, some architectures to, you know, design around. I was just talking with an engineer at [company name] and they are really interested in understanding how to do various trace sampling architectures, right? Where we can really have like a whole, as you said, catalog these architectures to show people. And we all of our companies, like you know, when I was at [another company name], we had all these architectures internally that we would post blog posts about to share. +**Jacob:** Different companies have so many different needs, and so getting started is always going to be the most important thing for us to really nail. Uh, the next thing that I heard from when I did the event in New York is reference architectures and showing how we can do Hotel in all of these various environments, so that people can have some code samples to copy from, uh, some architectures to, you know, design around. I was just talking with an engineer at a company, and they are really interested in understanding how to do various trace sampling architectures, right? Where we can really have like a whole—as you said—catalog these architectures to show people. -**Jacob:** We have a lot of companies that are using hotel as part of the ingestion path as well, which involves a lot of this architectural work. Similarly, a lot of companies that are contributors to hotel also are obviously using the tooling themselves for their own company's observability. We should reach out to the people who are doing that and try to publish what are the reference architectures that hotel maintainers use ourselves, right? +**Jacob:** And we—all of our companies, like, you know, when I was at our flight, we had all these architectures internally that we would post blog posts about to share. We have a lot of companies that are using Hotel as part of the ingestion path as well, which involves a lot of this architectural work. Similarly, a lot of companies that are contributors to Hotel also are obviously using the tooling themselves for their own company's observability. We should reach out to the people who are doing that and try to publish what are the reference architectures that Hotel maintainers use ourselves, right? **Sophia:** I think that would be really useful for people who are trying to implement this right now. -**Reese:** Absolutely. +**Jacob:** Absolutely. -**Sophia:** And so are we planning to host them like in a repository or like on sites directly? +**Sophia:** For sure. Um, are we planning to host them like in a repository or like on sites directly? -**Jacob:** Oh, I think would be the best. I think, you know, we have documentation places, the operator group, and I think the idea of everything being the doc is going to be the way to go. +**Jacob:** Oh, I think would be the best. I think, you know, we have documentation places in the operator group, and I think the idea of everything being in the doc is going to be the way to go. **Reese:** Right. -**Jacob:** Like centralizing it and making sure that it's well organized, which is a lot of the work that you're doing, super valuable. +**Jacob:** Like centralizing it and making sure that it's well organized, which is a lot of the work that you're doing. Super valuable. **Sophia:** Of course. -**Jacob:** But I think it just—it needs to be there right now. I think it exists on a few different company blogs, a few different companies— +**Jacob:** But I think it just needs to be there right now. I think it exists on a few different company blogs, a few different companies. **Sophia:** Head scattered. -**Jacob:** —bringing that together I think would be really, really valuable. +**Jacob:** Bringing that together, I think would be really, really valuable. -**Reese:** One reason let's do it. +**Reese:** One reason, let's do it, right? There's so many companies like contributing. So there's definitely a bunch of references that people in our community could use. -**Jacob:** Right? There's so many companies like contributing. So there's definitely a bunch of references that people in our community could use. +**Jacob:** Yes. -**Sophia:** Yes. +**Sophia:** Yeah. -**Jacob:** Yeah. +**Jacob:** Absolutely. And so many people are—people are so friendly, right? Like back to that, like going to the Hotel booth. We've had so many people who have never come up to us before and are just asking great questions, and we have so many people who just, you know, we hang out there 'cause we love to talk to people about this. It's rare that you get to have this level of engagement with end users, right? -**Reese:** Absolutely. And so many people are, people are so friendly, right? Like back to that, like going to uh the hotel booth. We've had so many people who have never come up to us before and are just asking great questions, and we have so many people who just, you know, we hang out there because we love to talk to people about this. It's rare that you get to have this level of engagement with end users, right? Uh, that direct type of engagement is so valuable and so people coming by has been one of my favorite experiences of it always. I think the reason that I love to come to these is just getting to talk to people, hearing people's problems, digging in because then it helps us plan what I like what I need to do for the next two years, right? I can take that feedback on it. +**Jacob:** That direct type of engagement is so valuable, and so people coming by has been one of my favorite experiences of it always. I think the reason that I love to come to these is just getting to talk to people, hearing people's problems, digging in because then it helps us plan what I like what I need to do for the next two years, right? I can take that feedback on it. -**Sophia:** What, um, so in during your time here at CubeCon, like from the audience questions that you had at your talk and people who come through the OpenTelemetry observatory, do you have—are you seeing any like general trends as to like what people are leaning toward or needing more help with? +**Reese:** What, um, so during your time here at KubeCon, like from the audience questions that you had at your talk and people who come through the OpenTelemetry booth, do you have—are you seeing any like general trends as to like what people are leaning toward or needing more help with? -**Jacob:** Yeah. So I say the main thing that is of concern to me is around OpAmp, which has been getting a lot of—there are a lot of talks in OpAmp this CubeCon. I don't know if that if we've noticed that there were at least four that mentioned it, which is a lot. +**Jacob:** Yeah. So I say the main thing that is of concern to me is around OpAmp, which has been getting a lot of—there are a lot of talks in OpAmp this KubeCon. I don't know if that—if we've noticed that there were at least four that mentioned it, which is a lot. **Sophia:** Could you explain that a little bit more for us? @@ -216,7 +251,7 @@ In this live segment from KubeCon North America 2025, hosts Reese and Sophia dis **Sophia:** We're slowing down a little bit. -**Jacob:** So, OpAmp is the uh Open Agent Management Protocol, uh, part of the hotel project but is also designed to be in the same way that hotel is very vendor-neutral, it's neutral to agents as well. So, it's not just for the collector necessarily to be for a lot of different uh agents out there. The goal of the protocol is to enable things like uh component status reporting, health reporting, and remote configuration. And remote configuration is definitely the top of the town is what I would say. +**Jacob:** So, OpAmp is the Open Agent Management Protocol, uh, part of the Hotel project, but is also designed to be—in the same way that Hotel is very vendor neutral, it's neutral to agents as well. So, it's not just for the collector necessarily to be for a lot of different agents out there. The goal of the protocol is to enable things like component status reporting, health reporting, and remote configuration. And remote configuration is definitely the topic of the town is what I would say. **Sophia:** Yeah. Okay. Very cool. @@ -224,189 +259,205 @@ In this live segment from KubeCon North America 2025, hosts Reese and Sophia dis **Sophia:** Yeah. -**Jacob:** But it's different than a remote. What users really want is a way to push a rule to a collector and the collector accepts it and continues going. +**Jacob:** But it's different than a remote. What users really want is a way to push a rule to a collector, and the collector accepts it and continues going. **Sophia:** Yeah. -**Jacob:** So that's what a company like Bindplane has already built. Uh, and so what we want to do is take a lot of the lessons from what Bindplane has been successful with and obviously we're working with them as well. They are big contributors to uh, the idea is that we work all together to improve the provider posture here and improve the SDKs as well so that we can actually realize this vision that people can dynamically update their configurations wherever they might be. [laughter] Uh, but it's challenging. I mean that that's like a really tough topic. +**Jacob:** So that's what a company like BindPlane has already built. Uh, and so what we want to do is take a lot of the lessons from what BindPlane has been successful with, and obviously we're working with them as well. They are big contributors to uh—the idea is that we work all together to improve the provider posture here and improve the SDKs as well so that we can actually realize this vision that people can dynamically update their configurations wherever they might be. [laughter] Uh, but it's challenging. I mean, that's like a really tough topic. **Sophia:** Yeah. -**Jacob:** There's a lot of companies in hotel have done this work beyond Bindplane before. Elastic comes to mind. They have their own protocol for doing road configuration already. +**Jacob:** There's a lot of—many companies in Hotel have done this work beyond BindPlane before. Elastic comes to mind. + +**Sophia:** I work here. + +**Jacob:** Oh yeah. [laughter] + +**Jacob:** Elastic has uh their own protocol for doing remote configuration already. **Sophia:** Okay. Okay. -**Jacob:** We've talked with a few of their containers and there's definitely some like room for some good collaboration that we've talked about. +**Jacob:** We've talked with a few of their contributors, and there's definitely some room for some good collaboration that we've talked about. **Sophia:** Awesome. -**Jacob:** I think it's all to be seen, but this is definitely the thing that we hear a lot about. The thing that I hear a lot about on the operator side is in Kubernetes, how can I do uh, call it not just remote configuration but layered configuration. So what this means is how can I make it so that if I'm running in an enterprise company that I'm in a multi-tenant environment, I want like a base collector. I want to overlay. I want my teams to be able to overlay different pieces of config on top of that collector for their needs. +**Jacob:** I think it's all to be seen, but this is definitely the thing that we hear a lot about. The thing that I hear a lot about on the operator side is in Kubernetes, how can I do, uh, call it not just remote configuration but layered configuration? So what this means is how can I make it so that if I'm running in an enterprise company that I'm in a multi-tenant environment, I want like a base collector. I want to overlay. I want my teams to be able to overlay different pieces of config on top of that collector for their needs. -**Reese:** Right. +**Sophia:** Right. **Jacob:** It's a really complicated topic. -**Sophia:** Yeah, it's really— +**Sophia:** Yeah, it sounds like it. -**Jacob:** Sounds like it. And the thing that I sort of struggle with is how do we make this extensible and configurable but also simple to understand. The problem with this, that it starts getting into a lot of moving parts, and as soon as you start to introduce more moving parts is when complexity, and usually reliability, both complexity increases and reliability decreases, right? You need consistency and more moving parts, uh, will reduce that. It's sort of that—there's a great quote in reliability that I love. It's u—the most reliable system is the one that does—that never—that you, right? We know this quote. I'm totally botching it right now. [laughter] +**Jacob:** And the thing that I sort of struggle with is how do we make this extensible and configurable but also simple to understand. -**Sophia:** No, I hear it. I hear it. - -**Jacob:** The idea is that like [laughter] the most reliable system is the one that like doesn't exist, right? +**Jacob:** The problem with this is that it starts getting into a lot of moving parts, and as soon as you start to introduce more moving parts is when complexity and usually reliability—both complexity increases and reliability decreases, right? You need consistency, and more moving parts will reduce that. It's sort of that—there's a great quote in reliability that I love: The most reliable system is the one that does not exist. **Sophia:** Right. -**Jacob:** Because it's always achieving its goal of non-existing. And so if you want to really improve the reliability of the service, you delete it because then you don't have to care. Then it's at 100% because it's always giving you the same answers, which is nothing. - -**Sophia:** Yeah. [laughter] +**Jacob:** 'Cause it's always achieving its goal of non-existing. -**Jacob:** [laughter] Sorry. +**Sophia:** And so if you want to really improve the reliability of the service, you delete it. -**Sophia:** No, I'm here for it. I'm here for it. +**Jacob:** Because then you don't have to care, then it's at 100%. 'Cause it's always giving you the same answers, which is nothing. [laughter] -**Jacob:** Yeah. +**Sophia:** [laughter] Sorry. -**Reese:** Someone—I’ll say that to, I don’t know, someone at the Ozone booth and they'll be like, "You totally messed up." +**Jacob:** I'm here for it. I'm here for it. -**Sophia:** You know, [laughter] we'll have the correct throw in the shout out. +### [00:25:19] Guest introduction: Diana -**Reese:** Yeah. Thank you so much, Jacob. Um, we are going to bring on our next guest, Diana, in a moment. Um, and in the meantime, I really want to show off this baseball jersey. +**Reese:** Thank you so much, Jacob. Um, we are going to bring on our next guest, Diana, in a moment. Um, and in the meantime, I really want to show off this baseball jersey. **Sophia:** Looks really good. -**Reese:** But, um, our friends at Honeycomb have been kind enough to um, sponsor for the hotel community. Um, thank you, Austin Parker, for the design. Um, I would stand and show the back. It says uh the number 11 ACTUALLY I DON'T KNOW WHY IT SAYS 11 like [laughter] there. But yeah, it says maintainer on the back. Um, we'll have some pictures hopefully to show you. But yeah, you've got the logo over here and it says open too across the front. Very, very snazzy, very sharp. [laughter] Um, yeah. And it also comes in black, which I had the option, but I was like, the client looks sharp. Yeah. Um, and also, you can see the blue and the yellow very clearly. Yeah. Just wanted to show you a little bit of a conference fashion. Um, and then when we get um our second guest in a setup, I also want to show you her nails because they're amazing. +**Reese:** But, um, our friends at Honeycomb have been kind enough to um, sponsor for the Hotel community. Um, thank you, Austin Parker, for the design. Um, I would stand and show the back. It says, uh, the number 11. Actually, I don't know why it says 11 like [laughter] there. But yeah, it says maintainer on the back. Um, we'll have some pictures hopefully to show you. But yeah, you've got the logo over here, and it says OpenTelemetry across the front. Very, very snazzy, very sharp. [laughter] Um, yeah, and it also comes in black, which I had the option, but I was like, the client looks sharp. -**Sophia:** Oh yes. +**Sophia:** Yeah. -**Reese:** I don't know how close I can get. +**Reese:** Um, and also, you can see the blue and the yellow very clearly. Yeah. Just wanted to show you a little bit of conference fashion. -**Sophia:** Yeah. +**Sophia:** Um, and then when we get um our second guest in a setup, I also want to show you her nails because they're amazing. + +**Reese:** Oh, yes. + +**Sophia:** I don't know how close I can get. -**Reese:** And like I said, we have a group right here. I'm very excited that it's for today. So that's why we're able to, you know, have lunch on right now. But yeah, I brought my winter puppy out. Um, it was intense, but we made it through and we are now day two of the main event with one more day to go and I hope everyone is doing well. +**Reese:** Yeah. And like I said, we have a group right here. I'm very excited that it's for today. So that's why we're able to, you know, have lunch on right now. But yeah, I brought my winter puppy out. Um, it was intense, but we made it through and we are now day two of the main event with one more day to go, and I hope everyone is doing well. -**Sophia:** Staying hydrated. +**Sophia:** Staying hydrated. Staying hydrated is so large [laughter] and important. -**Reese:** Staying hydrated is so large [laughter] and important. And yeah, I think we're ready to speak to our next guest, Diana. +**Reese:** And yeah, I think we're ready to speak to our next guest, Diana. **Sophia:** Yes. Awesome. -[00:27:20] **Reese:** Diana, welcome. And thank you guys. Congratulations on uh winning the Open Salt Community Star 2025. She was one of several winners. Um, the only one I want to point out, but that's why it's so important. Um, Diana, can you please introduce yourself? And also, the earrings. +**Reese:** Diana, welcome. And thank you guys. Congratulations on uh winning the Open Source Community Star 2025. She was one of several winners. Um, the only one I want to point out, but that's why it's so important. Um, Diana, can you please introduce yourself? And also, the earrings. **Diana:** Yeah. Oh my gosh. Yes. Oh, those are gorgeous. Oh my goodness. Okay. Sorry. Continue. [laughter] -**Reese:** Like, hold up your treasury. - -**Diana:** Yeah. Yeah, I'm Diana. Uh, I'm a developer experience engineer at Parametric and I work all my life in engineering uh in observability the last couple of, like, five years. Uh, and I discovered like I really like going and explaining things to the community. So I had a chance like years back to be a speaker on one of the first conferences I've ever been in my life. Uh, I was super nervous but I liked it a lot and I saw like so many women on stage giving talks that I felt like I needed to do more. +**Diana:** Yeah. Yeah. I'm Diana. Uh, I'm a developer experience engineer at Parametric, and I work all my life in engineering, uh, in observability the last couple of like five years. Uh, and I discovered like I really like going and explaining things to the community. So I had a chance, like years back, to be a speaker on one of the first conferences I've ever been in my life. Uh, I was super nervous, but I liked it a lot, and I saw like so many women on stage giving talks that I felt like I needed to do more. **Sophia:** Of course. -[00:30:01] **Diana:** Uh, so I kind of did that for two years. Uh, and actually last year, like I was hearing so much about OpenTelemetry, I said, "Oh, I want to get involved. I want to do something." Uh, and I got lucky because in the same moment when I thought about it, the foundation was like, "Oh, we are looking for observability people, we are looking for uh, let's get involved, we want to create the first open and learn fundamentals." Uh, I got in, so it was like first start with the documentation, reading questions, questions about and that's like simple months into certain words. I knew it was finally polished. So that was like for me the kick into like sense and slowly I kind of like I was like, okay, I'm sorry things, how about I'm going to do my company? That was like uh for my current company actually. Um, so uh it was like really interesting because I wanted like from explaining OpenTelemetry to the entire company, getting software engineers involved like instrumenting some things in like SDKs, uh, testing everything. +### [00:29:04] Diana's localization efforts -**Reese:** Um, so yeah, production, so that was like a decision. +**Diana:** Uh, so I kind of did that for two years. Uh, and actually last year, like I was hearing so much about OpenTelemetry. I said, like, oh, I want to get involved. I want to do something. Uh, and I got lucky because in the same moment when I thought about it, the foundation was like, oh, we are looking for observability people. We are looking for, uh, let's get involved. We want to create the first OpenTelemetry Learn Fundamentals. Uh, I got in, so it was like first start with the documentation, reading questions, questions about and that's like simple months into certain words. I knew it was finally polished. So that was like for me the kick into like sense. -**Diana:** Uh, but that was like a really cool experience that I've been talking about since one of my talks and explaining how, you know, companies, some particular companies that to be honest they're not super big times to build [laughter] in takes a very important. So I need to be involved in the community. So for me that was like a very, very experience also like understand [laughter] yeah, I thought about myself. Okay, but if right now I'm all alone and I still want to contribute, how can I do it? And I saw translating documents. +**Diana:** And slowly I kind of like I was like, okay, I'm sorry things. How about I'm going to do my company? That was like uh for my current company actually. Um, so, uh, it was like really interesting because I wanted like from explaining OpenTelemetry to the entire company, getting software engineers involved, like instrumenting some things in like SDKs, uh, testing everything. Um, so yeah, production, so that was like decision. -**Sophia:** Oh, nice. That's interesting. +**Diana:** Uh, but that was like a really cool experience that I've been talking about since one of my talks and explaining how, you know, companies—some particular companies that to be honest, they're not super big, times to build [laughter] in takes a very important. So I need to be involved in the community. So for me that was like a very, very experience also like understand [laughter]. -**Diana:** I can do my own time. Start something, translations and yeah, can you um— +**Diana:** Yeah, I thought about myself. Okay, but if right now I'm all alone and I still want to contribute, how can I do it? And I saw translating documents. -**Reese:** So global organization, the projects or localization just refers to translating the documentation to different languages. +**Sophia:** Oh, nice. That's interesting. -**Diana:** Exactly. Yeah. So the idea that—okay, there may be a bit more um, other terms around localization. So it's not necessarily only the technical terms or the tech writing part, but there are like also like um how you're going to automate some CI, going to automate specific pipelines documentation. +**Diana:** I can do it my own time. Start something translations and yeah, can you um, so global organization the projects or localization just refers to translating the documentation to different languages? -**Reese:** Right. Right. +**Diana:** Exactly. Yeah. So the idea that, okay, there may be a bit more um other terms around localization. So it's not necessarily only the technical terms or the tech writing part, but there are like also like, um, how you're going to automate some CI, going to automate specific pipelines documentation. -**Diana:** Can also be that and also how are you communicating a specific message uh in a language, right? Interpretation of concepts and terms and technology in other languages. So I think that's how I see things. +**Sophia:** Right. + +**Diana:** Right. It can also be that and also how are you communicating a specific message in a language, right? Interpretation of concepts and terms and technology in other languages. So I think that's how I see things. **Sophia:** Well, you mentioned Spanish. Um, are you involved with other languages? Um, and can you also tell us like what other languages are being are for the project right now? -**Diana:** Yeah. Yeah. Yeah. Um, so when um back when I got involved, I didn't think of this year, uh, Spanish was available, obviously English, French. +**Diana:** Yeah. Yeah. Yeah. Um, so when um, back when I got involved, I didn't think of this year, uh, Spanish was available, obviously English, French. -**Reese:** Ah yes. +**Sophia:** Ah yes. **Diana:** And that was right. So from the beginning of the year up to now, I think three more languages emerged. So three more communities. Uh, I know for sure Bengali. -**Reese:** Oh nice. +**Sophia:** Oh nice. -**Diana:** And that's like so they really got together and they started to contribute more and more like their only technology. +**Diana:** And that's like so—they really got together, and they started to contribute more and more like their only technology. **Sophia:** Yeah. -**Diana:** Um, Ukrainian, that's locked up recently. So I know for sure they need more contributors. Uh, so somebody came with the idea and they started to do that. Um, and I started, uh, for my own language Romanian. I'm native to Romanian. Uh, I started like a couple of months back and I said, "Look, yeah, I think it's an amazing opportunity for my language to represent and for developers back home to understand a bit more language. It's going to really perspective like it is a really nice community." +**Diana:** Um, Ukrainian, that's locked up recently. So I know for sure they need more contributors. Uh, so somebody came with the idea, and they started to do that. + +**Diana:** Um, and I started, uh, for my own language, Romanian. I'm native to Romanian. Uh, I started like a couple of months back, and I said, "Look, yeah, I think it's an amazing opportunity for my language to represent and for developers back home to understand a bit more. Language is going to really perspective like it is a really nice community." -**Reese:** Yeah, that's really beautiful. +**Sophia:** Yeah, that's really beautiful. **Diana:** So like what is the hardest thing about localization that you find when you do that kind of work? -**Diana:** Yeah, I think um definitely translating specific terms into your own language or in that particular language, but giving it the flavor or giving it like [laughter] that really gets the essence so it doesn't have to sound like a very dry terminology and that word captures the essence of the meaning. So for example, I don't know about—if you translate really like [laughter] literally a word from English or another language, it might sound a bit weird. So you have to like, oh, does it sound really like this in that language? I have to like, you know, modify it a bit. I have to give it like a nice ring to it so like you can get it, you know, it's kind of like, uh, people like it immediately and they want to use it, right? Um, so that's I think that's the difficult part and also the difficult part is that there are many, many languages that without technical terms directly in English, right? +**Diana:** Yeah, I think, um, definitely translating specific terms into your own language or in that particular language, but giving it the flavor or giving it like [laughter] that really gets the essence so it doesn't have to sound like a very dry terminology and that word captures the essence of the meaning. So for example, I don't know about—if you translate really like [laughter] literally a word from English or another language, it might sound a bit weird. -**Sophia:** Right. +**Diana:** So you have to like, oh does it sound really like this in that language? I have to like, you know, modify it a bit. I have to give it like a nice ring to it so like you can get it, you know, it's kind of like, uh, people like it immediately and they want to use it, right? -**Diana:** And when you want to translate it in that language quite like the appropriate meaning is like, okay, I know you know uses that word in my language and they always something in English, right? Oh, that get immediately, you know, it doesn't sound strange, so those are like words basically. +**Sophia:** Um, so that's I think that's the difficult part, and also the difficult part is that there are many, many languages that without technical terms directly in English, right? -**Reese:** That's so wonderful. +**Diana:** And when you want to translate it in that language, quite like the appropriate meaning is like, okay, I know, you know, uses that word in my language, and they always something in English, right? Oh, that gets immediately, you know, it doesn't sound strange. So those are like words basically. + +**Sophia:** That's so wonderful. **Diana:** Yeah, it's really cool. -**Sophia:** Is there um like a way or um to kind of see, you know, the rate of adoption or involvement from the communities that now have these translations available? +**Sophia:** Is there, um, like a way or, um, to kind of see, you know, the rate of adoption or involvement from the communities that now have these translations available? + +**Diana:** Um, in terms of metrics or in terms of adoption, I think that we are, uh, kind of primitive. Um, and I think that you need to correlate it a bit with a ground meeting event or with some type of local community that you could, uh, speak about it on site, right? -**Diana:** Um, in terms of metrics or in terms of adoption, I think that we are uh kind of primitive. Um, and I think that you need to correlate it a bit with a ground meeting event or with some type of local community that you could uh speak about it on site, right? +**Diana:** Or this makes some noise back home, for example. Uh, and say look, we are starting this, we are doing this. We need more contributors. Uh, we need, you know, like the people from the local communities. So like that for each one of us is very much needed whether online or on site. -**Reese:** Or this makes some noise back home for example. +**Diana:** Uh, so for me, for example, I just started this with Romania. I started going like to people like I contributed before or knew from like events, and I'm also Romanian. I was like, can you promote it? Like we are like actually entitling our language to be heard in the open community, like really jumped into it, and yeah, for sure, let's do it. Like a lot of people started. It's really nice. -**Diana:** Uh, and say look, we are starting this, we are doing this. We need more contributors. Uh, we need, you know, like the people from the local communities. So like that for each one of us is very much needed whether online or on site. Uh, so for me, for example, I just started this with Romania. I started going like to people like I contributed before or knew from like events and I'm also Romanian. Around there, like can you put it, can you promote it? Like we are like actually entitling our language to be heard in the open community like really jumped into it and yeah, for sure, let's do it. +**Sophia:** I think as, you know, native English speakers, we get so used to everything being to English. -**Reese:** Like a lot of people started. It's really nice. +**Diana:** And honestly, you know, a lot of non-native English speakers, you know, they usually speak multiple languages. Um, and so I think it's really impressive, um, and awesome the work that you all are doing. -**Diana:** Yeah, I think as you know, native English speakers we get so used to everything being— +**Diana:** Yeah, I appreciate it. And also I think down the line people get motivated not only because of modernization but they start thinking, okay, do I like overall open source? Should I do something else maybe in another project? Does it help me in like my work projects, whatever I'm doing? -**Reese:** To English. +**Diana:** And I think this is how I practically, you know, promoted it to my community. I said if you guys like me, uh, to say like I'm doing an open source project as your contributor, like if your employer needs you to have this experience or maybe you want to go to a conference and talk about it. -**Diana:** And honestly, you know, a lot of um non-native English speakers, you know, they usually speak multiple languages. Um, and so I think it's really impressive um and awesome the work that you all are doing. +**Diana:** So all of these are very, very nice to have, you know, on your screen and, you know, different age groups, you know, there are university or later on in their career, and I think it's really important to keep motivation because it can be sometimes really hard to start in your free time or remind yourself that, you know, [laughter] you have an objective online. -**Diana:** Yeah, I appreciate it. And also I think down the line people get motivated not only because of modernization but they start thinking, okay, uh, do I like overall open source? Uh, should I do something else maybe in another project? Uh, does it help me in like my work projects, whatever I'm doing? Uh, and I think this is how I practically, you know, promoted it to my community. I said if you guys like me, uh to say like I'm doing an open source project as your contributor like if your employer needs you to have this experience or maybe you want to go to a conference and talk about it. So all of these are very, very nice to have, you know, on your screen and you know, different age groups, you know, there, university or later on their career and I think it's really important to keep motivation because it can be sometimes really hard to start in your free time or remind yourself that you know [laughter] you have an objective online. It's kind of like a gateway project into contributing to open source a little bit. [laughter] +**Diana:** It's kind of like a gateway project into contributing to open source a little bit. [laughter] -**Reese:** I always keep telling them that I didn't necessarily start with the organization and I share my new stuff here. I'll definitely go for example—I being an engineer, I still like doing engineering work. +**Sophia:** I always keep telling them that I didn't necessarily start with the organization, and I share my new stuff here. I'll definitely, for example, I—being an engineer, I still like doing engineering work. **Diana:** Oh yeah. -**Reese:** And I want to get involved in the codes part as well because I've been keeping an eye on everything that's happening talking like there are many, many things that for sure my company or other people will definitely find. +**Diana:** And I want to get involved in the code part as well because I've been keeping an eye on everything that's happening. Talking like there are many, many things that for sure my company or other people will definitely find. **Diana:** Um, so yeah, you know, one project or one S is not enough. -**Reese:** We're going to jump [laughter] into something else at some point and then the community especially is really, really and they keep talking about the same issues over and over again. +**Sophia:** We're going to jump [laughter] into something else at some point, and then the community especially is really, really—and they keep talking about the same issues over and over again. -**Diana:** Yeah. So just recently talking about I heard about the same kind of issues in Kubernetes. So I'm kind of like went back and forth because I'm kind of like struggling with similar issues. So I think it's very good to keep an open mind open and keep going. For me at least personally this is what motivates us going forward. You can diversify your contributions, not stop, and we always find something. +**Diana:** Yeah. So just recently talking about—I heard about the same kind of issues in Kubernetes. So I'm kind of like went back and forth because I'm kind of like struggling with similar issues. -**Reese:** Thank you. +**Diana:** So I think it's very good to keep an open mind open and keep going. For me, at least personally, this is what motivates us going forward. You can diversify your contributions, not stop, and we always find something. -**Diana:** Yeah, that's awesome. And then kind of pivoting a little bit. Um, I would love to know what you're looking forward to in the coming year for hotel. Like what are you excited about? What kind of trends you're seeing like implementation? +**Sophia:** Thank you. -**Diana:** Anything that has— +**Diana:** Yeah, that's awesome. -**Jacob:** So um, just like the last couple of weeks been to like specific events. I saw a lot of interesting talks about yeah obviously projector and uh I think these are two things that are really interesting at the moment. +**Sophia:** And then kind of pivoting a little bit, um, I would love to know what you're looking forward to in the coming year for Hotel. Like what are you excited about? What kind of trends are you seeing like implementation? -**Sophia:** So instrumentation, particular end users need a lot more best practices so people don't see users and say, "Oh, but we don't know how to do all this instrumentation or we don't know like which to add." I don't know like what's the easy stuff you can implement? Like can we can you add some like best practices? Can you, right? +**Diana:** Anything that has—so, um, just like the last couple of weeks been to like specific events. I saw a lot of interesting talks about, yeah, obviously projector, and, uh, I think these are two things that are really interesting at the moment. -**Diana:** You know, kind of like tell us, you know, like what would be like best starting point onboarding, let's say in this case. +### [00:41:30] Discussion on instrumentation best practices -**Jacob:** Uh, and I've been hearing like uh community and for OpenTelemetry. +**Diana:** So instrumentation, particular end users need a lot more best practices. So people don't see users and say, "Oh, but we don't know how to do all this instrumentation," or "We don't know like which to add." I don't know like, like what's the easy stuff you can implement? Like can we—can you add some like best practices? Can you, right? -**Reese:** Yeah. +**Diana:** You know, kind of like tell us, you know, like what would be like best starting point onboarding, let's say in this case? -**Diana:** You know, uh, like get them in. +**Sophia:** Uh, and I've been hearing like, uh, community and for OpenTelemetry. -**Sophia:** Yeah. [laughter] +**Diana:** Yeah. -**Reese:** I know that you're complaining, BUT TELL US more about your use. +**Sophia:** You know, get them in. -**Diana:** Right? +**Diana:** Yeah. [laughter] -**Sophia:** Yeah, that's so cool. +**Jacob:** I know that you're complaining, but tell us more about your use. -**Jacob:** Yeah. +**Sophia:** Right? + +**Diana:** That's so cool. + +**Sophia:** Yeah. Um, I just wanted to know, you said that you wanted to get involved in more of the engineering side, the coding side. -**Reese:** Um, I just wanted to know, you said that you wanted to get involved in more of the engineering side, the coding side. Is there somewhere specifically that you're looking forward to or some somewhere specifically that you would like to work in? Like anything exciting to you specifically? +**Diana:** Is there somewhere specifically that you're looking forward to or, or some somewhere specifically that you would like to work in? Like anything exciting to you specifically? **Diana:** Uh, for instance, I'm very interested in the river and the river. @@ -414,27 +465,27 @@ In this live segment from KubeCon North America 2025, hosts Reese and Sophia dis **Diana:** Uh, yeah. Well, I—I don't know. -**Reese:** No, right. Right. +**Sophia:** No, right. -**Diana:** You're getting [laughter] +**Diana:** Right? You're getting— -**Reese:** I think people just last [laughter] maintainers and people that got involved and all the idea about this convention around it. +**Diana:** I think people just last [laughter] maintainers and people that got involved and all the idea about this convention around it. -**Diana:** Uh, and although I—I’m not sure like how they are implementing it like practically, something that I want to do is stop from going to so many events like [laughter] my laptop stuff like this. +**Diana:** Uh, and although I—I’m not sure like how they are implementing it like practically, something that I want to do is stop from going to so many events like [laughter] my laptop, stuff like this. -**Reese:** So, um, I think it needs a bit of time and time and you know, with my team and my brother. +**Diana:** So, um, I think it needs a bit of time and time, and you know, with my team and my brother. -**Sophia:** That's awesome. I love it. +**Sophia:** That's awesome. I love it. Really cool. -**Reese:** Really cool. Awesome. Well, thank you so much, Diana. Congratulations again on your movie star award. It is so cute. And thank you so much, Sophia, for being my co-host. +**Reese:** Awesome. Well, thank you so much, Diana. Congratulations again on your Movie Star Award. It is so cute. And thank you so much, Sophia, for being my co-host. -**Sophia:** Yes. Thank you so much, Reese, for being the host host. +**Sophia:** Yes. Thank you so much, Reese, for being the host. -**Reese:** You're amazing. Our co-co-host [laughter] co-host. +**Reese:** You're amazing. Our co-co-host. [laughter] -**Sophia:** Um, yeah, we hope to see you at a future soon. Um, Amsterdam in EU and soon for our next year. Um, and we'll have some show up. So check us, check it out there and talk to us if you like please. [laughter] +**Sophia:** Um, yeah, we hope to see you at a future soon, um, in Amsterdam in the EU and soon for our next year. Um, and we'll have some show up. So check us, check it out there and talk to us if you like, please. [laughter] -**Reese:** See you everybody. Bye everybody. +**Reese:** See you, everybody. Bye, everybody. [music]