bun add @mentraos/sdk

// POST to https://translator-app.com/webhook/session-start
{
  sessionId: "session-789",
  userId: "[email protected]",
  startReason: "USER_REQUEST"
}

import { AppSession } from '@mentraos/sdk';

const session = new AppSession({
  packageName: 'com.translator.app',
  apiKey: process.env.MENTRAOS_API_KEY,
  cloudUrl: 'wss://cloud.mentraos.com/app-ws'
});

// Connect using the sessionId from webhook
await session.connect(sessionId, userId);

// App → Cloud (AppConnectionInit)
{
  type: "CONNECTION_INIT",
  packageName: "com.translator.app",
  sessionId: "session-789",
  apiKey: "app_api_key_here"
}

// Cloud → App (AppConnectionAck)
{
  type: "CONNECTION_ACK",
  settings: [/* app-specific settings */],
  mentraosSettings: { /* system settings */ },
  config: { /* app config */ },
  capabilities: {
    hasDisplay: true,
    hasMicrophone: true,
    // ... what the glasses can do
  }
}

// Subscribe to transcriptions in English and Spanish
await session.updateSubscriptions([
  {
    type: 'TRANSCRIPTION',
    config: {
      languages: ['en-US', 'es-ES'],
      interimResults: true
    }
  },
  {
    type: 'BUTTON_PRESS',
    config: {
      buttons: ['MAIN']  // Only care about main button
    }
  }
]);

class AppSession {
  // Connection management
  connect(sessionId: string, userId: string): Promise<void>
  disconnect(): void
  
  // Data subscriptions
  updateSubscriptions(subscriptions: SubscriptionConfig[]): Promise<void>
  
  // Display control
  layoutManager: LayoutManager
  
  // Settings
  settingsManager: SettingsManager
  
  // Hardware access
  modules: {
    camera: CameraModule
    audio: AudioModule
    location: LocationModule
  }
  
  // Event handling
  on(event: string, handler: Function): void
  off(event: string, handler: Function): void
}

// Simple text wall
await session.layouts.showTextWall("Hello World!");

// Double text wall (two text sections)
await session.layouts.showDoubleTextWall("Top Text", "Bottom Text");

// Reference card
await session.layouts.showReferenceCard({
  title: "Translation",
  text: "Spanish: Hola Mundo\nEnglish: Hello World"
});

// Dashboard card (left/right layout)
await session.layouts.showDashboardCard({
  left: "Label",
  right: "Value"
});

// Clear the display
await session.layouts.clearView();

// Custom bitmap display (for advanced use)
await session.layouts.showBitmapView({
  width: 400,
  height: 240,
  bitmap: bitmapData // Uint8Array of pixel data
});

// Listen for transcriptions
session.events.onTranscription((data) => {
  console.log(`User said: ${data.text} in ${data.transcribeLanguage}`);
  
  if (data.text.includes("translate")) {
    // Do translation logic
    translateAndDisplay(data.text);
  }
});

// Listen for button presses
session.events.onButtonPress((data) => {
  if (data.buttonId === 'main' && data.pressType === 'long') {
    // Toggle translation direction
    toggleLanguages();
  }
});

// Listen for photos
session.events.onPhotoTaken((data) => {
  // Process image for text recognition
  // data.photoData is an ArrayBuffer
  const text = await ocrProcess(data.photoData);
  await translateAndDisplay(text);
});

// Listen for head position
session.events.onHeadPosition((data) => {
  console.log(`Head position: ${data.position}`); // "up" or "down"
});

// Listen for location updates (through location module)
session.location.subscribeToStream({}, (data) => {
  console.log(`Location: ${data.lat}, ${data.lng}`);
});

// Listen for phone notifications
session.events.onPhoneNotifications((data) => {
  data.forEach(notification => {
    console.log(`Notification from ${notification.app}: ${notification.title}`);
  });
});

// Listen for app-to-app messages
session.events.onAppMessage((message) => {
  console.log(`Message from ${message.senderUserId}: ${message.payload}`);
});

// Note: The direct methods (session.onTranscription, etc.) are deprecated
// Always use session.events.* for event handling

// Get a single setting with optional default
const sourceLanguage = await session.settings.get('sourceLanguage', 'en-US');

// Check if a setting exists
const hasSourceLang = await session.settings.has('sourceLanguage');

// Listen for specific setting changes
session.settings.on('sourceLanguage', (newValue) => {
  console.log('Source language changed to:', newValue);
});

// Listen for any setting change
session.settings.onChange((key, newValue) => {
  console.log(`Setting ${key} changed to:`, newValue);
});

// Note: Settings schema is defined in your app's manifest, not in code
// Settings types from packages/sdk/src/types/enums.ts:
// - TOGGLE: boolean on/off
// - TEXT: text input
// - SELECT: dropdown selection
// - SLIDER: numeric slider
// - TEXTAREA: multi-line text
// - JSON: JSON object

// Request a photo
const photo = await session.camera.requestPhoto({
  metadata: { reason: 'text-recognition' }
});
// photo.photoData is an ArrayBuffer
// photo.mimeType is the image format (e.g., "image/jpeg")

// Start RTMP streaming
const streamResult = await session.camera.startStream({
  rtmpUrl: 'rtmp://your-server.com/live/stream-key'
});
// Returns { success: boolean, streamId?: string }

// Stop streaming
await session.camera.stopStream();

// Managed streaming (with HLS/DASH URLs)
const managedStream = await session.camera.startStream({
  managed: true,
  title: 'My Stream'
});
// Returns URLs for viewers: hlsUrl, dashUrl, etc.

// Play audio from URL
await session.audio.play('https://example.com/sound.mp3', {
  volume: 0.8  // Optional, 0-1
});

// Text-to-speech
await session.audio.speak('Hello world', {
  language: 'en-US',  // Optional
  voice: 'female'     // Optional
});

// Stop audio playback
await session.audio.stop();

// Subscribe to continuous location updates
await session.location.subscribeToStream({
  // Optional options
}, (location) => {
  console.log(`User is at ${location.lat}, ${location.lng}`);
  // accuracy field may be included if available
});

// Get the latest location once
const location = await session.location.getLatestLocation({
  // Optional options
});
console.log(`User is at ${location.lat}, ${location.lng}`);

// Unsubscribe from location updates
await session.location.unsubscribeFromStream();

// Write to dashboard
await session.dashboard.write({
  text: "3 new messages"
}, {
  // Optional targets array - defaults to ['dashboard']
  targets: ['dashboard', 'always_on']
});

// Listen for dashboard mode changes
session.dashboard.onModeChange((mode) => {
  console.log('Dashboard mode changed:', mode);
});

// For system/privileged apps only:
// Read dashboard content
const content = await session.dashboard.systemDashboard.read();

// Listen to dashboard content changes
session.dashboard.systemDashboard.onContentChange((content) => {
  console.log('Dashboard content:', content);
});

session.events.onDisconnected(() => {
  console.log('Lost connection, will auto-reconnect...');
});

session.events.onReconnected(() => {
  console.log('Back online!');
  // Refresh any state if needed
});

// Bad: Spamming display updates
for (let i = 0; i < 100; i++) {
  await session.layoutManager.showText(`Count: ${i}`); // Will be throttled!
}

// Good: Batch updates
const finalResult = calculateResult();
await session.layoutManager.showText(`Result: ${finalResult}`);

// The SDK uses ResourceTracker to automatically clean up
// But you can also manually clean up if needed:

// Disconnect when done
session.disconnect();

// Note: Event handlers are automatically cleaned up when session disconnects
// The ResourceTracker (packages/sdk/src/utils/resource-tracker.ts) handles:
// - Timers and intervals
// - Event listeners
// - WebSocket connections

import { 
  AppSession,
  TranscriptionData,
  ButtonPress,
  TextWall,
  ReferenceCard
} from '@mentraos/sdk';

// Everything is typed!
session.events.onTranscription((data: TranscriptionData) => {
  // TypeScript knows data.text exists
});

session.events.onTranscription(async (data) => {
  const command = parseCommand(data.text);
  
  switch (command.type) {
    case 'WEATHER':
      const weather = await getWeather();
      await session.layouts.showReferenceCard({
        title: "Weather",
        text: `${weather.temp}°F\n${weather.conditions}`
      });
      break;
    
    case 'REMINDER':
      await createReminder(command.text);
      await session.layouts.showTextWall("Reminder set!");
      break;
  }
});

// Update dashboard with live info
setInterval(async () => {
  const info = await fetchLiveData();
  
  await session.dashboard.write({
    text: `Stock: $${info.price} ${info.change > 0 ? '↑' : '↓'}`
  });
}, 60000); // Update every minute

let state = 'WAITING_FOR_INPUT';

session.events.onButtonPress(async (data) => {
  if (data.buttonId === 'main') {
    switch (state) {
      case 'WAITING_FOR_INPUT':
        await session.layouts.showTextWall("Listening...");
        state = 'LISTENING';
        break;
      
      case 'LISTENING':
        await session.layouts.showTextWall("Processing...");
        state = 'PROCESSING';
        processInput();
        break;
    }
  }
});

// Broadcast to all users with the same app
await session.broadcastToAppUsers({
  type: 'cursor_move',
  x: 100,
  y: 200
});

// Send direct message to specific user
await session.sendDirectMessage('[email protected]', {
  type: 'private_note',
  text: 'Check this out!'
});

// Discover other active users
const activeUsers = await session.discoverAppUsers();
console.log(`${activeUsers.users.length} users online`);

// Listen for messages from other app users
session.events.onAppMessage((message) => {
  console.log(`Message from ${message.senderUserId}: ${message.payload}`);
});