Usecase observe

// Discover product interaction elements
const productActions = await page.observe({
  instruction: "Find add to cart buttons, size selectors, and product images"
});

// Categorize actions by type
const cartButtons = productActions.filter(a => 
  a.description.toLowerCase().includes('cart')
);
const sizeOptions = productActions.filter(a => 
  a.description.toLowerCase().includes('size')
);

// Execute purchase workflow
if (sizeOptions.length > 0) {
  await page.act(sizeOptions[0]); // Select size first
}
if (cartButtons.length > 0) {
  await page.act(cartButtons[0]); // Then add to cart
}

// Analyze form structure before filling
const formElements = await page.observe({
  instruction: "Find form fields, validation messages, and submit buttons"
});

// Check for required fields
const requiredFields = formElements.filter(e => 
  e.description.includes('required') || e.description.includes('*')
);

console.log(`Found ${requiredFields.length} required fields to complete`);

// Fill form systematically
for (const field of requiredFields) {
  await page.act(field);
  // Add appropriate input based on field type
}

// Wait for and discover dynamically loaded content
await page.waitForLoadState('networkidle');

const dynamicElements = await page.observe({
  instruction: "Find newly loaded content, infinite scroll triggers, or loading indicators",
  domSettleTimeoutMs: 15000 // Wait longer for dynamic content
});

// Handle infinite scroll
const scrollTriggers = dynamicElements.filter(e => 
  e.description.toLowerCase().includes('load more') ||
  e.description.toLowerCase().includes('scroll')
);

if (scrollTriggers.length > 0) {
  await page.act(scrollTriggers[0]);
  // Recursively observe new content
  const newContent = await page.observe("Find additional items");
}

// Plan entire checkout flow upfront
async function planCheckoutWorkflow() {
  // Step 1: Cart page analysis
  await page.goto('/cart');
  const cartActions = await page.observe("Find checkout and cart modification options");
  
  // Step 2: Checkout page analysis  
  const checkoutButton = cartActions.find(a => a.description.includes('checkout'));
  if (checkoutButton) await page.act(checkoutButton);
  
  const checkoutActions = await page.observe("Find payment forms and shipping options");
  
  // Step 3: Plan execution order
  const shippingFields = checkoutActions.filter(a => a.description.includes('shipping'));
  const paymentFields = checkoutActions.filter(a => a.description.includes('payment'));
  const submitButton = checkoutActions.find(a => a.description.includes('complete order'));
  
  return { shippingFields, paymentFields, submitButton };
}

// Execute planned workflow
const workflow = await planCheckoutWorkflow();
// Fill shipping → payment → submit