Reliable queueing for
background work
at any scale

// User uploads image - you respond FAST, process later
app.post('/upload', upload.single('image'), async (req, res) => {
  const imageId = crypto.randomUUID();
  
  // Store original in YOUR S3/storage
  await s3.putObject({ Key: `temp/${imageId}`, Body: req.file.buffer });
  
  // Queue background job (Spooled just stores the job data)
  await fetch('https://api.spooled.cloud/api/v1/jobs', {
    method: 'POST',
    headers: { 
      'Authorization': 'Bearer sp_live_...',
      'Content-Type': 'application/json'
    },
    body: JSON.stringify({
      queue_name: 'image-processing',
      payload: { image_id: imageId, user_id: req.user.id }
    })
  });
  
  // Return immediately - user doesn't wait for processing
  res.json({ id: imageId, status: 'processing' });
});

// Your image worker (runs on YOUR server with Sharp installed)
const WORKER_ID = 'worker-1';

async function processImages() {
  const res = await fetch('https://api.spooled.cloud/api/v1/jobs/claim', {
    method: 'POST',
    headers: { 
      'Authorization': 'Bearer sp_live_...',
      'Content-Type': 'application/json'
    },
    body: JSON.stringify({ queue_name: 'image-processing', worker_id: WORKER_ID, limit: 5 })
  });
  const { jobs } = await res.json();
  
  for (const job of jobs) {
    // Get original from YOUR storage
    const original = await s3.getObject({ Key: `temp/${job.payload.image_id}` });
    
    // Process with Sharp (runs on YOUR server, uses YOUR CPU)
    const thumbnail = await sharp(original.Body)
      .resize(200, 200)
      .webp({ quality: 80 })
      .toBuffer();
    
    // Store in YOUR S3
    await s3.putObject({ 
      Key: `images/${job.payload.image_id}/thumb.webp`,
      Body: thumbnail
    });
    
    // Done!
    await fetch(`https://api.spooled.cloud/api/v1/jobs/${job.id}/complete`, {
      method: 'POST',
      headers: { 
        'Authorization': 'Bearer sp_live_...',
        'Content-Type': 'application/json'
      },
      body: JSON.stringify({ worker_id: WORKER_ID })
    });
  }
}

// Queue 100K notifications - Spooled handles reliability
async function announceNewFeature(userIds) {
  // Queue in batches
  for (let i = 0; i < userIds.length; i += 100) {
    const batch = userIds.slice(i, i + 100);
    
    // Each job = one user to notify
    for (const userId of batch) {
      await fetch('https://api.spooled.cloud/api/v1/jobs', {
        method: 'POST',
        headers: { 
          'Authorization': 'Bearer sp_live_...',
          'Content-Type': 'application/json'
        },
        body: JSON.stringify({
          queue_name: 'notifications',
          payload: { user_id: userId, template: 'new_feature' },
          idempotency_key: `notify-${userId}-feature-v2`
        })
      });
    }
  }
  console.log(`Queued ${userIds.length} notifications`);
}

// Notification worker - YOU pay for email/push services
const WORKER_ID = 'worker-1';

async function sendNotifications() {
  const res = await fetch('https://api.spooled.cloud/api/v1/jobs/claim', {
    method: 'POST',
    headers: { 
      'Authorization': 'Bearer sp_live_...',
      'Content-Type': 'application/json'
    },
    body: JSON.stringify({ queue_name: 'notifications', worker_id: WORKER_ID, limit: 50 })
  });
  const { jobs } = await res.json();
  
  for (const job of jobs) {
    const user = await db.users.get(job.payload.user_id);
    
    // Send via YOUR email service (Resend, SendGrid, etc.)
    // YOU pay Resend/SendGrid for each email sent
    await resend.emails.send({
      from: 'noreply@yourapp.com',
      to: user.email,
      subject: 'New Feature Available!',
      html: renderTemplate(job.payload.template)
    });
    
    await fetch(`https://api.spooled.cloud/api/v1/jobs/${job.id}/complete`, {
      method: 'POST',
      headers: { 
        'Authorization': 'Bearer sp_live_...',
        'Content-Type': 'application/json'
      },
      body: JSON.stringify({ worker_id: WORKER_ID })
    });
    
    await sleep(10);  // Rate limit: ~100 emails/sec
  }
}

// GitHub sends webhook to YOUR server
app.post('/webhooks/github', async (req, res) => {
  const event = req.headers['x-github-event'];
  const payload = req.body;
  
  if (event === 'push' && payload.ref === 'refs/heads/main') {
    // Queue deployment job
    await fetch('https://api.spooled.cloud/api/v1/jobs', {
      method: 'POST',
      headers: { 
        'Authorization': 'Bearer sp_live_...',
        'Content-Type': 'application/json'
      },
      body: JSON.stringify({
        queue_name: 'deployments',
        payload: {
          repo: payload.repository.clone_url,
          commit: payload.after,
          author: payload.pusher.name
        },
        idempotency_key: `deploy-${payload.after}`  // No duplicate deploys
      })
    });
  }
  
  res.status(200).send('OK');
});

// Deployment worker (runs on YOUR CI server or container)
const WORKER_ID = 'worker-1';

async function runDeployments() {
  const res = await fetch('https://api.spooled.cloud/api/v1/jobs/claim', {
    method: 'POST',
    headers: { 
      'Authorization': 'Bearer sp_live_...',
      'Content-Type': 'application/json'
    },
    body: JSON.stringify({ queue_name: 'deployments', worker_id: WORKER_ID, limit: 1 })
  });
  const { jobs } = await res.json();
  if (!jobs.length) return;
  
  const job = jobs[0];
  try {
    // YOUR server runs these commands
    await exec(`git clone --depth 1 ${job.payload.repo}`);
    await exec('npm ci && npm test && npm run build');
    await exec('kubectl apply -f k8s/');  // YOUR K8s cluster
    
    // Notify YOUR Slack workspace
    await fetch('https://slack.com/api/chat.postMessage', {
      headers: { 'Authorization': 'Bearer xoxb-YOUR-SLACK-TOKEN' },
      body: JSON.stringify({
        channel: '#deployments',
        text: `✅ Deployed ${job.payload.commit.slice(0,7)}`
      })
    });
    
    await fetch(`https://api.spooled.cloud/api/v1/jobs/${job.id}/complete`, {
      method: 'POST',
      headers: { 
        'Authorization': 'Bearer sp_live_...',
        'Content-Type': 'application/json'
      },
      body: JSON.stringify({ worker_id: WORKER_ID })
    });
  } catch (err) {
    await fetch(`https://api.spooled.cloud/api/v1/jobs/${job.id}/fail`, {
      method: 'POST',
      headers: { 
        'Authorization': 'Bearer sp_live_...',
        'Content-Type': 'application/json'
      },
      body: JSON.stringify({ worker_id: WORKER_ID, error: err.message })
    });
  }
}