Building a Real-Time Analytics Dashboard

An operations screen, refreshing every 2 seconds

Eight live metrics over the bdma.rides dataset you have been modelling since Module 2:

Modelled on Redis Labs' basic-analytics-dashboard-redis-bitmaps-nodejs, rebuilt on our ride-booking domain.

Four stages, one direction

Compute on write, not on read

db.rides.aggregate([
  { $match: { status: "Completed" } },
  { $group: { _id: "$vehicle_type",
      revenue: { $sum: "$booking_value" } } },
  { $sort: { revenue: -1 } },
  { $limit: 5 }
])

// once, as the booking lands
ZINCRBY dash:vehicle:revenue
        412.50 "Go Sedan"

// forever after:
ZREVRANGE dash:vehicle:revenue
          0 4 WITHSCORES

Which trade are you making?

The key schema IS the design document

Three decisions — and one trap

One module owns every key string

export const slug = (s) => s.toLowerCase().replace(/\s+/g, '-');

// "CID482910" -> 482910. Exactly what SETBIT needs.
export const customerOffset = (id) => Number.parseInt(id.slice(3), 10);

export const K = {
  totalBookings:      'dash:total:bookings',
  vehicleRevenue:     'dash:vehicle:revenue',
  pickupTop:          'dash:pickup:top',
  payment:            'dash:payment',
  status:    (s)    => `dash:status:${slug(s)}`,
  revenue:   (date) => `dash:revenue:${date}`,
  hour:      (h)    => `dash:hour:${h}`,
  customers: (date) => `dash:customers:${date}`,
};

One booking, one transaction

function queueBooking(tx, b) {
  const day  = b.date;                              // "2024-03-14"
  const hour = new Date(b.booked_at).getUTCHours(); // 18

  tx.incr(K.totalBookings);
  tx.incr(K.status(b.status));
  tx.incr(K.hour(hour));
  tx.zIncrBy(K.pickupTop, 1, b.route.pickup);
  tx.setBit(K.customers(day), customerOffset(b.customer_id), 1);
  tx.expire(K.customers(day), NINETY_DAYS, 'NX');   // NX: don't push the window

  if (typeof b.booking_value === 'number') {        // absent on cancellations
    tx.incrByFloat(K.revenue(day), b.booking_value);
    tx.zIncrBy(K.vehicleRevenue, b.booking_value, b.vehicle_type);
  }
  if (b.payment_method) tx.hIncrBy(K.payment, b.payment_method, 1);
}

export async function recordBooking(b) {
  const tx = redis.multi(); queueBooking(tx, b); return tx.exec();
}
export async function recordBatch(list) {
  const tx = redis.multi(); for (const b of list) queueBooking(tx, b);
  return tx.exec();                                 // one round trip, N bookings
}

Consistency and throughput — not rollback

Load 15,000 bookings, then check your work

// replay.js — batches of 500, then dribble the tail so the page moves
for (let i = 0; i < history.length; i += 500)
  await recordBatch(history.slice(i, i + 500));

GET dash:total:bookings                 # -> 15000
GET dash:status:completed               # -> 9343   (62.29%)
GET dash:status:cancelled-by-driver     # -> 2698   (17.99%)
GET dash:status:no-driver-found         # -> 1036    (6.91%)
GET dash:status:cancelled-by-customer   # -> 1022    (6.81%)
GET dash:status:incomplete              # ->  901    (6.01%)
GET dash:hour:18                        # -> 1373, the busiest hour
ZREVRANGE dash:vehicle:revenue 0 2 WITHSCORES
# Premier Sedan 536855 · Go Sedan 491473 · Uber XL 431927

The reader does no analytical work

app.get('/api/dashboard', async (req, res, next) => {
  try {
    const day = req.query.date ?? '2024-03-14';

    const [headline, statusCounts, hourCounts, vehicles, pickups,
           payment, actives] = await Promise.all([
      redis.mGet([K.totalBookings, K.revenue(day)]),
      redis.mGet(STATUSES.map((s) => K.status(s))),
      redis.mGet(HOURS.map((h) => K.hour(h))),
      redis.zRangeWithScores(K.vehicleRevenue, 0, 4, { REV: true }),
      redis.zRangeWithScores(K.pickupTop,      0, 4, { REV: true }),
      redis.hGetAll(K.payment),
      redis.bitCount(K.customers(day)),
    ]);

    res.json({ date: day, total_bookings: num(headline[0]),
      revenue_today: num(headline[1]), active_customers: actives,
      vehicle_revenue: vehicles.map(({ value, score }) =>
        ({ vehicle_type: value, revenue: score })), /* ... */ });
  } catch (err) { next(err); }
});

Five reads, none of which grow

Poll, repaint, nothing clever

async function refresh() {
  const res = await fetch('/api/dashboard?date=2024-03-14');
  if (!res.ok) return;                      // keep the last good paint
  const d = await res.json();

  total.textContent   = d.total_bookings.toLocaleString('en-IN');
  revenue.textContent = inr(d.revenue_today);
  actives.textContent = d.active_customers.toLocaleString('en-IN');
  vehicles.innerHTML  = d.vehicle_revenue
    .map((v) => `<li>${v.vehicle_type} — ${inr(v.revenue)}</li>`).join('');
}
refresh(); setInterval(refresh, 2000);

node api.js      # terminal 1 — http://localhost:3000
node replay.js   # terminal 2 — watch the numbers climb

Retention over a million customers — two commands

BITOP AND dash:retained:2024-03-13:2024-03-14 \
          dash:customers:2024-03-13 dash:customers:2024-03-14
BITCOUNT  dash:retained:2024-03-13:2024-03-14

Redis dashboard vs Atlas Charts

You get the metrics you chose in advance

Six things to carry forward

1. Compute on write, not on read. The read then costs the same at 15,000 bookings or 15,000,000.
2. The key schema is the design document. Eight keys, five types, one keys.js — Redis has no catalogue to recover it from.
3. Guard every optional field. 5,657 of 15,000 bookings have no booking_value; one NaN poisons a counter permanently.
4. MULTI/EXEC buys consistency and throughput, not rollback. Eight updates, one round trip, visible together.
5. Two commands compute retention over a million customers. BITOP AND + BITCOUNT, at ~125 KB per day.
6. You bought speed with flexibility. Keep the system of record for every question you did not anticipate — and reconcile against it on a schedule.