How Twitter Generates Your Home Feed in 5 Seconds

-- Feed query: pull tweets from all followed accounts
SELECT t.tweet_id, t.content, t.created_at, t.author_id
FROM tweets t
WHERE t.author_id IN (
  SELECT followed_id
  FROM follows
  WHERE follower_id = :user_id
)
ORDER BY t.created_at DESC
LIMIT 20;

// Fan-out worker (runs after every tweet)
async function fanOutTweet(tweetId, authorId) {
  // Get all followers of the author
  const followers = await db.query(
    'SELECT follower_id FROM follows WHERE followed_id = ?',
    [authorId]
  );

  // Push tweet ID to each follower's timeline list in Redis
  const pipeline = redis.pipeline();
  for (const { follower_id } of followers) {
    pipeline.lpush(`timeline:${follower_id}`, tweetId);
    pipeline.ltrim(`timeline:${follower_id}`, 0, 799); // keep 800 tweets
  }
  await pipeline.exec();
}

// Read feed (fast — just Redis)
async function getHomeFeed(userId, page = 0, pageSize = 20) {
  const start = page * pageSize;
  const end = start + pageSize - 1;

  // Step 1: Get tweet IDs from pre-built timeline
  const tweetIds = await redis.lrange(`timeline:${userId}`, start, end);

  // Step 2: Batch-fetch tweet details
  const tweets = await Promise.all(
    tweetIds.map(id => getTweetById(id))
  );

  return tweets;
}

// Hybrid fan-out: write side
const CELEBRITY_THRESHOLD = 1_000_000; // 1M followers

async function hybridFanOut(tweetId, authorId) {
  const followerCount = await getFollowerCount(authorId);

  if (followerCount < CELEBRITY_THRESHOLD) {
    // Regular user: push to all follower timelines (fan-out on write)
    await fanOutToFollowers(tweetId, authorId);
  } else {
    // Celebrity: only store tweet, skip fan-out (fan-out on read)
    // Followers will pull celebrity tweets at read time
    await storeInTweetsTable(tweetId, authorId);
  }
}

// Hybrid read: combine prebuilt timeline + celebrity tweets
async function getHomeFeed(userId) {
  // 1. Load pre-built timeline from Redis (regular accounts)
  const prebuiltIds = await redis.lrange(`timeline:${userId}`, 0, 199);

  // 2. Find celebrity accounts this user follows
  const celebFollows = await db.query(`
    SELECT f.followed_id
    FROM follows f
    JOIN users u ON u.id = f.followed_id
    WHERE f.follower_id = ?
      AND u.follower_count >= ?
    LIMIT 100
  `, [userId, CELEBRITY_THRESHOLD]);

  // 3. Fetch recent celebrity tweets (fan-out on read for celebrities)
  const celebTweetIds = await Promise.all(
    celebFollows.map(({ followed_id }) =>
      redis.lrange(`user_tweets:${followed_id}`, 0, 19)
    )
  );

  // 4. Merge all tweet IDs, fetch details, rank by time
  const allIds = [...new Set([...prebuiltIds, ...celebTweetIds.flat()])];
  const tweets = await batchFetchTweets(allIds);

  return tweets
    .sort((a, b) => b.created_at - a.created_at)
    .slice(0, 20);
}

// A production-ready simplified version using Redis + PostgreSQL

class TimelineService {
  constructor(redis, db) {
    this.redis = redis;
    this.db = db;
    this.CELEBRITY_THRESHOLD = 50_000;  // lower for smaller platforms
    this.TIMELINE_MAX_SIZE = 800;
  }

  async onTweetCreated(tweetId, authorId) {
    const followerCount = await this.getFollowerCount(authorId);

    if (followerCount >= this.CELEBRITY_THRESHOLD) {
      // Celebrity: only append to their own tweet list
      await this.redis.lpush(`user_tweets:${authorId}`, tweetId);
      await this.redis.ltrim(`user_tweets:${authorId}`, 0, 999);
      return;
    }

    // Regular: fan-out to all followers
    const followers = await this.db.query(
      'SELECT follower_id FROM follows WHERE followed_id = ?',
      [authorId]
    );

    if (followers.length === 0) return;

    // Batch Redis writes with pipelining
    const batchSize = 1000;
    for (let i = 0; i < followers.length; i += batchSize) {
      const batch = followers.slice(i, i + batchSize);
      const pipeline = this.redis.pipeline();
      for (const { follower_id } of batch) {
        const key = `timeline:${follower_id}`;
        pipeline.lpush(key, tweetId);
        pipeline.ltrim(key, 0, this.TIMELINE_MAX_SIZE - 1);
        pipeline.expire(key, 30 * 24 * 60 * 60); // 30 day TTL
      }
      await pipeline.exec();
    }
  }

  async getTimeline(userId, cursor = 0, limit = 20) {
    // 1. Pre-built timeline
    const prebuiltIds = await this.redis.lrange(
      `timeline:${userId}`, cursor, cursor + limit * 3 - 1
    );

    // 2. Celebrity merging
    const celebs = await this.getCelebrityFollows(userId);
    const celebIds = await Promise.all(
      celebs.map(id => this.redis.lrange(`user_tweets:${id}`, 0, 19))
    );

    // 3. Merge, deduplicate, sort, paginate
    const allIds = [...new Set([...prebuiltIds, ...celebIds.flat()])];
    return allIds.slice(0, limit);
  }
}