effect-patterns-making-http-requests

This skill provides 10 curated Effect-TS patterns for making http requests. Use this skill when working on tasks related to:

• making http requests
• Best practices in Effect-TS applications
• Real-world patterns and solutions

---

🟢 Beginner Patterns

Parse JSON Responses Safely

Rule: Always validate HTTP responses with Schema to catch API changes at runtime.

Good Example:

import { Effect, Console } from "effect"
import { Schema } from "effect"
import { HttpClient, HttpClientRequest, HttpClientResponse } from "@effect/platform"
import { NodeHttpClient, NodeRuntime } from "@effect/platform-node"

// ============================================
// 1. Define response schemas
// ============================================

const PostSchema = Schema.Struct({
  id: Schema.Number,
  title: Schema.String,
  body: Schema.String,
  userId: Schema.Number,
})

type Post = Schema.Schema.Type<typeof PostSchema>

const PostArraySchema = Schema.Array(PostSchema)

// ============================================
// 2. Fetch and validate single item
// ============================================

const getPost = (id: number) =>
  Effect.gen(function* () {
    const client = yield* HttpClient.HttpClient

    const response = yield* client.get(
      `https://jsonplaceholder.typicode.com/posts/${id}`
    )
    const json = yield* HttpClientResponse.json(response)

    // Validate against schema - fails if data doesn't match
    const post = yield* Schema.decodeUnknown(PostSchema)(json)

    return post
  })

// ============================================
// 3. Fetch and validate array
// ============================================

const getPosts = Effect.gen(function* () {
  const client = yield* HttpClient.HttpClient

  const response = yield* client.get(
    "https://jsonplaceholder.typicode.com/posts"
  )
  const json = yield* HttpClientResponse.json(response)

  // Validate array of posts
  const posts = yield* Schema.decodeUnknown(PostArraySchema)(json)

  return posts
})

// ============================================
// 4. Handle validation errors
// ============================================

const safeGetPost = (id: number) =>
  getPost(id).pipe(
    Effect.catchTag("ParseError", (error) =>
      Effect.gen(function* () {
        yield* Console.error(`Invalid response format: ${error.message}`)
        // Return a default or fail differently
        return yield* Effect.fail(new Error(`Post ${id} has invalid format`))
      })
    )
  )

// ============================================
// 5. Schema with optional fields
// ============================================

const UserSchema = Schema.Struct({
  id: Schema.Number,
  name: Schema.String,
  email: Schema.String,
  phone: Schema.optional(Schema.String),        // May not exist
  website: Schema.optional(Schema.String),
  company: Schema.optional(
    Schema.Struct({
      name: Schema.String,
      catchPhrase: Schema.optional(Schema.String),
    })
  ),
})

const getUser = (id: number) =>
  Effect.gen(function* () {
    const client = yield* HttpClient.HttpClient

    const response = yield* client.get(
      `https://jsonplaceholder.typicode.com/users/${id}`
    )
    const json = yield* HttpClientResponse.json(response)

    return yield* Schema.decodeUnknown(UserSchema)(json)
  })

// ============================================
// 6. Run examples
// ============================================

const program = Effect.gen(function* () {
  yield* Console.log("=== Validated Single Post ===")
  const post = yield* getPost(1)
  yield* Console.log(`Title: ${post.title}`)

  yield* Console.log("\n=== Validated Posts Array ===")
  const posts = yield* getPosts
  yield* Console.log(`Fetched ${posts.length} posts`)

  yield* Console.log("\n=== User with Optional Fields ===")
  const user = yield* getUser(1)
  yield* Console.log(`User: ${user.name}`)
  yield* Console.log(`Company: ${user.company?.name ?? "N/A"}`)
})

program.pipe(
  Effect.provide(NodeHttpClient.layer),
  NodeRuntime.runMain
)

Rationale:

Use Effect Schema to validate HTTP JSON responses, ensuring the data matches your expected types at runtime.

---

APIs can change without warning:

1. Fields disappear - Backend removes a field
2. Types change - String becomes number
3. Nulls appear - Required field becomes optional
4. New fields - Extra data you didn't expect

Schema validation catches these issues immediately.

---

---

Your First HTTP Request

Rule: Use @effect/platform HttpClient for type-safe HTTP requests with automatic error handling.

Good Example:

import { Effect, Console } from "effect"
import { HttpClient, HttpClientRequest, HttpClientResponse } from "@effect/platform"
import { NodeHttpClient, NodeRuntime } from "@effect/platform-node"

// ============================================
// 1. Simple GET request
// ============================================

const simpleGet = Effect.gen(function* () {
  const client = yield* HttpClient.HttpClient
  
  // Make a GET request
  const response = yield* client.get("https://jsonplaceholder.typicode.com/posts/1")
  
  // Get response as JSON
  const json = yield* HttpClientResponse.json(response)
  
  return json
})

// ============================================
// 2. GET with typed response
// ============================================

interface Post {
  id: number
  title: string
  body: string
  userId: number
}

const getPost = (id: number) =>
  Effect.gen(function* () {
    const client = yield* HttpClient.HttpClient
    const response = yield* client.get(
      `https://jsonplaceholder.typicode.com/posts/${id}`
    )
    const post = yield* HttpClientResponse.json(response) as Effect.Effect<Post>
    return post
  })

// ============================================
// 3. POST with body
// ============================================

const createPost = (title: string, body: string) =>
  Effect.gen(function* () {
    const client = yield* HttpClient.HttpClient
    
    const request = HttpClientRequest.post(
      "https://jsonplaceholder.typicode.com/posts"
    ).pipe(
      HttpClientRequest.jsonBody({ title, body, userId: 1 })
    )
    
    const response = yield* client.execute(yield* request)
    const created = yield* HttpClientResponse.json(response)
    
    return created
  })

// ============================================
// 4. Handle errors
// ============================================

const safeGetPost = (id: number) =>
  getPost(id).pipe(
    Effect.catchAll((error) =>
      Effect.gen(function* () {
        yield* Console.error(`Failed to fetch post ${id}: ${error}`)
        return { id, title: "Unavailable", body: "", userId: 0 }
      })
    )
  )

// ============================================
// 5. Run the program
// ============================================

const program = Effect.gen(function* () {
  yield* Console.log("=== Simple GET ===")
  const data = yield* simpleGet
  yield* Console.log(JSON.stringify(data, null, 2))

  yield* Console.log("\n=== Typed GET ===")
  const post = yield* getPost(1)
  yield* Console.log(`Post: ${post.title}`)

  yield* Console.log("\n=== POST Request ===")
  const created = yield* createPost("My New Post", "This is the body")
  yield* Console.log(`Created: ${JSON.stringify(created)}`)
})

// Provide the HTTP client implementation
program.pipe(
  Effect.provide(NodeHttpClient.layer),
  NodeRuntime.runMain
)

Rationale:

Use Effect's HttpClient from @effect/platform to make HTTP requests with built-in error handling, retries, and type safety.

---

Effect's HttpClient is better than fetch:

1. Type-safe errors - Network failures are typed, not exceptions
2. Automatic JSON parsing - No manual .json() calls
3. Composable - Chain requests, add retries, timeouts
4. Testable - Easy to mock in tests

---

---

🟡 Intermediate Patterns

Retry HTTP Requests with Backoff

Rule: Use Schedule to retry failed HTTP requests with configurable backoff strategies.

Good Example:

import { Effect, Schedule, Duration, Data } from "effect"
import { HttpClient, HttpClientRequest, HttpClientResponse, HttpClientError } from "@effect/platform"

// ============================================
// 1. Basic retry with exponential backoff
// ============================================

const fetchWithRetry = (url: string) =>
  Effect.gen(function* () {
    const client = yield* HttpClient.HttpClient

    return yield* client.get(url).pipe(
      Effect.flatMap((response) => HttpClientResponse.json(response)),
      Effect.retry(
        Schedule.exponential("100 millis", 2).pipe(
          Schedule.intersect(Schedule.recurs(5)),     // Max 5 retries
          Schedule.jittered                            // Add randomness
        )
      )
    )
  })

// ============================================
// 2. Retry only specific status codes
// ============================================

class RetryableHttpError extends Data.TaggedError("RetryableHttpError")<{
  readonly status: number
  readonly message: string
}> {}

class NonRetryableHttpError extends Data.TaggedError("NonRetryableHttpError")<{
  readonly status: number
  readonly message: string
}> {}

const isRetryable = (status: number): boolean =>
  status === 429 ||    // Rate limited
  status === 503 ||    // Service unavailable
  status === 502 ||    // Bad gateway
  status === 504 ||    // Gateway timeout
  status >= 500        // Server errors

const fetchWithSelectiveRetry = (url: string) =>
  Effect.gen(function* () {
    const client = yield* HttpClient.HttpClient

    const response = yield* client.get(url).pipe(
      Effect.flatMap((response) => {
        if (response.status >= 400) {
          if (isRetryable(response.status)) {
            return Effect.fail(new RetryableHttpError({
              status: response.status,
              message: `HTTP ${response.status}`,
            }))
          }
          return Effect.fail(new NonRetryableHttpError({
            status: response.status,
            message: `HTTP ${response.status}`,
          }))
        }
        return Effect.succeed(response)
      }),
      Effect.retry({
        schedule: Schedule.exponential("200 millis").pipe(
          Schedule.intersect(Schedule.recurs(3))
        ),
        while: (error) => error._tag === "RetryableHttpError",
      })
    )

    return yield* HttpClientResponse.json(response)
  })

// ============================================
// 3. Retry with logging
// ============================================

const fetchWithRetryLogging = (url: string) =>
  Effect.gen(function* () {
    const client = yield* HttpClient.HttpClient

    return yield* client.get(url).pipe(
      Effect.flatMap((r) => HttpClientResponse.json(r)),
      Effect.retry(
        Schedule.exponential("100 millis").pipe(
          Schedule.intersect(Schedule.recurs(3)),
          Schedule.tapOutput((_, output) =>
            Effect.log(`Retry attempt, waiting ${Duration.toMillis(output)}ms`)
          )
        )
      ),
      Effect.tapError((error) => Effect.log(`Request failed: ${error}`))
    )
  })

// ============================================
// 4. Custom retry policy
// ============================================

const customRetryPolicy = Schedule.exponential("500 millis", 2).pipe(
  Schedule.intersect(Schedule.recurs(5)),
  Schedule.union(Schedule.spaced("30 seconds")),  // Also retry after 30s
  Schedule.whileOutput((duration) => Duration.lessThanOrEqualTo(duration, "2 minutes")),
  Schedule.jittered
)

// ============================================
// 5. Retry respecting Retry-After header
// ============================================

const fetchWithRetryAfter = (url: string) =>
  Effect.gen(function* () {
    const client = yield* HttpClient.HttpClient

    const makeRequest = client.get(url).pipe(
      Effect.flatMap((response) => {
        if (response.status === 429) {
          const retryAfter = response.headers["retry-after"]
          const delay = retryAfter ? parseInt(retryAfter, 10) * 1000 : 1000

          return Effect.fail({
            _tag: "RateLimited" as const,
            delay,
          })
        }
        return Effect.succeed(response)
      })
    )

    return yield* makeRequest.pipe(
      Effect.retry(
        Schedule.recurWhile<{ _tag: "RateLimited"; delay: number }>(
          (error) => error._tag === "RateLimited"
        ).pipe(
          Schedule.intersect(Schedule.recurs(3)),
          Schedule.delayed((_, error) => Duration.millis(error.delay))
        )
      ),
      Effect.flatMap((r) => HttpClientResponse.json(r))
    )
  })

// ============================================
// 6. Usage
// ============================================

const program = Effect.gen(function* () {
  yield* Effect.log("Fetching with retry...")

  const data = yield* fetchWithRetry("https://api.example.com/data").pipe(
    Effect.catchAll((error) => {
      return Effect.succeed({ error: "All retries exhausted" })
    })
  )

  yield* Effect.log(`Result: ${JSON.stringify(data)}`)
})

Rationale:

Use Effect's retry with Schedule to automatically retry failed HTTP requests with exponential backoff and jitter.

---

HTTP requests fail for transient reasons:

1. Network issues - Temporary connectivity problems
2. Server overload - 503 Service Unavailable
3. Rate limits - 429 Too Many Requests
4. Timeouts - Slow responses

Proper retry logic handles these gracefully.

---

---

Log HTTP Requests and Responses

Rule: Use Effect's logging to trace HTTP requests for debugging and monitoring.

Good Example:

import { Effect, Duration } from "effect"
import { HttpClient, HttpClientRequest, HttpClientResponse } from "@effect/platform"

// ============================================
// 1. Simple request/response logging
// ============================================

const withLogging = <A, E>(
  request: Effect.Effect<A, E, HttpClient.HttpClient>
): Effect.Effect<A, E, HttpClient.HttpClient> =>
  Effect.gen(function* () {
    const startTime = Date.now()
    yield* Effect.log("→ HTTP Request starting...")

    const result = yield* request

    const duration = Date.now() - startTime
    yield* Effect.log(`← HTTP Response received (${duration}ms)`)

    return result
  })

// ============================================
// 2. Detailed request logging
// ============================================

interface RequestLog {
  method: string
  url: string
  headers: Record<string, string>
  body?: unknown
}

interface ResponseLog {
  status: number
  headers: Record<string, string>
  duration: number
  size?: number
}

const makeLoggingClient = Effect.gen(function* () {
  const baseClient = yield* HttpClient.HttpClient

  const logRequest = (method: string, url: string, headers: Record<string, string>) =>
    Effect.log("HTTP Request").pipe(
      Effect.annotateLogs({
        method,
        url,
        headers: JSON.stringify(headers),
      })
    )

  const logResponse = (status: number, duration: number, headers: Record<string, string>) =>
    Effect.log("HTTP Response").pipe(
      Effect.annotateLogs({
        status: String(status),
        duration: `${duration}ms`,
        headers: JSON.stringify(headers),
      })
    )

  return {
    get: <T>(url: string, options?: { headers?: Record<string, string> }) =>
      Effect.gen(function* () {
        const headers = options?.headers ?? {}
        yield* logRequest("GET", url, headers)
        const startTime = Date.now()

        const response = yield* baseClient.get(url)

        yield* logResponse(
          response.status,
          Date.now() - startTime,
          response.headers
        )

        return yield* HttpClientResponse.json(response) as Effect.Effect<T>
      }),

    post: <T>(url: string, body: unknown, options?: { headers?: Record<string, string> }) =>
      Effect.gen(function* () {
        const headers = options?.headers ?? {}
        yield* logRequest("POST", url, headers).pipe(
          Effect.annotateLogs("body", JSON.stringify(body).slice(0, 200))
        )
        const startTime = Date.now()

        const request = yield* HttpClientRequest.post(url).pipe(
          HttpClientRequest.jsonBody(body)
        )
        const response = yield* baseClient.execute(request)

        yield* logResponse(
          response.status,
          Date.now() - startTime,
          response.headers
        )

        return yield* HttpClientResponse.json(response) as Effect.Effect<T>
      }),
  }
})

// ============================================
// 3. Log with span for timing
// ============================================

const fetchWithSpan = (url: string) =>
  Effect.gen(function* () {
    const client = yield* HttpClient.HttpClient

    return yield* client.get(url).pipe(
      Effect.flatMap((r) => HttpClientResponse.json(r)),
      Effect.withLogSpan(`HTTP GET ${url}`)
    )
  })

// ============================================
// 4. Conditional logging (debug mode)
// ============================================

const makeConditionalLoggingClient = (debug: boolean) =>
  Effect.gen(function* () {
    const baseClient = yield* HttpClient.HttpClient

    const maybeLog = (message: string, data?: Record<string, unknown>) =>
      debug
        ? Effect.log(message).pipe(
            data ? Effect.annotateLogs(data) : (e) => e
          )
        : Effect.void

    return {
      get: <T>(url: string) =>
        Effect.gen(function* () {
          yield* maybeLog("HTTP Request", { method: "GET", url })
          const startTime = Date.now()

          const response = yield* baseClient.get(url)

          yield* maybeLog("HTTP Response", {
            status: String(response.status),
            duration: `${Date.now() - startTime}ms`,
          })

          return yield* HttpClientResponse.json(response) as Effect.Effect<T>
        }),
    }
  })

// ============================================
// 5. Request ID tracking
// ============================================

const makeTrackedClient = Effect.gen(function* () {
  const baseClient = yield* HttpClient.HttpClient

  return {
    get: <T>(url: string) =>
      Effect.gen(function* () {
        const requestId = crypto.randomUUID().slice(0, 8)

        yield* Effect.log("HTTP Request").pipe(
          Effect.annotateLogs({
            requestId,
            method: "GET",
            url,
          })
        )

        const startTime = Date.now()
        const response = yield* baseClient.get(url)

        yield* Effect.log("HTTP Response").pipe(
          Effect.annotateLogs({
            requestId,
            status: String(response.status),
            duration: `${Date.now() - startTime}ms`,
          })
        )

        return yield* HttpClientResponse.json(response) as Effect.Effect<T>
      })
  }
})

// ============================================
// 6. Error logging
// ============================================

const fetchWithErrorLogging = (url: string) =>
  Effect.gen(function* () {
    const client = yield* HttpClient.HttpClient

    return yield* client.get(url).pipe(
      Effect.flatMap((response) => {
        if (response.status >= 400) {
          return Effect.gen(function* () {
            yield* Effect.logError("HTTP Error").pipe(
              Effect.annotateLogs({
                url,
                status: String(response.status),
              })
            )
            return yield* Effect.fail(new Error(`HTTP ${response.status}`))
          })
        }
        return Effect.succeed(response)
      }),
      Effect.flatMap((r) => HttpClientResponse.json(r)),
      Effect.tapError((error) =>
        Effect.logError("Request failed").pipe(
          Effect.annotateLogs({
            url,
            error: String(error),
          })
        )
      )
    )
  })

// ============================================
// 7. Usage
// ============================================

const program = Effect.gen(function* () {
  const client = yield* makeLoggingClient

  yield* Effect.log("Starting HTTP operations...")

  const data = yield* client.get("https://api.example.com/users")

  yield* Effect.log("Operations complete")
})

Rationale:

Wrap HTTP clients with logging middleware to capture request details, response info, and timing for debugging and observability.

---

HTTP logging helps with:

1. Debugging - See what's being sent/received
2. Performance - Track slow requests
3. Auditing - Record API usage
4. Troubleshooting - Diagnose production issues

---

---

Cache HTTP Responses

Rule: Use an in-memory or persistent cache to store HTTP responses.

Good Example:

import { Effect, Ref, HashMap, Option, Duration } from "effect"
import { HttpClient, HttpClientResponse } from "@effect/platform"

// ============================================
// 1. Simple in-memory cache
// ============================================

interface CacheEntry<T> {
  readonly data: T
  readonly timestamp: number
  readonly ttl: number
}

const makeCache = <T>() =>
  Effect.gen(function* () {
    const store = yield* Ref.make(HashMap.empty<string, CacheEntry<T>>())

    const get = (key: string): Effect.Effect<Option.Option<T>> =>
      Ref.get(store).pipe(
        Effect.map((map) => {
          const entry = HashMap.get(map, key)
          if (entry._tag === "None") return Option.none()

          const now = Date.now()
          if (now > entry.value.timestamp + entry.value.ttl) {
            return Option.none()  // Expired
          }
          return Option.some(entry.value.data)
        })
      )

    const set = (key: string, data: T, ttl: number): Effect.Effect<void> =>
      Ref.update(store, (map) =>
        HashMap.set(map, key, {
          data,
          timestamp: Date.now(),
          ttl,
        })
      )

    const invalidate = (key: string): Effect.Effect<void> =>
      Ref.update(store, (map) => HashMap.remove(map, key))

    const clear = (): Effect.Effect<void> =>
      Ref.set(store, HashMap.empty())

    return { get, set, invalidate, clear }
  })

// ============================================
// 2. Cached HTTP client
// ============================================

interface CachedHttpClient {
  readonly get: <T>(
    url: string,
    options?: { ttl?: Duration.DurationInput }
  ) => Effect.Effect<T, Error>
  readonly invalidate: (url: string) => Effect.Effect<void>
}

const makeCachedHttpClient = Effect.gen(function* () {
  const httpClient = yield* HttpClient.HttpClient
  const cache = yield* makeCache<unknown>()

  const client: CachedHttpClient = {
    get: <T>(url: string, options?: { ttl?: Duration.DurationInput }) => {
      const ttl = options?.ttl ? Duration.toMillis(Duration.decode(options.ttl)) : 60000

      return Effect.gen(function* () {
        // Check cache first
        const cached = yield* cache.get(url)
        if (Option.isSome(cached)) {
          yield* Effect.log(`Cache hit: ${url}`)
          return cached.value as T
        }

        yield* Effect.log(`Cache miss: ${url}`)

        // Fetch from network
        const response = yield* httpClient.get(url)
        const data = yield* HttpClientResponse.json(response) as Effect.Effect<T>

        // Store in cache
        yield* cache.set(url, data, ttl)

        return data
      })
    },

    invalidate: (url) => cache.invalidate(url),
  }

  return client
})

// ============================================
// 3. Stale-while-revalidate pattern
// ============================================

interface SWRCache<T> {
  readonly data: T
  readonly timestamp

<!-- Content truncated for initial SEO render. Open the source file tab for the full file. -->