Go Colly and Cloudflare Turnstile: A Scraper Tutorial

package main

import (
  "bytes"
  "crypto/rand"
  "encoding/json"
  "errors"
  "fmt"
  "log"
  "net/http"
  "os"
  "time"

  "github.com/gocolly/colly/v2"
)

var (
  apiURL = os.Getenv("ZEROCAPTCHA_API")
  apiKey = os.Getenv("ZEROCAPTCHA_KEY")
  api    = &http.Client{Timeout: 15 * time.Second}
)

type reply struct {
  ErrorID   int    `json:"errorId"`
  ErrorCode string `json:"errorCode"`
  TaskID    string `json:"taskId"`
  Status    string `json:"status"`
  Solution  struct {
    Token string `json:"token"`
  } `json:"solution"`
}

func call(method string, body map[string]any, idempotencyKey string) (reply, error) {
  body["clientKey"] = apiKey
  payload, err := json.Marshal(body)
  if err != nil {
    return reply{}, err
  }
  req, err := http.NewRequest(http.MethodPost, apiURL+"/"+method, bytes.NewReader(payload))
  if err != nil {
    return reply{}, err
  }
  req.Header.Set("Content-Type", "application/json")
  if idempotencyKey != "" {
    req.Header.Set("Idempotency-Key", idempotencyKey)
  }
  resp, err := api.Do(req)
  if err != nil {
    return reply{}, err
  }
  defer resp.Body.Close()
  var r reply
  if err := json.NewDecoder(resp.Body).Decode(&r); err != nil {
    return reply{}, fmt.Errorf("%s: %w", method, err)
  }
  if r.ErrorID != 0 {
    return r, fmt.Errorf("%s: %s", method, r.ErrorCode)
  }
  return r, nil
}

func solveTurnstile(pageURL, sitekey, action, cdata string) (string, error) {
  // The widget's data-action and data-cdata go in metadata, only when it sets them: many sites
  // check both when they verify the token.
  metadata := map[string]string{}
  if action != "" {
    metadata["action"] = action
  }
  if cdata != "" {
    metadata["cdata"] = cdata
  }
  task := map[string]any{"type": "TurnstileTaskProxyless", "websiteURL": pageURL, "websiteKey": sitekey, "metadata": metadata}
  // One Idempotency-Key per task: a retried create with it returns the same task.
  created, err := call("createTask", map[string]any{"task": task}, rand.Text())
  if err != nil {
    return "", err
  }
  deadline := time.Now().Add(180 * time.Second)
  for time.Now().Before(deadline) {
    time.Sleep(2 * time.Second)
    result, err := call("getTaskResult", map[string]any{"taskId": created.TaskID}, "")
    if err != nil {
      return "", err
    }
    if result.Status == "ready" {
      return result.Solution.Token, nil
    }
  }
  return "", errors.New("no token within 180 seconds")
}

func main() {
  c := colly.NewCollector(colly.AllowedDomains("shop.example.com"), colly.Async(true))
  if err := c.Limit(&colly.LimitRule{DomainGlob: "*", Parallelism: 4}); err != nil {
    log.Fatal(err)
  }

  c.OnHTML("form#search", func(e *colly.HTMLElement) {
    sitekey := e.ChildAttr("[data-sitekey]", "data-sitekey")
    if sitekey == "" {
      log.Printf("no Cloudflare Turnstile widget on %s", e.Request.URL)
      return
    }
    token, err := solveTurnstile(e.Request.URL.String(), sitekey,
      e.ChildAttr("[data-sitekey]", "data-action"), e.ChildAttr("[data-sitekey]", "data-cdata"))
    if err != nil {
      log.Print(err)
      return
    }
    target := e.Request.AbsoluteURL(e.Attr("action"))
    if err := c.Post(target, map[string]string{"q": "running shoes", "cf-turnstile-response": token}); err != nil {
      log.Print(err)
    }
  })

  c.OnHTML(".result a[href]", func(e *colly.HTMLElement) {
    fmt.Println(e.Text, e.Request.AbsoluteURL(e.Attr("href")))
  })

  if err := c.Visit("https://shop.example.com/search"); err != nil {
    log.Fatal(err)
  }
  c.Wait()
}