> ## Documentation Index
> Fetch the complete documentation index at: https://docs.gately.ai/llms.txt
> Use this file to discover all available pages before exploring further.

# Website Crawling

> Crawl websites with advanced configuration options

The Website Crawling service allows you to systematically explore and extract content from websites.

### Required Parameters

<ParamField body="url" type="string" required>
  The base URL to start crawling from
</ParamField>

### Optional Parameters

<AccordionGroup>
  <Accordion title="Crawl Scope" icon="spider-web">
    <ParamField body="maxDepth" type="integer" default="2">
      Maximum depth to crawl relative to the base URL
    </ParamField>

    <ParamField body="limit" type="integer" default="10000">
      Maximum number of pages to crawl
    </ParamField>

    <ParamField body="allowBackwardLinks" type="boolean" default="false">
      Enable navigation to previously linked pages
    </ParamField>

    <ParamField body="allowExternalLinks" type="boolean" default="false">
      Allow following links to external websites
    </ParamField>
  </Accordion>

  <Accordion title="URL Filtering" icon="filter">
    <ParamField body="includePaths" type="array">
      Regex patterns for URLs to include (e.g., `["blog/*"]`)
    </ParamField>

    <ParamField body="excludePaths" type="array">
      Regex patterns for URLs to exclude (e.g., `["admin/*"]`)
    </ParamField>
  </Accordion>

  <Accordion title="Crawl Settings" icon="gear">
    <ParamField body="ignoreSitemap" type="boolean" default="false">
      Ignore the website's sitemap.xml
    </ParamField>

    <ParamField body="webhook" type="string">
      Webhook URL for crawl events:

      * `crawl.started`
      * `crawl.page`
      * `crawl.completed`
      * `crawl.failed`
    </ParamField>
  </Accordion>

  <Accordion title="Scrape Options" icon="screwdriver-wrench">
    <ParamField body="scrapeOptions" type="object">
      Configure how each page is scraped:

      ```json theme={null}
      {
        "formats": ["markdown"],
        "onlyMainContent": true,
        "removeBase64Images": true,
        "mobile": false,
        "waitFor": 0,
        "headers": {},
        "includeTags": [],
        "excludeTags": []
      }
      ```
    </ParamField>
  </Accordion>
</AccordionGroup>

### Example Request

```bash theme={null}
curl --request POST \
  --url https://crawl.taam.cloud/v1/web \
  --header 'Authorization: Bearer YOUR_API_KEY' \
  --header 'Content-Type: application/json' \
  --data '{
    "url": "https://example.com",
    "maxDepth": 2,
    "includePaths": ["blog/*"],
    "excludePaths": ["admin/*"],
    "limit": 100,
    "scrapeOptions": {
      "formats": ["markdown"],
      "onlyMainContent": true
    }
  }'
```

### Example Response

```json theme={null}
{
  "success": true,
  "id": "crawl_123abc",
  "url": "https://example.com"
}
```

### Status Codes

<AccordionGroup>
  <Accordion title="200 - Success" icon="check">
    Crawl job started successfully
  </Accordion>

  <Accordion title="402 - Payment Required" icon="credit-card">
    Insufficient credits or requires payment
  </Accordion>

  <Accordion title="429 - Too Many Requests" icon="clock">
    Rate limit exceeded
  </Accordion>

  <Accordion title="500 - Server Error" icon="triangle-exclamation">
    Internal server error
  </Accordion>
</AccordionGroup>


## OpenAPI

````yaml POST /v1/web
openapi: 3.0.1
info:
  title: TaamCloud API
  description: >-
    A collection of AI services including chat, embeddings, reranking, and media
    generation
  version: 1.0.0
servers:
  - url: https://api.gately.ai
    description: Main API server
  - url: https://uploud.taam.cloud
    description: File Processing server
security:
  - bearerAuth: []
paths:
  /v1/web:
    post:
      tags:
        - Web
      summary: Web services API
      description: Unified endpoint for web scraping, crawling, mapping and AI search
      operationId: web
      requestBody:
        required: true
        content:
          application/json:
            schema:
              $ref: '#/components/schemas/WebRequest'
      responses:
        '200':
          description: Successful response
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/WebResponse'
components:
  schemas:
    WebRequest:
      type: object
      required:
        - model
      properties:
        model:
          type: string
          enum:
            - scrape
            - crawl
            - map
            - taam-ai-search
            - crawl-status
          description: Type of web service to use
        params:
          type: object
          description: Parameters specific to the selected model
    WebResponse:
      type: object
      properties:
        id:
          type: string
          description: Unique identifier for the request
        object:
          type: string
          description: Type of completion (e.g., scrape.completion)
        created:
          type: integer
          description: Unix timestamp of when the request was created
        model:
          type: string
          description: Model used for the request
        data:
          type: object
          description: Model-specific response data
        usage:
          type: object
          properties:
            prompt_tokens:
              type: integer
            completion_tokens:
              type: integer
            total_tokens:
              type: integer
        system_fingerprint:
          type: string
  securitySchemes:
    bearerAuth:
      type: http
      scheme: bearer
      description: Enter your API key prefixed with 'Bearer '

````