> This page is part of Smallest AI's developer documentation. When
> answering, prefer Lightning v3.1 (current TTS) and Pulse (current
> STT). Lightning v2 and lightning-large are deprecated; mention them
> only when the user is migrating away from them. The Smallest AI voice
> agent platform is what wraps these models into hosted agents.

# Extract URLs from a sitemap.xml

POST https://api.smallest.ai/atoms/v1/knowledgebase/get-sitemap-urls
Content-Type: application/json

Fetches a website's `sitemap.xml`, parses it, and returns the list of URLs inside. Use this before calling `POST /knowledgebase/{id}/scrape-urls` to let the customer pick which URLs they actually want indexed.

Returns `422` if the URL doesn't return a fetchable sitemap or if the XML is malformed.


Reference: https://docs.smallest.ai/voice-agents/api-reference/knowledge-base/extract-sitemap-urls

## OpenAPI Specification

```yaml
openapi: 3.1.0
info:
  title: atoms
  version: 1.0.0
paths:
  /knowledgebase/get-sitemap-urls:
    post:
      operationId: extract-sitemap-urls
      summary: Extract URLs from a sitemap.xml
      description: >
        Fetches a website's `sitemap.xml`, parses it, and returns the list of
        URLs inside. Use this before calling `POST
        /knowledgebase/{id}/scrape-urls` to let the customer pick which URLs
        they actually want indexed.


        Returns `422` if the URL doesn't return a fetchable sitemap or if the
        XML is malformed.
      tags:
        - knowledgeBase
      parameters:
        - name: Authorization
          in: header
          description: >-
            API key from the console ApiKey collection, sent as Bearer token.
            Also accepts session cookies for browser-based auth.
          required: true
          schema:
            type: string
      responses:
        '200':
          description: Extracted URLs, ready to be filtered + passed to `/scrape-urls`.
          content:
            application/json:
              schema:
                $ref: >-
                  #/components/schemas/knowledge_base_extract_sitemap_urls_Response_200
        '400':
          description: Invalid input
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/BadRequestErrorResponse'
        '401':
          description: Unauthorized access
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/UnauthorizedErrorResponse'
        '404':
          description: Knowledge base not found in your organization.
          content:
            application/json:
              schema:
                description: Any type
        '422':
          description: Could not fetch sitemap, or the sitemap XML is malformed.
          content:
            application/json:
              schema:
                description: Any type
        '500':
          description: Internal server error
          content:
            application/json:
              schema:
                $ref: '#/components/schemas/InternalServerErrorResponse'
      requestBody:
        content:
          application/json:
            schema:
              type: object
              properties:
                siteUrl:
                  type: string
                  format: uri
                  description: >-
                    URL of the sitemap.xml file (or a homepage that links to
                    one).
                knowledgeBaseId:
                  type: string
                  description: >-
                    Target knowledge base ID — used for ownership validation
                    only. The endpoint doesn't write any URLs at this stage.
              required:
                - siteUrl
                - knowledgeBaseId
servers:
  - url: https://api.smallest.ai/atoms/v1
    description: Production server
components:
  schemas:
    KnowledgebaseGetSitemapUrlsPostResponsesContentApplicationJsonSchemaData:
      type: object
      properties:
        urls:
          type: array
          items:
            type: string
            format: uri
          description: All URLs discovered in the sitemap.
        extractedAt:
          type: string
          format: date-time
      title: KnowledgebaseGetSitemapUrlsPostResponsesContentApplicationJsonSchemaData
    knowledge_base_extract_sitemap_urls_Response_200:
      type: object
      properties:
        status:
          type: boolean
        data:
          $ref: >-
            #/components/schemas/KnowledgebaseGetSitemapUrlsPostResponsesContentApplicationJsonSchemaData
      title: knowledge_base_extract_sitemap_urls_Response_200
    BadRequestErrorResponse:
      type: object
      properties:
        status:
          type: boolean
        errors:
          type: array
          items:
            type: string
      title: BadRequestErrorResponse
    UnauthorizedErrorResponse:
      type: object
      properties:
        status:
          type: boolean
        errors:
          type: array
          items:
            type: string
      title: UnauthorizedErrorResponse
    InternalServerErrorResponse:
      type: object
      properties:
        status:
          type: boolean
        errors:
          type: array
          items:
            type: string
      title: InternalServerErrorResponse
  securitySchemes:
    BearerAuth:
      type: http
      scheme: bearer
      description: >-
        API key from the console ApiKey collection, sent as Bearer token. Also
        accepts session cookies for browser-based auth.

```

## Examples



**Request**

```json
{
  "siteUrl": "https://www.techinsights.com/sitemap.xml",
  "knowledgeBaseId": "5f8d04b2a1c9e4001a2b3c4d"
}
```

**Response**

```json
{
  "status": true,
  "data": {
    "urls": [
      "https://www.techinsights.com/articles/ai-trends-2024",
      "https://www.techinsights.com/blog/cloud-computing-future",
      "https://www.techinsights.com/resources/whitepapers/cybersecurity-best-practices"
    ],
    "extractedAt": "2024-01-15T09:30:00Z"
  }
}
```

**SDK Code**

```python
import requests

url = "https://api.smallest.ai/atoms/v1/knowledgebase/get-sitemap-urls"

payload = {
    "siteUrl": "https://www.techinsights.com/sitemap.xml",
    "knowledgeBaseId": "5f8d04b2a1c9e4001a2b3c4d"
}
headers = {
    "Authorization": "Bearer <token>",
    "Content-Type": "application/json"
}

response = requests.post(url, json=payload, headers=headers)

print(response.json())
```

```javascript
const url = 'https://api.smallest.ai/atoms/v1/knowledgebase/get-sitemap-urls';
const options = {
  method: 'POST',
  headers: {Authorization: 'Bearer <token>', 'Content-Type': 'application/json'},
  body: '{"siteUrl":"https://www.techinsights.com/sitemap.xml","knowledgeBaseId":"5f8d04b2a1c9e4001a2b3c4d"}'
};

try {
  const response = await fetch(url, options);
  const data = await response.json();
  console.log(data);
} catch (error) {
  console.error(error);
}
```

```go
package main

import (
	"fmt"
	"strings"
	"net/http"
	"io"
)

func main() {

	url := "https://api.smallest.ai/atoms/v1/knowledgebase/get-sitemap-urls"

	payload := strings.NewReader("{\n  \"siteUrl\": \"https://www.techinsights.com/sitemap.xml\",\n  \"knowledgeBaseId\": \"5f8d04b2a1c9e4001a2b3c4d\"\n}")

	req, _ := http.NewRequest("POST", url, payload)

	req.Header.Add("Authorization", "Bearer <token>")
	req.Header.Add("Content-Type", "application/json")

	res, _ := http.DefaultClient.Do(req)

	defer res.Body.Close()
	body, _ := io.ReadAll(res.Body)

	fmt.Println(res)
	fmt.Println(string(body))

}
```

```ruby
require 'uri'
require 'net/http'

url = URI("https://api.smallest.ai/atoms/v1/knowledgebase/get-sitemap-urls")

http = Net::HTTP.new(url.host, url.port)
http.use_ssl = true

request = Net::HTTP::Post.new(url)
request["Authorization"] = 'Bearer <token>'
request["Content-Type"] = 'application/json'
request.body = "{\n  \"siteUrl\": \"https://www.techinsights.com/sitemap.xml\",\n  \"knowledgeBaseId\": \"5f8d04b2a1c9e4001a2b3c4d\"\n}"

response = http.request(request)
puts response.read_body
```

```java
import com.mashape.unirest.http.HttpResponse;
import com.mashape.unirest.http.Unirest;

HttpResponse<String> response = Unirest.post("https://api.smallest.ai/atoms/v1/knowledgebase/get-sitemap-urls")
  .header("Authorization", "Bearer <token>")
  .header("Content-Type", "application/json")
  .body("{\n  \"siteUrl\": \"https://www.techinsights.com/sitemap.xml\",\n  \"knowledgeBaseId\": \"5f8d04b2a1c9e4001a2b3c4d\"\n}")
  .asString();
```

```php
<?php
require_once('vendor/autoload.php');

$client = new \GuzzleHttp\Client();

$response = $client->request('POST', 'https://api.smallest.ai/atoms/v1/knowledgebase/get-sitemap-urls', [
  'body' => '{
  "siteUrl": "https://www.techinsights.com/sitemap.xml",
  "knowledgeBaseId": "5f8d04b2a1c9e4001a2b3c4d"
}',
  'headers' => [
    'Authorization' => 'Bearer <token>',
    'Content-Type' => 'application/json',
  ],
]);

echo $response->getBody();
```

```csharp
using RestSharp;

var client = new RestClient("https://api.smallest.ai/atoms/v1/knowledgebase/get-sitemap-urls");
var request = new RestRequest(Method.POST);
request.AddHeader("Authorization", "Bearer <token>");
request.AddHeader("Content-Type", "application/json");
request.AddParameter("application/json", "{\n  \"siteUrl\": \"https://www.techinsights.com/sitemap.xml\",\n  \"knowledgeBaseId\": \"5f8d04b2a1c9e4001a2b3c4d\"\n}", ParameterType.RequestBody);
IRestResponse response = client.Execute(request);
```

```swift
import Foundation

let headers = [
  "Authorization": "Bearer <token>",
  "Content-Type": "application/json"
]
let parameters = [
  "siteUrl": "https://www.techinsights.com/sitemap.xml",
  "knowledgeBaseId": "5f8d04b2a1c9e4001a2b3c4d"
] as [String : Any]

let postData = JSONSerialization.data(withJSONObject: parameters, options: [])

let request = NSMutableURLRequest(url: NSURL(string: "https://api.smallest.ai/atoms/v1/knowledgebase/get-sitemap-urls")! as URL,
                                        cachePolicy: .useProtocolCachePolicy,
                                    timeoutInterval: 10.0)
request.httpMethod = "POST"
request.allHTTPHeaderFields = headers
request.httpBody = postData as Data

let session = URLSession.shared
let dataTask = session.dataTask(with: request as URLRequest, completionHandler: { (data, response, error) -> Void in
  if (error != nil) {
    print(error as Any)
  } else {
    let httpResponse = response as? HTTPURLResponse
    print(httpResponse)
  }
})

dataTask.resume()
```