IndexNow API Deep Dive: Mastering JSON Payloads & Key Verifications

https://api.indexnow.org/indexnow

host:
  - Must be a valid hostname
  - No protocol (https://)
  - No path (/)
  - No port (:8080)
  - Examples: "example.com", "subdomain.example.com"
  
key:
  - Minimum 8 characters
  - Recommended: 32-64 hexadecimal characters
  - Can contain: a-z, A-Z, 0-9, -, _
  - Must match the content of your key file exactly
  
keyLocation:
  - Must use HTTPS protocol
  - Must be accessible without authentication
  - Must return Content-Type: text/plain
  - Must return HTTP 200 OK
  - File content must match the key parameter exactly
  
urlList:
  - Array of 1 to 10,000 strings
  - Each must be a valid absolute URL
  - Each must use HTTPS protocol
  - Each must belong to the specified host
  - Maximum 2,048 characters per URL
  - Must be URL-encoded if containing special characters

{
  "host": "example.com",
  "key": "8f7d6e5c4b3a2918f7d6e5c4b3a2918f",
  "keyLocation": "https://example.com/8f7d6e5c4b3a2918f7d6e5c4b3a2918f.txt",
  "urlList": [
    "https://example.com/page1",
    "https://example.com/page2",
    "https://example.com/page3"
  ]
}

// WRONG: Host includes protocol
{
  "host": "https://example.com",
  "key": "abc123",
  "keyLocation": "https://example.com/abc123.txt",
  "urlList": ["https://example.com/page"]
}

// WRONG: URL uses HTTP instead of HTTPS
{
  "host": "example.com",
  "key": "abc123",
  "keyLocation": "https://example.com/abc123.txt",
  "urlList": ["http://example.com/page"]
}

// WRONG: URL belongs to different host
{
  "host": "example.com",
  "key": "abc123",
  "keyLocation": "https://example.com/abc123.txt",
  "urlList": ["https://other-site.com/page"]
}

// WRONG: Empty urlList
{
  "host": "example.com",
  "key": "abc123",
  "keyLocation": "https://example.com/abc123.txt",
  "urlList": []
}

GET https://api.indexnow.org/indexnow?url={URL}&key={KEY}

# Submit a single URL via GET
curl -G "https://api.indexnow.org/indexnow" \
  --data-urlencode "url=https://example.com/new-post" \
  --data-urlencode "key=abc123def456789ghi012jkl345mno67"

POST https://api.indexnow.org/indexnow HTTP/1.1
Content-Type: application/json; charset=utf-8

{
  "host": "example.com",
  "key": "YOUR_KEY",
  "keyLocation": "https://example.com/YOUR_KEY.txt",
  "urlList": ["https://example.com/page"]
}

curl -X POST "https://api.indexnow.org/indexnow" \
  -H "Content-Type: application/json; charset=utf-8" \
  -d '{
    "host": "example.com",
    "key": "abc123def456789ghi012jkl345mno67",
    "keyLocation": "https://example.com/abc123def456789ghi012jkl345mno67.txt",
    "urlList": [
      "https://example.com/blog/post-1",
      "https://example.com/blog/post-2",
      "https://example.com/products/widget"
    ]
  }'

import requests

payload = {
    "host": "example.com",
    "key": "abc123def456789ghi012jkl345mno67",
    "keyLocation": "https://example.com/abc123def456789ghi012jkl345mno67.txt",
    "urlList": [
        "https://example.com/blog/post-1",
        "https://example.com/blog/post-2"
    ]
}

response = requests.post(
    "https://api.indexnow.org/indexnow",
    json=payload,
    headers={"Content-Type": "application/json; charset=utf-8"}
)

print(f"Status: {response.status_code}")
print(f"Response: {response.text}")

const payload = {
  host: "example.com",
  key: "abc123def456789ghi012jkl345mno67",
  keyLocation: "https://example.com/abc123def456789ghi012jkl345mno67.txt",
  urlList: [
    "https://example.com/blog/post-1",
    "https://example.com/blog/post-2"
  ]
};

fetch("https://api.indexnow.org/indexnow", {
  method: "POST",
  headers: {
    "Content-Type": "application/json; charset=utf-8"
  },
  body: JSON.stringify(payload)
})
.then(response => {
  console.log(`Status: ${response.status}`);
  return response.text();
})
.then(text => console.log(`Response: ${text}`));

import requests

def submit_large_url_list(urls, host, key, key_location):
    chunk_size = 10000
    total_submitted = 0
    
    for i in range(0, len(urls), chunk_size):
        chunk = urls[i:i + chunk_size]
        
        payload = {
            "host": host,
            "key": key,
            "keyLocation": key_location,
            "urlList": chunk
        }
        
        response = requests.post(
            "https://api.indexnow.org/indexnow",
            json=payload,
            headers={"Content-Type": "application/json; charset=utf-8"}
        )
        
        if response.status_code == 200:
            total_submitted += len(chunk)
            print(f"Batch {i//chunk_size + 1}: Submitted {len(chunk)} URLs")
        else:
            print(f"Batch {i//chunk_size + 1} failed: HTTP {response.status_code}")
            # Log failed URLs for retry
            
    return total_submitted

# Usage
urls = [f"https://example.com/page-{i}" for i in range(25000)]
submit_large_url_list(urls, "example.com", "your-key", "https://example.com/your-key.txt")

import time
import requests
from typing import List

class IndexNowClient:
    def __init__(self, host: str, key: str, key_location: str):
        self.host = host
        self.key = key
        self.key_location = key_location
        self.endpoint = "https://api.indexnow.org/indexnow"
    
    def submit(self, urls: List[str], max_retries: int = 3) -> dict:
        payload = {
            "host": self.host,
            "key": self.key,
            "keyLocation": self.key_location,
            "urlList": urls
        }
        
        for attempt in range(max_retries + 1):
            try:
                response = requests.post(
                    self.endpoint,
                    json=payload,
                    headers={"Content-Type": "application/json; charset=utf-8"},
                    timeout=30
                )
                
                if response.status_code == 200:
                    return {
                        "success": True,
                        "status": response.status_code,
                        "urls_submitted": len(urls)
                    }
                
                elif response.status_code == 429:
                    # Rate limited—back off
                    wait_time = 2 ** attempt * 5  # 5, 10, 20 seconds
                    print(f"Rate limited. Waiting {wait_time}s before retry...")
                    time.sleep(wait_time)
                    continue
                
                elif response.status_code in [500, 502, 503, 504]:
                    # Server error—retry
                    if attempt < max_retries:
                        wait_time = 2 ** attempt * 10  # 10, 20, 40 seconds
                        print(f"Server error {response.status_code}. Retrying in {wait_time}s...")
                        time.sleep(wait_time)
                        continue
                
                elif response.status_code == 403:
                    # Key validation failed—do not retry
                    return {
                        "success": False,
                        "status": response.status_code,
                        "error": "Key validation failed. Check key file.",
                        "urls_submitted": 0
                    }
                
                elif response.status_code == 400:
                    # Bad request—do not retry
                    return {
                        "success": False,
                        "status": response.status_code,
                        "error": f"Bad request: {response.text}",
                        "urls_submitted": 0
                    }
                
                else:
                    return {
                        "success": False,
                        "status": response.status_code,
                        "error": f"Unexpected status: {response.status_code}",
                        "urls_submitted": 0
                    }
                    
            except requests.RequestException as e:
                if attempt < max_retries:
                    wait_time = 2 ** attempt * 5
                    print(f"Request failed: {e}. Retrying in {wait_time}s...")
                    time.sleep(wait_time)
                else:
                    return {
                        "success": False,
                        "error": str(e),
                        "urls_submitted": 0
                    }
        
        return {
            "success": False,
            "error": "Max retries exceeded",
            "urls_submitted": 0
        }

Attempt 1: Immediate submission
  └── If 429 or 5xx → Wait 5 seconds → Retry

Attempt 2: After 5-second delay
  └── If 429 or 5xx → Wait 10 seconds → Retry

Attempt 3: After 10-second delay
  └── If 429 or 5xx → Wait 20 seconds → Retry

Attempt 4: After 20-second delay
  └── If 429 or 5xx → Wait 60 seconds → Retry

Attempt 5: After 60-second delay
  └── If still failing → Queue for later, alert admin

// JavaScript retry implementation
async function submitWithRetry(urls, maxRetries = 4) {
  const payload = {
    host: 'example.com',
    key: process.env.INDEXNOW_KEY,
    keyLocation: `https://example.com/${process.env.INDEXNOW_KEY}.txt`,
    urlList: urls
  };
  
  for (let attempt = 0; attempt <= maxRetries; attempt++) {
    try {
      const response = await fetch('https://api.indexnow.org/indexnow', {
        method: 'POST',
        headers: { 'Content-Type': 'application/json' },
        body: JSON.stringify(payload)
      });
      
      if (response.status === 200) {
        return { success: true, attempt: attempt + 1 };
      }
      
      if (response.status === 403) {
        // Do not retry key validation failures
        throw new Error(`Key validation failed: ${await response.text()}`);
      }
      
      if (response.status === 400) {
        // Do not retry bad requests
        throw new Error(`Bad request: ${await response.text()}`);
      }
      
      if (response.status === 429 || response.status >= 500) {
        if (attempt < maxRetries) {
          const delay = Math.pow(2, attempt) * 5000; // 5s, 10s, 20s, 40s
          console.log(`Attempt ${attempt + 1} failed with ${response.status}. Retrying in ${delay}ms...`);
          await new Promise(resolve => setTimeout(resolve, delay));
          continue;
        }
      }
      
      throw new Error(`Unexpected status: ${response.status}`);
      
    } catch (error) {
      if (attempt === maxRetries) {
        throw new Error(`Max retries exceeded: ${error.message}`);
      }
    }
  }
}

import time
from collections import deque

class RateLimitedIndexNowClient:
    def __init__(self, host, key, key_location, max_requests_per_minute=60):
        self.host = host
        self.key = key
        self.key_location = key_location
        self.max_requests = max_requests_per_minute
        self.requests = deque()
    
    def _check_rate_limit(self):
        now = time.time()
        # Remove requests older than 60 seconds
        while self.requests and self.requests[0] < now - 60:
            self.requests.popleft()
        
        if len(self.requests) >= self.max_requests:
            sleep_time = 60 - (now - self.requests[0])
            if sleep_time > 0:
                print(f"Rate limit reached. Sleeping {sleep_time:.1f}s...")
                time.sleep(sleep_time)
    
    def submit(self, urls):
        self._check_rate_limit()
        self.requests.append(time.time())
        
        import requests
        payload = {
            "host": self.host,
            "key": self.key,
            "keyLocation": self.key_location,
            "urlList": urls
        }
        
        return requests.post(
            "https://api.indexnow.org/indexnow",
            json=payload,
            headers={"Content-Type": "application/json; charset=utf-8"}
        )

def validate_payload(payload):
    # Validate host
    if not payload.get('host') or '/' in payload['host'] or ':' in payload['host']:
        raise ValueError("Invalid host format")
    
    # Validate key
    if len(payload.get('key', '')) < 8:
        raise ValueError("Key must be at least 8 characters")
    
    # Validate keyLocation
    if not payload.get('keyLocation', '').startswith('https://'):
        raise ValueError("keyLocation must use HTTPS")
    
    # Validate urlList
    urls = payload.get('urlList', [])
    if not urls or len(urls) > 10000:
        raise ValueError("urlList must contain 1-10,000 URLs")
    
    for url in urls:
        if not url.startswith('https://'):
            raise ValueError(f"URL must use HTTPS: {url}")
        if len(url) > 2048:
            raise ValueError(f"URL too long: {url}")
        if payload['host'] not in url:
            raise ValueError(f"URL does not match host: {url}")
    
    return True

#!/usr/bin/env python3
import os
import time
import secrets
import logging
import requests
from pathlib import Path
from typing import List, Optional

logging.basicConfig(level=logging.INFO)
logger = logging.getLogger('indexnow')

class IndexNowClient:
    def __init__(self, host: str, key: Optional[str] = None, 
                 key_file_dir: Optional[str] = None):
        self.host = host
        self.key = key or self._generate_key()
        self.key_location = f"https://{host}/{self.key}.txt"
        self.endpoint = "https://api.indexnow.org/indexnow"
        self.key_file_dir = Path(key_file_dir or os.getcwd())
        
        self._ensure_key_file()
        logger.info(f"IndexNow client initialized for {host}")
    
    def _generate_key(self) -> str:
        return secrets.token_hex(32)
    
    def _ensure_key_file(self):
        file_path = self.key_file_dir / f"{self.key}.txt"
        if not file_path.exists():
            file_path.write_text(self.key)
            logger.info(f"Created key file: {file_path}")
    
    def submit(self, urls: List[str], max_retries: int = 3) -> dict:
        if not urls:
            return {"success": False, "error": "No URLs provided"}
        
        if len(urls) > 10000:
            raise ValueError("Maximum 10,000 URLs per request")
        
        payload = {
            "host": self.host,
            "key": self.key,
            "keyLocation": self.key_location,
            "urlList": urls
        }
        
        for attempt in range(max_retries + 1):
            try:
                response = requests.post(
                    self.endpoint,
                    json=payload,
                    headers={"Content-Type": "application/json; charset=utf-8"},
                    timeout=30
                )
                
                if response.status_code == 200:
                    logger.info(f"Submitted {len(urls)} URLs successfully")
                    return {
                        "success": True,
                        "status": response.status_code,
                        "urls_submitted": len(urls),
                        "attempts": attempt + 1
                    }
                
                elif response.status_code == 429:
                    if attempt < max_retries:
                        delay = 2 ** attempt * 5
                        logger.warning(f"Rate limited. Retrying in {delay}s...")
                        time.sleep(delay)
                        continue
                
                elif response.status_code >= 500:
                    if attempt < max_retries:
                        delay = 2 ** attempt * 10
                        logger.warning(f"Server error {response.status_code}. Retrying in {delay}s...")
                        time.sleep(delay)
                        continue
                
                else:
                    return {
                        "success": False,
                        "status": response.status_code,
                        "error": response.text,
                        "urls_submitted": 0
                    }
                    
            except requests.RequestException as e:
                if attempt < max_retries:
                    delay = 2 ** attempt * 5
                    logger.error(f"Request failed: {e}. Retrying in {delay}s...")
                    time.sleep(delay)
                else:
                    return {
                        "success": False,
                        "error": str(e),
                        "urls_submitted": 0
                    }
        
        return {"success": False, "error": "Max retries exceeded", "urls_submitted": 0}
    
    def submit_single(self, url: str) -> dict:
        return self.submit([url])

# Usage example
if __name__ == "__main__":
    client = IndexNowClient("example.com")
    
    # Submit single URL
    result = client.submit_single("https://example.com/new-post")
    print(result)
    
    # Submit batch
    urls = [f"https://example.com/page-{i}" for i in range(100)]
    result = client.submit(urls)
    print(result)

#!/usr/bin/env node
const axios = require('axios');
const crypto = require('crypto');
const fs = require('fs').promises;
const path = require('path');

class IndexNowClient {
  constructor(options = {}) {
    this.host = options.host;
    this.key = options.key || this.generateKey();
    this.keyLocation = `https://${this.host}/${this.key}.txt`;
    this.endpoint = options.endpoint || 'https://api.indexnow.org/indexnow';
    this.keyFileDir = options.keyFileDir || process.cwd();
    this.logger = options.logger || console;
    
    this.ensureKeyFile().catch(err => this.logger.error(err));
  }
  
  generateKey() {
    return crypto.randomBytes(32).toString('hex');
  }
  
  async ensureKeyFile() {
    const filePath = path.join(this.keyFileDir, `${this.key}.txt`);
    try {
      await fs.access(filePath);
    } catch {
      await fs.writeFile(filePath, this.key);
      this.logger.log(`Created key file: ${filePath}`);
    }
  }
  
  async submit(urls, maxRetries = 3) {
    const urlArray = Array.isArray(urls) ? urls : [urls];
    
    if (urlArray.length === 0) {
      return { success: false, error: 'No URLs provided' };
    }
    
    if (urlArray.length > 10000) {
      throw new Error('Maximum 10,000 URLs per request');
    }
    
    const payload = {
      host: this.host,
      key: this.key,
      keyLocation: this.keyLocation,
      urlList: urlArray
    };
    
    for (let attempt = 0; attempt <= maxRetries; attempt++) {
      try {
        const response = await axios.post(this.endpoint, payload, {
          headers: { 'Content-Type': 'application/json; charset=utf-8' },
          timeout: 30000
        });
        
        if (response.status === 200) {
          this.logger.log(`Submitted ${urlArray.length} URLs successfully`);
          return {
            success: true,
            status: response.status,
            urlsSubmitted: urlArray.length,
            attempts: attempt + 1
          };
        }
        
      } catch (error) {
        const status = error.response?.status;
        
        if (status === 429 || status >= 500) {
          if (attempt < maxRetries) {
            const delay = Math.pow(2, attempt) * 5000;
            this.logger.warn(`Attempt ${attempt + 1} failed (${status}). Retrying in ${delay}ms...`);
            await new Promise(resolve => setTimeout(resolve, delay));
            continue;
          }
        }
        
        return {
          success: false,
          status: status,
          error: error.response?.data || error.message,
          urlsSubmitted: 0
        };
      }
    }
    
    return { success: false, error: 'Max retries exceeded', urlsSubmitted: 0 };
  }
  
  async submitSingle(url) {
    return this.submit([url]);
  }
}

// Usage
const client = new IndexNowClient({ host: 'example.com' });

client.submitSingle('https://example.com/new-post')
  .then(result => console.log(result));

client.submit([
  'https://example.com/page-1',
  'https://example.com/page-2'
]).then(result => console.log(result));