SpiderForce4AI Documentation

# Server Configuration
    PORT=3004
    NODE_ENV=production
    MAX_RETRIES=2
    PAGE_TIMEOUT=30000
    MAX_CONCURRENT_PAGES=10

    # Cleaning Configuration
    AGGRESSIVE_CLEANING=true
    REMOVE_IMAGES=false

    # Dynamic Content Handling
    MIN_CONTENT_LENGTH=500
    SCROLL_WAIT_TIME=200

// SpiderForce4AI Tool for n8n
  // This tool processes URLs through SpiderForce4AI and returns markdown content.
  // The input comes as 'query' parameter containing a URL
  
  const SPIDERFORCE_BASE_URL = 'http://localhost:3004'; // or your cloud deployment instance URL
  
  try {
      // Validate input
      if (!query || !query.startsWith('http')) {
          return 'Error: Invalid URL provided. URL must start with http:// or https://';
      }
  
      const options = {
          url: `${SPIDERFORCE_BASE_URL}/convert`,
          method: 'POST',
          headers: {
              'Content-Type': 'application/json'
          },
          body: {
              url: query,
             // targetSelectors: ['.main-content', 'article', '#content'],
             // removeSelectors: ['.ads', '.nav', '.footer', '.header']
          }
      };
  
      const response = await this.helpers.httpRequest(options);
      
      // Return markdown content from response
      return response.markdown || response.toString();
  
  } catch (error) {
      return `Error processing URL: ${error.message}`;
  }

openapi: 3.0.0
  info:
    title: SpiderForce4AI Web to Markdown Converter
    description: Convert a web page to Markdown using Dify
    author: Piotr Tamulewicz
    version: 1.0.0
  
  servers:
    - url: http://localhost:3004
  
  paths:
    /convert:
      get:
        summary: Convert a web page to Markdown
        description: Retrieves the content of a web page and converts it to Markdown format
        parameters:
          - name: url
            in: query
            description: The URL of the web page to convert
            required: true
            schema:
              type: string
              format: uri
          - name: targetSelectors
            in: query
            description: CSS selectors to target specific content (comma-separated)
            required: false
            schema:
              type: string
          - name: removeSelectors
            in: query  
            description: CSS selectors to remove unwanted content (comma-separated)
            required: false
            schema:
              type: string
        responses:
          '200':
            description: Successful conversion
            content:
              text/markdown:
                schema:
                  type: string
          '400':
            description: Bad request (missing or invalid URL)
          '500':
            description: Internal server error

from spiderforce4ai import SpiderForce4AI, CrawlConfig
import requests
from pathlib import Path

# Define extraction template structure
extraction_template = """
{
    "Title": "Extract the main title of the content",
    "MetaDescription": "Extract a search-friendly meta description (under 160 characters)",
    "KeyPoints": ["Extract 3-5 key points from the content"],
    "Categories": ["Extract relevant categories for this content"],
    "ReadingTimeMinutes": "Estimate reading time in minutes"
}
"""

# Define a custom webhook function
def post_extraction_webhook(extraction_result):
    """Send extraction results to a webhook and return transformed data."""
    # Add custom fields or transform the data as needed
    payload = {
        "url": extraction_result.get("url", ""),
        "title": extraction_result.get("Title", ""),
        "description": extraction_result.get("MetaDescription", ""),
        "key_points": extraction_result.get("KeyPoints", []),
        "categories": extraction_result.get("Categories", []),
        "reading_time": extraction_result.get("ReadingTimeMinutes", ""),
        "processed_at": extraction_result.get("timestamp", "")
    }
    
    # Send to webhook (example using a different webhook than the main one)
    try:
        response = requests.post(
            "https://webhook.site/your-extraction-webhook-id",
            json=payload,
            headers={
                "Authorization": "Bearer extraction-token",
                "Content-Type": "application/json"
            },
            timeout=10
        )
        print(f"Extraction webhook sent: Status {response.status_code}")
    except Exception as e:
        print(f"Extraction webhook error: {str(e)}")
    
    # Return the transformed data (will be stored in result.extraction_result)
    return payload

# Initialize crawler
spider = SpiderForce4AI("http://localhost:3004")

# Configure with post-extraction and webhooks
config = CrawlConfig(
    # Basic crawling settings
    target_selector="article",
    remove_selectors=[".ads", ".navigation", ".comments"],
    max_concurrent_requests=5,
    
    # Regular webhook for crawl results
    webhook_url="https://webhook.site/your-crawl-webhook-id",
    webhook_headers={
        "Authorization": "Bearer crawl-token",
        "Content-Type": "application/json"
    },
    
    # Post-extraction LLM processing
    post_extraction_agent={
        "model": "gpt-4-turbo",  # Or another compatible model
        "api_key": "your-api-key-here",
        "max_tokens": 1000,
        "temperature": 0.3,
        "response_format": "json_object",  # Request JSON response format
        "messages": [
            {
                "role": "system",
                "content": f"Extract the following information from the content. Return ONLY valid JSON, no explanations:\n\n{extraction_template}"
            },
            {
                "role": "user",
                "content": "{here_markdown_content}"  # Will be replaced with actual content
            }
        ]
    },
    # Save combined extraction results
    post_extraction_agent_save_to_file="extraction_results.json",
    # Custom function to transform and send extraction webhook
    post_agent_transformer_function=post_extraction_webhook
)

# Crawl a sitemap with both regular and post-extraction webhooks
results = spider.crawl_sitemap_server_parallel("https://petertam.pro/sitemap.xml", config)