How to Build a Competitive Intelligence Dashboard

How to Build a Competitive Intelligence Dashboard with Python

Competitive intelligence means knowing what your competitors are doing -- their pricing, product launches, marketing moves, customer sentiment, and market positioning -- before they know you're watching.

This tutorial shows how to build a competitive intelligence dashboard using Python and SearchHive's APIs to automatically collect competitor data from multiple sources and surface actionable insights.

Key Takeaways

• A competitive intelligence dashboard combines web scraping, search monitoring, and data analysis
• SearchHive provides three APIs that cover the full pipeline: SwiftSearch (discover), ScrapeForge (collect), DeepDive (extract)
• Python + a simple data store (SQLite) is enough for a powerful in-house dashboard
• Track competitor pricing, feature changes, reviews, and search rankings automatically
• Start free with 500 credits -- no infrastructure to manage

Prerequisites

• Python 3.8+
• Libraries: requests, sqlite3 (built-in), schedule (pip install schedule)
• SearchHive API key (get one free)
• A list of competitors to monitor

Step 1: Define Your Monitoring Framework

A good competitive dashboard tracks these data points:

# Configuration - customize for your market
CONFIG = {
    "competitors": ["competitor-a.com", "competitor-b.com", "competitor-c.io"],
    "keywords": [
        "project management software",
        "team collaboration tool",
        "remote work platform"
    ],
    "review_sources": ["trustpilot.com/review/", "g2.com/products/"],
    "scan_interval_hours": 6,
    "data_retention_days": 90
}

Step 2: Set Up the Data Store

SQLite handles this perfectly -- no external database needed:

import sqlite3
from datetime import datetime, timedelta

def init_db(db_path="competitive_intel.db"):
    # Initialize the SQLite database
    conn = sqlite3.connect(db_path)
    c = conn.cursor()
    
    # Competitor pricing snapshots
    c.execute('''
        CREATE TABLE IF NOT EXISTS pricing_snapshots (
            id INTEGER PRIMARY KEY AUTOINCREMENT,
            competitor TEXT,
            plan_name TEXT,
            price TEXT,
            features TEXT,
            scraped_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
        )
    ''')
    
    # Search ranking positions
    c.execute('''
        CREATE TABLE IF NOT EXISTS search_rankings (
            id INTEGER PRIMARY KEY AUTOINCREMENT,
            keyword TEXT,
            competitor TEXT,
            position INTEGER,
            title TEXT,
            url TEXT,
            scraped_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
        )
    ''')
    
    # Competitor feature/product changes
    c.execute('''
        CREATE TABLE IF NOT EXISTS product_changes (
            id INTEGER PRIMARY KEY AUTOINCREMENT,
            competitor TEXT,
            page_url TEXT,
            change_type TEXT,
            old_content TEXT,
            new_content TEXT,
            detected_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
        )
    ''')
    
    # Review scores
    c.execute('''
        CREATE TABLE IF NOT EXISTS review_scores (
            id INTEGER PRIMARY KEY AUTOINCREMENT,
            competitor TEXT,
            source TEXT,
            rating REAL,
            review_count INTEGER,
            scraped_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
        )
    ''')
    
    conn.commit()
    conn.close()
    print("Database initialized.")

init_db()

Step 3: Scrape Competitor Pricing Pages

import requests
import json

API_KEY = "your_api_key"
BASE = "https://api.searchhive.dev/v1"

def scrape_pricing_page(competitor_url):
    # Scrape a competitor's pricing page and extract plan data
    response = requests.post(
        f"{BASE}/scrape",
        headers={"Authorization": f"Bearer {API_KEY}"},
        json={
            "url": competitor_url,
            "render_js": True,
            "format": "markdown"
        }
    )
    response.raise_for_status()
    content = response.json()["markdown"]
    
    # Extract structured pricing data
    extract_response = requests.post(
        f"{BASE}/deepdive",
        headers={"Authorization": f"Bearer {API_KEY}"},
        json={
            "content": content,
            "extract": [
                "plan_name",
                "monthly_price",
                "annual_price",
                "key_features",
                "free_tier_details"
            ]
        }
    )
    extract_response.raise_for_status()
    return extract_response.json()["data"]

def save_pricing_snapshot(competitor, pricing_data, db_path="competitive_intel.db"):
    # Save pricing data to the database
    conn = sqlite3.connect(db_path)
    c = conn.cursor()
    
    for plan in pricing_data:
        c.execute(
            "INSERT INTO pricing_snapshots (competitor, plan_name, price, features) VALUES (?, ?, ?, ?)",
            (competitor, plan.get("plan_name", ""), 
             plan.get("monthly_price", ""), 
             json.dumps(plan.get("key_features", []), ensure_ascii=False))
        )
    
    conn.commit()
    conn.close()

# Run pricing scan
for competitor in CONFIG["competitors"]:
    print(f"Scanning pricing for {competitor}...")
    try:
        data = scrape_pricing_page(f"https://{competitor}/pricing")
        save_pricing_snapshot(competitor, data)
        print(f"  Found {len(data)} plans")
    except Exception as e:
        print(f"  Error: {e}")

Step 4: Track Search Engine Rankings

def track_search_rankings(keywords, competitors, db_path="competitive_intel.db"):
    # Track where competitors rank for target keywords
    conn = sqlite3.connect(db_path)
    c = conn.cursor()
    
    for keyword in keywords:
        print(f"Searching: {keyword}")
        response = requests.get(
            f"{BASE}/search",
            headers={"Authorization": f"Bearer {API_KEY}"},
            params={"q": keyword, "num": 20}
        )
        results = response.json().get("results", [])
        
        for position, result in enumerate(results, 1):
            url = result.get("url", "")
            # Check if this result belongs to a tracked competitor
            for comp in competitors:
                if comp in url:
                    c.execute(
                        "INSERT INTO search_rankings (keyword, competitor, position, title, url) VALUES (?, ?, ?, ?, ?)",
                        (keyword, comp, position, result.get("title", ""), url)
                    )
                    print(f"  {comp} ranks #{position} for '{keyword}'")
    
    conn.commit()
    conn.close()

track_search_rankings(CONFIG["keywords"], CONFIG["competitors"])

Step 5: Detect Product Page Changes

import hashlib

def detect_page_changes(competitors, db_path="competitive_intel.db"):
    # Detect changes on competitor product/landing pages
    conn = sqlite3.connect(db_path)
    c = conn.cursor()
    
    # Create hash tracking table
    c.execute('''
        CREATE TABLE IF NOT EXISTS page_hashes (
            competitor TEXT PRIMARY KEY,
            content_hash TEXT,
            updated_at TIMESTAMP
        )
    ''')
    conn.commit()
    
    pages_to_monitor = [
        (comp, f"https://{comp}")
        for comp in competitors
    ]
    
    for competitor, url in pages_to_monitor:
        try:
            response = requests.post(
                f"{BASE}/scrape",
                headers={"Authorization": f"Bearer {API_KEY}"},
                json={"url": url, "render_js": True, "format": "text"}
            )
            content = response.json().get("text", "")
            content_hash = hashlib.md5(content.encode()).hexdigest()
            
            # Check for previous hash
            c.execute(
                "SELECT content_hash FROM page_hashes WHERE competitor = ?",
                (competitor,)
            )
            row = c.fetchone()
            
            if row and row[0] != content_hash:
                print(f"CHANGE DETECTED: {competitor}")
                c.execute(
                    "INSERT INTO product_changes (competitor, page_url, change_type) VALUES (?, ?, ?)",
                    (competitor, url, "content_modified")
                )
            
            # Upsert current hash
            c.execute(
                "INSERT OR REPLACE INTO page_hashes (competitor, content_hash, updated_at) VALUES (?, ?, ?)",
                (competitor, content_hash, datetime.now().isoformat())
            )
            
        except Exception as e:
            print(f"  Error scanning {competitor}: {e}")
    
    conn.commit()
    conn.close()

Step 6: Generate Dashboard Reports

def generate_report(db_path="competitive_intel.db"):
    # Generate a text-based competitive intelligence report
    conn = sqlite3.connect(db_path)
    c = conn.cursor()
    
    report = []
    report.append("=" * 60)
    report.append("COMPETITIVE INTELLIGENCE REPORT")
    report.append(f"Generated: {datetime.now().strftime('%Y-%m-%d %H:%M')}")
    report.append("=" * 60)
    
    # Latest pricing
    report.append("\n--- LATEST PRICING SNAPSHOT ---")
    c.execute('''
        SELECT competitor, plan_name, price, scraped_at 
        FROM pricing_snapshots 
        WHERE scraped_at > datetime('now', '-7 days')
        ORDER BY competitor, scraped_at DESC
    ''')
    for row in c.fetchall():
        report.append(f"  {row[0]} | {row[1]}: {row[2]} ({row[3]})")
    
    # Search ranking summary
    report.append("\n--- SEARCH RANKING SUMMARY (LAST 7 DAYS) ---")
    c.execute('''
        SELECT keyword, competitor, AVG(position) as avg_pos, COUNT(*) as appearances
        FROM search_rankings
        WHERE scraped_at > datetime('now', '-7 days')
        GROUP BY keyword, competitor
        ORDER BY avg_pos ASC
    ''')
    for row in c.fetchall():
        report.append(f"  '{row[0]}' | {row[1]}: avg #{row[2]:.1f} ({row[3]} appearances)")
    
    # Recent changes
    report.append("\n--- RECENT PRODUCT CHANGES ---")
    c.execute('''
        SELECT competitor, page_url, change_type, detected_at
        FROM product_changes
        WHERE detected_at > datetime('now', '-14 days')
        ORDER BY detected_at DESC
    ''')
    for row in c.fetchall():
        report.append(f"  {row[0]}: {row[2]} on {row[1]} ({row[3]})")
    
    conn.close()
    return "\n".join(report)

# Generate and print
report = generate_report()
print(report)

Step 7: Automate with Scheduled Scans

import schedule
import time

def run_full_scan():
    # Run all competitive intelligence scans
    print(f"\n[{datetime.now()}] Starting full competitive scan...")
    
    for competitor in CONFIG["competitors"]:
        try:
            data = scrape_pricing_page(f"https://{competitor}/pricing")
            save_pricing_snapshot(competitor, data)
        except Exception as e:
            print(f"  Pricing scan error for {competitor}: {e}")
    
    track_search_rankings(CONFIG["keywords"], CONFIG["competitors"])
    detect_page_changes(CONFIG["competitors"])
    
    report = generate_report()
    with open(f"reports/intel_{datetime.now():%Y%m%d_%H%M}.txt", "w") as f:
        f.write(report)
    
    print("Scan complete.")

# Schedule every 6 hours
schedule.every(6).hours.do(run_full_scan)

# Or run once immediately
run_full_scan()

Common Issues and Solutions

Issue: Competitor uses aggressive bot protection

SearchHive's ScrapeForge handles most anti-bot measures through proxy rotation and stealth headers. For particularly difficult sites, the premium_proxy option routes through residential proxies.

Issue: Pricing tables not extracting cleanly

Some pricing pages use complex CSS grid layouts. Switch DeepDive's format to html instead of markdown for better structure preservation, or add specific CSS selectors if the page is predictable.

Issue: Too many false-positive change detections

Compare page content at the section level rather than the full page hash. Extract key sections (hero text, feature list) and hash those individually.

Next Steps

• Add automated alerts (email/Slack) when competitor pricing changes or rankings shift significantly
• Visualize trends over time with matplotlib or export to Google Sheets
• Combine with /blog/scrape-trustpilot-reviews-brand-monitoring for review monitoring
• For SEO competitive analysis, see /compare/serpapi

Build your competitive intelligence dashboard with 500 free credits. No credit card needed -- just an API key and this tutorial.