Scraping Xymon for Timestamp & Specific Keywords and Sending Logs to Splunk

1. Install Required Python Libraries

pip install requests beautifulsoup4 splunk-sdk

2. Python Script to Extract Timestamp & File Names and Send to Splunk

import requests

from bs4 import BeautifulSoup

import json

import splunklib.client as client

import splunklib.helpers as helpers

import logging

import re

# Setup logging

logging.basicConfig(filename="xymon_scraper.log", level=logging.INFO, format="%(asctime)s - %(levelname)s - %(message)s")

# Xymon Web Interface URL

XYMON_URL = "http://<xymon-server>/xymon-cgi/bb-hostsvc.sh?HOST=all"

# Xymon Authentication (if required)

XYMON_USERNAME = "your_xymon_user"

XYMON_PASSWORD = "your_xymon_password"

# Splunk Credentials

SPLUNK_HOST = "your-splunk-server"

SPLUNK_PORT = 8089

SPLUNK_USERNAME = "admin"

SPLUNK_PASSWORD = "yourpassword"

SPLUNK_INDEX = "xymon_logs"

# Define filename pattern to extract (modify as needed)

FILENAME_PATTERN = r"([a-zA-Z0-9_-]+\.log)"

# Function to fetch Xymon data

def fetch_xymon_data():

    try:

        session = requests.Session()

        auth = (XYMON_USERNAME, XYMON_PASSWORD) if XYMON_USERNAME else None

        response = session.get(XYMON_URL, auth=auth, timeout=10)

        if response.status_code == 200:

            logging.info("Successfully fetched Xymon data")

            return response.text, response.headers.get("Date")

        else:

            logging.error(f"Failed to fetch Xymon data. Status code: {response.status_code}")

            return None, None

    except Exception as e:

        logging.error(f"Error fetching Xymon data: {str(e)}")

        return None, None

# Function to extract timestamps & specific filenames from Xymon

def parse_xymon_data(html_data, timestamp):

    soup = BeautifulSoup(html_data, "html.parser")

    logs = []

    for link in soup.find_all("a"):

        service_text = link.text.strip()

        # Extract file names based on pattern

        filename_match = re.search(FILENAME_PATTERN, service_text)

        if filename_match:

            log_entry = {

                "filename": filename_match.group(0),

                "timestamp": timestamp

            }

            logs.append(log_entry)

    logging.info(f"Extracted {len(logs)} logs with filenames from Xymon")

    return logs

# Function to send logs to Splunk

def send_to_splunk(logs):

    try:

        service = client.connect(

            host=SPLUNK_HOST,

            port=SPLUNK_PORT,

            username=SPLUNK_USERNAME,

            password=SPLUNK_PASSWORD

        )

        for log in logs:

            event = json.dumps(log)

            helpers.send_data(service, event, host=SPLUNK_HOST, index=SPLUNK_INDEX)

        logging.info(f"Successfully sent {len(logs)} logs to Splunk")

    except Exception as e:

        logging.error(f"Error sending logs to Splunk: {str(e)}")

# Main function

def main():

    html_data, timestamp = fetch_xymon_data()

    if html_data and timestamp:

        logs = parse_xymon_data(html_data, timestamp)

        if logs:

            send_to_splunk(logs)

        else:

            logging.warning("No relevant logs extracted from Xymon")

    else:

        logging.warning("No data fetched from Xymon")

if __name__ == "__main__":

    main()