1. Install Required Python Libraries
pip install requests beautifulsoup4 splunk-sdk
2. Python Script to Extract Timestamp & File Names and Send to Splunk
import requests
from bs4 import BeautifulSoup
import json
import splunklib.client as client
import splunklib.helpers as helpers
import logging
import re
# Setup logging
logging.basicConfig(filename="xymon_scraper.log", level=logging.INFO, format="%(asctime)s - %(levelname)s - %(message)s")
# Xymon Web Interface URL
XYMON_URL = "http://<xymon-server>/xymon-cgi/bb-hostsvc.sh?HOST=all"
# Xymon Authentication (if required)
XYMON_USERNAME = "your_xymon_user"
XYMON_PASSWORD = "your_xymon_password"
# Splunk Credentials
SPLUNK_HOST = "your-splunk-server"
SPLUNK_PORT = 8089
SPLUNK_USERNAME = "admin"
SPLUNK_PASSWORD = "yourpassword"
SPLUNK_INDEX = "xymon_logs"
# Define filename pattern to extract (modify as needed)
FILENAME_PATTERN = r"([a-zA-Z0-9_-]+\.log)"
# Function to fetch Xymon data
def fetch_xymon_data():
try:
session = requests.Session()
auth = (XYMON_USERNAME, XYMON_PASSWORD) if XYMON_USERNAME else None
response = session.get(XYMON_URL, auth=auth, timeout=10)
if response.status_code == 200:
logging.info("Successfully fetched Xymon data")
return response.text, response.headers.get("Date")
else:
logging.error(f"Failed to fetch Xymon data. Status code: {response.status_code}")
return None, None
except Exception as e:
logging.error(f"Error fetching Xymon data: {str(e)}")
return None, None
# Function to extract timestamps & specific filenames from Xymon
def parse_xymon_data(html_data, timestamp):
soup = BeautifulSoup(html_data, "html.parser")
logs = []
for link in soup.find_all("a"):
service_text = link.text.strip()
# Extract file names based on pattern
filename_match = re.search(FILENAME_PATTERN, service_text)
if filename_match:
log_entry = {
"filename": filename_match.group(0),
"timestamp": timestamp
}
logs.append(log_entry)
logging.info(f"Extracted {len(logs)} logs with filenames from Xymon")
return logs
# Function to send logs to Splunk
def send_to_splunk(logs):
try:
service = client.connect(
host=SPLUNK_HOST,
port=SPLUNK_PORT,
username=SPLUNK_USERNAME,
password=SPLUNK_PASSWORD
)
for log in logs:
event = json.dumps(log)
helpers.send_data(service, event, host=SPLUNK_HOST, index=SPLUNK_INDEX)
logging.info(f"Successfully sent {len(logs)} logs to Splunk")
except Exception as e:
logging.error(f"Error sending logs to Splunk: {str(e)}")
# Main function
def main():
html_data, timestamp = fetch_xymon_data()
if html_data and timestamp:
logs = parse_xymon_data(html_data, timestamp)
if logs:
send_to_splunk(logs)
else:
logging.warning("No relevant logs extracted from Xymon")
else:
logging.warning("No data fetched from Xymon")
if __name__ == "__main__":
main()
