363 lines
15 KiB
Python
363 lines
15 KiB
Python
#!/usr/bin/env python3
|
|
"""
|
|
X86 Instruction Scraper
|
|
Combines mnemonic discovery from scraper.py with batch processing from populate_json.py
|
|
Scrapes x86 instruction information from https://www.felixcloutier.com/x86/
|
|
and saves it to a local JSON file for offline use.
|
|
"""
|
|
|
|
import requests
|
|
from bs4 import BeautifulSoup
|
|
import json
|
|
import time
|
|
import re
|
|
import os
|
|
from typing import Dict, Optional, List, Tuple
|
|
|
|
def fetch_all_mnemonics() -> Tuple[List[str], Dict[str, str]]:
|
|
"""
|
|
Fetch all mnemonics from the main x86 reference page tables
|
|
|
|
Returns:
|
|
Tuple of (mnemonics_list, mnemonic_to_url_mapping)
|
|
"""
|
|
print("Fetching all mnemonics from https://www.felixcloutier.com/x86/")
|
|
|
|
try:
|
|
response = requests.get("https://www.felixcloutier.com/x86/", timeout=15)
|
|
response.raise_for_status()
|
|
|
|
soup = BeautifulSoup(response.content, 'html.parser')
|
|
|
|
# Find all tables containing instructions
|
|
tables = soup.find_all('table')
|
|
print(f"Found {len(tables)} tables on the page")
|
|
|
|
all_mnemonics = set()
|
|
mnemonic_to_url = {} # Maps mnemonic to its actual URL path
|
|
|
|
for i, table in enumerate(tables):
|
|
print(f"Processing table {i+1}/{len(tables)}")
|
|
|
|
# Find all rows in the table
|
|
rows = table.find_all('tr')
|
|
|
|
# Skip header row and process data rows
|
|
for row in rows[1:]: # Skip first row (header)
|
|
cells = row.find_all(['td', 'th'])
|
|
if cells:
|
|
# First cell should contain the mnemonic
|
|
mnemonic_cell = cells[0]
|
|
|
|
# Extract mnemonic from the cell (could be a link)
|
|
mnemonic_link = mnemonic_cell.find('a')
|
|
if mnemonic_link:
|
|
# Extract mnemonic from the link text
|
|
mnemonic_text = mnemonic_link.get_text().strip()
|
|
# Extract the href for the URL
|
|
href = mnemonic_link.get('href', '')
|
|
if href.startswith('/x86/'):
|
|
url_path = href[5:] # Remove '/x86/' prefix
|
|
else:
|
|
url_path = href
|
|
else:
|
|
# Extract mnemonic from cell text
|
|
mnemonic_text = mnemonic_cell.get_text().strip()
|
|
url_path = mnemonic_text.lower()
|
|
|
|
if mnemonic_text:
|
|
# Clean up the mnemonic text
|
|
# Remove any extra whitespace or special characters
|
|
mnemonic_text = mnemonic_text.split()[0] if mnemonic_text.split() else ""
|
|
|
|
# Handle special cases like "GETSEC[CAPABILITIES]" -> "CAPABILITIES"
|
|
if '[' in mnemonic_text and ']' in mnemonic_text:
|
|
# Extract the part inside brackets
|
|
match = re.search(r'\[([^\]]+)\]', mnemonic_text)
|
|
if match:
|
|
extracted_mnemonic = match.group(1)
|
|
all_mnemonics.add(extracted_mnemonic.upper())
|
|
mnemonic_to_url[extracted_mnemonic.upper()] = url_path
|
|
|
|
# Handle cases like "VMLAUNCH/VMRESUME" -> both instructions
|
|
elif '/' in mnemonic_text:
|
|
for part in mnemonic_text.split('/'):
|
|
clean_part = part.strip()
|
|
if clean_part and clean_part.replace('_', '').isalnum():
|
|
all_mnemonics.add(clean_part.upper())
|
|
mnemonic_to_url[clean_part.upper()] = url_path
|
|
elif ':' in mnemonic_text:
|
|
# Handle cases like "VMLAUNCH:VMRESUME" -> both instructions
|
|
for part in mnemonic_text.split(':'):
|
|
clean_part = part.strip()
|
|
if clean_part and clean_part.replace('_', '').isalnum():
|
|
all_mnemonics.add(clean_part.upper())
|
|
mnemonic_to_url[clean_part.upper()] = url_path
|
|
else:
|
|
# Regular single mnemonic
|
|
if mnemonic_text and mnemonic_text.replace('_', '').isalnum():
|
|
all_mnemonics.add(mnemonic_text.upper())
|
|
mnemonic_to_url[mnemonic_text.upper()] = url_path
|
|
|
|
# Convert to sorted list
|
|
mnemonics_list = sorted(list(all_mnemonics))
|
|
|
|
print(f"Successfully extracted {len(mnemonics_list)} unique mnemonics")
|
|
print(f"Sample mnemonics: {mnemonics_list[:10]}")
|
|
print(f"Sample URL mappings:")
|
|
for mnemonic in mnemonics_list[:5]:
|
|
print(f" {mnemonic} -> {mnemonic_to_url.get(mnemonic, 'N/A')}")
|
|
|
|
return mnemonics_list, mnemonic_to_url
|
|
|
|
except requests.RequestException as e:
|
|
print(f"Error fetching main page: {e}")
|
|
print("Falling back to hardcoded list...")
|
|
fallback_mnemonics = get_fallback_mnemonics()
|
|
fallback_mapping = {m: m.lower() for m in fallback_mnemonics}
|
|
return fallback_mnemonics, fallback_mapping
|
|
except Exception as e:
|
|
print(f"Error parsing main page: {e}")
|
|
print("Falling back to hardcoded list...")
|
|
fallback_mnemonics = get_fallback_mnemonics()
|
|
fallback_mapping = {m: m.lower() for m in fallback_mnemonics}
|
|
return fallback_mnemonics, fallback_mapping
|
|
|
|
def get_fallback_mnemonics() -> List[str]:
|
|
"""
|
|
Fallback list of mnemonics in case web scraping fails
|
|
"""
|
|
return [
|
|
# Core common instructions
|
|
"MOV", "LEA", "PUSH", "POP", "XCHG", "MOVSX", "MOVZX", "CVTSI2SD", "CVTSD2SI",
|
|
"ADD", "SUB", "MUL", "DIV", "INC", "DEC", "AND", "OR", "XOR", "NOT", "SHL", "SHR",
|
|
"JMP", "CALL", "RET", "LOOP", "JNZ", "JZ", "JE", "JNE", "JL", "JLE", "JG", "JGE",
|
|
"CMP", "TEST", "NEG", "CLD", "STD", "NOP", "HLT", "INT", "IRET", "LEAVE",
|
|
# Add more common ones
|
|
"FADD", "FSUB", "FMUL", "FDIV", "FLD", "FST", "FSTP"
|
|
]
|
|
|
|
def scrape_instruction_info(instruction: str, url_path: str) -> Optional[Dict]:
|
|
"""
|
|
Scrape instruction information from felixcloutier.com
|
|
"""
|
|
try:
|
|
url = f"https://www.felixcloutier.com/x86/{url_path}"
|
|
print(f"Scraping: {instruction} from {url}")
|
|
|
|
response = requests.get(url, timeout=10)
|
|
response.raise_for_status()
|
|
|
|
soup = BeautifulSoup(response.content, 'html.parser')
|
|
|
|
# Extract the title (instruction name)
|
|
title_elem = soup.find('h1')
|
|
instruction_title = title_elem.get_text().strip() if title_elem else ""
|
|
|
|
# Extract Opcode table and relevant details
|
|
opcode = ""
|
|
instruction_description = ""
|
|
|
|
table = soup.find('table')
|
|
if table:
|
|
rows = table.find_all('tr')
|
|
# Look for rows that mention our specific instruction
|
|
instruction_row = None
|
|
for row in rows[1:]: # Skip header row
|
|
cells = row.find_all(['td', 'th'])
|
|
if len(cells) > 1:
|
|
# Check if this row contains our instruction
|
|
instruction_cell = cells[1].get_text().strip()
|
|
if instruction.upper() in instruction_cell.upper():
|
|
instruction_row = row
|
|
break
|
|
|
|
# If we found a specific row for this instruction, use it
|
|
if instruction_row:
|
|
cells = instruction_row.find_all(['td', 'th'])
|
|
if len(cells) > 0:
|
|
opcode = cells[0].get_text().strip()
|
|
if len(cells) > 6:
|
|
instruction_description = cells[6].get_text().strip()
|
|
elif len(cells) > 5:
|
|
instruction_description = cells[5].get_text().strip()
|
|
elif len(cells) > 4:
|
|
instruction_description = cells[4].get_text().strip()
|
|
else:
|
|
# Fall back to first data row
|
|
if len(rows) > 1:
|
|
cells = rows[1].find_all(['td', 'th'])
|
|
if len(cells) > 0:
|
|
opcode = cells[0].get_text().strip()
|
|
if len(cells) > 6:
|
|
instruction_description = cells[6].get_text().strip()
|
|
elif len(cells) > 5:
|
|
instruction_description = cells[5].get_text().strip()
|
|
|
|
# Extract the operation section
|
|
operation_section = ""
|
|
operation_header = soup.find('h2', id='operation')
|
|
if operation_header:
|
|
# Get all pre tags until the next h2
|
|
operation_parts = []
|
|
for sibling in operation_header.find_next_siblings():
|
|
if sibling.name == 'h2':
|
|
break
|
|
if sibling.name == 'pre':
|
|
# Preserve original formatting and indentation
|
|
operation_parts.append(sibling.get_text())
|
|
operation_section = '\n\n'.join(operation_parts)
|
|
|
|
# Extract the description section
|
|
description_section = ""
|
|
description_header = soup.find('h2', id='description')
|
|
if description_header:
|
|
# Get all paragraphs until the next h2
|
|
description_parts = []
|
|
for sibling in description_header.find_next_siblings():
|
|
if sibling.name == 'h2':
|
|
break
|
|
if sibling.name == 'p':
|
|
description_parts.append(sibling.get_text().strip())
|
|
description_section = '\n'.join(description_parts)
|
|
|
|
return {
|
|
'instruction': instruction.upper(),
|
|
'title': instruction_title,
|
|
'opcode': opcode,
|
|
'description': description_section,
|
|
'operation': operation_section,
|
|
'url': url
|
|
}
|
|
|
|
except requests.RequestException as e:
|
|
print(f"Error fetching {instruction}: {e}")
|
|
return None
|
|
except Exception as e:
|
|
print(f"Error parsing {instruction}: {e}")
|
|
return None
|
|
|
|
def load_existing_data(filename: str) -> Dict:
|
|
"""Load existing JSON data if it exists"""
|
|
if os.path.exists(filename):
|
|
try:
|
|
with open(filename, 'r', encoding='utf-8') as f:
|
|
return json.load(f)
|
|
except:
|
|
return {}
|
|
return {}
|
|
|
|
def save_data(data: Dict, filename: str):
|
|
"""Save data to JSON file"""
|
|
with open(filename, 'w', encoding='utf-8') as f:
|
|
json.dump(data, f, indent=2, ensure_ascii=False)
|
|
|
|
def main():
|
|
"""Main function to scrape all instructions and save to JSON file"""
|
|
print("=== Merged X86 Instruction Scraper ===")
|
|
|
|
# Load existing data
|
|
output_file = "../syntaxes/x86_instructions.json"
|
|
existing_data = load_existing_data(output_file)
|
|
print(f"Loaded {len(existing_data)} existing instructions")
|
|
|
|
# Fetch all mnemonics from the website
|
|
print("\n" + "=" * 60)
|
|
print("PHASE 1: Fetching all mnemonics from the website")
|
|
print("=" * 60)
|
|
|
|
all_mnemonics, mnemonic_to_url = fetch_all_mnemonics()
|
|
print(f"Total mnemonics discovered: {len(all_mnemonics)}")
|
|
|
|
if not all_mnemonics:
|
|
print("No mnemonics found! Exiting.")
|
|
return
|
|
|
|
# Save the discovered mnemonics to a file for reference
|
|
with open("discovered_mnemonics.txt", "w") as f:
|
|
for mnemonic in all_mnemonics:
|
|
f.write(f"{mnemonic}\n")
|
|
print(f"Saved discovered mnemonics to: discovered_mnemonics.txt")
|
|
|
|
# Save the URL mappings for reference
|
|
with open("mnemonic_url_mappings.txt", "w") as f:
|
|
for mnemonic in all_mnemonics:
|
|
url_path = mnemonic_to_url.get(mnemonic, mnemonic.lower())
|
|
f.write(f"{mnemonic} -> {url_path}\n")
|
|
print(f"Saved URL mappings to: mnemonic_url_mappings.txt")
|
|
|
|
# Filter out already processed mnemonics
|
|
remaining_mnemonics = [m for m in all_mnemonics if m.lower() not in existing_data]
|
|
print(f"Remaining mnemonics to process: {len(remaining_mnemonics)}")
|
|
|
|
if not remaining_mnemonics:
|
|
print("All mnemonics already processed!")
|
|
return
|
|
|
|
print("\n" + "=" * 60)
|
|
print("PHASE 2: Scraping detailed information for each instruction")
|
|
print("=" * 60)
|
|
|
|
# Process in batches
|
|
batch_size = 50
|
|
successful_scrapes = 0
|
|
failed_scrapes = 0
|
|
|
|
for i in range(0, len(remaining_mnemonics), batch_size):
|
|
batch = remaining_mnemonics[i:i+batch_size]
|
|
print(f"\n=== Processing batch {i//batch_size + 1} ({len(batch)} instructions) ===")
|
|
|
|
for j, instruction in enumerate(batch):
|
|
print(f"[{i+j+1}/{len(remaining_mnemonics)}] Processing: {instruction}")
|
|
|
|
# Get URL path
|
|
url_path = mnemonic_to_url.get(instruction, instruction.lower())
|
|
|
|
# Scrape instruction info
|
|
info = scrape_instruction_info(instruction, url_path)
|
|
if info:
|
|
existing_data[instruction.lower()] = info
|
|
successful_scrapes += 1
|
|
print(f" ✓ Successfully scraped: {instruction}")
|
|
else:
|
|
failed_scrapes += 1
|
|
print(f" ✗ Failed to scrape: {instruction}")
|
|
|
|
# No current rate limit but uncomment if needed
|
|
# time.sleep(0.3)
|
|
|
|
# Save after each batch
|
|
save_data(existing_data, output_file)
|
|
print(f"Saved batch to {output_file}")
|
|
|
|
# Pause between batches
|
|
if i + batch_size < len(remaining_mnemonics):
|
|
print(f"Pausing 3 seconds before next batch...")
|
|
time.sleep(3)
|
|
|
|
print(f"\n" + "=" * 60)
|
|
print("SCRAPING COMPLETE")
|
|
print("=" * 60)
|
|
print(f"Successfully scraped: {successful_scrapes}")
|
|
print(f"Failed to scrape: {failed_scrapes}")
|
|
print(f"Total instructions in database: {len(existing_data)}")
|
|
if remaining_mnemonics:
|
|
print(f"Success rate: {successful_scrapes / len(remaining_mnemonics) * 100:.1f}%")
|
|
|
|
# Show file size
|
|
if os.path.exists(output_file):
|
|
file_size = os.path.getsize(output_file)
|
|
print(f"Final file size: {file_size / (1024*1024):.1f} MB")
|
|
|
|
# Show statistics
|
|
if existing_data:
|
|
print(f"\nSample instruction data (first entry):")
|
|
first_key = next(iter(existing_data))
|
|
sample = existing_data[first_key]
|
|
print(f" Instruction: {sample['instruction']}")
|
|
print(f" Title: {sample['title']}")
|
|
print(f" Opcode: {sample['opcode']}")
|
|
print(f" Description: {sample['description'][:100]}...")
|
|
|
|
if __name__ == "__main__":
|
|
main() |