local file mnemonics info instead of fetching them

This commit is contained in:
kapitche 2025-07-04 00:12:59 -04:00
parent 24560b4c5c
commit 437bbf0c9d
7 changed files with 10108 additions and 116 deletions

4
.gitignore vendored
View file

@ -1,2 +1,4 @@
node_modules
*.vsix
*.vsix
/scrapper/discovered_mnemonics.txt
/scrapper/mnemonic_url_mappings.txt

View file

@ -2,7 +2,7 @@
"name": "x86-assembly-syntax",
"displayName": "Universal Assembly Syntax Highlighting",
"description": "Syntax highlighting extension for x86, x64 and ARM assembly both for Intel and AT&T syntax",
"version": "1.2.2",
"version": "1.2.3",
"engines": {
"vscode": "^1.75.0"
},
@ -93,10 +93,6 @@
"highlighting"
],
"homepage": "https://github.com/haxo-games/x86-x64-syntax-highlighting#readme",
"dependencies": {
"axios": "^1.7.7",
"cheerio": "^1.0.0"
},
"devDependencies": {
"vscode": "^1.1.34"
}

View file

@ -0,0 +1,3 @@
requests>=2.25.1
beautifulsoup4>=4.9.3
lxml>=4.6.3

363
scrapper/scraper.py Normal file
View file

@ -0,0 +1,363 @@
#!/usr/bin/env python3
"""
X86 Instruction Scraper
Combines mnemonic discovery from scraper.py with batch processing from populate_json.py
Scrapes x86 instruction information from https://www.felixcloutier.com/x86/
and saves it to a local JSON file for offline use.
"""
import requests
from bs4 import BeautifulSoup
import json
import time
import re
import os
from typing import Dict, Optional, List, Tuple
def fetch_all_mnemonics() -> Tuple[List[str], Dict[str, str]]:
"""
Fetch all mnemonics from the main x86 reference page tables
Returns:
Tuple of (mnemonics_list, mnemonic_to_url_mapping)
"""
print("Fetching all mnemonics from https://www.felixcloutier.com/x86/")
try:
response = requests.get("https://www.felixcloutier.com/x86/", timeout=15)
response.raise_for_status()
soup = BeautifulSoup(response.content, 'html.parser')
# Find all tables containing instructions
tables = soup.find_all('table')
print(f"Found {len(tables)} tables on the page")
all_mnemonics = set()
mnemonic_to_url = {} # Maps mnemonic to its actual URL path
for i, table in enumerate(tables):
print(f"Processing table {i+1}/{len(tables)}")
# Find all rows in the table
rows = table.find_all('tr')
# Skip header row and process data rows
for row in rows[1:]: # Skip first row (header)
cells = row.find_all(['td', 'th'])
if cells:
# First cell should contain the mnemonic
mnemonic_cell = cells[0]
# Extract mnemonic from the cell (could be a link)
mnemonic_link = mnemonic_cell.find('a')
if mnemonic_link:
# Extract mnemonic from the link text
mnemonic_text = mnemonic_link.get_text().strip()
# Extract the href for the URL
href = mnemonic_link.get('href', '')
if href.startswith('/x86/'):
url_path = href[5:] # Remove '/x86/' prefix
else:
url_path = href
else:
# Extract mnemonic from cell text
mnemonic_text = mnemonic_cell.get_text().strip()
url_path = mnemonic_text.lower()
if mnemonic_text:
# Clean up the mnemonic text
# Remove any extra whitespace or special characters
mnemonic_text = mnemonic_text.split()[0] if mnemonic_text.split() else ""
# Handle special cases like "GETSEC[CAPABILITIES]" -> "CAPABILITIES"
if '[' in mnemonic_text and ']' in mnemonic_text:
# Extract the part inside brackets
match = re.search(r'\[([^\]]+)\]', mnemonic_text)
if match:
extracted_mnemonic = match.group(1)
all_mnemonics.add(extracted_mnemonic.upper())
mnemonic_to_url[extracted_mnemonic.upper()] = url_path
# Handle cases like "VMLAUNCH/VMRESUME" -> both instructions
elif '/' in mnemonic_text:
for part in mnemonic_text.split('/'):
clean_part = part.strip()
if clean_part and clean_part.replace('_', '').isalnum():
all_mnemonics.add(clean_part.upper())
mnemonic_to_url[clean_part.upper()] = url_path
elif ':' in mnemonic_text:
# Handle cases like "VMLAUNCH:VMRESUME" -> both instructions
for part in mnemonic_text.split(':'):
clean_part = part.strip()
if clean_part and clean_part.replace('_', '').isalnum():
all_mnemonics.add(clean_part.upper())
mnemonic_to_url[clean_part.upper()] = url_path
else:
# Regular single mnemonic
if mnemonic_text and mnemonic_text.replace('_', '').isalnum():
all_mnemonics.add(mnemonic_text.upper())
mnemonic_to_url[mnemonic_text.upper()] = url_path
# Convert to sorted list
mnemonics_list = sorted(list(all_mnemonics))
print(f"Successfully extracted {len(mnemonics_list)} unique mnemonics")
print(f"Sample mnemonics: {mnemonics_list[:10]}")
print(f"Sample URL mappings:")
for mnemonic in mnemonics_list[:5]:
print(f" {mnemonic} -> {mnemonic_to_url.get(mnemonic, 'N/A')}")
return mnemonics_list, mnemonic_to_url
except requests.RequestException as e:
print(f"Error fetching main page: {e}")
print("Falling back to hardcoded list...")
fallback_mnemonics = get_fallback_mnemonics()
fallback_mapping = {m: m.lower() for m in fallback_mnemonics}
return fallback_mnemonics, fallback_mapping
except Exception as e:
print(f"Error parsing main page: {e}")
print("Falling back to hardcoded list...")
fallback_mnemonics = get_fallback_mnemonics()
fallback_mapping = {m: m.lower() for m in fallback_mnemonics}
return fallback_mnemonics, fallback_mapping
def get_fallback_mnemonics() -> List[str]:
"""
Fallback list of mnemonics in case web scraping fails
"""
return [
# Core common instructions
"MOV", "LEA", "PUSH", "POP", "XCHG", "MOVSX", "MOVZX", "CVTSI2SD", "CVTSD2SI",
"ADD", "SUB", "MUL", "DIV", "INC", "DEC", "AND", "OR", "XOR", "NOT", "SHL", "SHR",
"JMP", "CALL", "RET", "LOOP", "JNZ", "JZ", "JE", "JNE", "JL", "JLE", "JG", "JGE",
"CMP", "TEST", "NEG", "CLD", "STD", "NOP", "HLT", "INT", "IRET", "LEAVE",
# Add more common ones
"FADD", "FSUB", "FMUL", "FDIV", "FLD", "FST", "FSTP"
]
def scrape_instruction_info(instruction: str, url_path: str) -> Optional[Dict]:
"""
Scrape instruction information from felixcloutier.com
"""
try:
url = f"https://www.felixcloutier.com/x86/{url_path}"
print(f"Scraping: {instruction} from {url}")
response = requests.get(url, timeout=10)
response.raise_for_status()
soup = BeautifulSoup(response.content, 'html.parser')
# Extract the title (instruction name)
title_elem = soup.find('h1')
instruction_title = title_elem.get_text().strip() if title_elem else ""
# Extract Opcode table and relevant details
opcode = ""
instruction_description = ""
table = soup.find('table')
if table:
rows = table.find_all('tr')
# Look for rows that mention our specific instruction
instruction_row = None
for row in rows[1:]: # Skip header row
cells = row.find_all(['td', 'th'])
if len(cells) > 1:
# Check if this row contains our instruction
instruction_cell = cells[1].get_text().strip()
if instruction.upper() in instruction_cell.upper():
instruction_row = row
break
# If we found a specific row for this instruction, use it
if instruction_row:
cells = instruction_row.find_all(['td', 'th'])
if len(cells) > 0:
opcode = cells[0].get_text().strip()
if len(cells) > 6:
instruction_description = cells[6].get_text().strip()
elif len(cells) > 5:
instruction_description = cells[5].get_text().strip()
elif len(cells) > 4:
instruction_description = cells[4].get_text().strip()
else:
# Fall back to first data row
if len(rows) > 1:
cells = rows[1].find_all(['td', 'th'])
if len(cells) > 0:
opcode = cells[0].get_text().strip()
if len(cells) > 6:
instruction_description = cells[6].get_text().strip()
elif len(cells) > 5:
instruction_description = cells[5].get_text().strip()
# Extract the operation section
operation_section = ""
operation_header = soup.find('h2', id='operation')
if operation_header:
# Get all pre tags until the next h2
operation_parts = []
for sibling in operation_header.find_next_siblings():
if sibling.name == 'h2':
break
if sibling.name == 'pre':
# Preserve original formatting and indentation
operation_parts.append(sibling.get_text())
operation_section = '\n\n'.join(operation_parts)
# Extract the description section
description_section = ""
description_header = soup.find('h2', id='description')
if description_header:
# Get all paragraphs until the next h2
description_parts = []
for sibling in description_header.find_next_siblings():
if sibling.name == 'h2':
break
if sibling.name == 'p':
description_parts.append(sibling.get_text().strip())
description_section = '\n'.join(description_parts)
return {
'instruction': instruction.upper(),
'title': instruction_title,
'opcode': opcode,
'description': description_section,
'operation': operation_section,
'url': url
}
except requests.RequestException as e:
print(f"Error fetching {instruction}: {e}")
return None
except Exception as e:
print(f"Error parsing {instruction}: {e}")
return None
def load_existing_data(filename: str) -> Dict:
"""Load existing JSON data if it exists"""
if os.path.exists(filename):
try:
with open(filename, 'r', encoding='utf-8') as f:
return json.load(f)
except:
return {}
return {}
def save_data(data: Dict, filename: str):
"""Save data to JSON file"""
with open(filename, 'w', encoding='utf-8') as f:
json.dump(data, f, indent=2, ensure_ascii=False)
def main():
"""Main function to scrape all instructions and save to JSON file"""
print("=== Merged X86 Instruction Scraper ===")
# Load existing data
output_file = "../syntaxes/x86_instructions.json"
existing_data = load_existing_data(output_file)
print(f"Loaded {len(existing_data)} existing instructions")
# Fetch all mnemonics from the website
print("\n" + "=" * 60)
print("PHASE 1: Fetching all mnemonics from the website")
print("=" * 60)
all_mnemonics, mnemonic_to_url = fetch_all_mnemonics()
print(f"Total mnemonics discovered: {len(all_mnemonics)}")
if not all_mnemonics:
print("No mnemonics found! Exiting.")
return
# Save the discovered mnemonics to a file for reference
with open("discovered_mnemonics.txt", "w") as f:
for mnemonic in all_mnemonics:
f.write(f"{mnemonic}\n")
print(f"Saved discovered mnemonics to: discovered_mnemonics.txt")
# Save the URL mappings for reference
with open("mnemonic_url_mappings.txt", "w") as f:
for mnemonic in all_mnemonics:
url_path = mnemonic_to_url.get(mnemonic, mnemonic.lower())
f.write(f"{mnemonic} -> {url_path}\n")
print(f"Saved URL mappings to: mnemonic_url_mappings.txt")
# Filter out already processed mnemonics
remaining_mnemonics = [m for m in all_mnemonics if m.lower() not in existing_data]
print(f"Remaining mnemonics to process: {len(remaining_mnemonics)}")
if not remaining_mnemonics:
print("All mnemonics already processed!")
return
print("\n" + "=" * 60)
print("PHASE 2: Scraping detailed information for each instruction")
print("=" * 60)
# Process in batches
batch_size = 50
successful_scrapes = 0
failed_scrapes = 0
for i in range(0, len(remaining_mnemonics), batch_size):
batch = remaining_mnemonics[i:i+batch_size]
print(f"\n=== Processing batch {i//batch_size + 1} ({len(batch)} instructions) ===")
for j, instruction in enumerate(batch):
print(f"[{i+j+1}/{len(remaining_mnemonics)}] Processing: {instruction}")
# Get URL path
url_path = mnemonic_to_url.get(instruction, instruction.lower())
# Scrape instruction info
info = scrape_instruction_info(instruction, url_path)
if info:
existing_data[instruction.lower()] = info
successful_scrapes += 1
print(f" ✓ Successfully scraped: {instruction}")
else:
failed_scrapes += 1
print(f" ✗ Failed to scrape: {instruction}")
# No current rate limit but uncomment if needed
# time.sleep(0.3)
# Save after each batch
save_data(existing_data, output_file)
print(f"Saved batch to {output_file}")
# Pause between batches
if i + batch_size < len(remaining_mnemonics):
print(f"Pausing 3 seconds before next batch...")
time.sleep(3)
print(f"\n" + "=" * 60)
print("SCRAPING COMPLETE")
print("=" * 60)
print(f"Successfully scraped: {successful_scrapes}")
print(f"Failed to scrape: {failed_scrapes}")
print(f"Total instructions in database: {len(existing_data)}")
if remaining_mnemonics:
print(f"Success rate: {successful_scrapes / len(remaining_mnemonics) * 100:.1f}%")
# Show file size
if os.path.exists(output_file):
file_size = os.path.getsize(output_file)
print(f"Final file size: {file_size / (1024*1024):.1f} MB")
# Show statistics
if existing_data:
print(f"\nSample instruction data (first entry):")
first_key = next(iter(existing_data))
sample = existing_data[first_key]
print(f" Instruction: {sample['instruction']}")
print(f" Title: {sample['title']}")
print(f" Opcode: {sample['opcode']}")
print(f" Description: {sample['description'][:100]}...")
if __name__ == "__main__":
main()

View file

@ -1,11 +1,14 @@
const vscode = require("vscode");
const axios = require("axios");
const cheerio = require("cheerio");
const list = require("./mnemonics.js");
const fs = require("fs");
const path = require("path");
let languageConfigDisposable = null;
let instructionData = null;
function activate(context) {
// Load instruction data from local JSON file
loadInstructionData();
// Set up initial language configuration
updateLanguageConfiguration();
@ -20,24 +23,18 @@ function activate(context) {
// Hover provider for mnemonics
const hoverProvider = vscode.languages.registerHoverProvider("x86asm", {
async provideHover(document, position, token) {
provideHover(document, position, token) {
const range = document.getWordRangeAtPosition(
position,
/\b[a-zA-Z0-9_]+\b/
);
const word = document.getText(range);
if (word) {
// Check if the word is a valid mnemonic
if (list.x86x64Mnemonics.includes(word.toUpperCase())) {
const instructionInfo = await fetchInstructionInfo(
word.toLowerCase()
);
if (instructionInfo) {
return new vscode.Hover(instructionInfo);
}
} else {
return undefined;
if (word && instructionData) {
// Check if the word is a valid mnemonic by looking it up in the instruction data
const instructionInfo = getInstructionInfo(word.toLowerCase());
if (instructionInfo) {
return new vscode.Hover(instructionInfo);
}
}
@ -68,6 +65,38 @@ function activate(context) {
);
}
function loadInstructionData() {
try {
const dataPath = path.join(
__dirname,
"..",
"syntaxes",
"x86_instructions.json"
);
if (fs.existsSync(dataPath)) {
const rawData = fs.readFileSync(dataPath, "utf8");
instructionData = JSON.parse(rawData);
console.log(
`Loaded ${
Object.keys(instructionData).length
} instructions from syntaxes database`
);
} else {
console.warn(
"syntaxes/x86_instructions.json not found. Please run the scraper.py script first."
);
vscode.window.showWarningMessage(
"syntaxes/x86_instructions.json not found. Please run the scraper.py script to generate the instruction database."
);
}
} catch (error) {
console.error("Error loading instruction data:", error);
vscode.window.showErrorMessage(
"Error loading instruction database: " + error.message
);
}
}
function updateLanguageConfiguration() {
// Dispose previous language configuration if it exists
if (languageConfigDisposable) {
@ -116,47 +145,35 @@ function updateLanguageConfiguration() {
);
}
async function fetchInstructionInfo(instruction) {
try {
const url = `https://www.felixcloutier.com/x86/${instruction}`;
const response = await axios.get(url);
const $ = cheerio.load(response.data);
// Extract the title (instruction name)
const instructionTitle = $("h1").text();
// Extract Opcode table and relevant details
const opcodeTable = $("table").first();
const opcode = opcodeTable.find("tr").eq(1).find("td").eq(0).text();
const instructionDescription = opcodeTable
.find("tr")
.eq(1)
.find("td")
.eq(5)
.text();
// Extract the operation section
const operationSection = $("#operation").next("pre").text();
// Extract the description section
const descriptionSection = $("#description").nextUntil("h2", "p").text();
// Combine the extracted information into a formatted output
let hoverContent = `**${instructionTitle}**\n\n`;
hoverContent += `**Opcode:** ${opcode}\n`;
hoverContent += `**Description:** ${instructionDescription}\n\n`;
hoverContent += `**Details:**\n${descriptionSection}\n\n`;
hoverContent += `**Operation:**\n${operationSection}`;
return hoverContent;
} catch (error) {
if (error.response && error.response.status === 404) {
console.warn(`Instruction info not found for: ${instruction}`);
} else {
console.error(`Failed to fetch instruction info: ${error.message}`);
}
return null; // Return null if fetching fails
function getInstructionInfo(instruction) {
if (!instructionData) {
return null;
}
const info = instructionData[instruction.toLowerCase()];
if (!info) {
return null;
}
// Format the hover content using the syntaxes/x86_instructions.json format
// Keep title as is since it usually doesn't need paragraph breaks
let hoverContent = `**${info.title}**\n\n`;
if (info.opcode) {
hoverContent += `**Opcode:** ${info.opcode}\n\n`;
}
if (info.description) {
// Replace single \n with double \n for proper markdown paragraph breaks
const formattedDescription = info.description.replace(/\n/g, "\n\n");
hoverContent += `**Description:** ${formattedDescription}\n\n`;
}
if (info.url) {
hoverContent += `[View Documentation](${info.url})`;
}
return hoverContent;
}
async function findDefinition(word) {

View file

@ -1,55 +0,0 @@
const x86x64Mnemonics = [
// Data Transfer
"MOV", "LEA", "PUSH", "POP", "XCHG", "MOVSX", "MOVZX", "CVTSI2SD", "CVTSD2SI",
"MOVD", "MOVQ", "MOVSS", "MOVSD", "MOVAPD", "MOVDQA", "MOVDQU",
"MOVNTI", "MOVNTQ", "MOVNTDQ", "MOVNTDQA", "MOVNTQ", "MOVNTDQ",
// Arithmetic and Logical
"ADD", "SUB", "MUL", "DIV", "INC", "DEC", "AND", "OR", "XOR", "NOT", "SHL", "SHR",
"SAL", "SAR", "SHRD", "SHLD", "IMUL", "IDIV", "IDIVQ", "MULSD", "DIVSD",
"ADDPD", "SUBPD", "MULPD", "DIVPD", "ANDPD", "ORPD", "XORPD",
"ADDPS", "SUBPS", "MULPS", "DIVPS", "ANDPS", "ORPS", "XORPS",
// Control Flow
"JMP", "Jcc", "CALL", "RET", "LOOP", "JNZ", "JZ", "JE", "JNE", "JL", "JLE",
"JG", "JGE", "JB", "JBE", "JA", "JAE", "LOOPNE", "LOOPE",
"LOOPNZ", "LOOPZ", "CMOVcc", "SETcc",
// Other
"CMP", "TEST", "NEG", "NOT", "CLD", "STD", "CWD", "CDQ", "CQO",
"LEA", "LES", "LDS", "LSS", "LFS", "LGS",
"LES", "LDS", "LSS", "LFS", "LGS",
"REP", "REPE", "REPNE", "REPZ", "REPNZ",
"BSF", "BSR", "BT", "BTC", "BTR", "BTS", "LEAVE",
// Floating-Point
"FADD", "FSUB", "FMUL", "FDIV", "FCHS", "FABS", "FCOM", "FCOMP", "FCOMPP",
"FADD", "FSUB", "FMUL", "FDIV", "FCHS", "FABS", "FCOM", "FCOMP", "FCOMPP",
"FST", "FSTPN", "FSTP", "FLD", "FLD1", "FLDPI", "FLDL2T", "FLDL2E",
"FLDCW", "FSTCW",
"FCOMI", "FCOMIP", "FUCOMI", "FUCOMIP",
// SIMD
"MOVDQA", "MOVDQU", "PSHUFD", "PSHUFLW", "PSHUFLH", "PSLLD", "PSLLQ", "PSLLW",
"PSRAD", "PSRAQ", "PSRAW", "PADDW", "PADDQ", "PADDUSB", "PADDUSW",
"PSUBW", "PSUBQ", "PSUBUSB", "PSUBUSW",
"PMADDWD", "PMULHW", "PMULLW",
"PMAXSW", "PMAXUW", "PMINSW", "PMINUW",
// AVX Instructions
"VMOVD", "VMOVQ", "VMOVSS", "VMOVSD", "VMOVAPD", "VMOVDQA", "VMOVDQU",
"VADDPD", "VADDPS", "VADDL", "VADDL2", "VADDL3", "VADDL4",
"VSUBPD", "VSUBPS", "VSUBL", "VSUBL2", "VSUBL3", "VSUBL4",
"VMULPD", "VMULPS", "VMULL", "VMULL2", "VMULL3", "VMULL4",
"VDIVPD", "VDIVPS", "VDIVL", "VDIVL2", "VDIVL3", "VDIVL4",
"VANDPD", "VANDPS", "VANDL", "VANDL2", "VANDL3", "VANDL4",
"VORPD", "VORPS", "VORL", "VORL2", "VORL3", "VORL4",
"VXORPD", "VXORPS", "VXORL", "VXORL2", "VXORL3", "VXORL4",
// AVX-512 Instructions
"VPMOVD", "VPMOVQ", "VPMOVSS", "VPMOVSD", "VPMOVAPD", "VPMOVDQA", "VPMOVDQU",
"VPADDPD", "VPADDPS", "VPADDL", "VPADDL2", "VPADDL3", "VPADDL4",
"VPSUBPD", "VPSUBPS", "VPSUBL", "VPSUBL2", "VPSUBL3", "VPSUBL4",
"VPMULPD", "VPMULPS", "VPMULL", "VPMULL2", "VPMULL3", "VPMULL4",
"VPDIVPD", "VPDIVPS", "VPDIVL", "VPDIVL2", "VPDIVL3", "VPDIVL4",
"VPANDPD", "VPANDPS", "VPANDL", "VPANDL2", "VPANDL3", "VPANDL4",
"VPORPD", "VPORPS", "VPORL", "VPORL2", "VPORL3", "VPORL4",
"VPXORPD", "VPXORPS", "VPXORL", "VPXORL2", "VPXORL3", "VPXORL4",
];
module.exports = {
x86x64Mnemonics
};

File diff suppressed because one or more lines are too long