1. 项目背景与核心价值最近在安全研究工作中经常需要批量分析CVE漏洞数据。手动从美国国家漏洞数据库NVD官网逐个查看漏洞信息效率太低于是我用Python写了个自动化采集脚本。这个爬虫可以自动抓取NVD的CVE分页列表提取关键字段并生成结构化数据文件对安全从业人员和漏洞研究人员特别实用。这个工具主要解决三个痛点自动获取完整的CVE漏洞清单避免人工逐页查看标准化提取关键字段CVSS评分、漏洞类型等支持多种输出格式CSV/Excel/JSON方便后续分析2. 技术方案设计2.1 目标网站分析NVD官网采用经典的分页结构每页显示20条CVE记录。通过Chrome开发者工具分析发现数据通过后端渲染没有直接API分页参数为startIndex每次递增20关键字段都包含在tr标签中有固定class命名2.2 技术选型# 主要依赖库 import requests from bs4 import BeautifulSoup import pandas as pd import json import time选择RequestsBeautifulSoup组合是因为NVD没有反爬机制不需要Selenium等重型工具页面结构规整XPath定位不如CSS选择器直观轻量级方案运行效率更高注意虽然NVD没有严格反爬但仍建议设置2-3秒的请求间隔避免给服务器造成负担3. 核心代码实现3.1 分页爬取逻辑BASE_URL https://nvd.nist.gov/vuln/search/results PARAMS { form_type: Basic, results_type: overview, search_type: all, startIndex: 0 } def crawl_nvd(max_pages10): all_data [] for page in range(max_pages): PARAMS[startIndex] page * 20 response requests.get(BASE_URL, paramsPARAMS) soup BeautifulSoup(response.text, html.parser) # 解析表格数据 rows soup.select(table[data-testidvulnerability-table] tr) for row in rows[1:]: # 跳过表头 data parse_row(row) all_data.append(data) time.sleep(2.5) # 礼貌爬取 return all_data3.2 关键字段解析def parse_row(row): return { CVE编号: row.select_one(th a).text.strip(), 发布日期: row.select_one(td[data-testidvuln-published-on]).text, 更新时间: row.select_one(td[data-testidvuln-last-modified-on]).text, CVSS评分: row.select_one(td[data-testidvuln-cvss3-panel-score]).text, 严重等级: row.select_one(td[data-testidvuln-cvss3-panel-severity]).text, 漏洞摘要: row.select_one(td[data-testidvuln-summary]).text }4. 数据存储实现4.1 CSV格式输出def save_csv(data, filenamenvd_data.csv): df pd.DataFrame(data) df.to_csv(filename, indexFalse, encodingutf-8-sig)4.2 Excel格式输出def save_excel(data, filenamenvd_data.xlsx): df pd.DataFrame(data) writer pd.ExcelWriter(filename, enginexlsxwriter) df.to_excel(writer, indexFalse) writer.close()4.3 JSON格式输出def save_json(data, filenamenvd_data.json): with open(filename, w, encodingutf-8) as f: json.dump(data, f, ensure_asciiFalse, indent2)5. 实战经验与优化建议5.1 性能优化技巧使用Session对象复用TCP连接session requests.Session() response session.get(BASE_URL, paramsPARAMS)启用gzip压缩减少传输量headers {Accept-Encoding: gzip}实现断点续爬import os if os.path.exists(progress.json): with open(progress.json) as f: start_page json.load(f)[last_page]5.2 常见问题处理字段缺失情况处理cvss_score row.select_one(td[data-testidvuln-cvss3-panel-score]) cvss_score cvss_score.text if cvss_score else N/A网络异常重试机制from tenacity import retry, stop_after_attempt retry(stopstop_after_attempt(3)) def safe_request(url): return requests.get(url, timeout10)反爬应对策略headers { User-Agent: Mozilla/5.0 (Windows NT 10.0; Win64; x64), Referer: https://nvd.nist.gov/ }6. 完整代码示例import requests from bs4 import BeautifulSoup import pandas as pd import json import time from datetime import datetime class NVDCrawler: def __init__(self): self.base_url https://nvd.nist.gov/vuln/search/results self.session requests.Session() self.session.headers.update({ User-Agent: Mozilla/5.0, Accept-Encoding: gzip }) def crawl(self, max_pages10): all_data [] params { form_type: Basic, results_type: overview, search_type: all, startIndex: 0 } for page in range(max_pages): params[startIndex] page * 20 try: response self.session.get(self.base_url, paramsparams) response.raise_for_status() data self.parse_page(response.text) all_data.extend(data) print(f已抓取第 {page1} 页累计 {len(all_data)} 条数据) time.sleep(2.5) except Exception as e: print(f第 {page1} 页抓取失败: {str(e)}) return all_data def parse_page(self, html): soup BeautifulSoup(html, html.parser) rows soup.select(table[data-testidvulnerability-table] tr)[1:] data [] for row in rows: try: item { CVE编号: self.get_text(row, th a), 发布日期: self.get_text(row, td[data-testidvuln-published-on]), 更新时间: self.get_text(row, td[data-testidvuln-last-modified-on]), CVSS评分: self.get_text(row, td[data-testidvuln-cvss3-panel-score], N/A), 严重等级: self.get_text(row, td[data-testidvuln-cvss3-panel-severity], N/A), 漏洞摘要: self.get_text(row, td[data-testidvuln-summary], ), 抓取时间: datetime.now().strftime(%Y-%m-%d %H:%M:%S) } data.append(item) except Exception as e: print(f解析行数据失败: {str(e)}) return data def get_text(self, element, selector, defaultNone): found element.select_one(selector) return found.text.strip() if found else default def save(self, data, formatcsv, filenameNone): if not filename: filename fnvd_data.{format.lower()} df pd.DataFrame(data) if format.lower() csv: df.to_csv(filename, indexFalse, encodingutf-8-sig) elif format.lower() excel: df.to_excel(filename, indexFalse) elif format.lower() json: df.to_json(filename, orientrecords, indent2) else: raise ValueError(不支持的格式) print(f数据已保存到 {filename}) # 使用示例 if __name__ __main__: crawler NVDCrawler() data crawler.crawl(max_pages5) # 抓取5页数据 crawler.save(data, formatcsv) crawler.save(data, formatexcel) crawler.save(data, formatjson)7. 扩展应用场景这个爬虫收集的数据可以用于漏洞趋势分析按时间统计高危漏洞数量厂商漏洞统计从CVE编号提取厂商前缀CVSS评分分布分析漏洞类型词云生成分析摘要关键词# 示例分析CVSS评分分布 import matplotlib.pyplot as plt df pd.DataFrame(data) df[CVSS评分] pd.to_numeric(df[CVSS评分], errorscoerce) df[CVSS评分].hist(bins10) plt.title(CVSS Score Distribution) plt.xlabel(Score) plt.ylabel(Count) plt.show()实际使用中发现几个实用技巧可以结合CVE详情页爬取更完整的漏洞信息添加自动邮件通知功能当发现特定高危漏洞时触发告警将数据导入Elasticsearch方便全文检索定期运行脚本实现漏洞监控
