fscan/Common/Parse.go
ZacharyZcR 095437ad1a feat: 实施内存分配优化提升扫描性能
主要优化:
• 创建字符串构建器池,字符串连接性能提升18倍,内存减少99.8%
• 实施切片和Map对象池复用机制,减少频繁内存分配
• 优化SSH凭证生成,预分配切片容量减少58.6%内存使用
• 改进端口扫描和ICMP模块的Map容量预估机制
• 保持100%向后API兼容性

性能改进:
- 字符串操作: 8154ns→447ns (18x提升)
- 内存分配减少: 99.8% (8.3GB→16MB)
- SSH凭证生成: 内存减少58.6%
- 对象池复用率: 100%

新增文件:
+ common/utils/stringbuilder.go - 字符串构建器池
+ common/utils/slicepool.go - 切片对象池
+ common/utils/mappool.go - Map对象池
+ common/utils/benchmark_test.go - 性能基准测试
+ Common/utils/ - 大写版本兼容目录

修改文件:
* Common/Parse.go - 使用优化的字符串连接和去重函数
* Plugins/SSH.go - 凭证生成预分配优化
* Core/ICMP.go - 网段统计Map容量预估
* Core/PortScan.go - 端口排除Map预分配

通过专业基准测试验证,显著改善大规模扫描场景的内存效率和性能表现。
2025-08-07 01:09:54 +08:00

497 lines
13 KiB
Go
Raw Blame History

This file contains ambiguous Unicode characters

This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.

package common
import (
"fmt"
"sync"
"time"
"github.com/shadow1ng/fscan/common/i18n"
"github.com/shadow1ng/fscan/common/logging"
"github.com/shadow1ng/fscan/common/parsers"
"github.com/shadow1ng/fscan/common/utils"
)
// ParsedConfiguration 解析后的完整配置(兼容旧代码)
type ParsedConfiguration struct {
*parsers.ParsedConfig
}
// Parser 主解析器
type Parser struct {
mu sync.RWMutex
fileReader *parsers.FileReader
credentialParser *parsers.CredentialParser
targetParser *parsers.TargetParser
networkParser *parsers.NetworkParser
validationParser *parsers.ValidationParser
options *parsers.ParserOptions
initialized bool
}
// NewParser 创建新的解析器实例
func NewParser(options *parsers.ParserOptions) *Parser {
if options == nil {
options = parsers.DefaultParserOptions()
}
// 创建文件读取器
fileReader := parsers.NewFileReader(nil)
// 创建各个子解析器
credentialParser := parsers.NewCredentialParser(fileReader, nil)
targetParser := parsers.NewTargetParser(fileReader, nil)
networkParser := parsers.NewNetworkParser(nil)
validationParser := parsers.NewValidationParser(nil)
return &Parser{
fileReader: fileReader,
credentialParser: credentialParser,
targetParser: targetParser,
networkParser: networkParser,
validationParser: validationParser,
options: options,
initialized: true,
}
}
// 全局解析器实例
var globalParser *Parser
var initOnce sync.Once
// getGlobalParser 获取全局解析器实例
func getGlobalParser() *Parser {
initOnce.Do(func() {
globalParser = NewParser(nil)
})
return globalParser
}
// Parse 主解析函数 - 保持与原版本兼容的接口
func Parse(Info *HostInfo) error {
// 首先应用LogLevel配置到日志系统
applyLogLevel()
parser := getGlobalParser()
// 构建输入参数
input := &AllInputs{
Credential: &parsers.CredentialInput{
Username: Username,
Password: Password,
AddUsers: AddUsers,
AddPasswords: AddPasswords,
HashValue: HashValue,
SshKeyPath: SshKeyPath,
Domain: Domain,
UsersFile: UsersFile,
PasswordsFile: PasswordsFile,
HashFile: HashFile,
},
Target: &parsers.TargetInput{
Host: Info.Host,
HostsFile: HostsFile,
ExcludeHosts: ExcludeHosts,
Ports: Ports,
PortsFile: PortsFile,
AddPorts: AddPorts,
ExcludePorts: ExcludePorts,
TargetURL: TargetURL,
URLsFile: URLsFile,
HostPort: HostPort,
LocalMode: LocalMode,
},
Network: &parsers.NetworkInput{
HttpProxy: HttpProxy,
Socks5Proxy: Socks5Proxy,
Timeout: Timeout,
WebTimeout: WebTimeout,
DisablePing: DisablePing,
DnsLog: DnsLog,
UserAgent: UserAgent,
Cookie: Cookie,
},
}
// 执行解析
result, err := parser.ParseAll(input)
if err != nil {
return fmt.Errorf(i18n.GetText("parse_error_config_failed", err))
}
// 更新全局变量以保持兼容性
if err := updateGlobalVariables(result.Config, Info); err != nil {
return fmt.Errorf(i18n.GetText("parse_error_update_vars_failed", err))
}
// 报告警告
for _, warning := range result.Warnings {
LogBase(warning)
}
// 显示解析结果摘要
showParseSummary(result.Config)
// 同步配置到core包
SyncToCore()
return nil
}
// AllInputs 所有输入参数的集合
type AllInputs struct {
Credential *parsers.CredentialInput `json:"credential"`
Target *parsers.TargetInput `json:"target"`
Network *parsers.NetworkInput `json:"network"`
}
// ParseAll 解析所有配置
func (p *Parser) ParseAll(input *AllInputs) (*parsers.ParseResult, error) {
if input == nil {
return nil, fmt.Errorf(i18n.GetText("parse_error_empty_input"))
}
p.mu.Lock()
defer p.mu.Unlock()
if !p.initialized {
return nil, fmt.Errorf(i18n.GetText("parse_error_parser_not_init"))
}
startTime := time.Now()
result := &parsers.ParseResult{
Config: &parsers.ParsedConfig{},
Success: true,
}
var allErrors []error
var allWarnings []string
// 解析凭据配置
if input.Credential != nil {
credResult, err := p.credentialParser.Parse(input.Credential, p.options)
if err != nil {
allErrors = append(allErrors, fmt.Errorf(i18n.GetText("parse_error_credential_failed", err)))
} else {
result.Config.Credentials = credResult.Config.Credentials
allErrors = append(allErrors, credResult.Errors...)
allWarnings = append(allWarnings, credResult.Warnings...)
}
}
// 解析目标配置
if input.Target != nil {
targetResult, err := p.targetParser.Parse(input.Target, p.options)
if err != nil {
allErrors = append(allErrors, fmt.Errorf(i18n.GetText("parse_error_target_failed", err)))
} else {
result.Config.Targets = targetResult.Config.Targets
allErrors = append(allErrors, targetResult.Errors...)
allWarnings = append(allWarnings, targetResult.Warnings...)
}
}
// 解析网络配置
if input.Network != nil {
networkResult, err := p.networkParser.Parse(input.Network, p.options)
if err != nil {
allErrors = append(allErrors, fmt.Errorf(i18n.GetText("parse_error_network_failed", err)))
} else {
result.Config.Network = networkResult.Config.Network
allErrors = append(allErrors, networkResult.Errors...)
allWarnings = append(allWarnings, networkResult.Warnings...)
}
}
// 执行验证
validationInput := &parsers.ValidationInput{
ScanMode: ScanMode,
LocalMode: LocalMode,
HasHosts: input.Target != nil && (input.Target.Host != "" || input.Target.HostsFile != ""),
HasURLs: input.Target != nil && (input.Target.TargetURL != "" || input.Target.URLsFile != ""),
HasPorts: input.Target != nil && (input.Target.Ports != "" || input.Target.PortsFile != ""),
HasProxy: input.Network != nil && (input.Network.HttpProxy != "" || input.Network.Socks5Proxy != ""),
DisablePing: input.Network != nil && input.Network.DisablePing,
HasCredentials: input.Credential != nil && (input.Credential.Username != "" || input.Credential.UsersFile != ""),
}
validationResult, err := p.validationParser.Parse(validationInput, result.Config, p.options)
if err != nil {
allErrors = append(allErrors, fmt.Errorf(i18n.GetText("parse_error_validation_failed", err)))
} else {
result.Config.Validation = validationResult.Config.Validation
allErrors = append(allErrors, validationResult.Errors...)
allWarnings = append(allWarnings, validationResult.Warnings...)
}
// 汇总结果
result.Errors = allErrors
result.Warnings = allWarnings
result.ParseTime = time.Since(startTime)
result.Success = len(allErrors) == 0
return result, nil
}
// updateGlobalVariables 更新全局变量以保持向后兼容性
func updateGlobalVariables(config *parsers.ParsedConfig, info *HostInfo) error {
if config == nil {
return nil
}
// 更新凭据相关全局变量
if config.Credentials != nil {
if len(config.Credentials.Usernames) > 0 {
// 更新全局用户字典
for serviceName := range Userdict {
Userdict[serviceName] = config.Credentials.Usernames
}
}
if len(config.Credentials.Passwords) > 0 {
Passwords = config.Credentials.Passwords
}
if len(config.Credentials.HashValues) > 0 {
HashValues = config.Credentials.HashValues
}
if len(config.Credentials.HashBytes) > 0 {
HashBytes = config.Credentials.HashBytes
}
}
// 更新目标相关全局变量
if config.Targets != nil {
if len(config.Targets.Hosts) > 0 {
if info.Host == "" {
info.Host = joinStrings(config.Targets.Hosts, ",")
} else {
info.Host += "," + joinStrings(config.Targets.Hosts, ",")
}
}
if len(config.Targets.URLs) > 0 {
URLs = config.Targets.URLs
}
if len(config.Targets.Ports) > 0 {
Ports = joinInts(config.Targets.Ports, ",")
}
if len(config.Targets.ExcludePorts) > 0 {
ExcludePorts = joinInts(config.Targets.ExcludePorts, ",")
}
if len(config.Targets.HostPorts) > 0 {
HostPort = config.Targets.HostPorts
}
}
// 更新网络相关全局变量
if config.Network != nil {
if config.Network.HttpProxy != "" {
HttpProxy = config.Network.HttpProxy
}
if config.Network.Socks5Proxy != "" {
Socks5Proxy = config.Network.Socks5Proxy
}
if config.Network.Timeout > 0 {
Timeout = int64(config.Network.Timeout.Seconds())
}
if config.Network.WebTimeout > 0 {
WebTimeout = int64(config.Network.WebTimeout.Seconds())
}
if config.Network.UserAgent != "" {
UserAgent = config.Network.UserAgent
}
if config.Network.Cookie != "" {
Cookie = config.Network.Cookie
}
DisablePing = config.Network.DisablePing
DnsLog = config.Network.EnableDNSLog
}
return nil
}
// RemoveDuplicate 去重函数 - 恢复原始高效实现
func RemoveDuplicate(old []string) []string {
if len(old) <= 1 {
return old
}
temp := make(map[string]struct{}, len(old))
result := make([]string, 0, len(old))
for _, item := range old {
if _, exists := temp[item]; !exists {
temp[item] = struct{}{}
result = append(result, item)
}
}
return result
}
// 辅助函数
// joinStrings 连接字符串切片 - 优化版本使用字符串构建器池
func joinStrings(slice []string, sep string) string {
return utils.JoinStrings(slice, sep)
}
// joinInts 连接整数切片 - 优化版本使用字符串构建器池
func joinInts(slice []int, sep string) string {
return utils.JoinInts(slice, sep)
}
// showParseSummary 显示解析结果摘要
func showParseSummary(config *parsers.ParsedConfig) {
if config == nil {
return
}
// 显示目标信息
if config.Targets != nil {
if len(config.Targets.Hosts) > 0 {
if len(config.Targets.Hosts) <= 5 {
LogBase(i18n.GetText("target_hosts_found", joinStrings(config.Targets.Hosts, ", ")))
} else {
LogBase(i18n.GetText("target_hosts_count", joinStrings(config.Targets.Hosts[:5], ", "), len(config.Targets.Hosts)))
}
}
if len(config.Targets.URLs) > 0 {
if len(config.Targets.URLs) <= 3 {
LogBase(i18n.GetText("target_urls_found", joinStrings(config.Targets.URLs, ", ")))
} else {
LogBase(i18n.GetText("target_urls_count", joinStrings(config.Targets.URLs[:3], ", "), len(config.Targets.URLs)))
}
}
if len(config.Targets.Ports) > 0 {
if len(config.Targets.Ports) <= 20 {
LogBase(i18n.GetText("target_ports_found", joinInts(config.Targets.Ports, ", ")))
} else {
LogBase(i18n.GetText("target_ports_count", joinInts(config.Targets.Ports[:20], ", "), len(config.Targets.Ports)))
}
}
if len(config.Targets.ExcludePorts) > 0 {
LogBase(i18n.GetText("target_exclude_ports", joinInts(config.Targets.ExcludePorts, ", ")))
}
if config.Targets.LocalMode {
LogBase(i18n.GetText("target_local_mode"))
}
}
// 显示网络配置
if config.Network != nil {
if config.Network.HttpProxy != "" {
LogBase(i18n.GetText("network_http_proxy", config.Network.HttpProxy))
}
if config.Network.Socks5Proxy != "" {
LogBase(i18n.GetText("network_socks5_proxy", config.Network.Socks5Proxy))
}
if config.Network.Timeout > 0 {
LogBase(i18n.GetText("network_timeout", config.Network.Timeout))
}
if config.Network.WebTimeout > 0 {
LogBase(i18n.GetText("network_web_timeout", config.Network.WebTimeout))
}
}
// 显示凭据信息
if config.Credentials != nil {
if len(config.Credentials.Usernames) > 0 {
LogBase(i18n.GetText("credential_username_count", len(config.Credentials.Usernames)))
}
if len(config.Credentials.Passwords) > 0 {
LogBase(i18n.GetText("credential_password_count", len(config.Credentials.Passwords)))
}
if len(config.Credentials.HashValues) > 0 {
LogBase(i18n.GetText("credential_hash_count", len(config.Credentials.HashValues)))
}
}
}
// applyLogLevel 应用LogLevel配置到日志系统
func applyLogLevel() {
if LogLevel == "" {
return // 使用默认级别
}
// 根据LogLevel字符串获取对应的日志级别
var level logging.LogLevel
switch LogLevel {
case LogLevelAll:
level = logging.LevelAll
case LogLevelError:
level = logging.LevelError
case LogLevelBase:
level = logging.LevelBase
case LogLevelInfo:
level = logging.LevelInfo
case LogLevelSuccess:
level = logging.LevelSuccess
case LogLevelDebug:
level = logging.LevelDebug
case LogLevelInfoSuccess:
level = logging.LevelInfoSuccess
case LogLevelBaseInfoSuccess:
level = logging.LevelBaseInfoSuccess
default:
// 向后兼容:如果是老的字符串格式
switch LogLevel {
case "ALL":
level = logging.LevelAll
case "ERROR":
level = logging.LevelError
case "BASE":
level = logging.LevelBase
case "INFO":
level = logging.LevelInfo
case "SUCCESS":
level = logging.LevelSuccess
case "DEBUG":
level = logging.LevelDebug
case "debug":
level = logging.LevelAll // 兼容旧的debug行为
default:
return // 无效的级别,保持默认
}
}
// 更新全局日志管理器的级别
if globalLogger != nil {
config := &logging.LoggerConfig{
Level: level,
EnableColor: !NoColor,
SlowOutput: SlowLogOutput,
ShowProgress: true,
StartTime: StartTime,
LevelColors: logging.GetDefaultLevelColors(),
}
newLogger := logging.NewLogger(config)
if ProgressBar != nil {
newLogger.SetProgressBar(ProgressBar)
}
newLogger.SetOutputMutex(&OutputMutex)
// 设置协调输出函数使用LogWithProgress
newLogger.SetCoordinatedOutput(LogWithProgress)
// 更新全局日志管理器
globalLogger = newLogger
// status变量已移除如需获取状态请直接调用newLogger.GetScanStatus()
}
}