1. 三层异步化
- Exec 方法立即返回
- addToAddressList 异步处理
- batchAddAddresses 异步批处理
2. 跳过耗时检查
- 移除 addressExists() 同步调用
- 依赖Mikrotik内置重复检查
3. 优化重试机制
- 重试次数:3 → 2
- 退避策略:指数 → 线性递增
- 总延迟大幅降低
4. 配置优化
- 连接超时:10s → 5s
- 关闭验证:verify_add: false
This commit is contained in:
parent
444c01d207
commit
c9c49f0827
3
dns.yaml
3
dns.yaml
@ -50,9 +50,10 @@ plugins:
|
|||||||
username: "admin"
|
username: "admin"
|
||||||
password: "szn0s!nw@pwd()"
|
password: "szn0s!nw@pwd()"
|
||||||
use_tls: false
|
use_tls: false
|
||||||
timeout: 10
|
timeout: 5 # 减少连接超时时间
|
||||||
address_list4: "gfw" # 改为 gfw,插件会自动创建这个地址列表
|
address_list4: "gfw" # 改为 gfw,插件会自动创建这个地址列表
|
||||||
mask4: 24
|
mask4: 24
|
||||||
comment: "amazon_domain"
|
comment: "amazon_domain"
|
||||||
timeout_addr: 86400
|
timeout_addr: 86400
|
||||||
cache_ttl: 86400
|
cache_ttl: 86400
|
||||||
|
verify_add: false # 关闭验证,提升性能
|
||||||
@ -248,7 +248,8 @@ func (p *mikrotikAddressListPlugin) Exec(_ context.Context, qCtx *query_context.
|
|||||||
// 检查连接是否正常
|
// 检查连接是否正常
|
||||||
if p.conn == nil {
|
if p.conn == nil {
|
||||||
p.log.Error("MikroTik connection is nil")
|
p.log.Error("MikroTik connection is nil")
|
||||||
return fmt.Errorf("mikrotik_addresslist: connection is nil")
|
// 不返回错误,避免影响DNS响应
|
||||||
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
r := qCtx.R()
|
r := qCtx.R()
|
||||||
@ -263,10 +264,13 @@ func (p *mikrotikAddressListPlugin) Exec(_ context.Context, qCtx *query_context.
|
|||||||
zap.String("qname", domain),
|
zap.String("qname", domain),
|
||||||
zap.Int("answer_count", len(r.Answer)))
|
zap.Int("answer_count", len(r.Answer)))
|
||||||
|
|
||||||
if err := p.addToAddressList(r, domain); err != nil {
|
// 异步处理Mikrotik操作,不阻塞DNS响应
|
||||||
|
go func(response *dns.Msg, domainName string) {
|
||||||
|
if err := p.addToAddressList(response, domainName); err != nil {
|
||||||
p.log.Error("failed to add addresses to MikroTik", zap.Error(err))
|
p.log.Error("failed to add addresses to MikroTik", zap.Error(err))
|
||||||
return fmt.Errorf("mikrotik_addresslist: %w", err)
|
// 不返回错误到主流程,避免影响DNS响应
|
||||||
}
|
}
|
||||||
|
}(r, domain)
|
||||||
}
|
}
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
@ -337,21 +341,31 @@ func (p *mikrotikAddressListPlugin) addToAddressList(r *dns.Msg, domain string)
|
|||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// 动态调整工作池大小
|
// 立即记录并启动异步处理,不等待任何操作
|
||||||
p.adjustWorkerPoolSize(len(addresses))
|
p.log.Debug("queuing addresses for async processing",
|
||||||
|
zap.Int("address_count", len(addresses)),
|
||||||
|
zap.String("domain", domain))
|
||||||
|
|
||||||
// 使用优化的批量处理
|
// 异步处理所有地址,包括工作池调整和批量处理
|
||||||
if err := p.batchAddAddresses(addresses, p.args.AddressList4, p.args.Mask4, domain); err != nil {
|
go func(addrs []netip.Addr, listName string, mask int, domainName string) {
|
||||||
return err
|
// 在异步线程中调整工作池大小
|
||||||
|
p.adjustWorkerPoolSize(len(addrs))
|
||||||
|
|
||||||
|
// 启动批量处理
|
||||||
|
if err := p.batchAddAddresses(addrs, listName, mask, domainName); err != nil {
|
||||||
|
p.log.Error("async batch processing failed", zap.Error(err))
|
||||||
}
|
}
|
||||||
|
|
||||||
// 记录缓存统计信息
|
// 记录缓存统计信息
|
||||||
total, valid := p.getCacheStats()
|
total, valid := p.getCacheStats()
|
||||||
p.log.Info("IPv4 addresses processed",
|
p.log.Debug("async processing stats",
|
||||||
zap.Int("processed_count", len(addresses)),
|
zap.Int("processed_count", len(addrs)),
|
||||||
zap.Int("cache_total", total),
|
zap.Int("cache_total", total),
|
||||||
zap.Int("cache_valid", valid))
|
zap.Int("cache_valid", valid),
|
||||||
|
zap.String("domain", domainName))
|
||||||
|
}(addresses, p.args.AddressList4, p.args.Mask4, domain)
|
||||||
|
|
||||||
|
// 立即返回,不等待任何异步操作
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -381,17 +395,9 @@ func (p *mikrotikAddressListPlugin) addAddressToMikrotik(addr netip.Addr, listNa
|
|||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// 缓存中没有,检查 MikroTik 中是否已存在
|
// 跳过耗时的 MikroTik 存在性检查,直接尝试添加
|
||||||
exists, err := p.addressExists(listName, cidrAddr)
|
// 依赖 Mikrotik 的内置重复检查和错误处理
|
||||||
if err != nil {
|
p.log.Debug("skipping existence check, will attempt direct add", zap.String("cidr", cidrAddr), zap.String("list", listName))
|
||||||
// 如果检查失败,可能是地址列表不存在,继续尝试添加
|
|
||||||
p.log.Debug("failed to check if address exists in MikroTik, will try to add anyway", zap.Error(err))
|
|
||||||
} else if exists {
|
|
||||||
// 地址已存在于 MikroTik 中,添加到缓存并跳过
|
|
||||||
p.log.Debug("address already exists in MikroTik, adding to cache", zap.String("cidr", cidrAddr), zap.String("list", listName))
|
|
||||||
p.addToCache(listName, cidrAddr)
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// 构造 RouterOS 参数,注意必须以 = 开头!
|
// 构造 RouterOS 参数,注意必须以 = 开头!
|
||||||
params := []string{
|
params := []string{
|
||||||
@ -422,9 +428,10 @@ func (p *mikrotikAddressListPlugin) addAddressToMikrotik(addr netip.Addr, listNa
|
|||||||
|
|
||||||
p.log.Debug("Add to list: ", zap.Strings("params", params))
|
p.log.Debug("Add to list: ", zap.Strings("params", params))
|
||||||
|
|
||||||
// 发送到 RouterOS,带智能重试机制
|
// 发送到 RouterOS,优化重试机制以减少延迟
|
||||||
maxRetries := 3
|
maxRetries := 2 // 减少重试次数
|
||||||
backoffDuration := 100 * time.Millisecond
|
backoffDuration := 50 * time.Millisecond // 减少退避时间
|
||||||
|
var err error // 声明 err 变量
|
||||||
|
|
||||||
for i := 0; i < maxRetries; i++ {
|
for i := 0; i < maxRetries; i++ {
|
||||||
// 使用读锁保护连接访问
|
// 使用读锁保护连接访问
|
||||||
@ -441,7 +448,7 @@ func (p *mikrotikAddressListPlugin) addAddressToMikrotik(addr netip.Addr, listNa
|
|||||||
p.mu.Unlock()
|
p.mu.Unlock()
|
||||||
p.log.Error("failed to reconnect", zap.Error(err))
|
p.log.Error("failed to reconnect", zap.Error(err))
|
||||||
time.Sleep(backoffDuration)
|
time.Sleep(backoffDuration)
|
||||||
backoffDuration *= 2 // 指数退避
|
backoffDuration += 25 * time.Millisecond // 线性增加,减少总延迟
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
conn = p.conn
|
conn = p.conn
|
||||||
@ -470,7 +477,7 @@ func (p *mikrotikAddressListPlugin) addAddressToMikrotik(addr netip.Addr, listNa
|
|||||||
|
|
||||||
// 指数退避
|
// 指数退避
|
||||||
time.Sleep(backoffDuration)
|
time.Sleep(backoffDuration)
|
||||||
backoffDuration *= 2
|
backoffDuration += 25 * time.Millisecond // 线性增加,减少总延迟
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -541,7 +548,7 @@ func (p *mikrotikAddressListPlugin) addressExists(listName, address string) (boo
|
|||||||
return exists, nil
|
return exists, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// batchAddAddresses 批量添加地址到MikroTik(批量操作优化)
|
// batchAddAddresses 批量添加地址到MikroTik(完全异步化)
|
||||||
func (p *mikrotikAddressListPlugin) batchAddAddresses(addresses []netip.Addr, listName string, mask int, domain string) error {
|
func (p *mikrotikAddressListPlugin) batchAddAddresses(addresses []netip.Addr, listName string, mask int, domain string) error {
|
||||||
if len(addresses) == 0 {
|
if len(addresses) == 0 {
|
||||||
return nil
|
return nil
|
||||||
@ -549,10 +556,13 @@ func (p *mikrotikAddressListPlugin) batchAddAddresses(addresses []netip.Addr, li
|
|||||||
|
|
||||||
// 分批处理,每批10个地址
|
// 分批处理,每批10个地址
|
||||||
batchSize := 10
|
batchSize := 10
|
||||||
var wg sync.WaitGroup
|
totalBatches := (len(addresses) + batchSize - 1) / batchSize
|
||||||
var mu sync.Mutex
|
|
||||||
var errors []error
|
p.log.Info("starting async batch processing",
|
||||||
successCount := 0
|
zap.Int("total_addresses", len(addresses)),
|
||||||
|
zap.Int("batch_size", batchSize),
|
||||||
|
zap.Int("total_batches", totalBatches),
|
||||||
|
zap.String("domain", domain))
|
||||||
|
|
||||||
for i := 0; i < len(addresses); i += batchSize {
|
for i := 0; i < len(addresses); i += batchSize {
|
||||||
end := i + batchSize
|
end := i + batchSize
|
||||||
@ -561,46 +571,46 @@ func (p *mikrotikAddressListPlugin) batchAddAddresses(addresses []netip.Addr, li
|
|||||||
}
|
}
|
||||||
|
|
||||||
batch := addresses[i:end]
|
batch := addresses[i:end]
|
||||||
wg.Add(1)
|
batchIndex := i/batchSize + 1
|
||||||
|
|
||||||
go func(batch []netip.Addr) {
|
|
||||||
defer wg.Done()
|
|
||||||
|
|
||||||
|
// 异步处理每个批次,不等待完成
|
||||||
|
go func(batch []netip.Addr, batchIdx int) {
|
||||||
// 获取工作池槽位
|
// 获取工作池槽位
|
||||||
select {
|
select {
|
||||||
case p.workerPool <- struct{}{}:
|
case p.workerPool <- struct{}{}:
|
||||||
defer func() { <-p.workerPool }()
|
defer func() { <-p.workerPool }()
|
||||||
default:
|
default:
|
||||||
p.log.Debug("worker pool full, processing batch directly")
|
p.log.Debug("worker pool full, processing batch directly",
|
||||||
|
zap.Int("batch", batchIdx))
|
||||||
}
|
}
|
||||||
|
|
||||||
|
successCount := 0
|
||||||
|
errorCount := 0
|
||||||
|
|
||||||
for _, addr := range batch {
|
for _, addr := range batch {
|
||||||
if err := p.addAddressToMikrotik(addr, listName, mask, domain); err != nil {
|
if err := p.addAddressToMikrotik(addr, listName, mask, domain); err != nil {
|
||||||
mu.Lock()
|
errorCount++
|
||||||
errors = append(errors, err)
|
p.log.Debug("failed to add address in batch",
|
||||||
mu.Unlock()
|
zap.String("addr", addr.String()),
|
||||||
|
zap.Int("batch", batchIdx),
|
||||||
|
zap.Error(err))
|
||||||
} else {
|
} else {
|
||||||
mu.Lock()
|
|
||||||
successCount++
|
successCount++
|
||||||
mu.Unlock()
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}(batch)
|
|
||||||
}
|
|
||||||
|
|
||||||
wg.Wait()
|
p.log.Debug("batch processing completed",
|
||||||
|
zap.Int("batch", batchIdx),
|
||||||
if len(errors) > 0 {
|
|
||||||
p.log.Error("batch processing completed with errors",
|
|
||||||
zap.Int("success_count", successCount),
|
zap.Int("success_count", successCount),
|
||||||
zap.Int("error_count", len(errors)),
|
zap.Int("error_count", errorCount),
|
||||||
zap.Error(errors[0]))
|
zap.String("domain", domain))
|
||||||
return errors[0]
|
}(batch, batchIndex)
|
||||||
}
|
}
|
||||||
|
|
||||||
p.log.Info("batch processing completed successfully",
|
// 立即返回,不等待批次处理完成
|
||||||
zap.Int("success_count", successCount),
|
p.log.Debug("all batches queued for async processing",
|
||||||
zap.Int("total_count", len(addresses)))
|
zap.Int("total_addresses", len(addresses)),
|
||||||
|
zap.String("domain", domain))
|
||||||
|
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|||||||
Loading…
Reference in New Issue
Block a user