Upload 10 files
Browse files- .github/workflows/release_binary.yml +26 -0
- .github/workflows/release_docker.yml +37 -0
- channel/channel.go +8 -0
- cron/timing.go +26 -0
- domain/entity.go +61 -0
- pkg/validate.go +25 -0
- service/balancer_deeplx.go +193 -0
- service/deeplx.go +76 -0
- service/scan.go +205 -0
- web/deeplx_handler.go +31 -0
.github/workflows/release_binary.yml
ADDED
|
@@ -0,0 +1,26 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
name: Release Go Binary
|
| 2 |
+
|
| 3 |
+
on:
|
| 4 |
+
release:
|
| 5 |
+
types: [ created ]
|
| 6 |
+
|
| 7 |
+
jobs:
|
| 8 |
+
releases-matrix:
|
| 9 |
+
name: Release Go Binary
|
| 10 |
+
runs-on: ubuntu-latest
|
| 11 |
+
strategy:
|
| 12 |
+
matrix:
|
| 13 |
+
goos: [ linux, windows, darwin ]
|
| 14 |
+
goarch: [ amd64, arm64 ]
|
| 15 |
+
exclude:
|
| 16 |
+
- goarch: arm64
|
| 17 |
+
goos: windows
|
| 18 |
+
steps:
|
| 19 |
+
- uses: actions/checkout@v4
|
| 20 |
+
- uses: wangyoucao577/go-release-action@v1
|
| 21 |
+
with:
|
| 22 |
+
github_token: ${{ secrets.MY_TOKEN }}
|
| 23 |
+
goos: ${{ matrix.goos }}
|
| 24 |
+
goarch: ${{ matrix.goarch }}
|
| 25 |
+
binary_name: "deeplx"
|
| 26 |
+
extra_files: url.txt
|
.github/workflows/release_docker.yml
ADDED
|
@@ -0,0 +1,37 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
name: ci
|
| 2 |
+
|
| 3 |
+
on:
|
| 4 |
+
push:
|
| 5 |
+
branches:
|
| 6 |
+
- "main"
|
| 7 |
+
|
| 8 |
+
jobs:
|
| 9 |
+
docker:
|
| 10 |
+
runs-on: ubuntu-latest
|
| 11 |
+
steps:
|
| 12 |
+
- name: Checkout
|
| 13 |
+
uses: actions/checkout@v4
|
| 14 |
+
- name: Set up QEMU
|
| 15 |
+
uses: docker/setup-qemu-action@v3
|
| 16 |
+
- name: Set up Docker Buildx
|
| 17 |
+
uses: docker/setup-buildx-action@v3
|
| 18 |
+
- name: Login to Docker Hub
|
| 19 |
+
uses: docker/login-action@v3
|
| 20 |
+
with:
|
| 21 |
+
username: ${{ secrets.DOCKERHUB_USERNAME }}
|
| 22 |
+
password: ${{ secrets.DOCKERHUB_TOKEN }}
|
| 23 |
+
- name: Build and push
|
| 24 |
+
uses: docker/build-push-action@v5
|
| 25 |
+
with:
|
| 26 |
+
context: .
|
| 27 |
+
push: true
|
| 28 |
+
file: ./Dockerfile
|
| 29 |
+
platforms: linux/arm64,linux/amd64 # 需要的平台
|
| 30 |
+
#tags: ${{ steps.meta.outputs.tags }}
|
| 31 |
+
tags: neccen/deeplx-local:latest
|
| 32 |
+
# - name: Update repo description
|
| 33 |
+
# uses: peter-evans/dockerhub-description@v3
|
| 34 |
+
# with:
|
| 35 |
+
# username: ${{ secrets.DOCKERHUB_USERNAME }}
|
| 36 |
+
# password: ${{ secrets.DOCKERHUB_TOKEN }}
|
| 37 |
+
# repository: neccen/deeplx-local
|
channel/channel.go
ADDED
|
@@ -0,0 +1,8 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
package channel
|
| 2 |
+
|
| 3 |
+
import "os"
|
| 4 |
+
|
| 5 |
+
var (
|
| 6 |
+
Restart = make(chan os.Signal, 1)
|
| 7 |
+
Quit = make(chan os.Signal, 1)
|
| 8 |
+
)
|
cron/timing.go
ADDED
|
@@ -0,0 +1,26 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
package cron
|
| 2 |
+
|
| 3 |
+
import (
|
| 4 |
+
"deeplx-local/channel"
|
| 5 |
+
"log"
|
| 6 |
+
"time"
|
| 7 |
+
)
|
| 8 |
+
|
| 9 |
+
func StartTimer(t time.Duration, f func()) {
|
| 10 |
+
go func() {
|
| 11 |
+
for {
|
| 12 |
+
now := time.Now()
|
| 13 |
+
next := now.Add(t)
|
| 14 |
+
next = time.Date(next.Year(), next.Month(), next.Day(), 0, 0, 0, 0, next.Location())
|
| 15 |
+
timer := time.NewTimer(next.Sub(now))
|
| 16 |
+
select {
|
| 17 |
+
case <-channel.Quit:
|
| 18 |
+
log.Println("定时任务已退出")
|
| 19 |
+
timer.Stop()
|
| 20 |
+
return
|
| 21 |
+
case <-timer.C:
|
| 22 |
+
f()
|
| 23 |
+
}
|
| 24 |
+
}
|
| 25 |
+
}()
|
| 26 |
+
}
|
domain/entity.go
ADDED
|
@@ -0,0 +1,61 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
package domain
|
| 2 |
+
|
| 3 |
+
type TranslateRequest struct {
|
| 4 |
+
Text string `json:"text"`
|
| 5 |
+
SourceLang string `json:"source_lang"`
|
| 6 |
+
TargetLang string `json:"target_lang"`
|
| 7 |
+
}
|
| 8 |
+
|
| 9 |
+
// TranslateResponse
|
| 10 |
+
//
|
| 11 |
+
// {
|
| 12 |
+
// "code": 200,
|
| 13 |
+
// "id": 4548710002,
|
| 14 |
+
// "data": "我爱你",
|
| 15 |
+
// "alternatives": [
|
| 16 |
+
// "我爱你们",
|
| 17 |
+
// "我爱您",
|
| 18 |
+
// "我爱妳"
|
| 19 |
+
// ]
|
| 20 |
+
// }
|
| 21 |
+
type TranslateResponse struct {
|
| 22 |
+
Code int `json:"code"`
|
| 23 |
+
Data string `json:"data"`
|
| 24 |
+
}
|
| 25 |
+
|
| 26 |
+
type YingTuResponse struct {
|
| 27 |
+
Code int `json:"code"`
|
| 28 |
+
Data struct {
|
| 29 |
+
Total int `json:"total"`
|
| 30 |
+
Arr []YingTuResponseArr `json:"arr"`
|
| 31 |
+
ConsumeQuota string `json:"consume_quota"`
|
| 32 |
+
RestQuota string `json:"rest_quota"`
|
| 33 |
+
}
|
| 34 |
+
}
|
| 35 |
+
|
| 36 |
+
type YingTuResponseArr struct {
|
| 37 |
+
Url string `json:"url"`
|
| 38 |
+
}
|
| 39 |
+
|
| 40 |
+
type Quake360Response struct {
|
| 41 |
+
Code int `json:"code"`
|
| 42 |
+
Message string `json:"message"`
|
| 43 |
+
Data []Quake360ResponseData `json:"data"`
|
| 44 |
+
}
|
| 45 |
+
|
| 46 |
+
type Quake360ResponseData struct {
|
| 47 |
+
Domain string `json:"domain"`
|
| 48 |
+
Id string `json:"id"` // hpjx.e.eceping.net_443_tcp 或者 116.204.90.243_2001_tcp
|
| 49 |
+
}
|
| 50 |
+
|
| 51 |
+
type Quake360UserInfoResponse struct {
|
| 52 |
+
Code int `json:"code"`
|
| 53 |
+
Message string `json:"message"`
|
| 54 |
+
Data Quake360UserInfoResponseData `json:"data"`
|
| 55 |
+
}
|
| 56 |
+
|
| 57 |
+
type Quake360UserInfoResponseData struct {
|
| 58 |
+
MonthRemainingCredit int `json:"month_remaining_credit"`
|
| 59 |
+
FreeQueryApiCount int `json:"free_query_api_count"`
|
| 60 |
+
MobilePhone string `json:"mobile_phone"`
|
| 61 |
+
}
|
pkg/validate.go
ADDED
|
@@ -0,0 +1,25 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
package pkg
|
| 2 |
+
|
| 3 |
+
import (
|
| 4 |
+
"deeplx-local/domain"
|
| 5 |
+
"github.com/imroc/req/v3"
|
| 6 |
+
)
|
| 7 |
+
|
| 8 |
+
const validResp = "我爱你"
|
| 9 |
+
|
| 10 |
+
var validReq = domain.TranslateRequest{
|
| 11 |
+
Text: "I love you",
|
| 12 |
+
SourceLang: "EN",
|
| 13 |
+
TargetLang: "ZH",
|
| 14 |
+
}
|
| 15 |
+
|
| 16 |
+
// CheckURLAvailability 检查URL是否可用
|
| 17 |
+
func CheckURLAvailability(client *req.Client, url string) (bool, error) {
|
| 18 |
+
var result domain.TranslateResponse
|
| 19 |
+
response, err := client.R().SetBody(&validReq).SetSuccessResult(&result).Post(url)
|
| 20 |
+
if err != nil {
|
| 21 |
+
return false, err
|
| 22 |
+
}
|
| 23 |
+
defer response.Body.Close()
|
| 24 |
+
return validResp == result.Data, nil
|
| 25 |
+
}
|
service/balancer_deeplx.go
ADDED
|
@@ -0,0 +1,193 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
package service
|
| 2 |
+
|
| 3 |
+
import (
|
| 4 |
+
"context"
|
| 5 |
+
"deeplx-local/domain"
|
| 6 |
+
"deeplx-local/pkg"
|
| 7 |
+
"github.com/imroc/req/v3"
|
| 8 |
+
lop "github.com/samber/lo/parallel"
|
| 9 |
+
"github.com/sourcegraph/conc/pool"
|
| 10 |
+
"github.com/sourcegraph/conc/stream"
|
| 11 |
+
"log"
|
| 12 |
+
"regexp"
|
| 13 |
+
"strings"
|
| 14 |
+
"sync/atomic"
|
| 15 |
+
"time"
|
| 16 |
+
)
|
| 17 |
+
|
| 18 |
+
const (
|
| 19 |
+
maxLength = 4096
|
| 20 |
+
maxFailures = 3 //最大健康检查错误次数
|
| 21 |
+
)
|
| 22 |
+
|
| 23 |
+
type Server struct {
|
| 24 |
+
URL string
|
| 25 |
+
Weight int64
|
| 26 |
+
CurrentWeight int64
|
| 27 |
+
isAvailable bool
|
| 28 |
+
failureCount int
|
| 29 |
+
}
|
| 30 |
+
|
| 31 |
+
type LoadBalancer struct {
|
| 32 |
+
Servers []*Server
|
| 33 |
+
re *regexp.Regexp
|
| 34 |
+
client *req.Client
|
| 35 |
+
index uint32
|
| 36 |
+
unavailableServers []*Server // 不可用的服务器
|
| 37 |
+
healthCheck *time.Ticker // 健康检查定时器
|
| 38 |
+
}
|
| 39 |
+
|
| 40 |
+
// NewLoadBalancer 负载均衡
|
| 41 |
+
func NewLoadBalancer(vlist *[]string) TranslateService {
|
| 42 |
+
servers := lop.Map(*vlist, func(item string, index int) *Server {
|
| 43 |
+
return &Server{URL: item, Weight: 1, CurrentWeight: 1, isAvailable: true}
|
| 44 |
+
})
|
| 45 |
+
lb := &LoadBalancer{
|
| 46 |
+
Servers: servers,
|
| 47 |
+
client: req.NewClient().SetTimeout(2 * time.Second),
|
| 48 |
+
re: regexp.MustCompile(`[^.!?。!?]+[.!?。!?]`), //还有一种方式是 [^.!?。!?\s]+[.!?。!?]?\s* 这样能分割得更细小,但感觉没必要
|
| 49 |
+
unavailableServers: make([]*Server, 0),
|
| 50 |
+
healthCheck: time.NewTicker(time.Minute),
|
| 51 |
+
}
|
| 52 |
+
go lb.startHealthCheck() // 开启定时健康检查
|
| 53 |
+
return lb
|
| 54 |
+
}
|
| 55 |
+
|
| 56 |
+
func (lb *LoadBalancer) GetTranslateData(trReq domain.TranslateRequest) domain.TranslateResponse {
|
| 57 |
+
text := trReq.Text
|
| 58 |
+
textLength := len(text)
|
| 59 |
+
|
| 60 |
+
if textLength <= maxLength {
|
| 61 |
+
return lb.sendRequest(trReq)
|
| 62 |
+
}
|
| 63 |
+
|
| 64 |
+
var textParts []string
|
| 65 |
+
var currentPart string
|
| 66 |
+
|
| 67 |
+
sentences := lb.re.FindAllString(text, -1)
|
| 68 |
+
|
| 69 |
+
for _, sentence := range sentences {
|
| 70 |
+
if len(currentPart)+len(sentence) <= maxLength {
|
| 71 |
+
currentPart += sentence
|
| 72 |
+
} else {
|
| 73 |
+
textParts = append(textParts, currentPart)
|
| 74 |
+
currentPart = sentence
|
| 75 |
+
}
|
| 76 |
+
}
|
| 77 |
+
|
| 78 |
+
if currentPart != "" {
|
| 79 |
+
textParts = append(textParts, currentPart)
|
| 80 |
+
}
|
| 81 |
+
|
| 82 |
+
var results = make([]string, 0, len(textParts))
|
| 83 |
+
s := stream.New()
|
| 84 |
+
|
| 85 |
+
for _, part := range textParts {
|
| 86 |
+
s.Go(func() stream.Callback {
|
| 87 |
+
transReq := domain.TranslateRequest{
|
| 88 |
+
Text: part,
|
| 89 |
+
SourceLang: trReq.SourceLang,
|
| 90 |
+
TargetLang: trReq.TargetLang,
|
| 91 |
+
}
|
| 92 |
+
res := lb.sendRequest(transReq)
|
| 93 |
+
return func() {
|
| 94 |
+
results = append(results, res.Data)
|
| 95 |
+
}
|
| 96 |
+
})
|
| 97 |
+
}
|
| 98 |
+
|
| 99 |
+
s.Wait()
|
| 100 |
+
|
| 101 |
+
return domain.TranslateResponse{
|
| 102 |
+
Code: 200,
|
| 103 |
+
Data: strings.Join(results, ""),
|
| 104 |
+
}
|
| 105 |
+
}
|
| 106 |
+
|
| 107 |
+
func (lb *LoadBalancer) sendRequest(trReq domain.TranslateRequest) domain.TranslateResponse {
|
| 108 |
+
ctx, cancelFunc := context.WithTimeout(context.Background(), time.Second*5)
|
| 109 |
+
defer cancelFunc()
|
| 110 |
+
resultChan := make(chan domain.TranslateResponse, 5)
|
| 111 |
+
|
| 112 |
+
contextPool := pool.New().WithContext(ctx).WithMaxGoroutines(5)
|
| 113 |
+
for i := 0; i < 5; i++ {
|
| 114 |
+
contextPool.Go(func(ctx context.Context) error {
|
| 115 |
+
server := lb.getServer()
|
| 116 |
+
var trResult domain.TranslateResponse
|
| 117 |
+
response, err := lb.client.R().
|
| 118 |
+
SetContext(ctx).
|
| 119 |
+
SetBody(trReq).
|
| 120 |
+
SetSuccessResult(&trResult).
|
| 121 |
+
Post(server.URL)
|
| 122 |
+
|
| 123 |
+
if err != nil {
|
| 124 |
+
return err
|
| 125 |
+
}
|
| 126 |
+
response.Body.Close()
|
| 127 |
+
|
| 128 |
+
if trResult.Code == 200 && len(trResult.Data) > 0 {
|
| 129 |
+
resultChan <- trResult
|
| 130 |
+
cancelFunc()
|
| 131 |
+
} else {
|
| 132 |
+
server.isAvailable = false
|
| 133 |
+
lb.unavailableServers = append(lb.unavailableServers, server)
|
| 134 |
+
}
|
| 135 |
+
return nil
|
| 136 |
+
})
|
| 137 |
+
}
|
| 138 |
+
|
| 139 |
+
//go func() {
|
| 140 |
+
// _ = contextPool.Wait()
|
| 141 |
+
// if _, ok := <-resultChan; !ok { // 如果通道已经关闭,直接返回
|
| 142 |
+
// return
|
| 143 |
+
// }
|
| 144 |
+
// close(resultChan)
|
| 145 |
+
//}()
|
| 146 |
+
|
| 147 |
+
select {
|
| 148 |
+
case r := <-resultChan:
|
| 149 |
+
defer func() {
|
| 150 |
+
cancelFunc()
|
| 151 |
+
close(resultChan)
|
| 152 |
+
}()
|
| 153 |
+
return r
|
| 154 |
+
case <-ctx.Done():
|
| 155 |
+
close(resultChan)
|
| 156 |
+
log.Println("all requests failed")
|
| 157 |
+
}
|
| 158 |
+
return domain.TranslateResponse{}
|
| 159 |
+
}
|
| 160 |
+
|
| 161 |
+
func (lb *LoadBalancer) getServer() *Server {
|
| 162 |
+
index := atomic.AddUint32(&lb.index, 1) - 1
|
| 163 |
+
server := lb.Servers[index%uint32(len(lb.Servers))]
|
| 164 |
+
|
| 165 |
+
for !server.isAvailable {
|
| 166 |
+
index = atomic.AddUint32(&lb.index, 1) - 1
|
| 167 |
+
server = lb.Servers[index%uint32(len(lb.Servers))]
|
| 168 |
+
}
|
| 169 |
+
return server
|
| 170 |
+
}
|
| 171 |
+
|
| 172 |
+
func (lb *LoadBalancer) startHealthCheck() {
|
| 173 |
+
for range lb.healthCheck.C {
|
| 174 |
+
for i := 0; i < len(lb.unavailableServers); i++ {
|
| 175 |
+
server := lb.unavailableServers[i]
|
| 176 |
+
flag, _ := pkg.CheckURLAvailability(lb.client, server.URL)
|
| 177 |
+
if flag {
|
| 178 |
+
server.isAvailable = true
|
| 179 |
+
server.failureCount = 0
|
| 180 |
+
copy(lb.unavailableServers[i:], lb.unavailableServers[i+1:])
|
| 181 |
+
lb.unavailableServers = lb.unavailableServers[:len(lb.unavailableServers)-1]
|
| 182 |
+
i--
|
| 183 |
+
} else {
|
| 184 |
+
server.failureCount++
|
| 185 |
+
if server.failureCount >= maxFailures {
|
| 186 |
+
copy(lb.unavailableServers[i:], lb.unavailableServers[i+1:])
|
| 187 |
+
lb.unavailableServers = lb.unavailableServers[:len(lb.unavailableServers)-1]
|
| 188 |
+
i--
|
| 189 |
+
}
|
| 190 |
+
}
|
| 191 |
+
}
|
| 192 |
+
}
|
| 193 |
+
}
|
service/deeplx.go
ADDED
|
@@ -0,0 +1,76 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
package service
|
| 2 |
+
|
| 3 |
+
import (
|
| 4 |
+
"context"
|
| 5 |
+
"deeplx-local/domain"
|
| 6 |
+
"github.com/imroc/req/v3"
|
| 7 |
+
"github.com/sourcegraph/conc/pool"
|
| 8 |
+
"log"
|
| 9 |
+
"math/rand"
|
| 10 |
+
"time"
|
| 11 |
+
)
|
| 12 |
+
|
| 13 |
+
type TranslateService interface {
|
| 14 |
+
GetTranslateData(trReq domain.TranslateRequest) domain.TranslateResponse
|
| 15 |
+
}
|
| 16 |
+
|
| 17 |
+
type DeepLXService struct {
|
| 18 |
+
validList *[]string
|
| 19 |
+
randNum *rand.Rand
|
| 20 |
+
client *req.Client
|
| 21 |
+
}
|
| 22 |
+
|
| 23 |
+
func NewDeepLXService(vlist *[]string) TranslateService {
|
| 24 |
+
return &DeepLXService{
|
| 25 |
+
validList: vlist,
|
| 26 |
+
randNum: rand.New(rand.NewSource(time.Now().UnixNano())),
|
| 27 |
+
client: req.NewClient().SetTimeout(2 * time.Second),
|
| 28 |
+
}
|
| 29 |
+
}
|
| 30 |
+
|
| 31 |
+
func (d *DeepLXService) GetTranslateData(trReq domain.TranslateRequest) domain.TranslateResponse {
|
| 32 |
+
ctx, cancelFunc := context.WithCancel(context.Background())
|
| 33 |
+
defer cancelFunc()
|
| 34 |
+
resultChan := make(chan domain.TranslateResponse, 5)
|
| 35 |
+
|
| 36 |
+
contextPool := pool.New().WithContext(ctx).WithMaxGoroutines(5)
|
| 37 |
+
for i := 0; i < 5; i++ {
|
| 38 |
+
contextPool.Go(func(ctx context.Context) error {
|
| 39 |
+
url := (*d.validList)[d.randNum.Intn(len(*d.validList))]
|
| 40 |
+
var trResult domain.TranslateResponse
|
| 41 |
+
response, err := d.client.R().
|
| 42 |
+
SetContext(ctx).
|
| 43 |
+
SetBody(trReq).
|
| 44 |
+
SetSuccessResult(&trResult).
|
| 45 |
+
Post(url)
|
| 46 |
+
|
| 47 |
+
if err != nil {
|
| 48 |
+
return err
|
| 49 |
+
}
|
| 50 |
+
response.Body.Close()
|
| 51 |
+
|
| 52 |
+
if trResult.Code == 200 && len(trResult.Data) > 0 {
|
| 53 |
+
resultChan <- trResult
|
| 54 |
+
cancelFunc()
|
| 55 |
+
}
|
| 56 |
+
return nil
|
| 57 |
+
})
|
| 58 |
+
}
|
| 59 |
+
|
| 60 |
+
go func() {
|
| 61 |
+
_ = contextPool.Wait()
|
| 62 |
+
if _, ok := <-resultChan; !ok { // 如果通道已经关闭,直接返回
|
| 63 |
+
return
|
| 64 |
+
}
|
| 65 |
+
close(resultChan)
|
| 66 |
+
}()
|
| 67 |
+
|
| 68 |
+
select {
|
| 69 |
+
case r := <-resultChan:
|
| 70 |
+
defer cancelFunc()
|
| 71 |
+
return r
|
| 72 |
+
case <-ctx.Done():
|
| 73 |
+
log.Println("all requests failed")
|
| 74 |
+
}
|
| 75 |
+
return domain.TranslateResponse{}
|
| 76 |
+
}
|
service/scan.go
ADDED
|
@@ -0,0 +1,205 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
package service
|
| 2 |
+
|
| 3 |
+
import (
|
| 4 |
+
"deeplx-local/domain"
|
| 5 |
+
"fmt"
|
| 6 |
+
"github.com/imroc/req/v3"
|
| 7 |
+
lop "github.com/samber/lo/parallel"
|
| 8 |
+
"log"
|
| 9 |
+
"net/url"
|
| 10 |
+
"strconv"
|
| 11 |
+
"strings"
|
| 12 |
+
"time"
|
| 13 |
+
)
|
| 14 |
+
|
| 15 |
+
const Quake360Page = "200"
|
| 16 |
+
const YingTuPageSize = "100"
|
| 17 |
+
|
| 18 |
+
type ScanService interface {
|
| 19 |
+
Scan() []string
|
| 20 |
+
}
|
| 21 |
+
|
| 22 |
+
type YingTuScanService struct {
|
| 23 |
+
client *req.Client
|
| 24 |
+
searchParam string
|
| 25 |
+
apiKey string
|
| 26 |
+
}
|
| 27 |
+
|
| 28 |
+
func NewYingTuScanService(client *req.Client, apikey string) ScanService {
|
| 29 |
+
return &YingTuScanService{
|
| 30 |
+
client,
|
| 31 |
+
"KHdlYi5ib2R5PT0iRGVlcEwgRnJlZSBBUEksIERldmVsb3BlZCBieSBzamxsZW8gYW5kIG1pc3N1by4gR28gdG8gL3RyYW5zbGF0ZSB3aXRoIFBPU1QuIGh0dHA6Ly9naXRodWIuY29tL093Ty1OZXR3b3JrL0RlZXBMWCIpJiYoaXAuY291bnRyeT09IuS4reWbvSIp",
|
| 32 |
+
apikey,
|
| 33 |
+
}
|
| 34 |
+
}
|
| 35 |
+
|
| 36 |
+
func (y *YingTuScanService) Scan() []string {
|
| 37 |
+
startDate, endDate := getStartDateAndEndDate()
|
| 38 |
+
address := buildHunterAPIAddress(y.apiKey, y.searchParam, startDate, endDate, 1)
|
| 39 |
+
var yingtuResp domain.YingTuResponse
|
| 40 |
+
|
| 41 |
+
response, err := y.client.
|
| 42 |
+
EnableInsecureSkipVerify().
|
| 43 |
+
R().
|
| 44 |
+
SetSuccessResult(&yingtuResp).
|
| 45 |
+
Get(address)
|
| 46 |
+
if err != nil {
|
| 47 |
+
log.Println("鹰图爬取 deeplx ip 失败:", err)
|
| 48 |
+
return nil
|
| 49 |
+
}
|
| 50 |
+
defer response.Body.Close()
|
| 51 |
+
|
| 52 |
+
if yingtuResp.Code != 200 || yingtuResp.Data.Total == 0 {
|
| 53 |
+
log.Println("鹰图爬取 deeplx ip 失败")
|
| 54 |
+
return nil
|
| 55 |
+
|
| 56 |
+
}
|
| 57 |
+
urls := lop.Map(yingtuResp.Data.Arr, func(item domain.YingTuResponseArr, _ int) string {
|
| 58 |
+
return item.Url
|
| 59 |
+
})
|
| 60 |
+
|
| 61 |
+
log.Printf("鹰图爬取 deeplx ip 成功,共爬取 %d 条数据,本次查询 %s ,当前 %s \n",
|
| 62 |
+
len(urls), yingtuResp.Data.ConsumeQuota, yingtuResp.Data.RestQuota)
|
| 63 |
+
|
| 64 |
+
return urls
|
| 65 |
+
|
| 66 |
+
}
|
| 67 |
+
|
| 68 |
+
// buildHunterAPIAddress 构建鹰图查询地址
|
| 69 |
+
func buildHunterAPIAddress(apiKey, searchParam, startDate, endDate string, page int) string {
|
| 70 |
+
baseURL := "https://hunter.qianxin.com/openApi/search"
|
| 71 |
+
params := url.Values{}
|
| 72 |
+
params.Set("api-key", apiKey)
|
| 73 |
+
params.Set("search", searchParam)
|
| 74 |
+
params.Set("page", strconv.Itoa(page))
|
| 75 |
+
params.Set("page_size", YingTuPageSize)
|
| 76 |
+
params.Set("is_web", "1")
|
| 77 |
+
params.Set("port_filter", "false")
|
| 78 |
+
params.Set("status_code", "200")
|
| 79 |
+
params.Set("start_time", startDate)
|
| 80 |
+
params.Set("end_time", endDate)
|
| 81 |
+
|
| 82 |
+
address := fmt.Sprintf("%s?%s", baseURL, params.Encode())
|
| 83 |
+
return address
|
| 84 |
+
}
|
| 85 |
+
|
| 86 |
+
func getStartDateAndEndDate() (string, string) {
|
| 87 |
+
now := time.Now()
|
| 88 |
+
year, month, day := now.Date()
|
| 89 |
+
endDate := fmt.Sprintf("%d-%02d-%02d", year, month, day)
|
| 90 |
+
// 上一月
|
| 91 |
+
lastMonth := now.AddDate(0, 0, -30)
|
| 92 |
+
year, month, day = lastMonth.Date()
|
| 93 |
+
startDate := fmt.Sprintf("%d-%02d-%02d", year, month, day)
|
| 94 |
+
return startDate, endDate
|
| 95 |
+
}
|
| 96 |
+
|
| 97 |
+
type Quake360ScanService struct {
|
| 98 |
+
client *req.Client
|
| 99 |
+
apiKey string
|
| 100 |
+
searchParam string
|
| 101 |
+
}
|
| 102 |
+
|
| 103 |
+
func NewQuake360ScanService(client *req.Client, apiKey string) ScanService {
|
| 104 |
+
return &Quake360ScanService{client: client, apiKey: apiKey, searchParam: "response:\"DeepL Free API, Developed by sjlleo and missuo. Go to /translate with POST. http://github.com/OwO-Network/DeepLX\" AND country: \"China\""}
|
| 105 |
+
}
|
| 106 |
+
|
| 107 |
+
func (q *Quake360ScanService) GetCredit() bool {
|
| 108 |
+
const address = "https://quake.360.net/api/v3/user/info"
|
| 109 |
+
var userInfo domain.Quake360UserInfoResponse
|
| 110 |
+
response, err := q.client.R().
|
| 111 |
+
SetHeader("X-QuakeToken", q.apiKey).
|
| 112 |
+
SetSuccessResult(&userInfo).
|
| 113 |
+
SetHeader("Content-Type", "application/json").
|
| 114 |
+
Get(address)
|
| 115 |
+
|
| 116 |
+
if err != nil {
|
| 117 |
+
log.Println("360用户详情接口请求失败:", err)
|
| 118 |
+
return false
|
| 119 |
+
}
|
| 120 |
+
|
| 121 |
+
defer response.Body.Close()
|
| 122 |
+
|
| 123 |
+
if userInfo.Code != 0 {
|
| 124 |
+
log.Println("360用户详情接口请求失败", userInfo.Message)
|
| 125 |
+
return false
|
| 126 |
+
}
|
| 127 |
+
|
| 128 |
+
// 判断月度免费次数 或 月度剩余积分 是否大于0
|
| 129 |
+
if userInfo.Data.FreeQueryApiCount > 0 || userInfo.Data.MonthRemainingCredit > 0 {
|
| 130 |
+
log.Printf("360用户 %s 详情接口请求成功,月度免费次数剩余:%d,月度剩余积分%d \n",
|
| 131 |
+
userInfo.Data.MobilePhone, userInfo.Data.FreeQueryApiCount, userInfo.Data.MonthRemainingCredit)
|
| 132 |
+
return true
|
| 133 |
+
}
|
| 134 |
+
|
| 135 |
+
return false
|
| 136 |
+
|
| 137 |
+
}
|
| 138 |
+
|
| 139 |
+
func (q *Quake360ScanService) Scan() []string {
|
| 140 |
+
// 无信用点直接返回 nil
|
| 141 |
+
if hasCredit := q.GetCredit(); !hasCredit {
|
| 142 |
+
return nil
|
| 143 |
+
}
|
| 144 |
+
|
| 145 |
+
const address = "https://quake.360.net/api/v3/search/quake_service"
|
| 146 |
+
reqParam := make(map[string]string)
|
| 147 |
+
reqParam["query"] = q.searchParam
|
| 148 |
+
reqParam["size"] = Quake360Page
|
| 149 |
+
reqParam["start"] = "0"
|
| 150 |
+
var quakeResp domain.Quake360Response
|
| 151 |
+
response, err := q.client.R().
|
| 152 |
+
SetHeader("X-QuakeToken", q.apiKey).
|
| 153 |
+
SetHeader("Content-Type", "application/json").
|
| 154 |
+
SetBody(reqParam).
|
| 155 |
+
SetSuccessResult(&quakeResp).
|
| 156 |
+
Post(address)
|
| 157 |
+
|
| 158 |
+
if err != nil {
|
| 159 |
+
log.Println("360爬取 deeplx ip 失败:", err)
|
| 160 |
+
return nil
|
| 161 |
+
}
|
| 162 |
+
defer response.Body.Close()
|
| 163 |
+
if quakeResp.Code != 0 {
|
| 164 |
+
log.Println("360爬取 deeplx ip 失败", quakeResp.Message)
|
| 165 |
+
return nil
|
| 166 |
+
}
|
| 167 |
+
|
| 168 |
+
urls := lop.Map(quakeResp.Data, func(item domain.Quake360ResponseData, _ int) string {
|
| 169 |
+
return getQuakeScanUrl(item)
|
| 170 |
+
})
|
| 171 |
+
return urls
|
| 172 |
+
}
|
| 173 |
+
|
| 174 |
+
func getQuakeScanUrl(data domain.Quake360ResponseData) string {
|
| 175 |
+
if data.Domain != "" {
|
| 176 |
+
return data.Domain
|
| 177 |
+
}
|
| 178 |
+
split := strings.Split(data.Id, "_") // 形如 116.204.90.243_2001_tcp
|
| 179 |
+
// 去掉最后一个元素
|
| 180 |
+
split = split[:len(split)-1]
|
| 181 |
+
// 拼接ip和端口
|
| 182 |
+
return strings.Join(split, ":")
|
| 183 |
+
}
|
| 184 |
+
|
| 185 |
+
// CombinedScanService 聚合扫描服务
|
| 186 |
+
type CombinedScanService struct {
|
| 187 |
+
scanServices []ScanService
|
| 188 |
+
}
|
| 189 |
+
|
| 190 |
+
// NewCombinedScanService 创建聚合扫描服务
|
| 191 |
+
func NewCombinedScanService(scanServices ...ScanService) *CombinedScanService {
|
| 192 |
+
return &CombinedScanService{scanServices: scanServices}
|
| 193 |
+
}
|
| 194 |
+
|
| 195 |
+
// Scan 聚合扫描服务
|
| 196 |
+
func (c *CombinedScanService) Scan() []string {
|
| 197 |
+
var combinedResults []string
|
| 198 |
+
for _, service := range c.scanServices {
|
| 199 |
+
results := service.Scan()
|
| 200 |
+
combinedResults = append(combinedResults, results...)
|
| 201 |
+
log.Printf("%T Scan Results: %d\n", service, len(results))
|
| 202 |
+
}
|
| 203 |
+
|
| 204 |
+
return combinedResults
|
| 205 |
+
}
|
web/deeplx_handler.go
ADDED
|
@@ -0,0 +1,31 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
package web
|
| 2 |
+
|
| 3 |
+
import (
|
| 4 |
+
"deeplx-local/domain"
|
| 5 |
+
"deeplx-local/service"
|
| 6 |
+
"github.com/gin-gonic/gin"
|
| 7 |
+
"net/http"
|
| 8 |
+
)
|
| 9 |
+
|
| 10 |
+
type DeepLXHandler struct {
|
| 11 |
+
service service.TranslateService
|
| 12 |
+
}
|
| 13 |
+
|
| 14 |
+
func NewDeepLXHandler(service service.TranslateService) *DeepLXHandler {
|
| 15 |
+
return &DeepLXHandler{service: service}
|
| 16 |
+
}
|
| 17 |
+
|
| 18 |
+
func (d *DeepLXHandler) Translate(c *gin.Context) {
|
| 19 |
+
var request domain.TranslateRequest
|
| 20 |
+
if err := c.ShouldBindJSON(&request); err != nil {
|
| 21 |
+
c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()})
|
| 22 |
+
return
|
| 23 |
+
}
|
| 24 |
+
|
| 25 |
+
translatedText := d.service.GetTranslateData(request)
|
| 26 |
+
c.JSON(http.StatusOK, translatedText)
|
| 27 |
+
}
|
| 28 |
+
|
| 29 |
+
func (d *DeepLXHandler) RegisterRoutes(engine *gin.Engine) {
|
| 30 |
+
engine.POST("/translate", d.Translate)
|
| 31 |
+
}
|