gcli2api / internal /syncer /syncer_test.go
a3216's picture
chore: 同步到上游 1.12.0-panel + 凭证同步/独立启动器/保活
6d60378 verified
Raw History Blame Contribute Delete
13.9 kB
package syncer
import (
"context"
"encoding/base64"
"encoding/json"
"fmt"
"net/http"
"net/http/httptest"
"os"
"path/filepath"
"strings"
"sync"
"testing"
"time"
)
// fakeBucket 是一个内存版 HF Bucket:只实现同步用到的两个只读端点。
type fakeBucket struct {
mu sync.Mutex
files map[string][]byte
mtime map[string]time.Time
}
func newFakeBucket() *fakeBucket {
return &fakeBucket{files: map[string][]byte{}, mtime: map[string]time.Time{}}
}
func (f *fakeBucket) put(path string, content []byte, mtime time.Time) {
f.mu.Lock()
defer f.mu.Unlock()
f.files[path] = content
f.mtime[path] = mtime
}
func (f *fakeBucket) get(path string) ([]byte, bool) {
f.mu.Lock()
defer f.mu.Unlock()
b, ok := f.files[path]
return b, ok
}
func (f *fakeBucket) handler(bucket string) http.Handler {
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
switch {
case strings.HasPrefix(r.URL.Path, "/api/buckets/"):
f.mu.Lock()
type entry struct {
Type string `json:"type"`
Path string `json:"path"`
Size int64 `json:"size"`
Mtime string `json:"mtime"`
XetHash string `json:"xetHash"`
}
out := []entry{}
for p, b := range f.files {
out = append(out, entry{
Type: "file", Path: p, Size: int64(len(b)),
Mtime: f.mtime[p].UTC().Format(time.RFC3339Nano), XetHash: "xet-" + p,
})
}
f.mu.Unlock()
_ = json.NewEncoder(w).Encode(out)
case strings.HasPrefix(r.URL.Path, "/buckets/"+bucket+"/resolve/"):
// Go 已把 %2F 解码回 '/',直接剥前缀取路径。
p := strings.TrimPrefix(r.URL.Path, "/buckets/"+bucket+"/resolve/")
b, ok := f.get(p)
if !ok {
http.Error(w, "not found", http.StatusNotFound)
return
}
_, _ = w.Write(b)
default:
http.NotFound(w, r)
}
})
}
// fakeWriter 记录写入,避免测试真的碰网络/子进程。
type fakeWriter struct {
mu sync.Mutex
put map[string][]byte
batches int
failOn string
}
func (f *fakeWriter) Name() string { return "fake" }
func (f *fakeWriter) Put(_ context.Context, path string, content []byte) error {
f.mu.Lock()
defer f.mu.Unlock()
if f.failOn == path {
return fmt.Errorf("boom")
}
if f.put == nil {
f.put = map[string][]byte{}
}
f.put[path] = content
return nil
}
func (f *fakeWriter) PutBatch(_ context.Context, files []PushFile, _ bool) error {
f.mu.Lock()
defer f.mu.Unlock()
f.batches++
if f.put == nil {
f.put = map[string][]byte{}
}
for _, fl := range files {
f.put[fl.Path] = fl.Content
}
return nil
}
func (f *fakeWriter) Delete(context.Context, string) error { return nil }
func (f *fakeWriter) written(path string) ([]byte, bool) {
f.mu.Lock()
defer f.mu.Unlock()
b, ok := f.put[path]
return b, ok
}
// authDoc 造一个嵌套形态的凭证文件(网关 auth.Parse 认的形态)。
func authDoc(uid, nick, at, rt string, exp int64) []byte {
return []byte(fmt.Sprintf(
`{"account":{"uid":%q,"nickname":%q},"auth":{"accessToken":%q,"refreshToken":%q,"expiresAt":%d}}`,
uid, nick, at, rt, exp))
}
// newTestEngine 起一个假桶 + 假写入通道的 Engine。
func newTestEngine(t *testing.T, fb *fakeBucket, local map[string][]byte) (*Engine, string, string) {
t.Helper()
dir := t.TempDir()
authDir := filepath.Join(dir, "auths")
dataDir := filepath.Join(dir, "data")
for rel, content := range local {
fp := filepath.Join(dir, rel)
if err := os.MkdirAll(filepath.Dir(fp), 0o755); err != nil {
t.Fatal(err)
}
if err := os.WriteFile(fp, content, 0o600); err != nil {
t.Fatal(err)
}
}
ts := httptest.NewServer(fb.handler("u/b"))
t.Cleanup(ts.Close)
e, err := New(Config{
Bucket: "u/b", Endpoint: ts.URL,
AuthDir: authDir, DataDir: dataDir,
IncludeData: []string{"state.json"},
Writer: "relay", RelayURL: "http://127.0.0.1:1", // 占位,测试里换成 fakeWriter
Timeout: "10s",
})
if err != nil {
t.Fatalf("New: %v", err)
}
return e, authDir, dataDir
}
// TestPullDownloadsRemoteOnly 本地首次启动:桶里独有的凭证要落到本地 auths/。
func TestPullDownloadsRemoteOnly(t *testing.T) {
fb := newFakeBucket()
fb.put("auths/workbuddy-u1.json", authDoc("u1", "n1", "at1", "rt1", 5000), time.Now())
e, authDir, _ := newTestEngine(t, fb, nil)
fw := &fakeWriter{}
e.writer = fw
p, res, err := e.Sync(context.Background(), ModePull, false)
if err != nil {
t.Fatalf("Sync: %v", err)
}
if res.Downloaded != 1 || res.Failed != 0 {
t.Fatalf("res=%+v plan=%+v", res, p.Actions)
}
raw, err := os.ReadFile(filepath.Join(authDir, "workbuddy-u1.json"))
if err != nil {
t.Fatalf("下载后本地文件不存在: %v", err)
}
if string(raw) != string(authDoc("u1", "n1", "at1", "rt1", 5000)) {
t.Errorf("内容不一致: %s", raw)
}
}
// TestPushUploadsLocalOnly 本地新增的账号要推到远端。
func TestPushUploadsLocalOnly(t *testing.T) {
fb := newFakeBucket()
e, _, _ := newTestEngine(t, fb, map[string][]byte{
"auths/workbuddy-u2.json": authDoc("u2", "n2", "at2", "rt2", 9000),
})
fw := &fakeWriter{}
e.writer = fw
_, res, err := e.Sync(context.Background(), ModePush, false)
if err != nil {
t.Fatalf("Sync: %v", err)
}
if res.Uploaded != 1 || res.Failed != 0 {
t.Fatalf("res=%+v", res)
}
got, ok := fw.written("auths/workbuddy-u2.json")
if !ok {
t.Fatal("远端没收到上传")
}
if string(got) != string(authDoc("u2", "n2", "at2", "rt2", 9000)) {
t.Errorf("上传内容不一致: %s", got)
}
}
// TestConflictNewerAccessTokenWins 两端都有同一账号且都刷新过 → 采用 expiresAt 更晚的一方,
// 并把赢家回灌到输家。这是"防止凭证不同步"的核心行为。
func TestConflictNewerAccessTokenWins(t *testing.T) {
fb := newFakeBucket()
// 远端更新(expiresAt 9000),本地陈旧(expiresAt 1000)。
fb.put("auths/workbuddy-u3.json", authDoc("u3", "n3", "remote-at", "remote-rt", 9000), time.Now())
e, authDir, _ := newTestEngine(t, fb, map[string][]byte{
"auths/workbuddy-u3.json": authDoc("u3", "n3", "local-at", "local-rt", 1000),
})
fw := &fakeWriter{}
e.writer = fw
p, res, err := e.Sync(context.Background(), ModeSync, false)
if err != nil {
t.Fatalf("Sync: %v", err)
}
if len(p.Conflicts()) != 1 {
t.Fatalf("应识别出 1 个冲突: %+v", p.Actions)
}
if res.Downloaded != 1 || res.Uploaded != 0 {
t.Fatalf("应由远端覆盖本地: res=%+v", res)
}
raw, _ := os.ReadFile(filepath.Join(authDir, "workbuddy-u3.json"))
if !strings.Contains(string(raw), "remote-at") {
t.Errorf("本地未被远端覆盖: %s", raw)
}
}
// TestConflictLocalNewerPushes 反向:本地更新 → 推到远端。
func TestConflictLocalNewerPushes(t *testing.T) {
fb := newFakeBucket()
fb.put("auths/workbuddy-u4.json", authDoc("u4", "n4", "remote-at", "remote-rt", 1000), time.Now())
e, authDir, _ := newTestEngine(t, fb, map[string][]byte{
"auths/workbuddy-u4.json": authDoc("u4", "n4", "local-at", "local-rt", 9000),
})
fw := &fakeWriter{}
e.writer = fw
_, res, err := e.Sync(context.Background(), ModeSync, false)
if err != nil {
t.Fatalf("Sync: %v", err)
}
if res.Uploaded != 1 || res.Downloaded != 0 {
t.Fatalf("应由本地覆盖远端: res=%+v", res)
}
got, _ := fw.written("auths/workbuddy-u4.json")
if !strings.Contains(string(got), "local-at") {
t.Errorf("远端未被本地覆盖: %s", got)
}
raw, _ := os.ReadFile(filepath.Join(authDir, "workbuddy-u4.json"))
if !strings.Contains(string(raw), "local-at") {
t.Errorf("本地被误改: %s", raw)
}
}
// TestIdenticalIsNoop 两端一致时不应产生任何读写。
func TestIdenticalIsNoop(t *testing.T) {
doc := authDoc("u5", "n5", "at", "rt", 5000)
fb := newFakeBucket()
fb.put("auths/workbuddy-u5.json", doc, time.Now())
e, _, _ := newTestEngine(t, fb, map[string][]byte{"auths/workbuddy-u5.json": doc})
fw := &fakeWriter{}
e.writer = fw
p, res, err := e.Sync(context.Background(), ModeSync, false)
if err != nil {
t.Fatalf("Sync: %v", err)
}
if p.Changes() != 0 || res.Uploaded != 0 || res.Downloaded != 0 {
t.Fatalf("一致时应无动作: plan=%+v res=%+v", p.Actions, res)
}
}
// TestExcludedPathsNeverSynced config.json 与 request-logs 绝不能进同步范围。
func TestExcludedPathsNeverSynced(t *testing.T) {
fb := newFakeBucket()
fb.put("config.json", []byte(`{"listen":":7861","api_key":"relay-secret"}`), time.Now())
fb.put("data/request-logs/requests-2026-10-02.jsonl", []byte("{}\n"), time.Now())
fb.put("data/state.json", []byte(`{"accounts":{}}`), time.Now())
e, authDir, dataDir := newTestEngine(t, fb, nil)
fw := &fakeWriter{}
e.writer = fw
p, res, err := e.Sync(context.Background(), ModeSync, false)
if err != nil {
t.Fatalf("Sync: %v", err)
}
if res.Downloaded != 1 { // 只有 data/state.json
t.Fatalf("应只下载 state.json: res=%+v plan=%+v", res, p.Actions)
}
for _, a := range p.Actions {
if a.Path == "config.json" || strings.HasPrefix(a.Path, "data/request-logs/") {
t.Errorf("被排除的路径出现在计划里: %+v", a)
}
}
if _, err := os.Stat(filepath.Join(authDir, "..", "config.json")); err == nil {
t.Error("config.json 被写到了本地")
}
if _, err := os.Stat(filepath.Join(dataDir, "request-logs")); err == nil {
t.Error("request-logs 被同步了")
}
}
// TestDryRunNeverWrites dry-run(status)不得产生任何落盘或上传,
// 但**必须**照常把差异算出来——漏算差异会让 status 骗人。
func TestDryRunNeverWrites(t *testing.T) {
fb := newFakeBucket()
fb.put("auths/workbuddy-u6.json", authDoc("u6", "n6", "at", "rt", 9000), time.Now())
e, authDir, _ := newTestEngine(t, fb, map[string][]byte{
"auths/workbuddy-u6.json": authDoc("u6", "n6", "old", "old", 1),
})
fw := &fakeWriter{}
e.writer = fw
plan, res, err := e.Sync(context.Background(), ModeSync, true)
if err != nil {
t.Fatalf("Sync: %v", err)
}
if plan.Changes() != 1 {
t.Fatalf("dry-run 也必须报出这 1 处差异: %+v", plan.Actions)
}
if res.Downloaded != 1 { // 计入"将会做",但没真做
t.Fatalf("dry-run 应计入计划动作: %+v", res)
}
if len(fw.put) != 0 || fw.batches != 0 {
t.Errorf("dry-run 不应上传: %+v", fw.put)
}
raw, _ := os.ReadFile(filepath.Join(authDir, "workbuddy-u6.json"))
if !strings.Contains(string(raw), "old") {
t.Errorf("dry-run 不应改本地: %s", raw)
}
}
// TestApplyNoWriterFailsLoudly 没有写入通道时必须明确失败,不能假装成功。
func TestApplyNoWriterFailsLoudly(t *testing.T) {
fb := newFakeBucket()
e, _, _ := newTestEngine(t, fb, map[string][]byte{
"auths/workbuddy-u7.json": authDoc("u7", "n7", "at", "rt", 9000),
})
e.writer = nil
_, res, err := e.Sync(context.Background(), ModePush, false)
if err != nil {
t.Fatalf("Sync 本身不应报错: %v", err)
}
if res.Failed != 1 || res.Uploaded != 0 {
t.Fatalf("应报告 1 个失败: %+v", res)
}
}
// TestNextLink 分页 Link 头解析(含相对 URL 与多段)。
func TestNextLink(t *testing.T) {
const ep = "https://huggingface.co"
cases := []struct{ in, want string }{
{"", ""},
{`<https://huggingface.co/api/buckets/u/b/tree?cursor=2>; rel="next"`, "https://huggingface.co/api/buckets/u/b/tree?cursor=2"},
{`</api/buckets/u/b/tree?cursor=3>; rel="next"`, ep + "/api/buckets/u/b/tree?cursor=3"},
{`<https://x/y>; rel="prev", </api/buckets/u/b/tree?c=9>; rel="next"`, ep + "/api/buckets/u/b/tree?c=9"},
{`<https://x/y>; rel="prev"`, ""},
{"garbage", ""},
}
for _, tc := range cases {
if got := nextLink(tc.in, ep); got != tc.want {
t.Errorf("nextLink(%q) = %q, want %q", tc.in, got, tc.want)
}
}
}
// TestLocalPathRejectsTraversal localPath 必须挡住越界路径。
func TestLocalPathRejectsTraversal(t *testing.T) {
e := &Engine{cfg: Config{AuthDir: "/tmp/a", DataDir: "/tmp/d", IncludeData: []string{"state.json"}}}
for _, bad := range []string{"config.json", "../evil.json", "auths/../../evil.json", "other/x.json"} {
if _, err := e.localPath(bad); err == nil {
t.Errorf("localPath(%q) 应报错", bad)
}
}
if fp, err := e.localPath("auths/workbuddy-1.json"); err != nil || !strings.HasSuffix(fp, "workbuddy-1.json") {
t.Errorf("localPath 正常路径失败: %q %v", fp, err)
}
}
// TestJSONOutputNeverLeaksCredentialMaterial 安全回归:FileState.Content 绝不能被序列化出去。
//
// Action 会被 `wb2api-sync -json` 整个打印,Content 一旦进 JSON,`status -json`
// 就会把 accessToken / refreshToken 明文(原始与 base64 两种形态)灌进终端、
// CI 日志或 issue 附件——等于自己把凭证泄露出去。实测早期版本正是这样泄露的。
func TestJSONOutputNeverLeaksCredentialMaterial(t *testing.T) {
const secret = "SUPER-SECRET-REFRESH-TOKEN-VALUE"
doc := authDoc("u1", "nick", "access-"+secret, "refresh-"+secret, 9000)
st := newFileState("auths/workbuddy-u1.json", doc, time.Now(), catAuth)
if st.Content == nil {
t.Fatal("夹具前提不成立:Content 应为原始字节")
}
if !strings.Contains(string(st.Content), secret) {
t.Fatal("夹具前提不成立:内容里应含 secret")
}
raw, err := json.Marshal(Action{Op: OpDownload, Path: st.Path, Category: catAuth, Local: st, Remote: st})
if err != nil {
t.Fatal(err)
}
got := string(raw)
if strings.Contains(got, secret) {
t.Fatalf("JSON 里出现了凭证明文: %s", got)
}
if strings.Contains(strings.ToLower(got), `"content"`) {
t.Errorf("JSON 里出现了 content 字段: %s", got)
}
b64 := base64.StdEncoding.EncodeToString(doc)
if len(b64) > 24 && strings.Contains(got, b64[:24]) {
t.Errorf("JSON 里出现了 content 的 base64 前缀: %s", got)
}
// 该有的诊断字段不能因为脱敏一起丢掉。
if !strings.Contains(got, st.SHA256) || !strings.Contains(got, "u1") {
t.Errorf("SHA256/UID 等诊断字段应保留: %s", got)
}
}