package pages import ( "bufio" "bytes" "code.gitea.io/sdk/gitea" "crypto/sha1" "fmt" "github.com/patrickmn/go-cache" "github.com/pkg/errors" "go.uber.org/zap" "io" "net/http" "slices" "strconv" "strings" "sync" "time" ) type DomainCache struct { ttl time.Duration *cache.Cache mutexes sync.Map } func (c *DomainCache) Close() error { if c.Cache != nil { c.Cache.Flush() } return nil } type DomainConfig struct { FetchTime int64 //上次刷新时间 PageDomain PageDomain Exists bool // 当前项目是否为 Pages FileCache *cache.Cache // 文件缓存 CNAME []string // 重定向地址 SHA string // 缓存 SHA DATE time.Time // 文件提交时间 BasePath string // 根目录 Topics map[string]bool // 存储库标记 Index string //默认页面 NotFound string //不存在页面 } func (receiver *DomainConfig) Close() error { receiver.FileCache.Flush() return nil } func (receiver *DomainConfig) IsRoutePage() bool { return receiver.Topics["routes-history"] || receiver.Topics["routes-hash"] } func NewDomainCache(ttl time.Duration, refreshTtl time.Duration) DomainCache { c := cache.New(refreshTtl, 2*refreshTtl) c.OnEvicted(func(_ string, i interface{}) { config := i.(*DomainConfig) if config != nil { err := config.Close() if err != nil { return } } }) return DomainCache{ ttl: ttl, Cache: c, mutexes: sync.Map{}, } } func fetch(client *GiteaConfig, domain *PageDomain, result *DomainConfig) error { branches, resp, err := client.Client.ListRepoBranches(domain.Owner, domain.Repo, gitea.ListRepoBranchesOptions{ ListOptions: gitea.ListOptions{ PageSize: 999, }, }) // 缓存 404 内容 if resp != nil && resp.StatusCode >= 400 && resp.StatusCode < 500 { result.Exists = false return nil } if err != nil { return err } topics, resp, err := client.Client.ListRepoTopics(domain.Owner, domain.Repo, gitea.ListRepoTopicsOptions{ ListOptions: gitea.ListOptions{ PageSize: 999, }, }) if err != nil { return err } branchIndex := slices.IndexFunc(branches, func(x *gitea.Branch) bool { return x.Name == domain.Branch }) if branchIndex == -1 { return errors.Wrap(ErrorNotFound, "branch not found") } currentSHA := branches[branchIndex].Commit.ID commitTime := branches[branchIndex].Commit.Timestamp result.Topics = make(map[string]bool) for _, topic := range topics { result.Topics[strings.ToLower(topic)] = true } if result.SHA == currentSHA { // 历史缓存一致,跳过 result.FetchTime = time.Now().UnixMilli() return nil } // 清理历史缓存 if result.SHA != currentSHA { if result.FileCache != nil { result.FileCache.Flush() } result.SHA = currentSHA result.DATE = commitTime } //查询是否为仓库 result.Exists, err = client.FileExists(domain, result.BasePath+"/index.html") if err != nil { return err } if !result.Exists { return nil } result.Index = "index.html" //############# 处理 404 if result.IsRoutePage() { result.NotFound = "/index.html" } else { notFound, err := client.FileExists(domain, result.BasePath+"/404.html") if err != nil { return err } if notFound { result.NotFound = "/404.html" } } // ############ 拉取 CNAME cname, err := client.ReadStringRepoFile(domain, "/CNAME") if err != nil && !errors.Is(err, ErrorNotFound) { // ignore not fond error return err } else if cname != "" { // 清理重定向 result.CNAME = make([]string, 0) scanner := bufio.NewScanner(strings.NewReader(cname)) for scanner.Scan() { alias := scanner.Text() alias = strings.TrimSpace(alias) alias = strings.TrimPrefix(strings.TrimPrefix(alias, "https://"), "http://") alias = strings.Split(alias, "/")[0] if len(strings.TrimSpace(alias)) > 0 { result.CNAME = append(result.CNAME, alias) } } } result.FetchTime = time.Now().UnixMilli() return nil } func (receiver *DomainConfig) tag(path string) string { return fmt.Sprintf("%x", sha1.Sum([]byte( fmt.Sprintf("%s|%s|%s", receiver.SHA, receiver.PageDomain.Key(), path)))) } func (receiver *DomainConfig) withNotFoundPage( client *GiteaConfig, response *FakeResponse, ) error { if receiver.NotFound == "" { // 没有默认页面 return ErrorNotFound } notFound, _ := receiver.FileCache.Get(receiver.NotFound) if notFound == nil { // 不存在 notfound domain := &receiver.PageDomain fileContext, err := client.OpenFileContext(domain, receiver.BasePath+receiver.NotFound) if errors.Is(err, ErrorNotFound) { //缓存 not found 不存在 receiver.FileCache.Set(receiver.NotFound, make([]byte, 0), cache.DefaultExpiration) return err } else if err != nil { return err } length, _ := strconv.Atoi(fileContext.Header.Get("Content-Length")) if length > client.CacheMaxSize { client.Logger.Debug("default page too large.") response.Body = fileContext.Body response.CacheModeIgnore() } else { // 保存缓存 client.Logger.Debug("create default error page.") defer fileContext.Body.Close() defBuf, _ := io.ReadAll(fileContext.Body) receiver.FileCache.Set(receiver.NotFound, defBuf, cache.DefaultExpiration) response.Body = NewByteBuf(defBuf) response.CacheModeMiss() } response.ContentTypeExt(receiver.NotFound) response.Length(length) } else { notFound := notFound.([]byte) if len(notFound) == 0 { // 不存在 NotFound return ErrorNotFound } response.Length(len(notFound)) response.Body = NewByteBuf(notFound) response.CacheModeHit() } client.Logger.Debug("use cache error page.") response.ContentTypeExt(receiver.NotFound) if receiver.IsRoutePage() { response.StatusCode = http.StatusOK } else { response.StatusCode = http.StatusNotFound } return nil } // todo: 读写加锁 func (receiver *DomainConfig) getCachedData( client *GiteaConfig, path string, ) (*FakeResponse, error) { result := NewFakeResponse() if strings.HasSuffix(path, "/") { path = path + "index.html" } for k, v := range client.CustomHeaders { result.SetHeader(k, v) } result.ETag(receiver.tag(path)) result.ContentTypeExt(path) cacheBuf, _ := receiver.FileCache.Get(path) // 使用缓存内容 if cacheBuf != nil { cacheBuf := cacheBuf.([]byte) if len(cacheBuf) == 0 { //使用 NotFound 内容 client.Logger.Debug("location not found ,", zap.Any("path", path)) return result, receiver.withNotFoundPage(client, result) } else { // 使用缓存 client.Logger.Debug("location use cache ,", zap.Any("path", path)) result.Body = ByteBuf{ bytes.NewBuffer(cacheBuf), } result.Length(len(cacheBuf)) result.CacheModeHit() return result, nil } } else { // 添加缓存 client.Logger.Debug("location add cache ,", zap.Any("path", path)) domain := *(&receiver.PageDomain) domain.Branch = receiver.SHA fileContext, err := client.OpenFileContext(&domain, receiver.BasePath+path) if err != nil && !errors.Is(err, ErrorNotFound) { return nil, err } else if errors.Is(err, ErrorNotFound) { client.Logger.Debug("location not found and src not found,", zap.Any("path", path)) // 不存在且源不存在 receiver.FileCache.Set(path, make([]byte, 0), cache.DefaultExpiration) return result, receiver.withNotFoundPage(client, result) } else { // 源存在,执行缓存 client.Logger.Debug("location found and set cache,", zap.Any("path", path)) length, _ := strconv.Atoi(fileContext.Header.Get("Content-Length")) if length > client.CacheMaxSize { client.Logger.Debug("location too large , skip cache.", zap.Any("path", path)) // 超过大小,回源 result.Body = fileContext.Body result.Length(length) result.CacheModeIgnore() return result, nil } else { client.Logger.Debug("location saved,", zap.Any("path", path)) // 未超过大小,缓存 body, _ := io.ReadAll(fileContext.Body) receiver.FileCache.Set(path, body, cache.DefaultExpiration) result.Body = NewByteBuf(body) result.Length(len(body)) result.CacheModeMiss() return result, nil } } } } func (receiver *DomainConfig) Copy( client *GiteaConfig, path string, writer http.ResponseWriter, _ *http.Request, ) (bool, error) { fakeResp, err := receiver.getCachedData(client, path) if err != nil { return false, err } for k, v := range fakeResp.Header { for _, s := range v { writer.Header().Add(k, s) } } writer.Header().Add("Pages-Server-Hash", receiver.SHA) writer.Header().Add("Last-Modified", receiver.DATE.UTC().Format(http.TimeFormat)) writer.WriteHeader(fakeResp.StatusCode) defer fakeResp.Body.Close() _, err = io.Copy(writer, fakeResp.Body) return true, err } // FetchRepo 拉取 Repo 信息 func (c *DomainCache) FetchRepo(client *GiteaConfig, domain *PageDomain) (*DomainConfig, bool, error) { nextTime := time.Now().UnixMilli() - c.ttl.Milliseconds() cacheKey := domain.Key() result, _ := c.Get(cacheKey) if result != nil { result := result.(*DomainConfig) if nextTime > result.FetchTime { // 刷新旧的缓存 result = nil } else { return result, true, nil } } if result == nil { lock := c.Lock(domain) defer lock() if result, find := c.Get(cacheKey); find { return result.(*DomainConfig), true, nil } result = &DomainConfig{ PageDomain: *domain, FileCache: cache.New(c.ttl, c.ttl*2), } if err := fetch(client, domain, result.(*DomainConfig)); err != nil { return nil, false, err } err := c.Add(cacheKey, result, cache.DefaultExpiration) if err != nil { return nil, false, err } return result.(*DomainConfig), false, nil } else { return result.(*DomainConfig), true, nil } } func (c *DomainCache) Lock(any *PageDomain) func() { return c.LockAny(any.Key()) } func (c *DomainCache) LockAny(any string) func() { value, _ := c.mutexes.LoadOrStore(any, &sync.Mutex{}) mtx := value.(*sync.Mutex) mtx.Lock() return func() { mtx.Unlock() } }