all_user/web_hgdjb_server
3
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

init: 初始化 web_hgdjb_server 项目

Browse Source 
Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>
This commit is contained in:
root
2026-11-07 10:47:32 +08:00
co-authored by Claude Opus 4.7
commit f13d33af4d
707 changed files with 147705 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
web/skd/job/llmstxt/llmstxt.go
+386
View File
@@ -0,0 +1,386 @@
package llmstxt
import (
"context"
"fmt"
"os"
"path/filepath"
"sync"
"videoweb/pub/common/log"
"videoweb/pub/common/utils"
"videoweb/pub/pubdef"
"videoweb/web/service/srv_cfg"
)
const (
maxRecentToolsInBrief = 50 // llms.txt 中最新工具条数上限
maxRecentArticlesBrief = 50 // llms.txt 中最新文章条数上限
maxRecentInFull = 500 // llms-full.txt 中最新工具/文章条数上限
)
var generateMu sync.Mutex
// GenerateLLMSTxt 生成 llms.txt（摘要）和 llms-full.txt（详细）
func GenerateLLMSTxt() {
generateMu.Lock()
defer generateMu.Unlock()
ctx := context.Background()
config, serr := srv_cfg.GetSystemConfig(ctx)
if serr != nil || config == nil {
log.Error("GenerateLLMSTxt GetSystemConfig", log.Any("err", serr))
return
}
//baseURL := strings.TrimSuffix(config.WebAddr, "/")
//categories, subCategories, cateNameMap, subCateNameMap := loadCategories(ctx)
//recentTools := loadRecentTools(ctx, maxRecentToolsInBrief)
//recentArticles := loadRecentArticles(ctx, maxRecentArticlesBrief)
//
//var brief strings.Builder
//writeBrief(&brief, config.WebTitle, config.WebDescription, baseURL, config.WebKeyworld, recentTools, recentArticles)
//
//if err := writeTextFile("llms.txt", brief.String()); err != nil {
// log.Error("GenerateLLMSTxt write llms.txt", log.Any("err", err))
// return
//}
//if err := commitFile("llms.txt"); err != nil {
// log.Error("GenerateLLMSTxt commit llms.txt", log.Any("err", err))
// return
//}
//
//allTools := loadRecentTools(ctx, maxRecentInFull)
//allArticles := loadRecentArticles(ctx, maxRecentInFull)
//
//var full strings.Builder
//writeFull(&full, config.WebTitle, config.WebDescription, baseURL, config.WebKeyworld, categories, subCategories, cateNameMap, subCateNameMap, allTools, allArticles)
//
//if err := writeTextFile("llms-full.txt", full.String()); err != nil {
// log.Error("GenerateLLMSTxt write llms-full.txt", log.Any("err", err))
// return
//}
//if err := commitFile("llms-full.txt"); err != nil {
// log.Error("GenerateLLMSTxt commit llms-full.txt", log.Any("err", err))
// return
//}
//log.Info("GenerateLLMSTxt done",
// log.Any("categories", len(categories)),
// log.Any("subCategories", len(subCategories)),
// log.Any("tools", len(allTools)),
// log.Any("articles", len(allArticles)),
//)
}
// ── 数据加载 ──────────────────────────────────────────────────
//
//func loadCategories(ctx context.Context) (
// categories toolcategorymod.ToolCategoryModelList,
// subCategories toolsubcategorymod.ToolSubCategoryModelList,
// cateNameMap map[int64]string,
// subCateNameMap map[int64]string,
//) {
// cateFilter := bson.M{
// toolcategorymod.FN.ToolCategory.Status: 1,
// toolcategorymod.FN.ToolCategory.Deleted: bson.M{"$ne": 1},
// }
// cateOpt := options.Find().SetSort(bson.D{
// {Key: toolcategorymod.FN.ToolCategory.Sort, Value: 1},
// {Key: toolcategorymod.FN.ToolCategory.Id, Value: 1},
// })
// categories, _ = toolcategorymod.FindMany(ctx, cateFilter, cateOpt)
//
// subFilter := bson.M{
// toolsubcategorymod.FN.ToolSubCategory.Status: 1,
// toolsubcategorymod.FN.ToolSubCategory.Deleted: bson.M{"$ne": 1},
// }
// subOpt := options.Find().SetSort(bson.D{
// {Key: toolsubcategorymod.FN.ToolSubCategory.Sort, Value: 1},
// {Key: toolsubcategorymod.FN.ToolSubCategory.Id, Value: 1},
// })
// subCategories, _ = toolsubcategorymod.FindMany(ctx, subFilter, subOpt)
//
// cateNameMap = make(map[int64]string, len(categories))
// for _, c := range categories {
// cateNameMap[c.Id] = c.Name
// }
// subCateNameMap = make(map[int64]string, len(subCategories))
// for _, s := range subCategories {
// subCateNameMap[s.Id] = s.Name
// }
// return categories, subCategories, cateNameMap, subCateNameMap
//}
//
//func loadRecentTools(ctx context.Context, limit int64) toolmod.ToolModelList {
// opt := options.Find().
// SetSort(bson.D{
// {Key: toolmod.FN.Tool.Sort, Value: 1},
// {Key: toolmod.FN.Tool.Hot, Value: -1},
// {Key: toolmod.FN.Tool.CreatedAt, Value: -1},
// }).
// SetLimit(limit)
// filter := bson.M{
// toolmod.FN.Tool.ReviewStatus: 1,
// toolmod.FN.Tool.Slug: bson.M{"$nin": []interface{}{"", nil}},
// }
// list, _ := toolmod.FindMany(ctx, filter, opt)
// return list
//}
//
//func loadRecentArticles(ctx context.Context, limit int64) articlemod.ArticleModelList {
// opt := options.Find().
// SetSort(bson.D{{Key: articlemod.FN.Article.CreatedAt, Value: -1}}).
// SetLimit(limit)
// filter := bson.M{
// articlemod.FN.Article.ReviewStatus: 1,
// articlemod.FN.Article.Slug: bson.M{"$nin": []interface{}{"", nil}},
// }
// list, _ := articlemod.FindMany(ctx, filter, opt)
// return list
//}
//
//// ── 内容构建 ──────────────────────────────────────────────────
//
//func writeBrief(
// b *strings.Builder,
// title, desc, baseURL string,
// keywords []string,
// recentTools toolmod.ToolModelList,
// recentArticles articlemod.ArticleModelList,
//) {
// writeHeader(b, title, desc, keywords)
// writeImportantPages(b, baseURL)
//
// if len(recentTools) > 0 {
// fmt.Fprintf(b, "## 最新 AI 工具\n\n")
// for _, t := range recentTools {
// if t == nil || t.Slug == "" {
// continue
// }
// summary := t.Intro
// if summary == "" {
// summary = t.Desc
// }
// if summary == "" {
// summary = t.Title
// }
// fmt.Fprintf(b, "- [%s](%s): %s\n", t.Title, toolDetailURL(baseURL, t.Slug), truncate(summary, 100))
// }
// b.WriteString("\n")
// }
//
// if len(recentArticles) > 0 {
// fmt.Fprintf(b, "## 最新 AI 资讯\n\n")
// for _, a := range recentArticles {
// if a == nil || a.Slug == "" {
// continue
// }
// summary := a.Intro
// if summary == "" {
// summary = a.Title
// }
// fmt.Fprintf(b, "- [%s](%s): %s\n", a.Title, articleDetailURL(baseURL, a.Slug), truncate(summary, 100))
// }
// b.WriteString("\n")
// }
//}
//
//func GetCategorySlug(id int64, categories toolcategorymod.ToolCategoryModelList) string {
// if id <= 0 {
// return ""
// }
// if len(categories) == 0 {
// return ""
// }
// for _, category := range categories {
// if category.Id == id {
// return category.Slug
// }
// }
// return ""
//}
//func GetSubCategorySlug(id int64, categories toolsubcategorymod.ToolSubCategoryModelList) string {
// if id <= 0 {
// return ""
// }
// if len(categories) == 0 {
// return ""
// }
// for _, category := range categories {
// if category.Id == id {
// return category.Slug
// }
// }
// return ""
//}
//func writeFull(
// b *strings.Builder,
// title, desc, baseURL string,
// keywords []string,
// categories toolcategorymod.ToolCategoryModelList,
// subCategories toolsubcategorymod.ToolSubCategoryModelList,
// cateNameMap, subCateNameMap map[int64]string,
// tools toolmod.ToolModelList,
// articles articlemod.ArticleModelList,
//) {
// writeHeader(b, title, desc, keywords)
// writeImportantPages(b, baseURL)
// writeCategorySections(b, baseURL, categories, subCategories, true)
//
// if len(tools) > 0 {
// fmt.Fprintf(b, "## AI 工具详情（最新 %d 条）\n\n", len(tools))
// for _, t := range tools {
// if t == nil || t.Slug == "" {
// continue
// }
// fmt.Fprintf(b, "### [%s](%s)\n\n", t.Title, toolDetailURL(baseURL, t.Slug))
//
// summary := t.Desc
// if summary == "" {
// summary = t.Intro
// }
// if summary != "" {
// fmt.Fprintf(b, "%s\n\n", truncate(summary, 500))
// }
// if subName := subCateNameMap[t.SubCateId]; subName != "" {
// fmt.Fprintf(b, "- 子分类：[%s](%s)\n", subName, subCategoryListURL(baseURL, GetCategorySlug(t.CategoryId, categories), GetSubCategorySlug(t.SubCateId, subCategories)))
// }
// if t.OfficialUrl != "" {
// fmt.Fprintf(b, "- 官网：%s\n", t.OfficialUrl)
// }
// if len(t.SeoInfo.Keyworld) > 0 {
// fmt.Fprintf(b, "- 关键词：%s\n", strings.Join(t.SeoInfo.Keyworld, "、"))
// }
// if !t.CreatedAt.IsZero() {
// fmt.Fprintf(b, "- 发布时间：%s\n", t.CreatedAt.Format("2006-01-02"))
// }
// b.WriteString("\n")
// }
// }
//
// if len(articles) > 0 {
// fmt.Fprintf(b, "## AI 资讯详情（最新 %d 条）\n\n", len(articles))
// for _, a := range articles {
// if a == nil || a.Slug == "" {
// continue
// }
// fmt.Fprintf(b, "### [%s](%s)\n\n", a.Title, articleDetailURL(baseURL, a.Slug))
//
// if a.Intro != "" {
// fmt.Fprintf(b, "%s\n\n", truncate(a.Intro, 500))
// }
// if a.Author != "" {
// fmt.Fprintf(b, "- 作者：%s\n", a.Author)
// }
// if len(a.SeoInfo.Keyworld) > 0 {
// fmt.Fprintf(b, "- 关键词：%s\n", strings.Join(a.SeoInfo.Keyworld, "、"))
// }
// if !a.CreatedAt.IsZero() {
// fmt.Fprintf(b, "- 发布时间：%s\n", a.CreatedAt.Format("2006-01-02"))
// }
// b.WriteString("\n")
// }
// }
//}
//
//func writeHeader(b *strings.Builder, title, desc string, keywords []string) {
// now := time.Now().Format("2006-01-02")
// fmt.Fprintf(b, "# %s\n\n", title)
// if desc != "" {
// fmt.Fprintf(b, "> %s\n\n", desc)
// }
// if len(keywords) > 0 {
// fmt.Fprintf(b, "> 关键词：%s\n\n", strings.Join(keywords, "、"))
// }
// fmt.Fprintf(b, "> 更新时间：%s\n\n", now)
//}
//
//func writeImportantPages(b *strings.Builder, baseURL string) {
// fmt.Fprintf(b, "## 重要页面\n\n")
// fmt.Fprintf(b, "- [首页](%s/): AI 工具导航主页\n", baseURL)
// fmt.Fprintf(b, "- [工具列表](%s/tool/list): 全部 AI 工具\n", baseURL)
// fmt.Fprintf(b, "- [热门推荐](%s/tool/hot): 热门 AI 工具\n", baseURL)
// fmt.Fprintf(b, "- [AI 资讯](%s/article/list): AI 行业资讯文章\n", baseURL)
// fmt.Fprintf(b, "- [搜索](%s/search): 搜索工具与资讯\n", baseURL)
// fmt.Fprintf(b, "- [关于我们](%s/about/): 站点介绍\n", baseURL)
// fmt.Fprintf(b, "- [Sitemap](%s/sitemap.xml): 网站地图\n\n", baseURL)
//}
//
//func writeCategorySections(
// b *strings.Builder,
// baseURL string,
// categories toolcategorymod.ToolCategoryModelList,
// subCategories toolsubcategorymod.ToolSubCategoryModelList,
// withSeo bool,
//) {
//
// if len(subCategories) > 0 {
// fmt.Fprintf(b, "## 工具子分类\n\n")
// for _, s := range subCategories {
// if s == nil {
// continue
// }
// subDesc := subCategoryDesc(s, withSeo)
// fmt.Fprintf(b, "- [%s](%s): %s\n", s.Name, subCategoryListURL(baseURL, GetCategorySlug(s.CategoryId, categories), GetSubCategorySlug(s.Id, subCategories)), subDesc)
// }
// b.WriteString("\n")
// }
//}
//
//func subCategoryDesc(s *toolsubcategorymod.ToolSubCategoryModel, withSeo bool) string {
// if withSeo && s.SeoInfo.Description != "" {
// return s.SeoInfo.Description
// }
// return s.Name + "相关 AI 工具"
//}
// ── URL 构建 ──────────────────────────────────────────────────
func toolDetailURL(baseURL, slug string) string {
return utils.JoinUrlPath(baseURL, fmt.Sprintf("/tool/details/%s", slug))
}
func articleDetailURL(baseURL, slug string) string {
return utils.JoinUrlPath(baseURL, fmt.Sprintf("/article/details/%s", slug))
}
func subCategoryListURL(baseURL string, categorySlug, subCateSlug string) string {
return utils.JoinUrlPath(baseURL, fmt.Sprintf("/classification/%s/%s", categorySlug, subCateSlug))
}
// ── 工具函数 ──────────────────────────────────────────────────
func truncate(s string, maxLen int) string {
runes := []rune(s)
if len(runes) <= maxLen {
return s
}
return string(runes[:maxLen]) + "…"
}
func writeTextFile(filename, content string) error {
if err := os.MkdirAll(pubdef.OutputDir, 0755); err != nil {
return fmt.Errorf("mkdir %s: %w", pubdef.OutputDir, err)
}
tmpPath := filepath.Join(pubdef.OutputDir, filename+".tmp")
f, err := os.Create(tmpPath)
if err != nil {
return fmt.Errorf("create %s: %w", tmpPath, err)
}
defer f.Close()
if _, err := f.WriteString(content); err != nil {
return fmt.Errorf("write %s: %w", filename, err)
}
return f.Sync()
}
func commitFile(filename string) error {
src := filepath.Join(pubdef.OutputDir, filename+".tmp")
dst := filepath.Join(pubdef.OutputDir, filename)
if err := os.Rename(src, dst); err != nil {
os.Remove(src)
return fmt.Errorf("rename %s: %w", filename, err)
}
return nil
}
web/skd/job/reportdata/event.go
+124
View File
@@ -0,0 +1,124 @@
package job
import (
"context"
"encoding/json"
"videoweb/pub/common/log"
"videoweb/pub/dataReport"
"videoweb/pub/models/eventmod"
"go.mongodb.org/mongo-driver/bson"
"go.mongodb.org/mongo-driver/mongo/options"
)
var isRuining = false
const (
maxRetryCount = 10 // 最大重试次数
maxReportNum = 100 // 最大上报数
)
func Report() {
//防止重入
if isRuining {
return
}
isRuining = true
defer func() {
isRuining = false
}()
ctx := context.Background()
opts := options.Find()
opts = opts.SetSort(bson.D{{Key: "_id", Value: 1}}).SetLimit(1000)
// 查询出一千条记录批量上传
list, stdErr := eventmod.FindManyEvent(ctx, bson.M{
"$or": []bson.M{
bson.M{
"status": eventmod.Default,
},
bson.M{
"status": eventmod.Failure,
"retryCount": bson.M{"$lte": maxRetryCount},
},
},
}, opts)
if stdErr != nil {
log.Error("Get event list error", log.E(stdErr))
return
}
if len(list) == 0 {
return
}
var eventList []*dataReport.CommonField
for _, v := range list {
event := &dataReport.CommonField{}
err := json.Unmarshal([]byte(v.Data), &event)
if err != nil {
log.Error("event report json.Unmarshal fail", log.Any("eventData", v.Data), log.E(err))
continue
}
eventList = append(eventList, event)
if len(eventList) < maxReportNum {
continue
}
// 批量上报
err = dataReport.BatchReport(ctx, eventList)
if err != nil {
log.Error("Batch report error", log.E(err))
// 将这些设置为失败
onReportEventFail(ctx, eventList)
} else {
onReportEventSuccess(ctx, eventList)
}
eventList = []*dataReport.CommonField{} // 重新变成0个
}
// 将剩余的批量上报
err := dataReport.BatchReport(ctx, eventList)
if err != nil {
log.Error("Batch report error", log.E(err))
// 将这些设置为失败
onReportEventFail(ctx, eventList)
} else {
onReportEventSuccess(ctx, eventList)
}
}
func onReportEventSuccess(ctx context.Context, eventList []*dataReport.CommonField) {
eventIds := []string{}
for _, v := range eventList {
eventIds = append(eventIds, v.EventID)
}
if len(eventIds) == 0 {
return
}
err := eventmod.UpdateManyEvent(ctx, bson.M{
"eventId": bson.M{"$in": eventIds},
}, bson.M{
"$set": bson.M{"status": eventmod.Success},
})
if err != nil {
log.Error("Update event fail", log.E(err))
return
}
}
func onReportEventFail(ctx context.Context, eventList []*dataReport.CommonField) {
eventIds := []string{}
for _, v := range eventList {
eventIds = append(eventIds, v.EventID)
}
if len(eventIds) == 0 {
return
}
err := eventmod.UpdateManyEvent(ctx, bson.M{
"eventId": bson.M{"$in": eventIds},
}, bson.M{
"$set": bson.M{"status": eventmod.Failure},
"$inc": bson.M{"retryCount": 1},
})
if err != nil {
log.Error("Update event fail", log.E(err))
return
}
}
web/skd/job/rss/rss.go
+310
View File
@@ -0,0 +1,310 @@
package rss
import (
"context"
"encoding/xml"
"fmt"
"os"
"path/filepath"
"strings"
"sync"
"time"
"videoweb/pub/common/log"
"videoweb/pub/common/netutil"
"videoweb/pub/common/utils"
"videoweb/pub/models/cfg"
"videoweb/pub/pubdef"
"videoweb/web/service/srv_cfg"
)
const (
maxRSSItems = 200 // RSS 条目上限
)
var generateMu sync.Mutex
// RSS 2.0 结构定义
type rssFeed struct {
XMLName xml.Name `xml:"rss"`
Version string `xml:"version,attr"`
Atom string `xml:"xmlns:atom,attr"`
Channel rssChannel `xml:"channel"`
}
type rssChannel struct {
Title string `xml:"title"`
Link string `xml:"link"`
Description string `xml:"description"`
Language string `xml:"language"`
LastBuildDate string `xml:"lastBuildDate"`
AtomLink atomLink `xml:"atom:link"`
Items []rssItem `xml:"item"`
}
type atomLink struct {
Href string `xml:"href,attr"`
Rel string `xml:"rel,attr"`
Type string `xml:"type,attr"`
}
type rssItem struct {
Title string `xml:"title"`
Link string `xml:"link"`
Description string `xml:"description"`
PubDate string `xml:"pubDate"`
GUID string `xml:"guid"`
Enclosure *rssEnclosure `xml:"enclosure,omitempty"`
}
type rssEnclosure struct {
URL string `xml:"url,attr"`
Type string `xml:"type,attr"`
Length string `xml:"length,attr"`
}
// rssEntry 内部排序用，合并文章与工具后统一输出
type rssEntry struct {
Id int64
Title string
Link string
Description string
PubAt time.Time
Cover string
}
func GenerateRSS() {
generateMu.Lock()
defer generateMu.Unlock()
ctx := context.Background()
config, serr := srv_cfg.GetSystemConfig(ctx)
if serr != nil {
log.Error("GenerateRSS GetSystemConfig", log.Any("err", serr))
return
}
if config == nil {
log.Error("GenerateRSS GetSystemConfig nil config")
return
}
//
//entries := loadRSSEntries(ctx, config.WebAddr, maxRSSItems)
//items := buildRSSItems(config, entries)
//feedURL := utils.JoinUrlPath(config.WebAddr, "/rss.xml")
//feed := rssFeed{
// Version: "2.0",
// Atom: "http://www.w3.org/2005/Atom",
// Channel: rssChannel{
// Title: config.WebTitle,
// Link: config.WebAddr,
// Description: channelDescription(config),
// Language: "zh-cn",
// LastBuildDate: time.Now().Format(time.RFC1123Z),
// AtomLink: atomLink{
// Href: feedURL,
// Rel: "self",
// Type: "application/rss+xml",
// },
// Items: items,
// },
//}
//const filename = "rss.xml"
//if err := writeRSSFile(filename, feed); err != nil {
// log.Error("GenerateRSS write", log.Any("err", err))
// return
//}
//
//src := filepath.Join(pubdef.OutputDir, filename+".tmp")
//dst := filepath.Join(pubdef.OutputDir, filename)
//if err := os.Rename(src, dst); err != nil {
// log.Error("GenerateRSS rename", log.Any("err", err))
// os.Remove(src)
// return
//}
//log.Info("GenerateRSS done", log.Any("items", len(items)))
}
func channelDescription(config *cfg.SysCfg) string {
if config.WebDescription != "" {
return config.WebDescription
}
return "AI 工具导航与资讯更新"
}
//
//func loadRSSEntries(ctx context.Context, baseURL string, limit int) []rssEntry {
// baseURL = strings.TrimSuffix(baseURL, "/")
// articleFilter := bson.M{
// articlemod.FN.Article.ReviewStatus: 1,
// articlemod.FN.Article.Slug: bson.M{"$nin": []interface{}{"", nil}},
// }
// articleOpt := options.Find().
// SetSort(bson.D{{Key: articlemod.FN.Article.CreatedAt, Value: -1}}).
// SetLimit(int64(limit))
// articles, _ := articlemod.FindMany(ctx, articleFilter, articleOpt)
//
// toolFilter := bson.M{
// toolmod.FN.Tool.ReviewStatus: 1,
// toolmod.FN.Tool.Slug: bson.M{"$nin": []interface{}{"", nil}},
// }
// toolOpt := options.Find().
// SetSort(bson.D{{Key: toolmod.FN.Tool.CreatedAt, Value: -1}}).
// SetLimit(int64(limit))
// tools, _ := toolmod.FindMany(ctx, toolFilter, toolOpt)
//
// entries := make([]rssEntry, 0, len(articles)+len(tools))
// for _, a := range articles {
// if a == nil || a.Slug == "" {
// continue
// }
// desc := a.Intro
// if desc == "" {
// desc = a.Title
// }
// entries = append(entries, rssEntry{
// Id: a.Id,
// Title: a.Title,
// Link: articleDetailURL(baseURL, a.Slug),
// Description: desc,
// PubAt: a.CreatedAt,
// Cover: a.Cover,
// })
// }
// for _, t := range tools {
// if t == nil || t.Slug == "" {
// continue
// }
// desc := t.Intro
// if desc == "" {
// desc = t.Desc
// }
// if desc == "" {
// desc = t.Title
// }
// cover := t.Logo
// if cover == "" {
// cover = t.Snapshot
// }
// entries = append(entries, rssEntry{
// Id: t.Id,
// Title: t.Title,
// Link: toolDetailURL(baseURL, t.Slug),
// Description: desc,
// PubAt: t.CreatedAt,
// Cover: cover,
// })
// }
//
// sort.Slice(entries, func(i, j int) bool {
// return entries[i].PubAt.After(entries[j].PubAt)
// })
// if len(entries) > limit {
// entries = entries[:limit]
// }
// return entries
//}
//
//func buildRSSItems(config *cfg.SysCfg, entries []rssEntry) []rssItem {
// logoURL := utils.JoinUrlPath(strings.TrimSuffix(config.WebAddr, "/"), "/static/images/logo.png")
// imgCdn := imageCdn(config)
//
// items := make([]rssItem, 0, len(entries))
// for _, e := range entries {
// if e.Link == "" {
// continue
// }
//
// pubDate := ""
// if !e.PubAt.IsZero() {
// pubDate = e.PubAt.Format(time.RFC1123Z)
// }
//
// item := rssItem{
// Title: e.Title,
// Link: e.Link,
// Description: e.Description,
// PubDate: pubDate,
// GUID: e.Link,
// }
//
// encURL, encType := resolveEnclosureURL(config, imgCdn, logoURL, e.Cover, e.Id)
// item.Enclosure = &rssEnclosure{
// URL: encURL,
// Type: encType,
// Length: "0",
// }
// items = append(items, item)
// }
// return items
//}
func imageCdn(config *cfg.SysCfg) string {
if config.ImgWebCdn != "" {
return config.ImgWebCdn
}
return config.UnencryptedCdn
}
func resolveEnclosureURL(config *cfg.SysCfg, imgCdn, logoURL, cover string, id int64) (url, mime string) {
mime = "image/jpeg"
if cover != "" && imgCdn != "" {
url = netutil.MergeURL(imgCdn, cover)
} else if len(config.SitemapDefaultImages) > 0 && imgCdn != "" {
img := config.SitemapDefaultImages[absModIndex(id, len(config.SitemapDefaultImages))]
url = netutil.MergeURL(imgCdn, img)
} else {
url = logoURL
mime = "image/png"
}
if strings.HasSuffix(strings.ToLower(url), ".png") {
mime = "image/png"
} else if strings.HasSuffix(strings.ToLower(url), ".webp") {
mime = "image/webp"
}
return url, mime
}
func toolDetailURL(baseURL, slug string) string {
return utils.JoinUrlPath(baseURL, fmt.Sprintf("/tool/details/%s", slug))
}
func articleDetailURL(baseURL, slug string) string {
return utils.JoinUrlPath(baseURL, fmt.Sprintf("/article/details/%s", slug))
}
func absModIndex(id int64, length int) int {
if length <= 0 {
return 0
}
idx := id % int64(length)
if idx < 0 {
idx = -idx
}
return int(idx)
}
func writeRSSFile(filename string, data interface{}) error {
if err := os.MkdirAll(pubdef.OutputDir, 0755); err != nil {
return fmt.Errorf("mkdir %s: %w", pubdef.OutputDir, err)
}
path := filepath.Join(pubdef.OutputDir, filename+".tmp")
file, err := os.Create(path)
if err != nil {
return fmt.Errorf("create %s: %w", filename, err)
}
defer file.Close()
if _, err := file.WriteString(xml.Header); err != nil {
return fmt.Errorf("write header %s: %w", filename, err)
}
encoder := xml.NewEncoder(file)
encoder.Indent("", " ")
if err := encoder.Encode(data); err != nil {
return fmt.Errorf("encode %s: %w", filename, err)
}
if err := encoder.Flush(); err != nil {
return fmt.Errorf("flush %s: %w", filename, err)
}
return file.Sync()
}
web/skd/job/seopush/seopush.go
+161
View File
@@ -0,0 +1,161 @@
// Package seopush 为搜索引擎推送做兜底：补推已上架短剧，并重试失败的任务。
package seopush
import (
"context"
"strings"
"sync"
"time"
"videoweb/pub/common/log"
"videoweb/pub/models/playletmod"
"videoweb/pub/models/seopushmod"
"videoweb/pub/pubdef"
"videoweb/web/service/srv_seopush"
"go.mongodb.org/mongo-driver/bson"
"go.mongodb.org/mongo-driver/mongo/options"
)
const (
backfillLimit = 100
retryLimit = 50
taskKeepDays = 30
)
var jobMu sync.Mutex
// Run 定时兜底任务
func Run() {
if !jobMu.TryLock() {
log.Warn("seopush: skip job, previous run still in progress")
return
}
defer jobMu.Unlock()
ctx := context.Background()
if !pubdef.Enabled() {
discardPendingTasks(ctx)
return
}
webAddr := srv_seopush.SEOBaseURL(ctx)
if webAddr == "" {
return
}
backfillVisiblePlaylets(ctx)
retryPendingTasks(ctx, webAddr)
cleanupExpiredTasks(ctx)
}
func discardPendingTasks(ctx context.Context) {
filter := bson.M{"$or": []bson.M{
{seopushmod.FN.SeoPushTask.GoogleStatus: seopushmod.PushStatusPending},
{seopushmod.FN.SeoPushTask.IndexNowStatus: seopushmod.PushStatusPending},
}}
list, serr := seopushmod.FindManySeoPushTasks(ctx, filter)
if serr != nil {
log.Warn("seopush: find pending tasks to discard failed", log.E(serr))
return
}
if len(list) == 0 {
return
}
ids := make([]int64, 0, len(list))
seen := make(map[int64]struct{}, len(list))
for _, task := range list {
if task == nil || task.VideoId <= 0 {
continue
}
if _, ok := seen[task.VideoId]; ok {
continue
}
seen[task.VideoId] = struct{}{}
ids = append(ids, task.VideoId)
}
count, serr := seopushmod.DelManySeoPushTask(ctx, filter)
if serr != nil {
log.Warn("seopush: discard pending tasks failed", log.E(serr))
return
}
if len(ids) > 0 {
if serr := playletmod.UpdateManyPlaylet(ctx,
bson.M{playletmod.FN.Playlet.Id: bson.M{"$in": ids}},
bson.M{"$set": bson.M{playletmod.FN.Playlet.IndexNow: int64(0)}}); serr != nil {
log.Warn("seopush: reset playlet indexNow failed", log.E(serr))
}
}
log.Info("seopush=skip=disabled", log.Any("discarded", count))
}
func backfillVisiblePlaylets(ctx context.Context) {
filter := bson.M{
playletmod.FN.Playlet.Status: playletmod.StatusUp,
"$or": []bson.M{
{playletmod.FN.Playlet.IndexNow: 0},
{playletmod.FN.Playlet.IndexNow: bson.M{"$exists": false}},
},
}
opt := options.Find().
SetSort(bson.M{playletmod.FN.Playlet.OnlineAt: -1}).
SetLimit(backfillLimit)
list, serr := playletmod.FindManyPlaylet(ctx, filter, opt)
if serr != nil {
log.Warn("seopush: backfill find playlets failed", log.E(serr))
return
}
if len(list) == 0 {
return
}
for _, item := range list {
if item == nil {
continue
}
srv_seopush.RegisterPlayletUpdated(ctx, item.Id, item.Status, item.OnlineAt)
}
log.Info("seopush: backfill registered", log.Any("count", len(list)))
}
func retryPendingTasks(ctx context.Context, webAddr string) {
conds := []bson.M{
{
seopushmod.FN.SeoPushTask.GoogleStatus: seopushmod.PushStatusPending,
seopushmod.FN.SeoPushTask.GoogleRetry: bson.M{"$lt": seopushmod.MaxPushRetry},
},
{
seopushmod.FN.SeoPushTask.IndexNowStatus: seopushmod.PushStatusPending,
seopushmod.FN.SeoPushTask.IndexNowRetry: bson.M{"$lt": seopushmod.MaxPushRetry},
},
}
opt := options.Find().
SetSort(bson.M{seopushmod.FN.SeoPushTask.CreatedAt: 1}).
SetLimit(retryLimit)
taskList, serr := seopushmod.FindManySeoPushTasks(ctx, bson.M{"$or": conds}, opt)
if serr != nil {
log.Warn("seopush: find pending tasks failed", log.E(serr))
return
}
if len(taskList) == 0 {
return
}
for _, task := range taskList {
if task == nil || strings.Contains(task.Url, "?") {
continue
}
srv_seopush.PushTask(ctx, webAddr, task)
}
log.Info("seopush: pending tasks processed", log.Any("count", len(taskList)))
}
func cleanupExpiredTasks(ctx context.Context) {
deadline := time.Now().AddDate(0, 0, -taskKeepDays)
count, serr := seopushmod.DelManySeoPushTask(ctx, bson.M{
seopushmod.FN.SeoPushTask.UpdatedAt: bson.M{"$lt": deadline},
})
if serr != nil {
log.Warn("seopush: cleanup expired tasks failed", log.E(serr))
return
}
if count > 0 {
log.Info("seopush: expired tasks cleaned", log.Any("count", count))
}
}
web/skd/job/transcode_video.go
+108
View File
@@ -0,0 +1,108 @@
package job
import (
"context"
"strings"
"sync/atomic"
"videoweb/pub/common/laosiji"
"videoweb/pub/common/log"
"videoweb/pub/models/videoinformationmod"
"videoweb/web/service/srv_videoinfo"
"go.mongodb.org/mongo-driver/bson"
"go.mongodb.org/mongo-driver/mongo/options"
)
const transcodeBatchSize = 5
const transcodeMaxFailCount = 3
var transcodeRunning int32
var transcodeCheckRunning int32
const transcodeCheckBatchSize = 100
// TranscodeVideoByWatchTimes 按观看次数挑选尚未转码成功的视频，批量提交 H265 转码。
func TranscodeVideoByWatchTimes() {
if !atomic.CompareAndSwapInt32(&transcodeRunning, 0, 1) {
log.Info("TranscodeVideoByWatchTimes skip: previous run still running")
return
}
defer atomic.StoreInt32(&transcodeRunning, 0)
ctx := context.Background()
if laosiji.LaosijiClient == nil || laosiji.LaosijiDomains == nil {
log.Warn("TranscodeVideoByWatchTimes skip: laosiji not ready")
return
}
if _, err := laosiji.LaosijiDomains.Get(ctx); err != nil {
log.Warn("TranscodeVideoByWatchTimes skip: domains not ready", log.E(err))
return
}
filter := bson.M{
videoinformationmod.FN.VideoInformation.HevcStatus: bson.M{
"$in": bson.A{videoinformationmod.HevcStatusNone, videoinformationmod.HevcStatusFailed},
},
videoinformationmod.FN.VideoInformation.HevcFailCount: bson.M{"$lt": transcodeMaxFailCount},
videoinformationmod.FN.VideoInformation.Video.Url: bson.M{"$ne": ""},
}
opts := options.Find().
SetSort(bson.D{{Key: videoinformationmod.FN.VideoInformation.Real.WatchTimes, Value: -1}}).
SetLimit(transcodeBatchSize)
videoList, serr := videoinformationmod.FindManyVideoInformations(ctx, filter, opts)
if serr != nil {
log.Error("TranscodeVideoByWatchTimes find video failed", log.Any("serr", serr))
return
}
if len(videoList) == 0 {
return
}
log.Info("TranscodeVideoByWatchTimes start", log.Any("count", len(videoList)))
for _, v := range videoList {
src := strings.TrimSpace(v.Video.Url)
if src == "" {
continue
}
if err := srv_videoinfo.SubmitTranscodeVideo(ctx, v.Id, src, v.Video.FileId); err != nil {
log.Warn("TranscodeVideoByWatchTimes submit failed",
log.Any("videoId", v.Id), log.Any("watchTimes", v.Real.WatchTimes), log.E(err))
}
}
log.Info("TranscodeVideoByWatchTimes done", log.Any("count", len(videoList)))
}
// CheckTranscodingResults 查询转码中任务并回写结果。
func CheckTranscodingResults() {
if !atomic.CompareAndSwapInt32(&transcodeCheckRunning, 0, 1) {
log.Info("CheckTranscodingResults skip: previous run still running")
return
}
defer atomic.StoreInt32(&transcodeCheckRunning, 0)
ctx := context.Background()
if laosiji.LaosijiClient == nil || laosiji.LaosijiDomains == nil {
return
}
filter := bson.M{videoinformationmod.FN.VideoInformation.HevcStatus: videoinformationmod.HevcStatusPending}
opts := options.Find().SetLimit(transcodeCheckBatchSize)
videoList, serr := videoinformationmod.FindManyVideoInformations(ctx, filter, opts)
if serr != nil {
log.Error("CheckTranscodingResults find video failed", log.Any("serr", serr))
return
}
if len(videoList) == 0 {
return
}
for _, v := range videoList {
src := strings.TrimSpace(v.Video.Url)
if src == "" {
continue
}
if err := srv_videoinfo.RefreshTranscodeVideo(ctx, v.Id, src, v.Video.FileId); err != nil {
log.Warn("CheckTranscodingResults refresh failed", log.Any("videoId", v.Id), log.E(err))
}
}
}
web/skd/job/vipexpiry/job.go
+45
View File
@@ -0,0 +1,45 @@
package vipexpiry
import (
"context"
"time"
"videoweb/pub/common/log"
"videoweb/pub/models/usermod"
"videoweb/pub/pubdef"
"go.mongodb.org/mongo-driver/bson"
)
var isRunning = false
// ResetExpiredVip 将 CurrentVipExpire 已过期但 VipType 仍非 None 的用户批量重置为普通用户。
// 同时清空 ShowCard* 展示字段，保证 DB 数据与实际状态一致。
func ResetExpiredVip() {
if isRunning {
return
}
isRunning = true
defer func() { isRunning = false }()
ctx := context.Background()
now := time.Now()
filter := bson.M{
usermod.FN.User.VipType: bson.M{"$gt": pubdef.VipTypeNone},
usermod.FN.User.CurrentVipExpire: bson.M{"$lt": now},
}
update := bson.M{
"$set": bson.M{
usermod.FN.User.VipType: pubdef.VipTypeNone,
usermod.FN.User.ShowCardId: int64(0),
usermod.FN.User.ShowCardName: "",
usermod.FN.User.UpdatedAt: now,
},
}
if serr := usermod.UpdateManyUser(ctx, filter, update); serr != nil {
log.Error("vipexpiry:ResetExpiredVip UpdateManyUser failed", log.Any("err", serr))
return
}
log.Info("vipexpiry:ResetExpiredVip done")
}