Initial commit: web_hghl_server

Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>
This commit is contained in:
2026-10-26 16:28:14 +08:00
co-authored by Claude Opus 4.7
commit 65fca60622
653 changed files with 135901 additions and 0 deletions
@@ -0,0 +1,325 @@
package srv_contentexcel
import (
"context"
"fmt"
"io"
"math/rand"
"strconv"
"strings"
"time"
"videoweb/pub/common/srvdef"
"videoweb/pub/models/videoinformationmod"
"videoweb/pub/pubdef"
"go.mongodb.org/mongo-driver/bson"
"go.mongodb.org/mongo-driver/mongo/options"
)
const queryBatchSize = 500
var importAddedTimeStart = time.Date(2025, 1, 1, 0, 0, 0, 0, time.Local)
func randomImportAddedTime(r *rand.Rand) time.Time {
now := time.Now()
if !now.After(importAddedTimeStart) {
return now
}
if r == nil {
r = rand.New(rand.NewSource(time.Now().UnixNano()))
}
delta := now.UnixNano() - importAddedTimeStart.UnixNano()
if delta <= 0 {
return now
}
return importAddedTimeStart.Add(time.Duration(r.Int63n(delta + 1)))
}
// Export 按条件导出视频 SEO 字段为 Excel。
func Export(ctx context.Context, req *ExportReq) ([]byte, string, *srvdef.SrvErr) {
contentType, serr := normalizeContentType(req.Type)
if serr != nil {
return nil, "", serr
}
dateFilter, serr := buildTimeRangeFilter(req.StartTime, req.EndTime)
if serr != nil {
return nil, "", serr
}
ids, serr := normalizeExportIDs(req.Ids)
if serr != nil {
return nil, "", serr
}
var rows []ContentRow
var err error
switch contentType {
case ContentTypeVideo:
rows, err = queryVideoRows(ctx, dateFilter, req.Status, ids)
}
if err != nil {
return nil, "", srvdef.New(srvdef.ExcelExportErr, err.Error())
}
data, err := buildWorkbook(rows)
if err != nil {
return nil, "", srvdef.New(srvdef.ExcelExportErr, err.Error())
}
filename := fmt.Sprintf("%s_content_%s.xlsx", contentType, time.Now().Format("20060102150405"))
return data, filename, nil
}
// Import 读取 Excel 并按 ID 批量更新标题、描述、关键词。
func Import(ctx context.Context, contentType string, reader io.Reader) (*ImportResult, *srvdef.SrvErr) {
normalizedType, serr := normalizeContentType(contentType)
if serr != nil {
return nil, serr
}
data, err := io.ReadAll(reader)
if err != nil {
return nil, srvdef.New(srvdef.ErrParamError, "读取上传文件失败")
}
rows, err := parseWorkbook(data)
if err != nil {
return nil, srvdef.New(srvdef.ErrParamError, err.Error())
}
result := &ImportResult{Total: len(rows)}
r := rand.New(rand.NewSource(time.Now().UnixNano()))
for _, row := range rows {
var updateErr *srvdef.SrvErr
switch normalizedType {
case ContentTypeVideo:
updateErr = updateVideoRow(ctx, row, r)
}
if updateErr != nil {
result.Failed++
result.Errors = append(result.Errors, fmt.Sprintf("ID=%d: %s", row.ID, updateErr.Tips))
continue
}
result.Success++
}
return result, nil
}
func normalizeContentType(raw string) (string, *srvdef.SrvErr) {
switch stringsTrimLower(raw) {
case ContentTypeVideo:
return stringsTrimLower(raw), nil
default:
return "", srvdef.New(srvdef.ErrParamError, "type 仅支持 video")
}
}
func stringsTrimLower(s string) string {
return strings.ToLower(strings.TrimSpace(s))
}
func buildTimeRangeFilter(startTime, endTime string) (bson.M, *srvdef.SrvErr) {
filter := bson.M{}
start, err := parseOptionalTime(startTime, false)
if err != nil {
return nil, srvdef.New(srvdef.ErrParamError, "startTime 格式错误,支持 2006-01-02 或 2006-01-02 15:04:05")
}
end, err := parseOptionalTime(endTime, true)
if err != nil {
return nil, srvdef.New(srvdef.ErrParamError, "endTime 格式错误,支持 2006-01-02 或 2006-01-02 15:04:05")
}
if start != nil && end != nil && start.After(*end) {
return nil, srvdef.New(srvdef.ErrParamError, "startTime 不能晚于 endTime")
}
if start != nil {
filter["$gte"] = *start
}
if end != nil {
filter["$lte"] = *end
}
if len(filter) == 0 {
return nil, nil
}
return filter, nil
}
func parseOptionalTime(raw string, endOfDay bool) (*time.Time, error) {
raw = strings.TrimSpace(raw)
if raw == "" {
return nil, nil
}
layouts := []string{
"2006-01-02 15:04:05",
"2006-01-02T15:04:05",
time.RFC3339,
}
for _, layout := range layouts {
if t, err := time.ParseInLocation(layout, raw, time.Local); err == nil {
return &t, nil
}
}
if t, err := time.ParseInLocation("2006-01-02", raw, time.Local); err == nil {
if endOfDay {
end := time.Date(t.Year(), t.Month(), t.Day(), 23, 59, 59, 0, t.Location())
return &end, nil
}
return &t, nil
}
return nil, fmt.Errorf("invalid time: %s", raw)
}
func applyTimeFilter(base bson.M, field string, rangeFilter bson.M) bson.M {
if len(rangeFilter) == 0 {
return base
}
base[field] = rangeFilter
return base
}
func applyStatusFilter(base bson.M, field string, status *pubdef.InformationStatus) bson.M {
if status == nil {
return base
}
base[field] = *status
return base
}
func applyIDsFilter(base bson.M, field string, ids []int64) bson.M {
if len(ids) == 0 {
return base
}
base[field] = bson.M{"$in": ids}
return base
}
// ParseExportIDs 解析导出接口的 ids 查询参数,支持 ids=1&ids=2 或 ids=1,2,3。
func ParseExportIDs(raw []string) ([]int64, *srvdef.SrvErr) {
if len(raw) == 0 {
return nil, nil
}
ids := make([]int64, 0, len(raw))
seen := make(map[int64]struct{}, len(raw))
for _, item := range raw {
item = strings.TrimSpace(item)
if item == "" {
continue
}
for _, part := range strings.Split(item, ",") {
part = strings.TrimSpace(part)
if part == "" {
continue
}
id, err := strconv.ParseInt(part, 10, 64)
if err != nil || id <= 0 {
return nil, srvdef.New(srvdef.ErrParamError, "ids 格式错误,需为正整数")
}
if _, ok := seen[id]; ok {
continue
}
seen[id] = struct{}{}
ids = append(ids, id)
}
}
return ids, nil
}
func normalizeExportIDs(raw []int64) ([]int64, *srvdef.SrvErr) {
if len(raw) == 0 {
return nil, nil
}
ids := make([]int64, 0, len(raw))
seen := make(map[int64]struct{}, len(raw))
for _, id := range raw {
if id <= 0 {
return nil, srvdef.New(srvdef.ErrParamError, "ids 格式错误,需为正整数")
}
if _, ok := seen[id]; ok {
continue
}
seen[id] = struct{}{}
ids = append(ids, id)
}
return ids, nil
}
func queryVideoRows(ctx context.Context, dateFilter bson.M, status *pubdef.InformationStatus, ids []int64) ([]ContentRow, error) {
filter := applyIDsFilter(applyStatusFilter(applyTimeFilter(bson.M{
videoinformationmod.FN.VideoInformation.MediaType: bson.M{
"$in": []videoinformationmod.MediaType{
videoinformationmod.MediaType_Video,
videoinformationmod.MediaType_AV,
},
},
}, videoinformationmod.FN.VideoInformation.CreatedAt, dateFilter),
videoinformationmod.FN.VideoInformation.Status, status),
videoinformationmod.FN.VideoInformation.Id, ids)
return queryRowsInBatches(ctx, func(skip int64) ([]ContentRow, bool, error) {
opt := options.Find().
SetSkip(skip).
SetLimit(queryBatchSize).
SetSort(bson.M{videoinformationmod.FN.VideoInformation.Id: 1}).
SetProjection(bson.M{
videoinformationmod.FN.VideoInformation.Id: 1,
videoinformationmod.FN.VideoInformation.Title: 1,
videoinformationmod.FN.VideoInformation.Desc: 1,
videoinformationmod.FN.VideoInformation.Keyworld: 1,
})
list, serr := videoinformationmod.FindManyVideoInformations(ctx, filter, opt)
if serr != nil {
return nil, false, fmt.Errorf("%s", serr.Tips)
}
rows := make([]ContentRow, 0, len(list))
for _, item := range list {
if item == nil {
continue
}
rows = append(rows, ContentRow{
ID: item.Id,
Title: item.Title,
Desc: item.Desc,
Keyworld: item.Keyworld,
})
}
return rows, int64(len(list)) == queryBatchSize, nil
})
}
func queryRowsInBatches(ctx context.Context, fetch func(skip int64) ([]ContentRow, bool, error)) ([]ContentRow, error) {
all := make([]ContentRow, 0)
var skip int64
for {
rows, hasMore, err := fetch(skip)
if err != nil {
return nil, err
}
all = append(all, rows...)
if !hasMore {
break
}
skip += queryBatchSize
}
return all, nil
}
func updateVideoRow(ctx context.Context, row ContentRow, r *rand.Rand) *srvdef.SrvErr {
now := time.Now()
addedTime := randomImportAddedTime(r)
update := bson.M{
"$set": bson.M{
videoinformationmod.FN.VideoInformation.Title: row.Title,
videoinformationmod.FN.VideoInformation.Desc: row.Desc,
videoinformationmod.FN.VideoInformation.Keyworld: row.Keyworld,
videoinformationmod.FN.VideoInformation.Status: pubdef.InformationStatusVerifyOK,
videoinformationmod.FN.VideoInformation.AddedTime: addedTime,
videoinformationmod.FN.VideoInformation.UpdatedAt: now,
},
}
return videoinformationmod.UpdateOneVideoInformation(ctx, bson.M{
videoinformationmod.FN.VideoInformation.Id: row.ID,
}, update)
}
+32
View File
@@ -0,0 +1,32 @@
package srv_contentexcel
import "videoweb/pub/pubdef"
const (
ContentTypeVideo = "video"
)
// ExportReq 导出查询参数。
type ExportReq struct {
Type string `form:"type" binding:"required"` // video
StartTime string `form:"startTime"` // 可选,按 createdAt 筛选,格式 2006-01-02 或 2006-01-02 15:04:05
EndTime string `form:"endTime"` // 可选,按 createdAt 筛选
Status *pubdef.InformationStatus `form:"status"` // 可选,0初始化 1上架 2下架,不传查全部
Ids []int64 // 可选,按 ID 筛选,支持 ids=1&ids=2 或 ids=1,2,3
}
// ImportResult 导入结果。
type ImportResult struct {
Total int `json:"total"`
Success int `json:"success"`
Failed int `json:"failed"`
Errors []string `json:"errors,omitempty"`
}
// ContentRow Excel 单行数据。
type ContentRow struct {
ID int64
Title string
Desc string
Keyworld []string
}
+160
View File
@@ -0,0 +1,160 @@
package srv_contentexcel
import (
"bytes"
"fmt"
"strconv"
"strings"
"github.com/xuri/excelize/v2"
)
const (
colID = "ID"
colTitle = "标题"
colDesc = "描述"
colKeyworld = "关键词"
colRedLineRemark = "备注(红线-建议下架)"
sheetName = "Sheet1"
excelHeaders = 4
colRedLineRemarkIdx = 4
)
var headerAliases = map[string]int{
"id": 0, colID: 0,
"title": 1, colTitle: 1,
"desc": 2, "description": 2, colDesc: 2,
"keyworld": 3, "keywords": 3, "keyword": 3, colKeyworld: 3,
colRedLineRemark: colRedLineRemarkIdx,
}
func buildWorkbook(rows []ContentRow) ([]byte, error) {
f := excelize.NewFile()
defer f.Close()
if err := f.SetSheetRow(sheetName, "A1", &[]string{colID, colTitle, colDesc, colKeyworld}); err != nil {
return nil, err
}
for i, row := range rows {
cell, err := excelize.CoordinatesToCellName(1, i+2)
if err != nil {
return nil, err
}
if err := f.SetSheetRow(sheetName, cell, &[]any{
row.ID,
row.Title,
row.Desc,
strings.Join(row.Keyworld, ","),
}); err != nil {
return nil, err
}
}
var buf bytes.Buffer
if err := f.Write(&buf); err != nil {
return nil, err
}
return buf.Bytes(), nil
}
func parseWorkbook(data []byte) ([]ContentRow, error) {
f, err := excelize.OpenReader(bytes.NewReader(data))
if err != nil {
return nil, fmt.Errorf("打开 Excel 失败: %w", err)
}
defer f.Close()
sheets := f.GetSheetList()
if len(sheets) == 0 {
return nil, fmt.Errorf("Excel 无工作表")
}
rows, err := f.GetRows(sheets[0])
if err != nil {
return nil, fmt.Errorf("读取 Excel 失败: %w", err)
}
if len(rows) <= 1 {
return nil, fmt.Errorf("Excel 无数据行")
}
colIndex := mapHeaderColumns(rows[0])
if len(colIndex) < excelHeaders {
return nil, fmt.Errorf("Excel 表头需包含: ID、标题、描述、关键词")
}
result := make([]ContentRow, 0, len(rows)-1)
for lineNo, row := range rows[1:] {
contentRow, ok, err := parseDataRow(row, colIndex, lineNo+2)
if err != nil {
return nil, err
}
if !ok {
continue
}
result = append(result, contentRow)
}
if len(result) == 0 {
return nil, fmt.Errorf("Excel 无有效数据行")
}
return result, nil
}
func mapHeaderColumns(header []string) map[int]int {
index := make(map[int]int, excelHeaders)
for i, name := range header {
key := strings.ToLower(strings.TrimSpace(name))
if col, ok := headerAliases[key]; ok {
index[col] = i
}
}
return index
}
func parseDataRow(row []string, colIndex map[int]int, lineNo int) (ContentRow, bool, error) {
if remarkIdx, ok := colIndex[colRedLineRemarkIdx]; ok {
if cellValue(row, remarkIdx) != "" {
return ContentRow{}, false, nil
}
}
idStr := cellValue(row, colIndex[0])
if idStr == "" {
return ContentRow{}, false, nil
}
id, err := strconv.ParseInt(idStr, 10, 64)
if err != nil || id <= 0 {
return ContentRow{}, false, fmt.Errorf("第 %d 行 ID 无效: %q", lineNo, idStr)
}
return ContentRow{
ID: id,
Title: cellValue(row, colIndex[1]),
Desc: cellValue(row, colIndex[2]),
Keyworld: parseKeywords(cellValue(row, colIndex[3])),
}, true, nil
}
func cellValue(row []string, index int) string {
if index < 0 || index >= len(row) {
return ""
}
return strings.TrimSpace(row[index])
}
func parseKeywords(raw string) []string {
raw = strings.TrimSpace(raw)
if raw == "" {
return nil
}
raw = strings.NewReplacer(",", ",", ";", ",", ";", ",").Replace(raw)
parts := strings.Split(raw, ",")
keywords := make([]string, 0, len(parts))
for _, part := range parts {
part = strings.TrimSpace(part)
if part != "" {
keywords = append(keywords, part)
}
}
return keywords
}
+201
View File
@@ -0,0 +1,201 @@
package srv_contentexcel
import (
"bytes"
"math/rand"
"testing"
"time"
"videoweb/pub/models/videoinformationmod"
"videoweb/pub/pubdef"
"go.mongodb.org/mongo-driver/bson"
"github.com/xuri/excelize/v2"
)
func TestBuildTimeRangeFilter(t *testing.T) {
t.Parallel()
filter, err := buildTimeRangeFilter("2026-01-01", "2026-01-31")
if err != nil {
t.Fatalf("buildTimeRangeFilter() err = %v", err)
}
if filter["$gte"] == nil || filter["$lte"] == nil {
t.Fatalf("buildTimeRangeFilter() = %#v, want range filter", filter)
}
empty, err := buildTimeRangeFilter("", "")
if err != nil {
t.Fatalf("buildTimeRangeFilter() empty err = %v", err)
}
if len(empty) != 0 {
t.Fatalf("buildTimeRangeFilter() empty = %#v, want nil/empty", empty)
}
}
func TestApplyStatusFilter(t *testing.T) {
t.Parallel()
base := bson.M{"a": 1}
if got := applyStatusFilter(base, "status", nil); len(got) != 1 {
t.Fatalf("nil status should not add filter: %#v", got)
}
status := pubdef.InformationStatusVerifyOK
got := applyStatusFilter(bson.M{}, videoinformationmod.FN.VideoInformation.Status, &status)
if got[videoinformationmod.FN.VideoInformation.Status] != pubdef.InformationStatusVerifyOK {
t.Fatalf("applyStatusFilter() status = %#v", got[videoinformationmod.FN.VideoInformation.Status])
}
start := time.Date(2026, 1, 1, 0, 0, 0, 0, time.Local)
end := time.Date(2026, 1, 31, 23, 59, 59, 0, time.Local)
dateFilter := bson.M{"$gte": start, "$lte": end}
videoFilter := applyIDsFilter(applyStatusFilter(
applyTimeFilter(bson.M{}, videoinformationmod.FN.VideoInformation.CreatedAt, dateFilter),
videoinformationmod.FN.VideoInformation.Status, &status,
), videoinformationmod.FN.VideoInformation.Id, []int64{1001, 1002})
if videoFilter[videoinformationmod.FN.VideoInformation.CreatedAt] == nil {
t.Fatal("expected createdAt filter")
}
if videoFilter[videoinformationmod.FN.VideoInformation.Status] != pubdef.InformationStatusVerifyOK {
t.Fatal("expected status filter")
}
if videoFilter[videoinformationmod.FN.VideoInformation.Id] == nil {
t.Fatal("expected id filter")
}
}
func TestParseExportIDs(t *testing.T) {
t.Parallel()
ids, err := ParseExportIDs([]string{"1001,1002", "1003"})
if err != nil {
t.Fatalf("ParseExportIDs() err = %v", err)
}
if len(ids) != 3 || ids[0] != 1001 || ids[1] != 1002 || ids[2] != 1003 {
t.Fatalf("ParseExportIDs() = %v", ids)
}
dup, err := ParseExportIDs([]string{"1001", "1001,1002"})
if err != nil {
t.Fatalf("ParseExportIDs() dup err = %v", err)
}
if len(dup) != 2 {
t.Fatalf("ParseExportIDs() dup = %v, want 2 unique ids", dup)
}
if _, err := ParseExportIDs([]string{"abc"}); err == nil {
t.Fatal("ParseExportIDs() invalid id should fail")
}
}
func TestParseKeywords(t *testing.T) {
t.Parallel()
got := parseKeywords("a,b, c;d")
want := []string{"a", "b", "c", "d"}
if len(got) != len(want) {
t.Fatalf("parseKeywords() = %v, want %v", got, want)
}
for i := range want {
if got[i] != want[i] {
t.Fatalf("parseKeywords()[%d] = %q, want %q", i, got[i], want[i])
}
}
}
func TestMapHeaderColumns(t *testing.T) {
t.Parallel()
index := mapHeaderColumns([]string{"ID", "标题", "描述", "关键词"})
for i := 0; i < excelHeaders; i++ {
if _, ok := index[i]; !ok {
t.Fatalf("missing column index %d", i)
}
}
}
func TestParseWorkbook(t *testing.T) {
t.Parallel()
rows := []ContentRow{{
ID: 1001,
Title: "标题A",
Desc: "描述A",
Keyworld: []string{"k1", "k2"},
}}
data, err := buildWorkbook(rows)
if err != nil {
t.Fatalf("buildWorkbook() err = %v", err)
}
parsed, err := parseWorkbook(data)
if err != nil {
t.Fatalf("parseWorkbook() err = %v", err)
}
if len(parsed) != 1 {
t.Fatalf("parseWorkbook() len = %d, want 1", len(parsed))
}
if parsed[0].ID != 1001 || parsed[0].Title != "标题A" {
t.Fatalf("parseWorkbook() = %+v", parsed[0])
}
}
func TestParseWorkbookSkipRedLineRemark(t *testing.T) {
t.Parallel()
f := excelize.NewFile()
defer f.Close()
if err := f.SetSheetRow(sheetName, "A1", &[]string{colID, colTitle, colDesc, colKeyworld, colRedLineRemark}); err != nil {
t.Fatalf("SetSheetRow header err = %v", err)
}
if err := f.SetSheetRow(sheetName, "A2", &[]any{1001, "标题A", "描述A", "k1", "建议下架"}); err != nil {
t.Fatalf("SetSheetRow row1 err = %v", err)
}
if err := f.SetSheetRow(sheetName, "A3", &[]any{1002, "标题B", "描述B", "k2", ""}); err != nil {
t.Fatalf("SetSheetRow row2 err = %v", err)
}
var buf bytes.Buffer
if err := f.Write(&buf); err != nil {
t.Fatalf("Write() err = %v", err)
}
parsed, err := parseWorkbook(buf.Bytes())
if err != nil {
t.Fatalf("parseWorkbook() err = %v", err)
}
if len(parsed) != 1 {
t.Fatalf("parseWorkbook() len = %d, want 1", len(parsed))
}
if parsed[0].ID != 1002 {
t.Fatalf("parseWorkbook() ID = %d, want 1002", parsed[0].ID)
}
}
func TestRandomImportAddedTime(t *testing.T) {
t.Parallel()
r := rand.New(rand.NewSource(1))
for i := 0; i < 20; i++ {
got := randomImportAddedTime(r)
if got.Before(importAddedTimeStart) {
t.Fatalf("randomImportAddedTime() = %v, before %v", got, importAddedTimeStart)
}
if got.After(time.Now()) {
t.Fatalf("randomImportAddedTime() = %v, after now", got)
}
}
}
func TestNormalizeContentType(t *testing.T) {
t.Parallel()
if _, err := normalizeContentType("video"); err != nil {
t.Fatalf("video should be valid: %v", err)
}
if _, err := normalizeContentType("post"); err == nil {
t.Fatal("post should be invalid")
}
}