Initial commit

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>

skd/service/export_task/advance_order_export.go

@@ -0,0 +1,102 @@
+package export_task
+
+import (
+	"91porn-server/common/log"
+	"91porn-server/models/v/advanceordermod"
+	"91porn-server/models/v/export_task_mod"
+	"encoding/json"
+	"fmt"
+	"time"
+
+	"go.mongodb.org/mongo-driver/bson"
+	"go.mongodb.org/mongo-driver/mongo/options"
+)
+
+func ExecExportAdvanceOrderTask(task *export_task_mod.ExportTask) (total int64, err error) {
+	arg := advanceordermod.QueryAllCond{}
+	err = json.Unmarshal([]byte(task.Param), &arg)
+	if err != nil {
+		return
+	}
+
+	var skip int64
+	var size int64 = 100
+	index := 1
+	path := fmt.Sprintf("./temp/%v%v", "预售订单列表-", task.ID.Hex())
+	zipName := fmt.Sprintf("%v_%v_预售订单列表_%v.zip", task.Admin, GetProName(), task.CreatedAt.Format("2006-01-02_15:04:05"))
+	var records []*advanceordermod.AdvanceOrderExport
+	excelList := []string{}
+	for {
+		fmt.Println("skip", skip, " now ", time.Now().Format("2006-01-02 15:04:05"))
+		opt := options.Find().SetSkip(skip).SetLimit(size).SetSort(bson.M{"createdAt": -1})
+		list, err := advanceordermod.QueryAllDocument(arg.Filter(), opt)
+		if err != nil {
+			log.Error("ExecExportAdvanceOrderTask advanceordermod.QueryAllDocument fail", log.E(err))
+			return total, err
+		}
+		for _, v := range list {
+			item := &advanceordermod.AdvanceOrderExport{
+				ID:            v.ID,
+				UID:           v.UID,
+				AdvOid:        v.AdvOid,
+				BalOid:        v.BalOid,
+				ProductID:     v.ProductID,
+				TotalAmount:   v.TotalAmount,
+				AdvanceAmount: v.AdvanceAmount,
+				BalanceAmount: v.BalanceAmount,
+				StartTime:     v.StartTime,
+				EndTime:       v.EndTime,
+				CreatedAt:     v.CreatedAt,
+			}
+			switch v.Status {
+			case advanceordermod.AdvanceProcessing:
+				item.Status = "预付中"
+			case advanceordermod.AdvanceSUCCESS:
+				item.Status = "预付成功"
+			case advanceordermod.BalanceProcessing:
+				item.Status = "尾款预付中"
+			case advanceordermod.BalanceSUCCESS:
+				item.Status = "尾款预付成功"
+			}
+			records = append(records, item)
+		}
+
+		var fileName = fmt.Sprintf("预售订单列表_%v.xlsx", index)
+		if len(records) >= maxDataNum {
+			// 获取到的数据已经达到了单文件最大限制数据量
+			filePath, err := SaveExcel(records, path, fileName)
+			if err != nil {
+				log.Error("ExecExportAdvanceOrderTask SaveExcel fail", log.E(err))
+				return total, err
+			}
+			// 重新归0
+			records = []*advanceordermod.AdvanceOrderExport{}
+			index = index + 1
+			excelList = append(excelList, filePath)
+			fmt.Println("index:", index)
+			total = total + int64(len(records))
+		} else if len(list) < int(size) {
+			if len(records) > 0 {
+				// 数据库已经取不到更多的数据了
+				filePath, err := SaveExcel(records, path, fileName)
+				if err != nil {
+					log.Error("ExecExportAdvanceOrderTask SaveExcel fail", log.E(err))
+					return total, err
+				}
+				excelList = append(excelList, filePath)
+				total = total + int64(len(records))
+			}
+			// 结束循环
+			break
+		}
+		skip = skip + size
+	}
+
+	// 发送到tg
+	err = SendTg(task.Admin, excelList, zipName)
+	if err != nil {
+		log.Error("ExecExportVideoTask SendTg fail", log.E(err))
+		return
+	}
+	return total, nil
+}

skd/service/export_task/base.go

@@ -0,0 +1,282 @@
+package export_task
+
+import (
+	"91porn-server/common/file"
+	"91porn-server/common/log"
+	commonWorker "91porn-server/common/worker"
+	"91porn-server/models/v/export_task_mod"
+	"fmt"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"reflect"
+	"time"
+
+	tgbotapi "github.com/go-telegram-bot-api/telegram-bot-api"
+	"github.com/pkg/errors"
+	"github.com/xuri/excelize/v2"
+	"go.mongodb.org/mongo-driver/bson"
+)
+
+const (
+	proName      = "91PORN"
+	tgToken      = "5852134617:AAEO0_GPZiUJS_LeR5dM7byEfOpnRg4dGqU"
+	maxDataNum   = 20000 // 每个excel存储的数据量
+	maxWorkerNum = 3
+	maxSize      = 1024 * 1024 * 20
+)
+
+var Bot *tgbotapi.BotAPI
+var worker *commonWorker.Worker
+
+func init() {
+	bot, err := tgbotapi.NewBotAPI(tgToken)
+	if err != nil || bot == nil {
+		log.Fatal("初始化tgbot失败", log.E(err))
+	}
+	Bot = bot
+	worker = commonWorker.NewWorker(maxWorkerNum)
+}
+
+var execExportTaskRuining bool
+
+func ExecExportTask() {
+	//防止重入
+	if execExportTaskRuining {
+		return
+	}
+	execExportTaskRuining = true
+	defer func() {
+		execExportTaskRuining = false
+	}()
+	// 获取任务列表
+	list, err := export_task_mod.GetTaskList()
+	if err != nil {
+		log.Error("ExecExportTask export_task_mod.GetTaskList fail", log.E(err))
+		return
+	}
+	for _, task := range list {
+		switch task.Type {
+		case export_task_mod.ExportUserTask:
+			CallExportFunc(task, ExecExportUserTask)
+		case export_task_mod.ExportVidTask:
+			CallExportFunc(task, ExecExportVideoTask)
+		case export_task_mod.ExportAdvanceOrderTask:
+			CallExportFunc(task, ExecExportAdvanceOrderTask)
+		case export_task_mod.ExportProductHistoryTask:
+			CallExportFunc(task, ExecExportProductHistoryTask)
+		}
+	}
+}
+
+func GetProName() string {
+	//if skdg.Conf.Base.Env == "test" {
+	//	return "测试环境_" + proName
+	//}
+	return proName
+}
+
+func CallExportFunc(task *export_task_mod.ExportTask, f func(t *export_task_mod.ExportTask) (int64, error)) {
+	if task == nil {
+		return
+	}
+	worker.Exec(func() {
+		// 已经执行过了
+		if task.Status != 0 {
+			return
+		}
+		//key := fmt.Sprintf("ExecExportTask:%v", task.ID.Hex())
+		//ok, err := skdg.Redis.SetNX(key, 1, time.Minute*30)
+		//if err != nil {
+		//	log.Error("ExecExportTask fail", log.E(err))
+		//	return
+		//}
+		//if !ok {
+		//	log.Info("ExportTask already executed")
+		//	return
+		//}
+		//defer skdg.Redis.Del(key)
+		start := time.Now().Unix()
+		total, err := f(task)
+		if err != nil {
+			log.Error("CallExportFunc fail", log.Any("task", task), log.E(err))
+			// 修改任务信息
+			export_task_mod.UpdateTask(task.ID, bson.M{
+				"status": 2,
+				"reason": err.Error(),
+			})
+			return
+		}
+		// 修改任务信息
+		export_task_mod.UpdateTask(task.ID, bson.M{
+			"status": 1,
+			"time":   time.Now().Unix() - start, // 耗时
+			"total":  total,
+		})
+	})
+	return
+}
+
+// SendTg 发送到tg
+func SendTg(admin string, excelList []string, zipFileName string) (err error) {
+	exportPath, err := filepath.Abs("temp")
+	if err != nil {
+		return
+	}
+	// 尝试创建
+	file.MakeDir(exportPath)
+	zipList, err := Zip(excelList, zipFileName)
+	if err != nil {
+		log.Error("SendTg Zip fail", log.Any("excelList", excelList), log.Any("zipFileName", zipFileName), log.E(err))
+		return
+	}
+	// 测试环境不需要发送到tg群
+	//if skdg.Conf.Base.Env == "test" {
+	//	return
+	//}
+	if len(zipList) == 0 {
+		return
+	}
+	var chatId int64 = -1003399433452
+
+	content := fmt.Sprintf("%v总共有%v个压缩包，请查收", admin, len(zipList))
+	_, err = Bot.Send(tgbotapi.NewMessage(chatId, content))
+	if err != nil {
+		log.Error("tgbot send fail", log.E(err))
+		return errors.Wrap(err, "发送TG消息失败")
+	}
+	for _, zipName := range zipList {
+		fileBytes, err := os.ReadFile("./temp/" + zipName)
+		if err != nil {
+			log.Error("ReadFile fail", log.Any("excelList", excelList), log.Any("zipFileName", zipFileName), log.E(err))
+			return errors.Wrap(err, "打开压缩文件失败")
+		}
+
+		// 创建要发送的文件
+		tFile := tgbotapi.FileBytes{Name: zipName, Bytes: fileBytes}
+
+		//生成csv发送到tg群
+		newDocument := tgbotapi.NewDocumentUpload(chatId, tFile)
+		_, err = Bot.Send(newDocument)
+		if err != nil {
+			log.Error("tgbot send fail", log.E(err))
+			return errors.Wrap(err, "发送TG消息失败")
+		}
+	}
+
+	return nil
+}
+
+func Zip(filePathList []string, zipFileName string) (zipList []string, err error) {
+	if len(filePathList) == 0 {
+		return
+	}
+	var currentSize int64
+	zipIndex := 1
+	currentZipName := fmt.Sprintf("%v_%v.zip", zipFileName, zipIndex)
+	args := []string{currentZipName}
+	for k, filePath := range filePathList {
+		// 获取文件大小
+		stat, err := os.Stat(filePath)
+		if err != nil {
+			return nil, err
+		}
+		currentSize = currentSize + stat.Size()
+		args = append(args, filePath)
+		// 判断当前所有文件加起来是否已经超过20M，超过就直接压缩,否则就接着等待压缩
+		if currentSize < maxSize && k != (len(filePathList)-1) {
+			continue
+		}
+		cmd1 := exec.Command("zip", args...)
+		out1, err := cmd1.CombinedOutput()
+		if err != nil {
+			return nil, err
+		}
+		fmt.Println(string(out1))
+		// 移动文件到目录里
+		cmd2 := exec.Command("mv", currentZipName, "./temp")
+		if err != nil {
+			return nil, err
+		}
+		out2, err := cmd2.CombinedOutput()
+		if err != nil {
+			return nil, err
+		}
+		fmt.Println(string(out2))
+		zipList = append(zipList, currentZipName)
+		// 重置
+		zipIndex = zipIndex + 1
+		currentZipName = fmt.Sprintf("%v_%v.zip", zipFileName, zipIndex)
+		args = []string{currentZipName}
+		currentSize = 0
+	}
+
+	return
+}
+
+// SaveExcel 保存数据到excel中
+func SaveExcel(records interface{}, path string, fileName string) (filePath string, err error) {
+
+	sheet := "sheet1"
+	xlsx := excelize.NewFile()       // new file
+	index, _ := xlsx.NewSheet(sheet) // new sheet
+	xlsx.SetActiveSheet(index)       // set active (default) sheet
+	t := reflect.TypeOf(records)
+	if t.Kind() != reflect.Slice {
+		panic("records must be slice")
+	}
+
+	s := reflect.ValueOf(records)
+	for i := 0; i < s.Len(); i++ {
+		elem := s.Index(i).Interface()
+		elemType := reflect.TypeOf(elem)
+		elemValue := reflect.ValueOf(elem)
+		if elemType.Kind() == reflect.Ptr {
+			elemType = elemType.Elem()
+			elemValue = elemValue.Elem()
+		}
+		if elemType.Kind() != reflect.Struct {
+			panic("record in slice must be a struct")
+		}
+		k := 0
+		for j := 0; j < elemType.NumField(); j++ {
+			field := elemType.Field(j)
+			tag := field.Tag.Get("xlsx")
+			if tag == "-" {
+				continue
+			}
+			if tag == "" {
+				tag = field.Name
+			}
+			column, _ := excelize.ColumnNumberToName(k + 1)
+			k++
+			name := tag
+			// 设置表头
+			if i == 0 {
+				_ = xlsx.SetCellValue(sheet, fmt.Sprintf("%s%d", column, i+1), name)
+			}
+			// 判断是否是时间类型
+
+			colValue := elemValue.Field(j).Interface()
+			if colTime, ok := colValue.(time.Time); ok {
+				colValue = colTime.Format("2006-01-02 15:04:05")
+			}
+			// 设置内容
+			_ = xlsx.SetCellValue(sheet, fmt.Sprintf("%s%d", column, i+2), colValue)
+		}
+	}
+	// 保存的目录看是否存在不存在就创建
+	_, err = os.Stat(path)
+	if os.IsNotExist(err) {
+		err = os.MkdirAll(path, os.ModePerm)
+		if err != nil {
+			return "", err
+		}
+	}
+	filePath = path + "/" + fileName
+	err = xlsx.SaveAs(filePath)
+	if err != nil {
+		return "", err
+	}
+	return filePath, nil
+}

skd/service/export_task/product_history_export.go

@@ -0,0 +1,115 @@
+package export_task
+
+import (
+	"91porn-server/common/log"
+	"91porn-server/models/v/advanceordermod"
+	"91porn-server/models/v/export_task_mod"
+	"91porn-server/models/v/prdcthsomod"
+	"91porn-server/models/v/productmod"
+	"encoding/json"
+	"fmt"
+	"time"
+
+	"go.mongodb.org/mongo-driver/bson"
+	"go.mongodb.org/mongo-driver/mongo/options"
+)
+
+func ExecExportProductHistoryTask(task *export_task_mod.ExportTask) (total int64, err error) {
+	arg := prdcthsomod.ProductHistoryQueryReq{}
+	err = json.Unmarshal([]byte(task.Param), &arg)
+	if err != nil {
+		return
+	}
+
+	var skip int64
+	var size int64 = 100
+	index := 1
+	path := fmt.Sprintf("./temp/%v%v", "会员卡购买记录-", task.ID.Hex())
+	zipName := fmt.Sprintf("%v_%v_会员卡购买记录_%v.zip", task.Admin, GetProName(), task.CreatedAt.Format("2006-01-02_15:04:05"))
+	var records []*prdcthsomod.ProductHistoryExport
+	excelList := []string{}
+	for {
+		fmt.Println("skip", skip, " now ", time.Now().Format("2006-01-02 15:04:05"))
+		opt := options.Find().SetSkip(skip).SetLimit(size).SetSort(bson.M{"createdAt": -1})
+		list, err := prdcthsomod.FindList(arg.GetCond(), opt)
+		if err != nil {
+			log.Error("ExecExportProductHistoryTask prdcthsomod.FindList fail", log.E(err))
+			return total, err
+		}
+		for _, v := range list {
+			item := &prdcthsomod.ProductHistoryExport{
+				ID:        v.ID,        // 交易订单号
+				UID:       v.UID,       // 用户id
+				ProductID: v.ProductID, // 会员卡id
+				Name:      v.Name,      // 商品名字
+				IsUpgrade: v.IsUpgrade, // 是否VIP升级
+				Amount:    v.Amount,    // 花费的余额
+				Income:    v.Income,    // 花费的收益
+				SysType:   v.SysType,   // 设备系统类型 ios pc android
+				CreatedAt: v.CreatedAt,
+			}
+			if v.ProductSnapShot != nil {
+				item.Duration = v.ProductSnapShot.Duration
+				item.VipLevel = v.ProductSnapShot.VipLevel
+				item.OriginalPrice = v.ProductSnapShot.OriginalPrice
+				if v.ProductSnapShot.DiscountedPriceIos != nil {
+					item.DiscountedPriceIos = *v.ProductSnapShot.DiscountedPriceIos
+				}
+				if v.ProductSnapShot.DiscountedPriceAnd != nil {
+					item.DiscountedPriceAnd = *v.ProductSnapShot.DiscountedPriceAnd
+				}
+			}
+			switch v.ProductType {
+			case productmod.VIP:
+				item.ProductType = "会员卡"
+			case productmod.AdvanceCard:
+				item.ProductType = "预售卡"
+			}
+			switch v.AdvanceOrderStatus {
+			case advanceordermod.AdvanceSUCCESS:
+				item.AdvanceOrderStatus = "预付成功"
+			case advanceordermod.BalanceSUCCESS:
+				item.AdvanceOrderStatus = "尾款支付成功"
+			}
+			records = append(records, item)
+		}
+
+		var fileName = fmt.Sprintf("预售订单列表_%v.xlsx", index)
+		if len(records) >= maxDataNum {
+			// 获取到的数据已经达到了单文件最大限制数据量
+			filePath, err := SaveExcel(records, path, fileName)
+			if err != nil {
+				log.Error("ExecExportProductHistoryTask SaveExcel fail", log.E(err))
+				return total, err
+			}
+			// 重新归0
+			records = []*prdcthsomod.ProductHistoryExport{}
+			index = index + 1
+			excelList = append(excelList, filePath)
+			fmt.Println("index:", index)
+			total = total + int64(len(records))
+		} else if len(list) < int(size) {
+			if len(records) > 0 {
+				// 数据库已经取不到更多的数据了
+				filePath, err := SaveExcel(records, path, fileName)
+				if err != nil {
+					log.Error("ExecExportProductHistoryTask SaveExcel fail", log.E(err))
+					return total, err
+				}
+				excelList = append(excelList, filePath)
+				total = total + int64(len(records))
+			}
+			// 结束循环
+			break
+		}
+		skip = skip + size
+	}
+
+	// 发送到tg
+	err = SendTg(task.Admin, excelList, zipName)
+	if err != nil {
+		log.Error("ExecExportProductHistoryTask SendTg fail", log.E(err))
+		return
+	}
+	return total, nil
+}

skd/service/export_task/user_export.go

@@ -0,0 +1,192 @@
+package export_task
+
+import (
+	"91porn-server/common"
+	"91porn-server/common/log"
+	"91porn-server/models/l/playlgmod"
+	"91porn-server/models/v/export_task_mod"
+	"91porn-server/models/v/usermod"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"go.mongodb.org/mongo-driver/bson"
+	"sync"
+)
+
+// ExecExportUserTask 执行用户导出任务
+func ExecExportUserTask(task *export_task_mod.ExportTask) (total int64, err error) {
+	type userListReq struct {
+		usermod.UserListSelector
+		common.StandQuery
+		IsPretendAcc *int `form:"isPretendAcc" json:"isPretendAcc"` //是否马甲账号
+	}
+	var arg userListReq
+	err = json.Unmarshal([]byte(task.Param), &arg)
+	if err != nil {
+		return
+	}
+	if arg.StartTime == nil || arg.EndTime == nil {
+		return 0, errors.New("没有选择时间")
+	}
+	cond, _ := common.StandQueryMap(arg.StandQuery, arg.UserListSelector)
+	//马甲账户
+	if arg.IsPretendAcc != nil && *arg.IsPretendAcc == 1 {
+		cond["devID"] = bson.M{"$regex": usermod.SystemDevIDPrex, "$options": "i"}
+	}
+	var skip int64
+	var size int64 = 500
+	records := []usermod.ExportUser{}
+
+	zipName := fmt.Sprintf("%v_%v_用户列表_%v.zip", task.Admin, GetProName(), task.CreatedAt.Format("2006-01-02_15:04:05"))
+	index := 1
+
+	path := fmt.Sprintf("./temp/%v%v", "用户列表-", task.ID.Hex())
+	excelList := []string{}
+	// 最后一条记录的时间
+	for {
+		userList, err := usermod.ExportFindMany(cond, skip, size)
+		if err != nil {
+			log.Error("ExecExportUserTask fail", log.E(err))
+			return total, err
+		}
+		videoCntMap := make(map[uint64]int)
+		unDealVideoCntMap := make(map[uint64]int)
+		// 针对91PORN做的优化，不然数据太多，导出数据文本太大，速度慢
+		//if arg.IsPretendAcc != nil && *arg.IsPretendAcc == 1 {
+		//	videoCntMap, unDealVideoCntMap, _ = vidmod.GetVideosByUIDs(getUIDs(userList))
+		//}
+		eUsers := encodeUsers(userList, videoCntMap, unDealVideoCntMap)
+
+		records = append(records, eUsers...)
+		var fileName = fmt.Sprintf("用户列表_%v.xlsx", index)
+		// 最多容纳5000条
+		if len(records) >= maxDataNum {
+			// 直接写入excel
+			filePath, err := SaveExcel(records, path, fileName)
+			if err != nil {
+				log.Error("ExecExportVideoTask SaveExcel fail", log.E(err))
+				return total, err
+			}
+			// 重置
+			records = []usermod.ExportUser{}
+			index = index + 1
+			excelList = append(excelList, filePath)
+			total = total + int64(len(records))
+		} else if len(userList) < int(size) {
+			if len(records) > 0 {
+				// 直接写入excel
+				filePath, err := SaveExcel(records, path, fileName)
+				if err != nil {
+					log.Error("ExecExportVideoTask SaveExcel fail", log.E(err))
+					return total, err
+				}
+				excelList = append(excelList, filePath)
+				total = total + int64(len(records))
+			}
+			// 已经没有更多数据了，直接返回
+			break
+		}
+
+		skip = skip + size
+	}
+	// 发送到tg
+	err = SendTg(task.Admin, excelList, zipName)
+	if err != nil {
+		log.Error("ExecExportUserTask SendTg fail", log.E(err))
+		return
+	}
+	return total, nil
+}
+
+func encodeUsers(users []*usermod.User, videoCntMap map[uint64]int, unDealVideoCntMap map[uint64]int) []usermod.ExportUser {
+	usersLen := len(users)
+	uids := make([]uint64, 0, usersLen)
+	uInfos := make([]usermod.ExportUser, 0, usersLen)
+	for _, u := range users {
+		if u != nil {
+			user := usermod.ExportUser{
+				UID: u.UID,
+				//DevID:          u.DevID,
+				DevType:    u.DevType,
+				RegisterIP: u.RegisterIP,
+				Mobile:     u.Mobile,
+				//Gender:         u.Gender,
+				Channel: u.DistrictCode,
+				//Name:           u.Name,
+				PromotionCode: u.PromCode,
+				//Summary:        u.Summary,
+				//Region:         u.Region,
+				//Birthday:       u.Birthday,
+				//VipLevel:       u.VipLevel,
+				VipExpireDate: u.VipExpireDate,
+				CreatedAt:     u.CreatedAt,
+				//MobileBindAt:   u.MobileBindAt,
+				//HasLocked:      u.HasLocked,
+				//HasBanned:      u.HasBanned,
+				//TotalVideoCnt:  videoCntMap[u.UID],
+				//UndealVideoCnt: unDealVideoCntMap[u.UID],
+				LastVisitAt: u.LastVisitAt,
+			}
+			uInfos = append(uInfos, user)
+			uids = append(uids, u.UID)
+		}
+	}
+	wCnt := setArray2Map(calWatchedVideoCnt(uids))
+	for i, u := range uInfos {
+		uInfos[i].WatchCount = wCnt[u.UID]
+	}
+	return uInfos
+}
+
+func getUIDs(users []*usermod.User) []uint64 {
+	uids := make([]uint64, 0, len(users))
+	for _, v := range users {
+		if v == nil {
+			continue
+		}
+		uids = append(uids, v.UID)
+	}
+	return uids
+}
+
+func setArray2Map(datas []playlgmod.UIDCount) map[uint64]int {
+	m := make(map[uint64]int)
+	for _, d := range datas {
+		m[d.UID] = d.Count
+	}
+	return m
+}
+
+func calWatchedVideoCnt(uids []uint64) []playlgmod.UIDCount {
+	le := len(uids)
+	if le < 1000 {
+		datas, _ := playlgmod.HasWatchedVideoCnt(uids)
+		return datas
+	}
+	//创建多个协程去拉取
+	const goRoutineCnt = 10
+	var wg sync.WaitGroup
+	var mData [goRoutineCnt][]playlgmod.UIDCount
+	size := le / goRoutineCnt
+	wg.Add(goRoutineCnt)
+	for i := 0; i < goRoutineCnt-1; i++ {
+		common.GoParam(i, func(i int) {
+			defer wg.Done()
+			begin := i * size
+			end := begin + size - 1
+			mData[i], _ = playlgmod.HasWatchedVideoCnt(uids[begin:end])
+		})
+	}
+	common.Go(func() {
+		defer wg.Done()
+		begin := (goRoutineCnt - 1) * size
+		end := le - 1
+		mData[goRoutineCnt-1], _ = playlgmod.HasWatchedVideoCnt(uids[begin:end])
+	})
+	wg.Wait()
+	datas := []playlgmod.UIDCount{}
+	for i := 0; i < goRoutineCnt; i++ {
+		datas = append(datas, mData[i]...)
+	}
+	return datas
+}

skd/service/export_task/video_export.go

@@ -0,0 +1,219 @@
+package export_task
+
+import (
+	"91porn-server/common/log"
+	"91porn-server/models/v/export_task_mod"
+	"91porn-server/models/v/modulevidmod"
+	"91porn-server/models/v/tagmod"
+	"91porn-server/models/v/vidmod"
+	"91porn-server/web/vidhelp"
+	"encoding/json"
+	"fmt"
+	"go.mongodb.org/mongo-driver/bson"
+	"go.mongodb.org/mongo-driver/bson/primitive"
+	"time"
+)
+
+func ExecExportVideoTask(task *export_task_mod.ExportTask) (total int64, err error) {
+	arg := vidmod.ListReq{}
+	err = json.Unmarshal([]byte(task.Param), &arg)
+	if err != nil {
+		return
+	}
+	cond, err := assembleConditions(arg)
+	if err != nil {
+		return
+	}
+	secID, cond, _, err := dealVidListCond(cond)
+	if err != nil {
+		return
+	}
+
+	var skip int64
+	var size int64 = 100
+	index := 1
+	path := fmt.Sprintf("./temp/%v%v", "帖子列表-", task.ID.Hex())
+	zipName := fmt.Sprintf("%v_%v_帖子列表_%v.zip", task.Admin, GetProName(), task.CreatedAt.Format("2006-01-02_15:04:05"))
+	var records []*vidmod.WebVideo
+	excelList := []string{}
+	for {
+		fmt.Println("skip", skip, " now ", time.Now().Format("2006-01-02 15:04:05"))
+		list, err := vidmod.ExportFindMany(cond, skip, size)
+		if err != nil {
+			log.Error("ExecExportVideoTask vidmod.ExportFindMany fail", log.E(err))
+			return total, err
+		}
+		infos, err := vidhelp.EncodeVideoInfo(list, secID)
+		if err != nil {
+			log.Error("ExecExportVideoTask vidhelp.EncodeVideoInfo fail", log.E(err))
+			return total, err
+		}
+		records = append(records, infos...)
+		var fileName = fmt.Sprintf("帖子列表_%v.xlsx", index)
+		if len(records) >= maxDataNum {
+			// 获取到的数据已经达到了单文件最大限制数据量
+			filePath, err := SaveExcel(records, path, fileName)
+			if err != nil {
+				log.Error("ExecExportVideoTask SaveExcel fail", log.E(err))
+				return total, err
+			}
+			// 重新归0
+			records = []*vidmod.WebVideo{}
+			index = index + 1
+			excelList = append(excelList, filePath)
+			fmt.Println("index:", index)
+			total = total + int64(len(records))
+		} else if len(list) < int(size) {
+			if len(records) > 0 {
+				// 数据库已经取不到更多的数据了
+				filePath, err := SaveExcel(records, path, fileName)
+				if err != nil {
+					log.Error("ExecExportVideoTask SaveExcel fail", log.E(err))
+					return total, err
+				}
+				excelList = append(excelList, filePath)
+				total = total + int64(len(records))
+			}
+			// 结束循环
+			break
+		}
+		skip = skip + size
+	}
+
+	// 发送到tg
+	err = SendTg(task.Admin, excelList, zipName)
+	if err != nil {
+		log.Error("ExecExportVideoTask SendTg fail", log.E(err))
+		return
+	}
+	return total, nil
+}
+
+func dealVidListCond(cond bson.M) (primitive.ObjectID, bson.M, []primitive.ObjectID, error) {
+	var secID primitive.ObjectID
+	var err error
+	sectionID, ok := cond["sectionID"]
+	if ok {
+		secID, err = primitive.ObjectIDFromHex(sectionID.(string))
+		if err != nil {
+			log.Error("primitive.ObjectIDFromHex", log.Any("sectionID", sectionID), log.E(err))
+			return secID, cond, nil, err
+		}
+		var flag bool
+		isSorted, ok := cond["isSortedUnderModule"]
+		if ok {
+			flag = isSorted.(bool)
+		}
+		videoIDs, err := modulevidmod.GetBySectionID(secID, flag)
+		if err != nil {
+			return secID, cond, nil, err
+		}
+		cond["_id"] = bson.M{"$in": videoIDs}
+		delete(cond, "sectionID")
+		delete(cond, "isSortedUnderModule")
+		return secID, cond, videoIDs, nil
+	}
+	delete(cond, "sectionID")
+	delete(cond, "isSortedUnderModule")
+	return secID, cond, nil, nil
+}
+
+func assembleConditions(req vidmod.ListReq) (m map[string]interface{}, err error) {
+	m = make(map[string]interface{})
+	if req.ShowType != nil {
+		m["showType"] = *req.ShowType
+	}
+	if req.Status != 4 {
+		if req.Status == 7 {
+			m["status"] = map[string]int{"$gt": 0}
+		} else {
+			m["status"] = req.Status
+		}
+	}
+	if req.IsFree == 0 {
+		m["coins"] = map[string]int{"$gt": 0}
+	}
+	if req.IsFree == 1 {
+		m["coins"] = map[string]int{"$eq": 0}
+	}
+	if req.IsUserUp == 2 {
+		m["publisherID"] = map[string]int{"$lt": 115000}
+	}
+	if req.IsUserUp == 1 {
+		m["publisherID"] = map[string]int{"$gt": 115000}
+	}
+	if len(req.Title) != 0 {
+		m["title"] = map[string]string{"$regex": req.Title, "$options": "i"}
+	}
+	if req.UID > 0 {
+		m["publisherID"] = req.UID
+	}
+	if req.Chosen == 1 {
+		m["chosen"] = true
+	}
+	if req.Chosen == 2 {
+		m["chosen"] = false
+	}
+	if req.FreeArea == 1 {
+		m["freeArea"] = true
+	}
+	if req.FreeArea == 2 {
+		m["freeArea"] = false
+	}
+	if !req.End.IsZero() {
+		m["createdAt"] = map[string]time.Time{"$gte": req.Start, "$lt": req.End}
+	}
+	if len(req.Tag) != 0 {
+		id, err := tagmod.GetTagIDByName(req.Tag)
+		if err != nil {
+			return m, err
+		}
+		m["tags"] = id
+	}
+	if len(req.ID) != 0 {
+		oid, err := primitive.ObjectIDFromHex(req.ID)
+		if err != nil {
+			return m, err
+		}
+		m["_id"] = oid
+	}
+	//通过初始价格判断是否是马甲账号
+	if req.IsPretendAcc == 1 {
+		m["coins"] = vidmod.PretendAccInitCoins
+	}
+	if req.IsPretendAcc == 2 {
+		//i := make(map[string]int64)
+		//i["$ne"] = vidmod.PretendAccInitCoins
+		m["coins"] = map[string]int64{"$ne": vidmod.PretendAccInitCoins}
+	}
+	m["deleteAt"] = bson.M{"$exists": false}
+	if req.NewsType != "" {
+		m["newsType"] = req.NewsType
+	}
+
+	if req.LiaoBaTop != nil {
+		m["liaoBaTop"] = *req.LiaoBaTop
+	}
+	if req.SectionID != "" {
+		m["sectionID"] = req.SectionID
+	}
+	m["isSortedUnderModule"] = req.IsSortedUnderModule
+	if req.IsRecommended != nil {
+		if *req.IsRecommended {
+			m["recoWeight"] = bson.M{"$gte": 0}
+		} else {
+			m["recoWeight"] = bson.M{"$lt": 0}
+		}
+	}
+	if req.Key == "likeRate" || req.Key == "purchaseRate" {
+		m[req.Key] = req.Value
+	}
+	if req.IsHappinessPlazaTop != nil && *req.IsHappinessPlazaTop {
+		if *req.IsHappinessPlazaTop {
+			m["happinessPlazaTop"] = bson.M{"$gt": 0}
+		} else {
+			m["happinessPlazaTop"] = bson.M{"$lte": 0}
+		}
+	}
+	return m, nil
+}