Claude Opus 5
@@ -0,0 +1,282 @@
|
||||
package export_task
|
||||
|
||||
import (
|
||||
"91porn-server/common/file"
|
||||
"91porn-server/common/log"
|
||||
commonWorker "91porn-server/common/worker"
|
||||
"91porn-server/models/v/export_task_mod"
|
||||
"fmt"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"reflect"
|
||||
"time"
|
||||
|
||||
tgbotapi "github.com/go-telegram-bot-api/telegram-bot-api"
|
||||
"github.com/pkg/errors"
|
||||
"github.com/xuri/excelize/v2"
|
||||
"go.mongodb.org/mongo-driver/bson"
|
||||
)
|
||||
|
||||
const (
|
||||
proName = "91PORN"
|
||||
tgToken = "5852134617:AAEO0_GPZiUJS_LeR5dM7byEfOpnRg4dGqU"
|
||||
maxDataNum = 20000 // 每个excel存储的数据量
|
||||
maxWorkerNum = 3
|
||||
maxSize = 1024 * 1024 * 20
|
||||
)
|
||||
|
||||
var Bot *tgbotapi.BotAPI
|
||||
var worker *commonWorker.Worker
|
||||
|
||||
func init() {
|
||||
bot, err := tgbotapi.NewBotAPI(tgToken)
|
||||
if err != nil || bot == nil {
|
||||
log.Fatal("初始化tgbot失败", log.E(err))
|
||||
}
|
||||
Bot = bot
|
||||
worker = commonWorker.NewWorker(maxWorkerNum)
|
||||
}
|
||||
|
||||
var execExportTaskRuining bool
|
||||
|
||||
func ExecExportTask() {
|
||||
//防止重入
|
||||
if execExportTaskRuining {
|
||||
return
|
||||
}
|
||||
execExportTaskRuining = true
|
||||
defer func() {
|
||||
execExportTaskRuining = false
|
||||
}()
|
||||
// 获取任务列表
|
||||
list, err := export_task_mod.GetTaskList()
|
||||
if err != nil {
|
||||
log.Error("ExecExportTask export_task_mod.GetTaskList fail", log.E(err))
|
||||
return
|
||||
}
|
||||
for _, task := range list {
|
||||
switch task.Type {
|
||||
case export_task_mod.ExportUserTask:
|
||||
CallExportFunc(task, ExecExportUserTask)
|
||||
case export_task_mod.ExportVidTask:
|
||||
CallExportFunc(task, ExecExportVideoTask)
|
||||
case export_task_mod.ExportAdvanceOrderTask:
|
||||
CallExportFunc(task, ExecExportAdvanceOrderTask)
|
||||
case export_task_mod.ExportProductHistoryTask:
|
||||
CallExportFunc(task, ExecExportProductHistoryTask)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func GetProName() string {
|
||||
//if skdg.Conf.Base.Env == "test" {
|
||||
// return "测试环境_" + proName
|
||||
//}
|
||||
return proName
|
||||
}
|
||||
|
||||
func CallExportFunc(task *export_task_mod.ExportTask, f func(t *export_task_mod.ExportTask) (int64, error)) {
|
||||
if task == nil {
|
||||
return
|
||||
}
|
||||
worker.Exec(func() {
|
||||
// 已经执行过了
|
||||
if task.Status != 0 {
|
||||
return
|
||||
}
|
||||
//key := fmt.Sprintf("ExecExportTask:%v", task.ID.Hex())
|
||||
//ok, err := skdg.Redis.SetNX(key, 1, time.Minute*30)
|
||||
//if err != nil {
|
||||
// log.Error("ExecExportTask fail", log.E(err))
|
||||
// return
|
||||
//}
|
||||
//if !ok {
|
||||
// log.Info("ExportTask already executed")
|
||||
// return
|
||||
//}
|
||||
//defer skdg.Redis.Del(key)
|
||||
start := time.Now().Unix()
|
||||
total, err := f(task)
|
||||
if err != nil {
|
||||
log.Error("CallExportFunc fail", log.Any("task", task), log.E(err))
|
||||
// 修改任务信息
|
||||
export_task_mod.UpdateTask(task.ID, bson.M{
|
||||
"status": 2,
|
||||
"reason": err.Error(),
|
||||
})
|
||||
return
|
||||
}
|
||||
// 修改任务信息
|
||||
export_task_mod.UpdateTask(task.ID, bson.M{
|
||||
"status": 1,
|
||||
"time": time.Now().Unix() - start, // 耗时
|
||||
"total": total,
|
||||
})
|
||||
})
|
||||
return
|
||||
}
|
||||
|
||||
// SendTg 发送到tg
|
||||
func SendTg(admin string, excelList []string, zipFileName string) (err error) {
|
||||
exportPath, err := filepath.Abs("temp")
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
// 尝试创建
|
||||
file.MakeDir(exportPath)
|
||||
zipList, err := Zip(excelList, zipFileName)
|
||||
if err != nil {
|
||||
log.Error("SendTg Zip fail", log.Any("excelList", excelList), log.Any("zipFileName", zipFileName), log.E(err))
|
||||
return
|
||||
}
|
||||
// 测试环境不需要发送到tg群
|
||||
//if skdg.Conf.Base.Env == "test" {
|
||||
// return
|
||||
//}
|
||||
if len(zipList) == 0 {
|
||||
return
|
||||
}
|
||||
var chatId int64 = -1003399433452
|
||||
|
||||
content := fmt.Sprintf("%v总共有%v个压缩包，请查收", admin, len(zipList))
|
||||
_, err = Bot.Send(tgbotapi.NewMessage(chatId, content))
|
||||
if err != nil {
|
||||
log.Error("tgbot send fail", log.E(err))
|
||||
return errors.Wrap(err, "发送TG消息失败")
|
||||
}
|
||||
for _, zipName := range zipList {
|
||||
fileBytes, err := os.ReadFile("./temp/" + zipName)
|
||||
if err != nil {
|
||||
log.Error("ReadFile fail", log.Any("excelList", excelList), log.Any("zipFileName", zipFileName), log.E(err))
|
||||
return errors.Wrap(err, "打开压缩文件失败")
|
||||
}
|
||||
|
||||
// 创建要发送的文件
|
||||
tFile := tgbotapi.FileBytes{Name: zipName, Bytes: fileBytes}
|
||||
|
||||
//生成csv发送到tg群
|
||||
newDocument := tgbotapi.NewDocumentUpload(chatId, tFile)
|
||||
_, err = Bot.Send(newDocument)
|
||||
if err != nil {
|
||||
log.Error("tgbot send fail", log.E(err))
|
||||
return errors.Wrap(err, "发送TG消息失败")
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func Zip(filePathList []string, zipFileName string) (zipList []string, err error) {
|
||||
if len(filePathList) == 0 {
|
||||
return
|
||||
}
|
||||
var currentSize int64
|
||||
zipIndex := 1
|
||||
currentZipName := fmt.Sprintf("%v_%v.zip", zipFileName, zipIndex)
|
||||
args := []string{currentZipName}
|
||||
for k, filePath := range filePathList {
|
||||
// 获取文件大小
|
||||
stat, err := os.Stat(filePath)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
currentSize = currentSize + stat.Size()
|
||||
args = append(args, filePath)
|
||||
// 判断当前所有文件加起来是否已经超过20M，超过就直接压缩,否则就接着等待压缩
|
||||
if currentSize < maxSize && k != (len(filePathList)-1) {
|
||||
continue
|
||||
}
|
||||
cmd1 := exec.Command("zip", args...)
|
||||
out1, err := cmd1.CombinedOutput()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
fmt.Println(string(out1))
|
||||
// 移动文件到目录里
|
||||
cmd2 := exec.Command("mv", currentZipName, "./temp")
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out2, err := cmd2.CombinedOutput()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
fmt.Println(string(out2))
|
||||
zipList = append(zipList, currentZipName)
|
||||
// 重置
|
||||
zipIndex = zipIndex + 1
|
||||
currentZipName = fmt.Sprintf("%v_%v.zip", zipFileName, zipIndex)
|
||||
args = []string{currentZipName}
|
||||
currentSize = 0
|
||||
}
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
// SaveExcel 保存数据到excel中
|
||||
func SaveExcel(records interface{}, path string, fileName string) (filePath string, err error) {
|
||||
|
||||
sheet := "sheet1"
|
||||
xlsx := excelize.NewFile() // new file
|
||||
index, _ := xlsx.NewSheet(sheet) // new sheet
|
||||
xlsx.SetActiveSheet(index) // set active (default) sheet
|
||||
t := reflect.TypeOf(records)
|
||||
if t.Kind() != reflect.Slice {
|
||||
panic("records must be slice")
|
||||
}
|
||||
|
||||
s := reflect.ValueOf(records)
|
||||
for i := 0; i < s.Len(); i++ {
|
||||
elem := s.Index(i).Interface()
|
||||
elemType := reflect.TypeOf(elem)
|
||||
elemValue := reflect.ValueOf(elem)
|
||||
if elemType.Kind() == reflect.Ptr {
|
||||
elemType = elemType.Elem()
|
||||
elemValue = elemValue.Elem()
|
||||
}
|
||||
if elemType.Kind() != reflect.Struct {
|
||||
panic("record in slice must be a struct")
|
||||
}
|
||||
k := 0
|
||||
for j := 0; j < elemType.NumField(); j++ {
|
||||
field := elemType.Field(j)
|
||||
tag := field.Tag.Get("xlsx")
|
||||
if tag == "-" {
|
||||
continue
|
||||
}
|
||||
if tag == "" {
|
||||
tag = field.Name
|
||||
}
|
||||
column, _ := excelize.ColumnNumberToName(k + 1)
|
||||
k++
|
||||
name := tag
|
||||
// 设置表头
|
||||
if i == 0 {
|
||||
_ = xlsx.SetCellValue(sheet, fmt.Sprintf("%s%d", column, i+1), name)
|
||||
}
|
||||
// 判断是否是时间类型
|
||||
|
||||
colValue := elemValue.Field(j).Interface()
|
||||
if colTime, ok := colValue.(time.Time); ok {
|
||||
colValue = colTime.Format("2006-01-02 15:04:05")
|
||||
}
|
||||
// 设置内容
|
||||
_ = xlsx.SetCellValue(sheet, fmt.Sprintf("%s%d", column, i+2), colValue)
|
||||
}
|
||||
}
|
||||
// 保存的目录看是否存在不存在就创建
|
||||
_, err = os.Stat(path)
|
||||
if os.IsNotExist(err) {
|
||||
err = os.MkdirAll(path, os.ModePerm)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
}
|
||||
filePath = path + "/" + fileName
|
||||
err = xlsx.SaveAs(filePath)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
return filePath, nil
|
||||
}
|
||||
Reference in New Issue
Block a user