fix:去重

EOEFANS · Feb 5, 2023 · d05d877 · d05d877
1 parent 18dcedf
commit d05d877
Show file tree

Hide file tree

Showing 2 changed files with 39 additions and 24 deletions.
diff --git a/internal/app/spider/picture.go b/internal/app/spider/picture.go
@@ -55,7 +55,7 @@ func (p *Picture) stop() error {
 }
 
 func (p *Picture) Run(ctx context.Context) error {
-	tk := time.NewTicker(60 * time.Minute)
+	tk := time.NewTicker(180 * time.Minute)
 	p.isRunning = true
 
 	go func() {

diff --git a/internal/repository/bilibili_picture.go b/internal/repository/bilibili_picture.go
@@ -6,7 +6,6 @@ import (
 
 	"git.vtb.link/eoefans/internal/app/api/idl"
 	"gorm.io/gorm"
-	"gorm.io/gorm/clause"
 )
 
 func NewBilibiliPicture(tx *gorm.DB) idl.BilibiliPictureRepository {
@@ -21,33 +20,49 @@ func (impl *BilibiliPictureMysqlImpl) Create(items []*idl.BilibiliDynamic) error
 	if len(items) == 0 {
 		return nil
 	}
-	picItems := make([]idl.BilibiliPicture, 0)
-	for i := range items {
-		for j := range items[i].Pictures {
-			imgAttr := idl.BilibiliPictureAttr{
-				Height: items[i].Pictures[j].Height,
-				Size:   items[i].Pictures[j].Size,
-				Width:  items[i].Pictures[j].Width,
+	return impl.tx.Transaction(func(_tx *gorm.DB) error {
+		ids := make([]uint64, 0)
+		for i := range items {
+			ids = append(ids, items[i].DynamicID)
+		}
+		var selectedIDs []uint64
+		err := _tx.Table(idl.BilibiliDynamic{}.TableName()).Where("dynamic_id in (?)", ids).Select("dynamic_id").Find(&selectedIDs).Error
+		if err != nil {
+			return err
+		}
+		filteritems := make([]*idl.BilibiliDynamic, 0)
+		for i := range items {
+			find := false
+			for j := range selectedIDs {
+				if items[i].DynamicID == selectedIDs[j] {
+					find = true
+					break
+				}
+			}
+			if !find {
+				filteritems = append(filteritems, items[i])
 			}
-			picItems = append(picItems, idl.BilibiliPicture{
-				DynamicID: items[i].DynamicID,
-				ImgSrc:    items[i].Pictures[j].ImgSrc,
-				ImgAttr:   imgAttr,
-			})
 		}
-	}
-	return impl.tx.Transaction(func(_tx *gorm.DB) error {
-		err := _tx.Table(idl.BilibiliDynamic{}.TableName()).Clauses(clause.OnConflict{
-			Columns:   []clause.Column{{Name: "dynamic_id"}},
-			DoUpdates: clause.AssignmentColumns([]string{"dynamic_id"}),
-		}).Create(&items).Error
+		err = _tx.Table(idl.BilibiliDynamic{}.TableName()).Create(&filteritems).Error
 		if err != nil {
 			return err
 		}
-		return _tx.Table(idl.BilibiliPicture{}.TableName()).Clauses(clause.OnConflict{
-			Columns:   []clause.Column{{Name: "img_src"}},
-			DoUpdates: clause.AssignmentColumns([]string{"img_src"}),
-		}).Create(&picItems).Error
+		picItems := make([]idl.BilibiliPicture, 0)
+		for i := range filteritems {
+			for j := range filteritems[i].Pictures {
+				imgAttr := idl.BilibiliPictureAttr{
+					Height: filteritems[i].Pictures[j].Height,
+					Size:   filteritems[i].Pictures[j].Size,
+					Width:  filteritems[i].Pictures[j].Width,
+				}
+				picItems = append(picItems, idl.BilibiliPicture{
+					DynamicID: filteritems[i].DynamicID,
+					ImgSrc:    filteritems[i].Pictures[j].ImgSrc,
+					ImgAttr:   imgAttr,
+				})
+			}
+		}
+		return _tx.Table(idl.BilibiliPicture{}.TableName()).Create(&picItems).Error
 	})
 }