diff --git a/internal/app/spider/picture.go b/internal/app/spider/picture.go index ec2e001..218223b 100644 --- a/internal/app/spider/picture.go +++ b/internal/app/spider/picture.go @@ -55,7 +55,7 @@ func (p *Picture) stop() error { } func (p *Picture) Run(ctx context.Context) error { - tk := time.NewTicker(60 * time.Minute) + tk := time.NewTicker(180 * time.Minute) p.isRunning = true go func() { diff --git a/internal/repository/bilibili_picture.go b/internal/repository/bilibili_picture.go index 1439e18..506fd47 100644 --- a/internal/repository/bilibili_picture.go +++ b/internal/repository/bilibili_picture.go @@ -6,7 +6,6 @@ import ( "git.vtb.link/eoefans/internal/app/api/idl" "gorm.io/gorm" - "gorm.io/gorm/clause" ) func NewBilibiliPicture(tx *gorm.DB) idl.BilibiliPictureRepository { @@ -21,33 +20,49 @@ func (impl *BilibiliPictureMysqlImpl) Create(items []*idl.BilibiliDynamic) error if len(items) == 0 { return nil } - picItems := make([]idl.BilibiliPicture, 0) - for i := range items { - for j := range items[i].Pictures { - imgAttr := idl.BilibiliPictureAttr{ - Height: items[i].Pictures[j].Height, - Size: items[i].Pictures[j].Size, - Width: items[i].Pictures[j].Width, + return impl.tx.Transaction(func(_tx *gorm.DB) error { + ids := make([]uint64, 0) + for i := range items { + ids = append(ids, items[i].DynamicID) + } + var selectedIDs []uint64 + err := _tx.Table(idl.BilibiliDynamic{}.TableName()).Where("dynamic_id in (?)", ids).Select("dynamic_id").Find(&selectedIDs).Error + if err != nil { + return err + } + filteritems := make([]*idl.BilibiliDynamic, 0) + for i := range items { + find := false + for j := range selectedIDs { + if items[i].DynamicID == selectedIDs[j] { + find = true + break + } + } + if !find { + filteritems = append(filteritems, items[i]) } - picItems = append(picItems, idl.BilibiliPicture{ - DynamicID: items[i].DynamicID, - ImgSrc: items[i].Pictures[j].ImgSrc, - ImgAttr: imgAttr, - }) } - } - return impl.tx.Transaction(func(_tx *gorm.DB) error { - err := _tx.Table(idl.BilibiliDynamic{}.TableName()).Clauses(clause.OnConflict{ - Columns: []clause.Column{{Name: "dynamic_id"}}, - DoUpdates: clause.AssignmentColumns([]string{"dynamic_id"}), - }).Create(&items).Error + err = _tx.Table(idl.BilibiliDynamic{}.TableName()).Create(&filteritems).Error if err != nil { return err } - return _tx.Table(idl.BilibiliPicture{}.TableName()).Clauses(clause.OnConflict{ - Columns: []clause.Column{{Name: "img_src"}}, - DoUpdates: clause.AssignmentColumns([]string{"img_src"}), - }).Create(&picItems).Error + picItems := make([]idl.BilibiliPicture, 0) + for i := range filteritems { + for j := range filteritems[i].Pictures { + imgAttr := idl.BilibiliPictureAttr{ + Height: filteritems[i].Pictures[j].Height, + Size: filteritems[i].Pictures[j].Size, + Width: filteritems[i].Pictures[j].Width, + } + picItems = append(picItems, idl.BilibiliPicture{ + DynamicID: filteritems[i].DynamicID, + ImgSrc: filteritems[i].Pictures[j].ImgSrc, + ImgAttr: imgAttr, + }) + } + } + return _tx.Table(idl.BilibiliPicture{}.TableName()).Create(&picItems).Error }) }