diff --git a/database/init.sql b/database/init.sql index b79bc8c..0314a35 100644 --- a/database/init.sql +++ b/database/init.sql @@ -126,4 +126,6 @@ create table bilibili_dynamics ( updated_at bigint unsigned not null comment '更新时间', index idx_dynamic_id(dynamic_id) comment '动态id索引', index idx_sent_at(sent_at) comment '发送时间索引' -)Engine=InnoDB comment '动态' charset 'utf8mb4'; \ No newline at end of file +)Engine=InnoDB comment '动态' charset 'utf8mb4'; + +alter table bilibili_dynamics add column topic_details json default null comment '动态的#xxx#' after pictures; \ No newline at end of file diff --git a/internal/app/api/idl/bilibili_picture.go b/internal/app/api/idl/bilibili_picture.go index fc04f2c..317532a 100644 --- a/internal/app/api/idl/bilibili_picture.go +++ b/internal/app/api/idl/bilibili_picture.go @@ -9,19 +9,20 @@ import ( // 图片来源于动态,以动态为单位 type BilibiliDynamic struct { - ID uint64 `gorm:"primarykey"` - UID uint64 `gorm:"column:uid"` - DynamicID uint64 `gorm:"column:dynamic_id"` - Pictures BilibiliDynamicPictures `gorm:"column:pictures;"` - TopicName string `gorm:"column:topic_name"` - TopicID uint64 `gorm:"column:topic_id"` - View uint64 `gorm:"column:view_nums"` - Repost uint64 `gorm:"column:repost"` - Comment uint64 `gorm:"column:comment_nums"` - Like uint64 `gorm:"column:favor"` - SentAt uint64 `gorm:"column:sent_at"` - CreatedAt uint64 `gorm:"autoCreateTime"` - UpdatedAt uint64 `gorm:"autoUpdateTime"` + ID uint64 `gorm:"primarykey"` + UID uint64 `gorm:"column:uid"` + DynamicID uint64 `gorm:"column:dynamic_id"` + Pictures BilibiliDynamicPictures `gorm:"column:pictures"` + TopicDetails *BilibiliDynamicTopicDetails `gorm:"topic_details"` + TopicName string `gorm:"column:topic_name"` + TopicID uint64 `gorm:"column:topic_id"` + View uint64 `gorm:"column:view_nums"` + Repost uint64 `gorm:"column:repost"` + Comment uint64 `gorm:"column:comment_nums"` + Like uint64 `gorm:"column:favor"` + SentAt uint64 `gorm:"column:sent_at"` + CreatedAt uint64 `gorm:"autoCreateTime"` + UpdatedAt uint64 `gorm:"autoUpdateTime"` } func (p BilibiliDynamicPictures) Value() (driver.Value, error) { @@ -42,6 +43,27 @@ func (c *BilibiliDynamicPictures) Scan(input interface{}) error { return nil } +func (d *BilibiliDynamicTopicDetails) Value() (driver.Value, error) { + if d == nil { + return nil, nil + } + return json.Marshal(d) +} + +func (d *BilibiliDynamicTopicDetails) Scan(input interface{}) error { + data, ok := input.([]byte) + if !ok { + return errors.New("invalid input in Scan") + } + result := BilibiliDynamicTopicDetails{} + err := json.Unmarshal(data, &result) + if err != nil { + return err + } + *d = result + return nil +} + type BilibiliPictureLatestReq struct { Page int `form:"page,default=1" binding:"omitempty,gt=0"` TopicID int `form:"topic_id"` @@ -83,6 +105,12 @@ type BilibiliDynamicPicture struct { ImgSrc string `json:"img_src"` } +type BilibiliDynamicTopicDetails []BilibiliDynamicTopicDetail +type BilibiliDynamicTopicDetail struct { + TopicID uint64 `json:"topic_id"` + TopicName string `json:"topic_name"` +} + func (BilibiliDynamic) TableName() string { return "bilibili_dynamics" } diff --git a/internal/app/spider/picture.go b/internal/app/spider/picture.go index 5c98d0b..82863aa 100644 --- a/internal/app/spider/picture.go +++ b/internal/app/spider/picture.go @@ -145,6 +145,16 @@ func (p *Picture) spider() error { continue } dynamic.Pictures = pictures + topicDetails := make(idl.BilibiliDynamicTopicDetails, 0) + for _, v := range v.Display.TopicInfo.TopicDetails { + topicDetails = append(topicDetails, idl.BilibiliDynamicTopicDetail{ + TopicID: v.TopicID, + TopicName: v.TopicName, + }) + } + if len(topicDetails) != 0 { + dynamic.TopicDetails = &topicDetails + } items = append(items, dynamic) default: continue diff --git a/internal/app/spider/picture_test.go b/internal/app/spider/picture_test.go index 29cf4a6..7622fbb 100644 --- a/internal/app/spider/picture_test.go +++ b/internal/app/spider/picture_test.go @@ -66,6 +66,12 @@ func TestInsertPicture(t *testing.T) { Comment: res.Card.Desc.Comment, Like: res.Card.Desc.Like, SentAt: res.Card.Desc.TimeStamp, + TopicDetails: &idl.BilibiliDynamicTopicDetails{ + { + TopicID: res.Card.Display.TopicInfo.TopicDetails[0].TopicID, + TopicName: res.Card.Display.TopicInfo.TopicDetails[0].TopicName, + }, + }, }} err = repository.NewBilibiliPicture(db).Create(items) if err != nil { diff --git a/internal/pkg/bilibili/video.go b/internal/pkg/bilibili/video.go index 27387c0..c8e0c1d 100644 --- a/internal/pkg/bilibili/video.go +++ b/internal/pkg/bilibili/video.go @@ -80,7 +80,18 @@ type DynamicCard struct { DynamicID uint64 `json:"dynamic_id"` TimeStamp uint64 `json:"timestamp"` } `json:"desc"` - Card string `json:"card"` + Card string `json:"card"` + Display struct { + TopicInfo struct { + TopicDetails DynamicTopicDetails `json:"topic_details"` + } `json:"topic_info"` + } `json:"display"` +} + +type DynamicTopicDetails []DynamicTopicDetail +type DynamicTopicDetail struct { + TopicID uint64 `json:"topic_id"` + TopicName string `json:"topic_name"` } // Card是json字符串,需要进一步解析