[feature]获取图片动态里的topic_details

EOEFANS · Jan 27, 2023 · 3fd19c2 · 3fd19c2
1 parent 9f28a74
commit 3fd19c2
Show file tree

Hide file tree

Showing 5 changed files with 72 additions and 15 deletions.
diff --git a/database/init.sql b/database/init.sql
@@ -126,4 +126,6 @@ create table bilibili_dynamics (
     updated_at bigint unsigned not null comment '更新时间',
     index idx_dynamic_id(dynamic_id) comment '动态id索引',
     index idx_sent_at(sent_at) comment '发送时间索引'
-)Engine=InnoDB comment '动态' charset 'utf8mb4';
+)Engine=InnoDB comment '动态' charset 'utf8mb4';
+
+alter table bilibili_dynamics add column topic_details json default null comment '动态的#xxx#' after pictures;
diff --git a/internal/app/api/idl/bilibili_picture.go b/internal/app/api/idl/bilibili_picture.go
@@ -9,19 +9,20 @@ import (
 
 // 图片来源于动态，以动态为单位
 type BilibiliDynamic struct {
-	ID        uint64                  `gorm:"primarykey"`
-	UID       uint64                  `gorm:"column:uid"`
-	DynamicID uint64                  `gorm:"column:dynamic_id"`
-	Pictures  BilibiliDynamicPictures `gorm:"column:pictures;"`
-	TopicName string                  `gorm:"column:topic_name"`
-	TopicID   uint64                  `gorm:"column:topic_id"`
-	View      uint64                  `gorm:"column:view_nums"`
-	Repost    uint64                  `gorm:"column:repost"`
-	Comment   uint64                  `gorm:"column:comment_nums"`
-	Like      uint64                  `gorm:"column:favor"`
-	SentAt    uint64                  `gorm:"column:sent_at"`
-	CreatedAt uint64                  `gorm:"autoCreateTime"`
-	UpdatedAt uint64                  `gorm:"autoUpdateTime"`
+	ID           uint64                       `gorm:"primarykey"`
+	UID          uint64                       `gorm:"column:uid"`
+	DynamicID    uint64                       `gorm:"column:dynamic_id"`
+	Pictures     BilibiliDynamicPictures      `gorm:"column:pictures"`
+	TopicDetails *BilibiliDynamicTopicDetails `gorm:"topic_details"`
+	TopicName    string                       `gorm:"column:topic_name"`
+	TopicID      uint64                       `gorm:"column:topic_id"`
+	View         uint64                       `gorm:"column:view_nums"`
+	Repost       uint64                       `gorm:"column:repost"`
+	Comment      uint64                       `gorm:"column:comment_nums"`
+	Like         uint64                       `gorm:"column:favor"`
+	SentAt       uint64                       `gorm:"column:sent_at"`
+	CreatedAt    uint64                       `gorm:"autoCreateTime"`
+	UpdatedAt    uint64                       `gorm:"autoUpdateTime"`
 }
 
 func (p BilibiliDynamicPictures) Value() (driver.Value, error) {
@@ -42,6 +43,27 @@ func (c *BilibiliDynamicPictures) Scan(input interface{}) error {
 	return nil
 }
 
+func (d *BilibiliDynamicTopicDetails) Value() (driver.Value, error) {
+	if d == nil {
+		return nil, nil
+	}
+	return json.Marshal(d)
+}
+
+func (d *BilibiliDynamicTopicDetails) Scan(input interface{}) error {
+	data, ok := input.([]byte)
+	if !ok {
+		return errors.New("invalid input in Scan")
+	}
+	result := BilibiliDynamicTopicDetails{}
+	err := json.Unmarshal(data, &result)
+	if err != nil {
+		return err
+	}
+	*d = result
+	return nil
+}
+
 type BilibiliPictureLatestReq struct {
 	Page    int `form:"page,default=1" binding:"omitempty,gt=0"`
 	TopicID int `form:"topic_id"`
@@ -83,6 +105,12 @@ type BilibiliDynamicPicture struct {
 	ImgSrc string  `json:"img_src"`
 }
 
+type BilibiliDynamicTopicDetails []BilibiliDynamicTopicDetail
+type BilibiliDynamicTopicDetail struct {
+	TopicID   uint64 `json:"topic_id"`
+	TopicName string `json:"topic_name"`
+}
+
 func (BilibiliDynamic) TableName() string {
 	return "bilibili_dynamics"
 }

diff --git a/internal/app/spider/picture.go b/internal/app/spider/picture.go
@@ -145,6 +145,16 @@ func (p *Picture) spider() error {
 						continue
 					}
 					dynamic.Pictures = pictures
+					topicDetails := make(idl.BilibiliDynamicTopicDetails, 0)
+					for _, v := range v.Display.TopicInfo.TopicDetails {
+						topicDetails = append(topicDetails, idl.BilibiliDynamicTopicDetail{
+							TopicID:   v.TopicID,
+							TopicName: v.TopicName,
+						})
+					}
+					if len(topicDetails) != 0 {
+						dynamic.TopicDetails = &topicDetails
+					}
 					items = append(items, dynamic)
 				default:
 					continue

diff --git a/internal/app/spider/picture_test.go b/internal/app/spider/picture_test.go
@@ -66,6 +66,12 @@ func TestInsertPicture(t *testing.T) {
 		Comment:   res.Card.Desc.Comment,
 		Like:      res.Card.Desc.Like,
 		SentAt:    res.Card.Desc.TimeStamp,
+		TopicDetails: &idl.BilibiliDynamicTopicDetails{
+			{
+				TopicID:   res.Card.Display.TopicInfo.TopicDetails[0].TopicID,
+				TopicName: res.Card.Display.TopicInfo.TopicDetails[0].TopicName,
+			},
+		},
 	}}
 	err = repository.NewBilibiliPicture(db).Create(items)
 	if err != nil {

diff --git a/internal/pkg/bilibili/video.go b/internal/pkg/bilibili/video.go
@@ -80,7 +80,18 @@ type DynamicCard struct {
 		DynamicID uint64      `json:"dynamic_id"`
 		TimeStamp uint64      `json:"timestamp"`
 	} `json:"desc"`
-	Card string `json:"card"`
+	Card    string `json:"card"`
+	Display struct {
+		TopicInfo struct {
+			TopicDetails DynamicTopicDetails `json:"topic_details"`
+		} `json:"topic_info"`
+	} `json:"display"`
+}
+
+type DynamicTopicDetails []DynamicTopicDetail
+type DynamicTopicDetail struct {
+	TopicID   uint64 `json:"topic_id"`
+	TopicName string `json:"topic_name"`
 }
 
 // Card是json字符串，需要进一步解析