spider/cmd/server/main.go

package main

import (
	"fmt"
	"log"
	"time"

	"spider/internal/config"
	"spider/internal/handler"
	"spider/internal/llm"
	"spider/internal/model"
	"spider/internal/plugin"
	"spider/internal/plugins/githubcollector"
	"spider/internal/plugins/tgcollector"
	"spider/internal/plugins/webcollector"
	"spider/internal/processor"
	"spider/internal/search"
	"spider/internal/store"
	"spider/internal/task"
	"spider/internal/telegram"

	"github.com/redis/go-redis/v9"
	"gorm.io/driver/mysql"
	"gorm.io/gorm"
)

func main() {
	// 1. Load config
	cfg, err := config.Load("configs/config.yaml")
	if err != nil {
		log.Fatalf("load config: %v", err)
	}

	// 2. Connect MySQL
	dsn := fmt.Sprintf("%s:%s@tcp(%s:%d)/%s?charset=utf8mb4&parseTime=True&loc=Local",
		cfg.MySQL.User, cfg.MySQL.Password, cfg.MySQL.Host, cfg.MySQL.Port, cfg.MySQL.Database)
	db, err := gorm.Open(mysql.Open(dsn), &gorm.Config{})
	if err != nil {
		log.Fatalf("connect mysql: %v", err)
	}

	// 3. AutoMigrate (5 tables)
	err = db.AutoMigrate(
		&model.Keyword{},
		&model.Channel{},
		&model.MerchantRaw{},
		&model.MerchantClean{},
		&model.TaskLog{},
	)
	if err != nil {
		log.Fatalf("automigrate: %v", err)
	}
	log.Println("MySQL tables migrated")

	// 4. Connect Redis
	rdb := redis.NewClient(&redis.Options{
		Addr:     fmt.Sprintf("%s:%d", cfg.Redis.Host, cfg.Redis.Port),
		Password: cfg.Redis.Password,
		DB:       cfg.Redis.DB,
	})
	log.Println("Redis connected")

	// 5. Initialize store
	s := store.New(db)

	// 6. Initialize external clients
	var llmClient *llm.Client
	if cfg.LLM.APIKey != "" {
		llmClient = llm.New(cfg.LLM.BaseURL, cfg.LLM.APIKey, cfg.LLM.Model, 30*time.Second)
	}

	var serperClient *search.SerperClient
	if cfg.Serper.APIKey != "" {
		serperClient = search.NewSerperClient(cfg.Serper.APIKey, cfg.Serper.ResultsPerPage, cfg.Serper.MaxPages)
	}

	tgAccounts := make([]telegram.Account, 0, len(cfg.Telegram.Accounts))
	for _, a := range cfg.Telegram.Accounts {
		tgAccounts = append(tgAccounts, telegram.Account{
			Phone:       a.Phone,
			SessionFile: a.SessionFile,
			AppID:       cfg.Telegram.AppID,
			AppHash:     cfg.Telegram.AppHash,
		})
	}
	tgManager := telegram.NewAccountManager(tgAccounts, rdb)

	// 7. Register plugins
	registry := plugin.NewRegistry()
	registry.Register(webcollector.New(serperClient))
	registry.Register(tgcollector.New(tgManager, llmClient, s))
	registry.Register(githubcollector.New(cfg.GitHub.Token, s))

	// 8. Initialize processor
	proc := processor.NewProcessor(s)

	// 9. Initialize task manager
	taskMgr := task.NewManager(db, rdb, registry, s, proc)

	// 10. Start HTTP server
	r := handler.SetupRouter(s, taskMgr)

	addr := handler.ServerAddr(cfg.Server.Port)
	log.Printf("Server starting on %s", addr)
	if err := r.Run(addr); err != nil {
		log.Fatalf("gin run: %v", err)
	}
}