package main import ( "fmt" "log" "time" "spider/internal/config" "spider/internal/handler" "spider/internal/llm" "spider/internal/model" "spider/internal/plugin" "spider/internal/plugins/githubcollector" "spider/internal/plugins/tgcollector" "spider/internal/plugins/webcollector" "spider/internal/processor" "spider/internal/search" "spider/internal/store" "spider/internal/task" "spider/internal/telegram" "github.com/redis/go-redis/v9" "gorm.io/driver/mysql" "gorm.io/gorm" ) func main() { // 1. Load config cfg, err := config.Load("configs/config.yaml") if err != nil { log.Fatalf("load config: %v", err) } // 2. Connect MySQL dsn := fmt.Sprintf("%s:%s@tcp(%s:%d)/%s?charset=utf8mb4&parseTime=True&loc=Local", cfg.MySQL.User, cfg.MySQL.Password, cfg.MySQL.Host, cfg.MySQL.Port, cfg.MySQL.Database) db, err := gorm.Open(mysql.Open(dsn), &gorm.Config{}) if err != nil { log.Fatalf("connect mysql: %v", err) } // 3. AutoMigrate (5 tables) err = db.AutoMigrate( &model.Keyword{}, &model.Channel{}, &model.MerchantRaw{}, &model.MerchantClean{}, &model.TaskLog{}, ) if err != nil { log.Fatalf("automigrate: %v", err) } log.Println("MySQL tables migrated") // 4. Connect Redis rdb := redis.NewClient(&redis.Options{ Addr: fmt.Sprintf("%s:%d", cfg.Redis.Host, cfg.Redis.Port), Password: cfg.Redis.Password, DB: cfg.Redis.DB, }) log.Println("Redis connected") // 5. Initialize store s := store.New(db) // 6. Initialize external clients var llmClient *llm.Client if cfg.LLM.APIKey != "" { llmClient = llm.New(cfg.LLM.BaseURL, cfg.LLM.APIKey, cfg.LLM.Model, 30*time.Second) } var serperClient *search.SerperClient if cfg.Serper.APIKey != "" { serperClient = search.NewSerperClient(cfg.Serper.APIKey, cfg.Serper.ResultsPerPage, cfg.Serper.MaxPages) } tgAccounts := make([]telegram.Account, 0, len(cfg.Telegram.Accounts)) for _, a := range cfg.Telegram.Accounts { tgAccounts = append(tgAccounts, telegram.Account{ Phone: a.Phone, SessionFile: a.SessionFile, AppID: cfg.Telegram.AppID, AppHash: cfg.Telegram.AppHash, }) } tgManager := telegram.NewAccountManager(tgAccounts, rdb) // 7. Register plugins registry := plugin.NewRegistry() registry.Register(webcollector.New(serperClient)) registry.Register(tgcollector.New(tgManager, llmClient, s)) registry.Register(githubcollector.New(cfg.GitHub.Token, s)) // 8. Initialize processor proc := processor.NewProcessor(s) // 9. Initialize task manager taskMgr := task.NewManager(db, rdb, registry, s, proc) // 10. Start HTTP server r := handler.SetupRouter(s, taskMgr) addr := handler.ServerAddr(cfg.Server.Port) log.Printf("Server starting on %s", addr) if err := r.Run(addr); err != nil { log.Fatalf("gin run: %v", err) } }