golang 实现漏洞监控02——github 历史漏洞信息爬取

func getGithubHeader() map[string]string{    token := cfg.Github.GithubToken    header := map[string]string{"Authorization":"token "+ token,}return header  }

url: https://docs.github.com/v3/search/title:"Search - GitHub Docs"description:"Use the REST API to search for specific items on GitHub."host: docs.github.comfavicon:/assets/cb-345/images/site/favicon.pngimage: https://github.githubassets.com/images/modules/open_graph/github-logo.png

func getCVEInfo(cve string)(exists bool, desc, cvss2, cvss3, cna string, err error){    url := nvdURL + cve      res, err := http.Get(url)if err !=nil{       fmt.Println("Error establishing connect:", err)returnfalse,"","","","", err  }    defer res.Body.Close()    doc, err := goquery.NewDocumentFromReader(res.Body)if err !=nil{       fmt.Println("Error parsing html:", err)returnfalse,"","","","", err  }//exists//cvss3 //cna//cvss2//descreturn exists, desc, cvss2, cvss3, cna,nil}

func fetchGithub(year, page int, wg *sync.WaitGroup, countchan chan struct{}) {      defer wg.Done()      /*    ...    data handle    ...    */       err = database.InsertGithubDB(cve, desc, date_published, cvss2, cvss3, cna, poc_url)         if err != nil {            fmt.Println("Error insert vul:", cve)            continue         }         countchan <- struct{}{}  //记录插入条数    }  }

func ConcurrentGitHubCrawler() {      var wg, mainWG sync.WaitGroup      var totalCount int       for year := startYear; year <= endYear; year++ {         mainWG.Add(1)         yearCount := 0         countChan := make(chan struct{})         for page := 1; page <= 10; page++ {            wg.Add(1)            go fetchGithub(year, page, &wg, countChan)         }         go func() {            wg.Wait()            close(countChan)            mainWG.Done()         }()         for range countChan {            yearCount++         }         totalCount += yearCount      }      mainWG.Wait()      fmt.Printf("Github: Successfully insert %d vul info", totalCount)  }