涉及问题

各种包例如：golang.org/x/net, golang/x/test 等的下载问题，其实 golang 在 github 上建立了一个镜像库，如 https://github.com/golang/net 即是 https://golang.org/x/net 的镜像库。

golang 与数据库交互

golang 文件读写

golang 多线程使用

golang 字符编码的转换

js 标签选择器

参考

系列文章 https://www.cnblogs.com/majianguo/p/8146110.html

start

js 选择器的问题

1）. 一个controller 可以定义多个OnHTML 回调函数 2）. "div > p" --> div 的直接子元素, "div p" --> div 下的所有子元素

golang数据库连接问题

参考： https://blog.csdn.net/webxscan/article/details/70174658

import ("database/sql"_ "github.com/go-sql-driver/mysql"
)//数据库配置
const (userName    = "root"password    = ""ip          = "127.0.0.1"port        = "3306"dbName      = "dbName"
)path := strings.Join([]string{userName, ":", password, "@tcp(",ip, ":", port, ")/", dbName, "?charset=utf8"}, "")DB, _ := sql.Open("mysql", path)
//验证连接
if errConn := DB.Ping(); errConn != nil{fmt.Println("open database fail")return
}
fmt.Println("connnect success")
defer DB.Close()stmt, err := DB.Prepare("insert into user(name age) values(?, ?)")
if err != nil {fmt.Println(err)
}
res, err := stmt.Exec("username", 18)
if err != nil {fmt.Println(err)
}
// 获取新插入行的id
fmt.Println(res.LastInsertId())

golang文件写入

var fileName = "./flag.txt"
var file *os.File
var err errorfunc main() {file = openFile(fileName)writeFile(file, "keep coding!!")
}func openFile(fileName string) *os.File {if checkFileIsExist(fileName) {//如果文件存在file, err = os.OpenFile(fileName, os.O_APPEND, 0666)} else {//创建文件file, err = os.Create(fileName)}check(err)return file
}func writeFile(file *os.File, content string) {writer := bufio.NewWriter(file)writer.WriteString(content)writer.WriteString("\r\n")writer.Flush()
}func check(e error) {if e != nil {panic(e)}
}func checkFileIsExist(filename string) bool {var exist = trueif _, err := os.Stat(filename); os.IsNotExist(err) {exist = false}return exist
}

golang字符编码的转换

抓取链家网数据时，网页本身是utf8编码，没有问题，可是抓取房天下的数据时，网页本身是gb2312，折腾了好一会， F**K

//src为要转换的字符串
func coverGBKToUTF8(src string) string {// 网上搜有说要调用translate函数的，实测不用return mahonia.NewDecoder("gbk").ConvertString(src)
}

golang多线程，控制消息同步

附上测试代码，便于理解

func main() {// 我们还可以创建一个带缓冲的channel：//c := make(chan int, 1024)// 从带缓冲的channel中读数据//for i:= range c {//}//此时，创建一个大小为1024的int类型的channel，即使没有读取方，写入方也可以一直往channel里写入，在缓冲区被填完之前都不会阻塞。chs := make([] chan int, 10)for i := 0; i < 10; i ++ {go func(i int) {chs[i] = make(chan int)count(chs[i], i)//fmt.Println("run thread ", i) // 打印要放在向信道发消息之前}(i)}for _, ch := range chs {value := <- chclose(ch)fmt.Println(value, " thread done")}fmt.Println("All done")
}func count(ch chan int, i int) {ch <- i // 向信道发消息的过程一定要放在协程内部，才不会被主进程阻塞
}

##最后，附上抓取放天下小区信息的源代码

package mainimport ("database/sql""fmt""github.com/PuerkitoBio/goquery""github.com/axgle/mahonia"_ "github.com/go-sql-driver/mysql""github.com/gocolly/colly""reflect""strconv""strings"
)//数据库配置
const (userName    = "root"password    = ""ip          = "127.0.0.1"port        = "3306"dbName      = "rixin"
)type XQinfo struct {name                stringaddr                stringarea                stringprice               stringimgs                stringpost_code           stringproperty_right      stringproperty_type       stringbuild_time          intdeveloper           stringbuild_type          stringbuild_area          stringbuild_struct        stringfloor_space         stringhouse_count         stringmanager_company     stringgreen_ratio         stringplot_ratio          stringproprety_fee        stringadditional_info     stringwater_supply        stringheat_supply         stringelec_supply         stringgas                 stringsecurity            stringenvironment         stringparking_space       stringother_info          stringdistrict            string      // 区
}var flagCh      = make(chan  int)
var infos       = make([]XQinfo, 0)
var count       = 1
var page        = 10func main() {path    := strings.Join([]string{userName, ":", password, "@tcp(",ip, ":", port, ")/", dbName, "?charset=utf8"}, "")max     := 10DB, _   := sql.Open("mysql", path)//验证连接if errConn := DB.Ping(); errConn != nil{fmt.Println("open database fail")return}fmt.Println("connnect success")defer DB.Close()for i := 0; i < max; i ++  {for j := 1; j <= page; j ++ {link := "https://tj.esf.fang.com/housing/__0_0_0_0_" + strconv.Itoa(i * 10 + j) + "_0_0_0/"go work(link, i * 10 + j)}}for {<- flagChcount++if count < max * page {fmt.Println("<- receive the " + strconv.Itoa(count) + " thread ending flag")} else {for _, info := range infos{insertDB(DB, info)}fmt.Println("insert DB == ", len(infos))break}}fmt.Println("All "+ strconv.Itoa(count) + " has done")
}func work(url string, page int) {c := colly.NewCollector()// 小区列表页c.OnHTML("body", func(e *colly.HTMLElement) {e.DOM.Find(".plotListwrap").Each(func(i int, selection *goquery.Selection) {info        := XQinfo{}link, _     := selection.Find(".plotListwrap > dt > a").Attr("href")// 行政区名称//district    := selection.Find(".plotListwrap > dd > p:nth-child(2) > a:nth-child(1)").Text();//info.district = districtdetailVisit(info, link)})})c.OnRequest(func(r *colly.Request) {fmt.Println("Visiting", r.URL.String())})c.OnScraped(func(response *colly.Response) {fmt.Println("the " + strconv.Itoa(page) + " thread sending end flag ->")flagCh <- 1})c.Visit(url)
}func detailVisit(info XQinfo, link string) {detailLink          := colly.NewCollector()detailController    := detailLink.Clone()detailLink.OnHTML("body", func(e *colly.HTMLElement) {// 详情链接link1, _    := e.DOM.Find("#kesfxqxq_A01_03_01 > a").Attr("href")// 图片地址img, _      := e.DOM.Find(".bannerbg_pos > a > img").Attr("src")info.imgs   = img// 价格//price       := e.DOM.Find(".prib").Text()//info.price  = price + "元/㎡   "detailController.OnHTML("body", func(e *colly.HTMLElement) {// 小区名称name        := e.DOM.Find(".ceninfo_sq > h1 > a").Text()info.name   = coverString(name)price       := e.DOM.Find(".detaiLtop > dl:nth-child(1) > dd > span").Text()info.price  = price + "元/㎡   "e.DOM.Find(".inforwrap").Each(func(i int, selection *goquery.Selection) {// 模块名称modelName := coverString(selection.Prev().Find("h3").Text())switch modelName {case "基本信息":dealInfo(selection, &info)case "配套设施":dealInfo(selection, &info)case "周边信息":selection.Find("dl dt").Each(func(_ int, otherSelect *goquery.Selection) {tab := coverString(otherSelect.Text())del := strings.Index(tab, "本段合作")if del == -1 {info.other_info = info.other_info + tab + "|"}})}})infos = append(infos, info)})// 访问小区详情detailController.Visit(link1)})detailLink.Visit(link)
}// 处理小区基础信息
func dealInfo(selection *goquery.Selection, info *XQinfo) {selection.Find("dl dd").Each(func(_ int, selectionbase *goquery.Selection) {setXQinfo(selectionbase, info)})selection.Find("dl dt").Each(func(_ int, selectionbase *goquery.Selection) {setXQinfo(selectionbase, info)})
}func setXQinfo(selectionbase *goquery.Selection, info *XQinfo) {orgKey  := coverString(selectionbase.Find("strong").Text())index   := strings.Index(orgKey, "：")var key stringif index > 0 {key = orgKey[:index]} else {key = orgKey}var value stringvar fullValue stringvalue,ok := selectionbase.Attr("title")if ok {value = coverString(value)} else {fullValue   = coverString(selectionbase.Text())value       = fullValue[strings.Index(fullValue, "：") + 3:]}switch key {case "小区地址":info.addr           = valuecase "所属区域":info.area           = valueinfo.district       = value[:strings.Index(value, " ")]case "邮编":info.post_code       = valuecase "产权描述":info.property_right  = valuecase "物业类别":info.property_type   = valuecase "建筑年代":year,_              := strconv.Atoi(string([]rune(value)[:4]))info.build_time      = yearcase "开 发 商":info.developer      = valuecase "建筑结构":info.build_struct    = valuecase "建筑类型":info.build_type      = valuecase "建筑面积":info.build_area      = valuecase "占地面积":info.floor_space     = valuecase "物业公司":info.manager_company  = valuecase "房屋总数":info.house_count     = valuecase "绿 化 率":info.green_ratio     = valuecase "容 积 率":info.plot_ratio      = valuecase "物 业 费":info.proprety_fee    = valuecase "附加信息":info.additional_info = valuecase "供水":info.water_supply    = valuecase "供暖":info.heat_supply     = valuecase "供电":info.elec_supply     = valuecase "燃气":info.gas            = valuecase "安全管理":info.security       = valuecase "卫生服务":info.environment    = valuecase "停 车 位":info.parking_space   = value}
}//src为要转换的字符串
func coverGBKToUTF8(src string) string {return mahonia.NewDecoder("gbk").ConvertString(src)
}func replaceNullHtml(src string) string {temp := strings.Replace(src, "聽", "", -1)temp = strings.Replace(temp, "小区网", "", -1)return temp
}func coverString(src string) string {return replaceNullHtml(coverGBKToUTF8(src))
}func insertDB(DB *sql.DB, info XQinfo) {t := reflect.TypeOf(info)v := reflect.ValueOf(info)sql1 := "insert into rx_xiaoqu_more("sql2 := ") values ("sql3 := ")"for i := 0; i < t.NumField(); i++ {sql1 = sql1 + t.Field(i).Nameif t.Field(i).Name == "build_time"  {sql2 = fmt.Sprintf("%s%s%d%s", sql2, "'", v.Field(i), "'")} else {sql2 = fmt.Sprintf("%s%s%s%s", sql2, "'", v.Field(i), "'")}if i != t.NumField() - 1 {sql1 = sql1 + ", "sql2 = sql2 + ", "}}stmt, err := DB.Prepare(sql1 + sql2 + sql3)if err != nil {fmt.Println(sql1 + sql2 + sql3)fmt.Println(err)}res, err := stmt.Exec()if err != nil {fmt.Println(sql1 + sql2 + sql3)fmt.Println(err)}fmt.Println(res.LastInsertId())
}

首次 golang爬虫插件gocolly/colly 使用经历相关推荐

Golang网络爬虫框架gocolly/colly（三）
熟悉了<Golang 网络爬虫框架gocolly/colly 一>和<Golang 网络爬虫框架gocolly/colly 二>之后就可以在网络上爬取大部分数据了.本文接下来将 ...
Golang 网络爬虫框架gocolly/colly
gocolly是Golang实现的网络爬虫框架,名列go版爬虫程序榜首. 安装 go get -u github.com/gocolly/colly/... 例子 import ( "fmt ...
Golang 网络爬虫框架gocolly/colly 四
爬虫靠演技,表演得越像浏览器,抓取数据越容易,这是我多年爬虫经验的感悟.回顾下个人的爬虫经历,共分三个阶段:第一阶段,09年左右开始接触爬虫,那时由于项目需要,要访问各大国际社交网站,Facebook ...
Golang爬虫框架 colly 简介
Golang爬虫框架 colly 简介 colly是一个采用Go语言编写的Web爬虫框架,旨在提供一个能够些任何爬虫/采集器/蜘蛛的简介模板,通过Colly.你可以轻松的从网站提取结构化数据,然后进行 ...
非零基础自学Golang 第17章 HTTP编程(上) 17.3 爬虫框架gocolly 17.3.1 gocolly简介
非零基础自学Golang 文章目录非零基础自学Golang 第17章 HTTP编程(上) 17.3 爬虫框架gocolly 17.3.1 gocolly简介第17章 HTTP编程(上) 17.3 ...
golang爬虫框架colly简单介绍
colly一款快速优雅的golang爬虫框架,简单易用,功能完备. colly 官网地址:http://go-colly.org/ colly github地址:github.com/gocolly/ ...
Go语言爬虫框架之Colly和Goquery
文章目录写在前面 Go语言爬虫框架之Colly和Goquery 网络爬虫爬虫的简单算法 Colly 开始 OnHTML OnRequest / OnResponse HTMLElement Bri ...
go 爬虫框架 - gocolly
colly 是 Go 实现的比较有名的一款爬虫框架,而且 Go 在高并发和分布式场景的优势也正是爬虫技术所需要的.它的主要特点是轻量.快速,设计非常优雅,并且分布式的支持也非常简单,易于扩展. 使用 ...
Golang爬虫终极杀器——Chromedp让你成为二维码登陆终结者（教程）
Golang爬虫终极杀器--Chromedp让你成为二维码登陆终结者(教程) Github源码 - chromedp 文章项目Gitee源码 1 Chromedp是什么 chromedp是一个更快.更 ...
Golang爬虫语言接入代理
golang语言也是爬虫中的一种框架语言.当然很多网络爬虫新手都会面临选择什么语言适合于爬虫.一般很多爬虫用户都会选择python和java框架语言来写爬虫程序从而进行采集数据.其实除了python和 ...

首次 golang爬虫插件gocolly/colly 使用经历

参考