library("XML")

#获取全部的链接
url <- 'http://www.csdn.net/tag/'
i_url_parse<-htmlParse(url,encoding="UTF-8")
xpath_1 <- "//div[@class='overflow']/a"
node<-getNodeSet(i_url_parse,xpath_1)for(j in 1:length(node)){    value2<-xmlGetAttr(node[[j]],name='href')print(value2)
}handler <- function(url){xpath_1 <- "//div[@class='line_list']/*"i_url_parse<-htmlParse(url,encoding="UTF-8");node<-getNodeSet(i_url_parse,xpath_1)#get all divfor(j in 1:length(node)){ value <- xmlGetAttr(node[[j]],name='class');if(value == 'tit_list'){tital <- xmlValue(node[[j]])}else if (value == 'dwon_words'){xpath_2 <- "//div[@class='dwon_words']/span[@class='tag_source']/a"node1 <-getNodeSet(i_url_parse,xpath_2)for(k in 1: length(node1)){value <- xmlGetAttr(node1[[k]],name='href');text <- xmlValue(node1[[k]]);}}}xpath_3<-"//div[@class='page_nav']/a[@class='pageliststy']"node3 <-getNodeSet(i_url_parse,xpath_3)for(m in 1: length(node3)){value1 <- xmlValue(node3[[m]])if(value1 == '下一页'){next_url = xmlGetAttr(node3[[m]],name='href');next_url <- paste('http://www.csdn.net',next_url,sep="")print(next_url)Sys.sleep(25)handler(next_url)}}}＃处理news
url <- 'http://www.csdn.net/tag/android/news'xpath_3<-"//div[@class='page_nav']/a[@class='pageliststy']"node3 <-getNodeSet(i_url_parse,xpath_3)for(m in 1: length(node3)){value1 <- xmlValue(node3[[m]])if(value1 == '下一页'){next_url = xmlGetAttr(node3[[m]],name='href');print(next_url)handler(next_url)}}

rm(list=ls())
gc()
library(bitops)
library(RCurl)
library(curl)
library(XML)
alphabet <- c("A","B","C","D","E","F","G","H","I","J","K","L","M","N","O","P","Q","R","S","T","U","V","W","X","Y","Z","#")
tags <- data.frame('字母分类具体标签'=NA)
temp <- data.frame('循环多个标签临时'=NA)
result <- data.frame('文章标题'=NA,'文章来源'=NA,'文章标签'=NA,'文章发布者'=NA,'字母分类'=NA,'字母分类具体标签'=NA)
x <- getURLContent("http://www.csdn.net/tag/",encoding="UTF-8")
url_tag <- htmlParse(x, encoding="UTF-8")
num <- 0
for (i in 1:27){xpath_tag <- paste("/html/body/div/div/ul/li[",i,"]/div/div/a",sep="")node_tag <- getNodeSet(url_tag, xpath_tag)m <- length(node_tag)print(paste(alphabet[i],"字母分类有",m,"个具体标签！",sep="   "))if(m==0){print(paste("没有找到",alphabet[i],"字母分类的具体标签相关节点！~~~",sep="   "))}else{for (j in 1:m){tags[j,] <- xmlValue(node_tag[[j]])k <- 0for(k in 1:10000){url <- paste("http://www.csdn.net/tag/",tags[j,1],"/news-",k,sep="")y <- getURLContent(url,encoding="UTF-8")url_news <- htmlParse(y, encoding="UTF-8")node_not_exists <- getNodeSet(url_news,"//div[@class='not_search']")if(length(node_not_exists)!=0){break}else{node_news <- getNodeSet(url_news, "//div[@class='line_list']")n <- length(node_news)if(n==0){print(paste("我,",alphabet[i],"字母分类具体标签",tags[j,1],"的资讯不存在！~~~",sep="   "))}else{print(paste(alphabet[i],"字母分类的具体标签",tags[j,1],"第",k,"页有",n,"条资讯！",sep="   "))for(p in 1:10){num <- num+1node_title <- getNodeSet(url_news,paste('/html/body/div/div/div[3]/div[2]/ul/li[',p,']/div/a',sep=""))node_source <- getNodeSet(url_news,paste('/html/body/div/div/div[3]/div[2]/ul/li[',p,']/div/div/span[1]/a',sep=""))node_tags <- getNodeSet(url_news,paste('/html/body/div/div/div[3]/div[2]/ul/li[',p,']/div/div/span[2]/a',sep=""))node_author <- getNodeSet(url_news,paste('/html/body/div/div/div[3]/div[2]/ul/li[',p,']/div/div/span[3]/a',sep=""))result[num,1] <- xmlValue(node_title[[1]])result[num,2] <- xmlValue(node_source[[1]])for(q in 1:length(node_tags)){temp[q,1] <- xmlValue(node_tags[[q]])}result[num,3] <- paste(temp[1],sep=",")result[num,4] <- xmlValue(node_author[[1]])result[num,5] <- alphabet[i]result[num,6] <- tags[j,1]}if(num <= 10){write.table(result[1:num,1:6],file=paste(alphabet[i],".r",sep=""),append=TRUE,col.names=TRUE)}else{write.table(result[(num-n+1):num,1:6],file=paste(alphabet[i],".r",sep=""),append=TRUE,col.names=FALSE)}Sys.sleep(2) #The time interval to suspend execution for, in seconds.rm(result)Sys.sleep(1) #The time interval to suspend execution for, in seconds.gc()result <- data.frame('文章标题'=NA,'文章来源'=NA,'文章标签'=NA,'文章发布者'=NA,'字母分类'=NA,'字母分类具体标签'=NA)print(paste(alphabet[i],"字母分类的具体标签",tags[j,1],"第",k,"页的",n,"条资讯已抓取完毕！",sep="   "))print(paste("字母分类i数字:",i,"字母分类的具体标签j数字:",j,"资讯页码k数字:",k,"资讯条目n数字:",n,"已抓取条数num数字:",num,sep=""))}}}}}
}