Socket编程Http下载的简单实现 - Mr.DejaVu - 博客园

<C/C++> Socket编程Http下载的简单实现

下载原理: 网上介绍很多,就是按照Http协议,使用Socket连接并发送请求头给Http服务器,若服务器正确响应,返回请求文件数据,接收并写文件保存.至于Http协议的请求头及响应头的格式,这里不再赘述,请Google之.

实现: 为此,我封装了一个HttpDownload类.先上代码...(基于WinSock,移植时请注意部分函数)

(HttpDownload.h)

 1 #ifndef HTTP_DOWNLOAD_H
 2 #define HTTP_DOWNLOAD_H
 3
 4 #include <cstdio>
 5 #include <string>
 6 #include <winsock2.h>
 7
 8 class HttpDownload {
 9 public:
10     HttpDownload(const char* hostAddr, const int port,
11                  const char* getPath, const char* saveFileName);
12     ~HttpDownload();
13     bool start();
14     void cancel();
15     void getPos(ULONGLONG& totalSize, ULONGLONG& downloadSize);
16 protected:
17     bool initSocket();                        //初始化Socket
18     bool sendRequest();                        //发送请求头
19     bool receiveData();                        //接收数据
20     bool closeTransfer();                    //关闭传输
21 private:
22     std::string m_hostAddr;                    //目标主机IP
23     int m_port;                                //HTTP端口号
24     std::string m_getPath;                    //目标文件相对路径
25     std::string m_saveFileName;                //保存文件路径
26     SOCKET m_sock;                            //Socket
27     FILE* m_fp;                                //保存文件指针
28     ULONGLONG m_fileTotalSize;                //目标文件总大小
29     ULONGLONG m_receivedDataSize;            //已接收数据大小
30     bool m_cancelFlag;                        //取消下载标记
31 };
32
33 #endif    //HTTP_DOWNLOAD_H

(HttpDownload.cpp)

  1 #include "HttpDownload.h"
  2
  3 #define BUFFER_SIZE 1024
  4
  5 HttpDownload::HttpDownload(const char* hostAddr, const int port, const char* getPath, const char* saveFileName)
  6 {
  7     m_hostAddr = hostAddr;
  8     m_port = port;
  9     m_getPath = getPath;
 10     m_saveFileName = saveFileName;
 11     m_sock = 0;
 12     m_fp = NULL;
 13     m_fileTotalSize = 1;    //没给0,因为分母
 14     m_receivedDataSize = 0;
 15     m_cancelFlag = false;
 16 }
 17
 18 HttpDownload::~HttpDownload()
 19 {
 20
 21 }
 22
 23 bool HttpDownload::initSocket()
 24 {
 25     m_sock = socket(AF_INET, SOCK_STREAM, 0);
 26     if (m_sock < 0)
 27         return false;
 28
 29     //设置Socket为非阻塞模式
 30     unsigned long mode = 1;
 31     if (ioctlsocket(m_sock, FIONBIO, &mode) < 0)
 32         return false;
 33
 34     if (m_hostAddr.empty())
 35         return false;
 36
 37     struct sockaddr_in destaddr;
 38     destaddr.sin_family = AF_INET;
 39     destaddr.sin_port = htons(m_port);
 40     destaddr.sin_addr.s_addr = inet_addr(m_hostAddr.c_str());
 41
 42     int nRet = connect(m_sock, (struct sockaddr*)&destaddr, sizeof(destaddr));
 43     if (nRet == 0)    //如果立即连接成功
 44         return true;
 45     //虽直接返回,但未立即成功,用select等待看socket是否可写来判断connect是否成功
 46     if (WSAGetLastError() != WSAEWOULDBLOCK)
 47         return false;
 48     int retryCount = 0;
 49     while(1)
 50     {
 51         fd_set writeSet, exceptSet;
 52         FD_ZERO(&writeSet);
 53         FD_SET(m_sock, &writeSet);
 54         exceptSet = writeSet;
 55
 56         struct timeval timeout;
 57         timeout.tv_sec = 3;
 58         timeout.tv_usec = 0;
 59
 60         int err = select((int)m_sock+1, NULL, &writeSet, &exceptSet, &timeout);
 61         if (err < 0)    //出错
 62             break;
 63         if (err == 0)    //超时
 64         {
 65             if (++retryCount > 10)    //重试10次
 66                 return false;
 67             continue;
 68         }
 69         if (FD_ISSET(m_sock, &writeSet))
 70             return true;
 71         if (FD_ISSET(m_sock, &exceptSet))
 72             break;
 73     }
 74     return false;
 75 }
 76
 77 bool HttpDownload::sendRequest()
 78 {
 79     if (m_getPath.empty())
 80         return false;
 81
 82     char requestHeader[256];
 83     //格式化请求头
 84     int len = sprintf(requestHeader,
 85         "GET %s HTTP/1.1\r\n"
 86         "Host: %s\r\n"
 87         "Range: bytes=%I64d-\r\n"    //从m_receivedDataSize位置开始
 88         "Connection: close\r\n"
 89         "\r\n",
 90         m_getPath.c_str(), m_hostAddr.c_str(), m_receivedDataSize);
 91
 92     int nSendBytes = 0;    //已发送字节数
 93     while(1)
 94     {
 95         fd_set writeSet;
 96         FD_ZERO(&writeSet);
 97         FD_SET(m_sock, &writeSet);
 98
 99         struct timeval timeout;
100         timeout.tv_sec = 3;
101         timeout.tv_usec = 0;
102
103         int err = select((int)m_sock+1, NULL, &writeSet, NULL, &timeout);
104         if (err < 0)
105             break;
106         if (err == 0)
107             continue;
108         int nBytes = send(m_sock, requestHeader+nSendBytes, len, 0);
109         if (nBytes < 0)
110         {
111             if (WSAGetLastError() != WSAEWOULDBLOCK)
112                 break;
113             nBytes = 0;
114         }
115         nSendBytes += nBytes;    //若一次未发完,累计,循环send
116         len -= nBytes;
117         if (len == 0)
118             return true;
119     }
120     return false;
121 }
122
123 bool HttpDownload::receiveData()
124 {
125     char responseHeader[BUFFER_SIZE] = {0};
126
127     struct timeval timeout;
128     timeout.tv_sec = 3;
129     timeout.tv_usec = 0;
130
131     //接收响应头
132     int retryCount = 0;
133     int nRecvBytes = 0;    //已接收字节数
134     while (1)
135     {
136         fd_set readSet;
137         FD_ZERO(&readSet);
138         FD_SET(m_sock, &readSet);
139         int nRet = select(m_sock+1, &readSet, NULL, NULL, &timeout);
140         if (nRet < 0)    //出错
141             return false;
142         if (nRet == 0)    //超时
143         {
144             if (++retryCount > 10)
145                 return false;
146             continue;
147         }
148         retryCount = 0;
149         if (recv(m_sock, responseHeader+nRecvBytes, 1, 0) <= 0)
150             return false;
151         nRecvBytes++;
152         if (nRecvBytes >= BUFFER_SIZE)
153             return false;
154         if (nRecvBytes >= 4 &&
155             responseHeader[nRecvBytes-4]=='\r' && responseHeader[nRecvBytes-3]=='\n' &&
156             responseHeader[nRecvBytes-2]=='\r' && responseHeader[nRecvBytes-1]=='\n')
157             break;
158     }
159     responseHeader[nRecvBytes] = '\0';
160
161     if (strncmp(responseHeader, "HTTP/", 5))
162         return false;
163     int status = 0;
164     float version = 0.0;
165     ULONGLONG startPos, endPos, totalLength;
166     startPos = endPos = totalLength = 0;
167     if (sscanf(responseHeader, "HTTP/%f %d ", &version, &status) != 2)
168         return false;
169     char* findStr = strstr(responseHeader, "Content-Range: bytes ");
170     if (findStr == NULL)
171         return false;
172     if (sscanf(findStr, "Content-Range: bytes %I64d-%I64d/%I64d",
173         &startPos, &endPos, &totalLength) != 3)
174         return false;
175     if (status != 200 && status != 206 || totalLength == 0)
176         return false;
177     if (m_fileTotalSize == 1)    //第一次获取HTTP响应头,保存目标文件总大小
178         m_fileTotalSize = totalLength;
179     if (m_receivedDataSize != startPos)
180         return false;
181
182     //接收目标文件数据
183     retryCount = 0;
184     while (1)
185     {
186         char buf[BUFFER_SIZE] = {0};
187         fd_set readSet;
188         FD_ZERO(&readSet);
189         FD_SET(m_sock, &readSet);
190
191         int nRet = select((int)m_sock+1, &readSet, NULL, NULL, &timeout);
192         if (nRet < 0)
193             break;
194         if (nRet == 0) {
195             if (++retryCount > 10)
196                 break;
197             continue;
198         }
199         int length = recv(m_sock, buf, BUFFER_SIZE, 0);
200         if(length < 0)    //出错
201             return false;
202         if (length == 0)    //socket被优雅关闭
203             return true;
204         size_t written = fwrite(buf, sizeof(char), length, m_fp);
205         if(written < length)
206             return false;
207         m_receivedDataSize += length;
208         if (m_receivedDataSize == m_fileTotalSize)    //文件接收完毕
209         {
210             return true;
211         }
212     }
213     return false;
214 }
215
216 bool HttpDownload::closeTransfer()
217 {
218     if (m_sock > 0) {
219         if (closesocket(m_sock) < 0)
220             return false;
221         m_sock = 0;
222     }
223     else
224         m_sock = 0;
225     return true;
226 }
227
228 bool HttpDownload::start()
229 {
230     m_fp = fopen(m_saveFileName.c_str(), "wb");    //创建文件
231     if (m_fp == NULL)
232         return false;
233     bool errFlag = false;
234     while(1)
235     {
236         if (!initSocket() || !sendRequest() || !receiveData())
237         {
238             if (m_cancelFlag)
239             {
240                 errFlag = true;
241                 break;
242             }
243             if (!closeTransfer())
244             {
245                 errFlag = true;
246                 break;
247             }
248             Sleep(1000);
249             continue;
250         }
251         break;
252     }
253     if(m_fp != NULL)
254     {
255         fclose(m_fp);
256         m_fp = NULL;
257     }
258     if (errFlag)
259         return false;
260     return true;
261 }
262
263 void HttpDownload::cancel()
264 {
265     m_cancelFlag = true;
266     closeTransfer();
267 }
268
269 void HttpDownload::getPos(ULONGLONG& totalSize, ULONGLONG& downloadSize)
270 {
271     totalSize = m_fileTotalSize;
272     downloadSize = m_receivedDataSize;
273 }

其中4个主要函数功能如下:

1) initSocket() : 初始化Socket->设置Socket为非阻塞模式->connect()

值得注意的是,这里我采用了非阻塞的select模型.相对来说非阻塞模式可以减少开销,增加错误控制能力,显得更灵活.因此,Line 42-73

connect之,因为非阻塞,立即返回,得到返回值判断,通常不会立即成功,然后

while(1) {

socket加入写描述符集

用select检测写写描述符集,延时3秒,可写就返回true

否则,重复10次(总共30秒),如果仍不成功,认为connect失败,返回false

}

2) sendRequest() : 格式化Http请求字符串->用send发送之.send依然使用非阻塞select模型判断,同上.

注意的是:这里Http请求字符串里的"Range: bytes="字段用m_receivedDataSize来格式化,此成员变量用于保存请求的目标文件开始位置.目的是为实现断点续传,若传输文件过程中网络异常后,可重新发送请求头,则只需从已下载位置之后继续传输.

3) receiveData() : recv接收Http响应头字符串->取出响应头中的信息(如文件大小)->若响应信息正确,开始recv接收目标文件数据->写文件

recv依然使用非阻塞select模型判断,同上.

4) closeTransfer() : 关闭套接字.(这里因在windows下,所以使用closesocket)

因为代码中多有注释,细节就不再多解释了.

另外,给出3个接口函数:

1) start() : 创建文件->分别依次调用initSocket(),sendRequest(),receiveData()->关闭文件,套接字

在一个循环,不断判断initSocket(),sendRequest(),receiveData()是否成功,若任一失败(网络异常),调用closeTransfer(),然后重新来,直到下载完毕或被cancel()中断

2) cancel() : 关闭套接字,并将m_cancelFlag置true

3) getPos() : 用于得到当前文件下载的进度

最后,附上源码,包含一个实现下载的控制台程序例子(MinGW编译),上图

(Main.cpp)

View Code

  1 #include <cstdio>
  2 #include "pthread.h"
  3 #include "HttpDownload.h"
  4 #include "InitWinSocket.h"
  5
  6 InitWinSocket init;
  7 const char* g_progName = NULL;
  8 const char* g_saveFileName = "savefilename";
  9
 10 void usage() {
 11     printf("Usage: %s http://www.xxx.com/filename %s\n", g_progName, g_saveFileName);
 12 }
 13
 14 void progressBar(float percent) {
 15     const int numTotal = 50;
 16     int numShow = (int)(numTotal * percent);
 17     if (numShow == 0)
 18         numShow = 1;
 19     if (numShow > numTotal)
 20         numShow = numTotal;
 21     char sign[numTotal+1] = {0};
 22     memset(sign, '=', numTotal);
 23     printf("\r%.2f%%\t[%-*.*s]", percent*100, numTotal, numShow, sign);
 24     fflush(stdout);
 25     if (numShow == numTotal)
 26         printf("\n");
 27 }
 28
 29 void parseURL(const char* url, char* hostAddr, int& port, char* getPath) {
 30     if (url == NULL || hostAddr == NULL || getPath == NULL)
 31         return;
 32     const char* temp = strstr(url, "http://");
 33     if (temp == NULL)
 34         return;
 35     const char* hostStart = temp + strlen("http://");
 36     const char* colon = strchr(hostStart, ':');
 37     if (colon != NULL)    //表示存在冒号,有端口号
 38         sscanf(hostStart, "%[^:]:%d%s", hostAddr, &port, getPath);
 39     else
 40         sscanf(hostStart, "%[^/]%s", hostAddr, getPath);
 41     //通过主机名转IP地址
 42     struct hostent* hostEntry;
 43     hostEntry = gethostbyname(hostAddr);
 44     if (hostEntry == NULL)
 45     {
 46         printf("Hostname not available!\n");
 47         return;
 48     }
 49     struct in_addr inAddr = {0};
 50     memcpy(&inAddr.s_addr, hostEntry->h_addr, sizeof(inAddr.s_addr));
 51     strcpy(hostAddr, inet_ntoa(inAddr));
 52 }
 53
 54 void* task(void* arg) {
 55     HttpDownload* pObj = (HttpDownload*)arg;
 56     if (pObj->start())
 57         return ((void*)1);
 58     else
 59         return ((void*)0);
 60 }
 61
 62 int main(int argc, char** argv) {
 63     g_progName = strrchr(argv[0], '\\');
 64     if (g_progName != NULL)
 65         g_progName += 1;
 66     else
 67         g_progName = argv[0];
 68     if (argc != 3 || strncmp(argv[1], "http://", strlen("http://")) != 0) {
 69         usage();
 70         return -1;
 71     }
 72     g_saveFileName = argv[2];
 73
 74     char hostAddr[256] = {0};
 75     int port = 80;
 76     char getPath[256] = {0};
 77     parseURL(argv[1], hostAddr, port, getPath);
 78
 79     HttpDownload obj(hostAddr, port, getPath, g_saveFileName);    //创建下载类对象
 80     pthread_t tid;
 81     int err = pthread_create(&tid, NULL, task, &obj);
 82     if (err != 0)
 83         printf("Start Download Failed!\n");
 84     else
 85         printf("Start Downloading...\n");
 86
 87     ULONGLONG totalSize = 1;
 88     ULONGLONG downloadSize = 0;
 89     float percent = 0;
 90     while (1) {
 91         obj.getPos(totalSize, downloadSize);
 92         percent = downloadSize/(float)totalSize;
 93         progressBar(percent);
 94         if (downloadSize == totalSize)
 95             break;
 96         Sleep(500);
 97     }
 98
 99     void* ret = NULL;
100     pthread_join(tid, &ret);
101     if (ret)
102         printf("Download Finished.\n");
103     else
104         printf("Download Failed!\n");
105     return 0;
106 }

附件: 源码下载