Libcurl HTTP 关于libcurl的相关函数介绍以及参数详见官方说明 curl HTTP Request 一个http请求包含方法、路径、http版本、请求包头请求方法 GET, HEAD, POST, PUT, DELETE, PATCH, OPTIONSGETcurl_easy_setopt(curl, CURLOPT_URL, "http://example.com?id=1")POSTcurl_easy_setopt(curl, CURLOPT_URL, "http://example.com"); curl_easy_setopt(curl, CURLOPT_POSTFIELDS, "username=admin&password=123");PUTcurl_easy_setopt(curl, CURLOPT_UPLOAD, 1L);DELETEcurl_easy_setopt(curl, CURLOPT_CUSTOMREQUEST, "DELETE"); curl_easy_setopt(curl, CURLOPT_URL, "https://example.com/file.txt");libcurl HTTP 请求包头 默认不设置的包头GET /file1.txt HTTP/1.1 Host: localhost Accept: */*POST /file1.txt HTTP/1.1 Host: localhost Accept: */* Content-Length: 6 Content-Type: application/x-www-form-urlencoded设置自定义包头添加一个包头 Name:Mikestruct curl_slist *list = NULL; list = curl_slist_append(list, "Name: Mike"); curl_easy_setopt(curl, CURLOPT_HTTPHEADER, list); curl_easy_perform(curl); curl_slist_free_all(list); /* free the list again */修改一个包头 Host: Alternativestruct curl_slist *list = NULL; list = curl_slist_append(list, "Host: Alternative"); curl_easy_setopt(curl, CURLOPT_HTTPHEADER, list); curl_easy_perform(curl); curl_slist_free_all(list); /* free the list again */删除一个包头struct curl_slist *list = NULL; list = curl_slist_append(list, "Accept:"); curl_easy_setopt(curl, CURLOPT_HTTPHEADER, list); curl_easy_perform(curl); curl_slist_free_all(list); /* free the list again */提供一个没有内容的包头struct curl_slist *list = NULL; list = curl_slist_append(list, "Moo;"); curl_easy_setopt(curl, CURLOPT_HTTPHEADER, list); curl_easy_perform(curl); curl_slist_free_all(list); /* free the list again */HTTP Response HTTP response code常见HTTP响应码Code Meaning1xx Transient code, a new one follows2xx Things are OK3xx The content is somewhere else4xx Failed because of a client problem5xx Failed because of a server problem获取相应状态码long code;curl_easy_getinfo(curl, CURLINFO_RESPONSE_CODE, &code);除了check curl请求的状态 check服务端返回过来的response statusCodeHTTP Range 如果客户端只希望从远程资源中获得前200个字节，或者在中间某个地方需要300个字节时//只取前两百个字节 curl_easy_setopt(curl, CURLOPT_RANGE, "0-199"); //从索引200开始的文件中的所有内容 curl_easy_setopt(curl, CURLOPT_RANGE, "200-"); //从索引0获取200字节，从索引1000获取200字节 curl_easy_setopt(curl, CURLOPT_RANGE, "0-199,1000-199");HTTP Authentication 用户名密码验证curl_easy_setopt(curl, CURLOPT_USERNAME, "joe"); curl_easy_setopt(curl, CURLOPT_PASSWORD, "secret");验证请求客户端并不能决定发送一个验证请求，当服务有资源需要被保护并且需要请求验证，服务器将会返回一个401和WWW-Authenticate: header，这个header包含了特定的验证方式curl_easy_setopt(curl, CURLOPT_HTTPAUTH, CURLAUTH_BASIC); curl_easy_setopt(curl, CURLOPT_HTTPAUTH, CURLAUTH_DIGEST); curl_easy_setopt(curl, CURLOPT_HTTPAUTH, CURLAUTH_NTLM); curl_easy_setopt(curl, CURLOPT_HTTPAUTH, CURLAUTH_NEGOTIATE);Cookie with libcurl cookie验证当启用cookie engine时 cookie将存储在相关联的句柄内存中读取cookiecurl_easy_setopt(easy, CURLOPT_COOKIEFILE, "cookies.txt");从服务端获取cookie并保存curl_easy_setopt(easy, CURLOPT_COOKIEJAR, "cookies.txt");稍后使用curl_easy_cleanup（）关闭easy句柄时，所有已知的cookie将被写入给定文件。该文件格式是浏览器曾经使用过的众所周知的“ Netscape cookie文件”格式。设置自定义cookie 不激活cookie enginecurl_easy_setopt(easy, CURLOPT_COOKIE, "name=daniel; present=yes;");在其中设置的字符串是将在HTTP请求中发送的原始字符串，并且应采用NAME = VALUE重复序列的格式； 包括分号分隔符cookie内存区存储大量cookieAdd a cookie to the cookie store#define SEP "\\t" /* Tab separates the fields */ char *my_cookie = "example.com" /* Hostname */ SEP "FALSE" /* Include subdomains */ SEP "/" /* Path */ SEP "FALSE" /* Secure */ SEP "0" /* Expiry in epoch time format. 0 == Session */ SEP "foo" /* Name */ SEP "bar"; /* Value */ curl_easy_setopt(curl, CURLOPT_COOKIELIST, my_cookie);Get all cookies from the cookie storestruct curl_slist *cookies curl_easy_getinfo(easy, CURLINFO_COOKIELIST, &cookies); /*这将返回指向cookie的链接列表的指针，并且每个cookie（再次）被指定为cookie文件格式的一行。该列表已分配给您，因此在应用程序处理完该信息后，请不要忘记调用curl_slist_free_all*/Cookie store commands//清除整个内存记录 curl_easy_setopt(curl, CURLOPT_COOKIELIST, "ALL"); //从内存中删除所有会话cookie（无有效期的cookie) curl_easy_setopt(curl, CURLOPT_COOKIELIST, "SESS"); //强制将所有cookie写入先前使用CURLOPT_COOKIEJAR指定的文件名 curl_easy_setopt(curl, CURLOPT_COOKIELIST, "FLUSH"); //从先前用CURLOPT_COOKIEFILE指定的文件名强制重新加载cookie curl_easy_setopt(curl, CURLOPT_COOKIELIST, "RELOAD"); DownloadGET方法请求资源easy = curl_easy_init(); curl_easy_setopt(easy, CURLOPT_URL, "http://example.com/"); curl_easy_perform(easy);如果使用其他方法请求资源向切随后换回get请求curl_easy_setopt(easy, CURLOPT_HTTPGET, 1L);下载相应的返回包头数据 用CURLOPT_HEADEReasy = curl_easy_init(); curl_easy_setopt(easy, CURLOPT_HEADER, 1L); curl_easy_setopt(easy, CURLOPT_URL, "http://example.com/"); curl_easy_perform(easy);或者将包头数据存储到文件easy = curl_easy_init(); FILE *file = fopen("headers", "wb"); curl_easy_setopt(easy, CURLOPT_HEADERDATA, file); curl_easy_setopt(easy, CURLOPT_URL, "http://example.com/"); curl_easy_perform(easy); fclose(file);在开发时设置详细模式，将同时显示发送到stderr的传出和传入标头：curl_easy_setopt(easy, CURLOPT_VERBOSE, 1L);获取一个页面存储到内存#include <stdio.h> #include <stdlib.h> #include <string.h> #include <curl/curl.h> struct MemoryStruct { char *memory; size_t size; }; static size_t WriteMemoryCallback(void *contents, size_t size, size_t nmemb, void *userp) { size_t realsize = size * nmemb; struct MemoryStruct *mem = (struct MemoryStruct *)userp; mem->memory = (char*)realloc(mem->memory, mem->size + realsize + 1); if(mem->memory == NULL) { return 0; } memcpy(&(mem->memory[mem->size]), contents, realsize); mem->size += realsize; mem->memory[mem->size] = 0; return realsize; } int main(void) { CURL *curl_handle; CURLcode res; struct MemoryStruct chunk; chunk.memory = malloc(1); chunk.size = 0; curl_global_init(CURL_GLOBAL_ALL); //整个程序全局只需要初始化一次 多次初始化会出问题 curl_handle = curl_easy_init(); curl_easy_setopt(curl_handle, CURLOPT_URL, "https://www.example.com/"); curl_easy_setopt(curl_handle, CURLOPT_WRITEFUNCTION, WriteMemoryCallback); curl_easy_setopt(curl_handle, CURLOPT_WRITEDATA, (void *)&chunk); //curl_easy_setopt(curl_handle, CURLOPT_USERAGENT, "libcurl-agent/1.0"); res = curl_easy_perform(curl_handle); if(res != CURLE_OK) { fprintf(stderr, "curl_easy_perform() failed: %s\n", curl_easy_strerror(res)); } else { printf("%lu bytes retrieved\n", (long)chunk.size); } curl_easy_cleanup(curl_handle); free(chunk.memory); curl_global_cleanup(); return 0; }获取一个文件流的回调函数//下载文件 static size_t Writeback(void* pBuffer, size_t nSize, size_t nMemByte, void* pParam) { //把下载到的数据以追加的方式写入文件(一定要有a，否则前面写入的内容就会被覆盖了) FILE* fp = NULL; fopen_s(&fp, "E:\\test_1.wav", "ab+"); size_t nWrite = fwrite(pBuffer, nSize, nMemByte, fp); fclose(fp); return nWrite; } //如果未设置此回调，则默认情况下libcurl使用'fwriteUploadloadhttp postHTTP请求将数据传递到服务器的标准方法，使用libcurl，通常可以提供该数据作为指针和长度curl_easy_setopt(easy, CURLOPT_POSTFIELDS, dataptr); curl_easy_setopt(easy, CURLOPT_POSTFIELDSIZE, (long)datalength);使用自定义回调函数将不断调用回调函数，直到所有数据都传输或者传输失败 如果未设置此回调，则默认情况下libcurl使用'fread'#include <stdio.h> #include <fcntl.h> #include <sys/stat.h> #include <curl/curl.h> //回调函数的签名 必须按照这个签名实现自己的回调函数 static size_t read_callback(void *ptr, size_t size, size_t nmemb, void *stream) { size_t retcode; curl_off_t nread; retcode = fread(ptr, size, nmemb, stream); nread = (curl_off_t)retcode; fprintf(stderr, "*** We read %" CURL_FORMAT_CURL_OFF_T " bytes from file\n", nread); return retcode; } int main(int argc, char **argv) { CURL *curl; CURLcode res; FILE * hd_src; struct stat file_info; char *file; char *url; if(argc < 3) return 1; file = argv[1]; url = argv[2]; stat(file, &file_info); hd_src = fopen(file, "rb"); curl_global_init(CURL_GLOBAL_ALL); curl = curl_easy_init(); if(curl) { /* we want to use our own read function */ curl_easy_setopt(curl, CURLOPT_READFUNCTION, read_callback); /* enable uploading */ curl_easy_setopt(curl, CURLOPT_UPLOAD, 1L); /* HTTP PUT please */ curl_easy_setopt(curl, CURLOPT_PUT, 1L); /* specify target URL, and note that this URL should include a file name, not only a directory */ curl_easy_setopt(curl, CURLOPT_URL, url); /* now specify which file to upload */ curl_easy_setopt(curl, CURLOPT_READDATA, hd_src); /* provide the size of the upload, we specicially typecast the value to curl_off_t since we must be sure to use the correct data size */ curl_easy_setopt(curl, CURLOPT_INFILESIZE_LARGE, (curl_off_t)file_info.st_size); /* Now run off and do what you've been told! */ res = curl_easy_perform(curl); /* Check for errors */ if(res != CURLE_OK) fprintf(stderr, "curl_easy_perform() failed: %s\n", curl_easy_strerror(res)); /* always cleanup */ curl_easy_cleanup(curl); } fclose(hd_src); /* close the local file */ curl_global_cleanup(); return 0; }HTTP PUT CURLOPT_INFILESIZE_LARGE设置上传的大小curl_easy_setopt(easy, CURLOPT_UPLOAD, 1L); curl_easy_setopt(easy, CURLOPT_INFILESIZE_LARGE, (curl_off_t) size); curl_easy_setopt(easy, CURLOPT_READFUNCTION, read_callback); curl_easy_setopt(easy, CURLOPT_URL, "https://example.com/handle/put"); /*如果不知道在传输之前不知道传输的大小，对于HTTP 1.0，您必须事先提供大小，HTTP 1.1 你可以添加 标头 Transfer-Encoding: chunked 对于HTTP 2及更高版本，则既不需要大小也不需要额外的标头*/

C++ redis connection pool 学习teamtalk服务端源码，把redis连接池的实现记录下,用到了hiredis三方库和头文件。整个redis缓存池是三个类实现的，redis-manager,redis-pool,redis-conn。{lamp/}main函数获取redis-manager对象实例CacheManager* CacheManager::getInstance() { if (!s_cache_manager) { s_cache_manager = new CacheManager(); if (s_cache_manager->Init()) { delete s_cache_manager; s_cache_manager = NULL; } } return s_cache_manager; } main() { //初始化redis CacheManager* pCacheManager = CacheManager::getInstance(); if (!pCacheManager) { log("CacheManager init failed"); return -1; } }通过redis-manager类init函数读取配置文件，创建对应的缓存池对象，并初始化缓存池，每个缓存池有一个缓存池名对应一个redis-pool指针加入到map中管理。int CacheManager::Init() { CConfigFileReader config_file("dbproxyserver.conf"); //CacheInstances=unread,group_set,token,sync,group_member char* cache_instances = config_file.GetConfigName("CacheInstances"); if (!cache_instances) { log("not configure CacheIntance"); return 1; } char host[64]; char port[64]; char db[64]; char maxconncnt[64]; CStrExplode instances_name(cache_instances, ','); for (uint32_t i = 0; i < instances_name.GetItemCnt(); i++) { char* pool_name = instances_name.GetItem(i); //printf("%s", pool_name); snprintf(host, 64, "%s_host", pool_name); snprintf(port, 64, "%s_port", pool_name); snprintf(db, 64, "%s_db", pool_name); snprintf(maxconncnt, 64, "%s_maxconncnt", pool_name); char* cache_host = config_file.GetConfigName(host); char* str_cache_port = config_file.GetConfigName(port); char* str_cache_db = config_file.GetConfigName(db); char* str_max_conn_cnt = config_file.GetConfigName(maxconncnt); if (!cache_host || !str_cache_port || !str_cache_db || !str_max_conn_cnt) { log("not configure cache instance: %s", pool_name); return 2; } CachePool* pCachePool = new CachePool(pool_name, cache_host, atoi(str_cache_port), atoi(str_cache_db), atoi(str_max_conn_cnt)); if (pCachePool->Init()) { log("Init cache pool failed"); return 3; } m_cache_pool_map.insert(make_pair(pool_name, pCachePool)); } return 0; }缓存池init的时候会创建redis连接对象，默认数量为2，并将创建的redis连接对象加入到缓存池的成员变量空闲redis连接数组中，等待被取用。int CachePool::Init() { for (int i = 0; i < m_cur_conn_cnt; i++) { CacheConn* pConn = new CacheConn(this); if (pConn->Init()) { delete pConn; return 1; } m_free_list.push_back(pConn); } log("cache pool: %s, list size: %lu", m_pool_name.c_str(), m_free_list.size()); return 0; }redis连接对象init的时候主要用于连接的初始化和重连操作，每次对redis增删改查的之前都会调用init这个函数，每4秒尝试重连一次。/* * redis初始化连接和重连操作，类似mysql_ping() */ int CacheConn::Init() { if (m_pContext) { return 0; } // 4s 尝试重连一次 uint64_t cur_time = (uint64_t)time(NULL); if (cur_time < m_last_connect_time + 4) { return 1; } m_last_connect_time = cur_time; // 200ms超时 struct timeval timeout = {0, 200000}; m_pContext = redisConnectWithTimeout(m_pCachePool->GetServerIP(), m_pCachePool->GetServerPort(), timeout); if (!m_pContext || m_pContext->err) { if (m_pContext) { log("redisConnect failed: %s", m_pContext->errstr); redisFree(m_pContext); m_pContext = NULL; } else { log("redisConnect failed"); } return 1; } redisReply* reply = (redisReply *)redisCommand(m_pContext, "SELECT %d", m_pCachePool->GetDBNum()); if (reply && (reply->type == REDIS_REPLY_STATUS) && (strncmp(reply->str, "OK", 2) == 0)) { freeReplyObject(reply); return 0; } else { log("select cache db failed"); return 2; } }整个初始化的过程大概结束，下面就是通过redis-manager从缓存池中获取释放redis连接的过程。缓存池名是通过配置文件中读入的，获取缓存池中的redis连接需要传入缓存池名，通过map获取到缓存池名对应的缓存池对象指针，并调用其成员函数getCacheConn获取redis连接。CacheConn* CacheManager::GetCacheConn(const char* pool_name) { map<string, CachePool*>::iterator it = m_cache_pool_map.find(pool_name); if (it != m_cache_pool_map.end()) { return it->second->GetCacheConn(); } else { return NULL; } }调用缓存池CachePool成员函数getCacheConn获取redis连接这块代码有可学之处，他用了pthread库里面的信号量来优雅地等待空闲连接数组中的连接对象，为了线程安全需要在访问空闲连接数组的时候加锁，不断判断数组中是否存在可用的redis连接对象，如果没有可用的连接，则将判断此时正在使用的redis连接对象个数是否超过指定最大值，超过则将用信号量阻塞，直到有空闲的连接被唤醒。如果此时连接数小于指定最大值，则将继续创建redis连接对象并初始化一个可用的redis连接对象加入到空闲redis连接数组中。如果数组中有可用的redis连接则将从数组头部取出一个元素并返回。CacheConn* CachePool::GetCacheConn() { m_free_notify.Lock(); while (m_free_list.empty()) { if (m_cur_conn_cnt >= m_max_conn_cnt) { m_free_notify.Wait(); } else { CacheConn* pCacheConn = new CacheConn(this); int ret = pCacheConn->Init(); if (ret) { log("Init CacheConn failed"); delete pCacheConn; m_free_notify.Unlock(); return NULL; } else { m_free_list.push_back(pCacheConn); m_cur_conn_cnt++; log("new cache connection: %s, conn_cnt: %d", m_pool_name.c_str(), m_cur_conn_cnt); } } } CacheConn* pConn = m_free_list.front(); m_free_list.pop_front(); m_free_notify.Unlock(); return pConn; }释放redis连接对象，传入需要释放的redis连接对象指针，加锁，遍历redis连接对象数组中是否存在该连接对象，不存在则将连接对象加入到该空闲redis连接数组中，信号量通知唤醒被阻塞的线程，redis连接数组中有空闲连接可用。void CachePool::RelCacheConn(CacheConn* pCacheConn) { m_free_notify.Lock(); list<CacheConn*>::iterator it = m_free_list.begin(); for (; it != m_free_list.end(); it++) { if (*it == pCacheConn) { break; } } if (it == m_free_list.end()) { m_free_list.push_back(pCacheConn); } m_free_notify.Signal(); m_free_notify.Unlock(); }

C++ thread pool 学习teamtalk服务端源码，记录下线程池的实现。主要是实现的类分为任务类(抽象类)、线程池类、工作线程类、线程同步类。 Task抽象任务类 虚基类为了能够处理不同任务#ifndef __TASK_H__ #define __TASK_H__ class CTask { public: CTask(){} virtual ~CTask(){} virtual void run() = 0; private: }; #endif /*defined(__TASK_H__) */线程池类threadPoolinit初始化并启动工作线程int CThreadPool::Init(uint32_t worker_size) { m_worker_size = worker_size; m_worker_list = new CWorkerThread [m_worker_size]; if (!m_worker_list) { return 1; } for (uint32_t i = 0; i < m_worker_size; i++) { m_worker_list[i].SetThreadIdx(i); m_worker_list[i].Start(); } return 0; }工作线程创建线程，遍历任务队列并处理。这里遍历任务队列的方式和遍历redis空闲连接队列一样，用一个锁和信号量优雅地等待任务。void* CWorkerThread::StartRoutine(void* arg) { CWorkerThread* pThread = (CWorkerThread*)arg; pThread->Execute(); return NULL; } void CWorkerThread::Start() { (void)pthread_create(&m_thread_id, NULL, StartRoutine, this); } void CWorkerThread::Execute() { while (true) { m_thread_notify.Lock(); // put wait in while cause there can be spurious wake up (due to signal/ENITR) while (m_task_list.empty()) { m_thread_notify.Wait(); } CTask* pTask = m_task_list.front(); m_task_list.pop_front(); m_thread_notify.Unlock(); pTask->run(); delete pTask; m_task_cnt++; //log("%d have the execute %d task\n", m_thread_idx, m_task_cnt); } }初始化工作做完说一下add task，通过线程池对象添加任务，根据线程在线程数组中的索引随机添加任务到一个随机线程中，调用工作线程的pushtask function,加锁并唤醒等待任务而阻塞的线程解锁。void CThreadPool::AddTask(CTask* pTask) { /* * select a random thread to push task * we can also select a thread that has less task to do * but that will scan the whole thread list and use thread lock to get each task size */ uint32_t thread_idx = random() % m_worker_size; m_worker_list[thread_idx].PushTask(pTask); } void CWorkerThread::PushTask(CTask* pTask) { m_thread_notify.Lock(); m_task_list.push_back(pTask); m_thread_notify.Signal(); m_thread_notify.Unlock(); }线程同步类通过封装pthread的互斥量和信号量实现构造函数负责初始化互斥量和信号量，析构函数负责destroy操作CThreadNotify::CThreadNotify() { pthread_mutexattr_init(&m_mutexattr); pthread_mutexattr_settype(&m_mutexattr, PTHREAD_MUTEX_RECURSIVE); pthread_mutex_init(&m_mutex, &m_mutexattr); pthread_cond_init(&m_cond, NULL); } CThreadNotify::~CThreadNotify() { pthread_mutexattr_destroy(&m_mutexattr); pthread_mutex_destroy(&m_mutex); pthread_cond_destroy(&m_cond); }实现线程同步lock、unlock、wait、signalvoid Lock() { pthread_mutex_lock(&m_mutex); } void Unlock() { pthread_mutex_unlock(&m_mutex); } void Wait() { pthread_cond_wait(&m_cond, &m_mutex); } void Signal() { pthread_cond_signal(&m_cond); }在dbproxy服务中用到了线程池，proxy连接类处理proxy任务时如下：static CThreadPool g_thread_pool; int init_proxy_conn(uint32_t thread_num) { //省略其余代码 g_thread_pool.Init(thread_num); } void CProxyConn::HandlePduBuf(uchar_t* pdu_buf, uint32_t pdu_len) { //省略其余代码 CTask* pTask = new CProxyTask(m_uuid, handler, pPdu); g_thread_pool.AddTask(pTask); }proxytask继承了task抽象类，并重写了run方法，使用task抽象类可以处理任何任务，代码质量很高。{lamp/}