对于同一个站点内部的页面应该设一个数据结构以保存,并且方便查找。这样,当爬虫系统得到了一个新的URL以后,可以快速的转换地址格式并进行连接。
void * urlparser(void *none){ Surl *url = NULL; Url *ourl = NULL; map::const_iterator itr; while(1) { pthread_mutex_lock(&sq_lock); while (surl_queue.empty()) { pthread_cond_wait(&sq_cond, &sq_lock); } url = surl_queue.front(); surl_queue.pop(); pthread_mutex_unlock(&sq_lock); ourl = surl2ourl(url); itr = host_ip_map.find(ourl->domain); if (itr == host_ip_map.end()) { /* not found */ /* dns resolve */ event_base * base = event_init(); evdns_init(); evdns_resolve_ipv4(ourl->domain, 0, dns_callback, ourl); event_dispatch(); event_base_free(base); } else { ourl->ip = strdup(itr->second.c_str()); push_ourlqueue(ourl); } } //evdns_base_free(dnsbase, 0); //event_base_free(base); return NULL;}