ter = array_value($config, 'filter');
    $arr = array_value($filter, $type);
    $enable = array_value($arr, 'enable');
    $wordarr = array_value($arr, 'keyword');

    if (0 == $enable || empty($wordarr)) return FALSE;

    foreach ($wordarr as $_keyword) {
        if (!$_keyword) continue;
        $r = strpos(strtolower($keyword), strtolower($_keyword));
        if (FALSE !== $r) {
            $error = $_keyword;
            return TRUE;
        }
    }
    return FALSE;
}

// return http://domain.com OR https://domain.com
function url_prefix()
{
    $http = ((isset($_SERVER['HTTPS']) && 'on' == $_SERVER['HTTPS']) || (isset($_SERVER['HTTP_X_FORWARDED_PROTO']) && $_SERVER['HTTP_X_FORWARDED_PROTO'] == 'https')) ? 'https://' : 'http://';
    return $http . $_SERVER['HTTP_HOST'];
}

// 唯一身份ID
function uniq_id()
{
    return uniqid(substr(md5(microtime(true) . mt_rand(1000, 9999)), 8, 8));
}

// 生成订单号 14位
function trade_no()
{
    $trade_no = str_replace('.', '', microtime(1));
    $strlen = mb_strlen($trade_no, 'UTF-8');
    $strlen = 14 - $strlen;
    $str = '';
    if ($strlen) {
        for ($i = 0; $i <= $strlen; $i++) {
            if ($i < $strlen) $str .= '0';
        }
    }
    return $trade_no . $str;
}

// 生成订单号 16位
function trade_no_16()
{
    $explode = explode(' ', microtime());
    $trade_no = $explode[1] . mb_substr($explode[0], 2, 6, 'UTF-8');
    return $trade_no;
}

// 当前年的天数
function date_year($time = NULL)
{
    $time = intval($time) ? $time : time();
    return date('L', $time) + 365;
}

// 当前年份中的第几天
function date_z($time = NULL)
{
    $time = intval($time) ? $time : time();
    return date('z', $time);
}

// 当前月份中的第几天，没有前导零 1 到 31
function date_j($time = NULL)
{
    $time = intval($time) ? $time : time();
    return date('j', $time);
}

// 当前月份中的第几天，有前导零的2位数字 01 到 31
function date_d($time = NULL)
{
    $time = intval($time) ? $time : time();
    return date('d', $time);
}

// 当前时间为星期中的第几天 数字表示 1表示星期一 到 7表示星期天
function date_w_n($time = NULL)
{
    $time = intval($time) ? $time : time();
    return date('N', $time);
}

// 当前日第几周
function date_d_w($time = NULL)
{
    $time = intval($time) ? $time : time();
    return date('W', $time);
}

// 当前几月 没有前导零1-12
function date_n($time = NULL)
{
    $time = intval($time) ? $time : time();
    return date('n', $time);
}

// 当前月的天数
function date_t($time = NULL)
{
    $time = intval($time) ? $time : time();
    return date('t', $time);
}

// 0 o'clock on the day
function clock_zero()
{
    return strtotime(date('Ymd'));
}

// 24 o'clock on the day
function clock_twenty_four()
{
    return strtotime(date('Ymd')) + 86400;
}

// 8点过期 / expired at 8 a.m.
function eight_expired($time = NULL)
{
    $time = intval($time) ? $time : time();
    // 当前时间大于8点则改为第二天8点过期
    $life = date('G') <= 8 ? (strtotime(date('Ymd')) + 28800 - $time) : clock_twenty_four() - $time + 28800;
    return $life;
}

// 24点过期 / expired at 24 a.m.
function twenty_four_expired($time = NULL)
{
    $time = intval($time) ? $time : time();
    $twenty_four = clock_twenty_four();
    $life = $twenty_four - $time;
    return $life;
}

/**
 * @param $url 提交地址
 * @param string $post POST数组 / 空为GET获取数据 / $post='GET'获取连续跳转最终URL
 * @param string $cookie cookie
 * @param int $timeout 超时
 * @param int $ms 设为1是毫秒
 * @return mixed    返回数据
 */
function https_request($url, $post = '', $cookie = '', $timeout = 30, $ms = 0)
{
    if (empty($url)) return FALSE;

    if (version_compare(PHP_VERSION, '5.2.3', '<')) {
        $ms = 0;
        $timeout = 30;
    }

    is_array($post) and $post = http_build_query($post);

    // 没有安装curl 使用http的形式，支持post
    if (!extension_loaded('curl')) {
        //throw new Exception('server not install CURL');
        if ($post) {
            return https_post($url, $post, $cookie, $timeout);
        } else {
            return http_get($url, $cookie, $timeout);
        }
    }

    is_array($cookie) and $cookie = http_build_query($cookie);
    $curl = curl_init();
    // 返回执行结果，不输出
    curl_setopt($curl, CURLOPT_RETURNTRANSFER, true);
    //php5.5跟php5.6中的CURLOPT_SAFE_UPLOAD的默认值不同
    if (class_exists('\CURLFile')) {
        curl_setopt($curl, CURLOPT_SAFE_UPLOAD, true);
    } else {
        defined('CURLOPT_SAFE_UPLOAD') and curl_setopt($curl, CURLOPT_SAFE_UPLOAD, false);
    }
    // 设定请求的RUL
    curl_setopt($curl, CURLOPT_URL, $url);
    // 设定返回信息中包含响应信息头
    if (ini_get('safe_mode') && ini_get('open_basedir')) {
        // $post参数必须为GET
        if ('GET' == $post) {
            // 安全模式时将头文件的信息作为数据流输出
            curl_setopt($curl, CURLOPT_HEADER, true);
            // 安全模式采用连续抓取
            curl_setopt($curl, CURLOPT_NOBODY, true);
        }
    } else {
        curl_setopt($curl, CURLOPT_HEADER, false);
        // 允许跳转10次
        curl_setopt($curl, CURLOPT_MAXREDIRS, 10);
        // 使用自动跳转，返回最后的Location
        curl_setopt($curl, CURLOPT_FOLLOWLOCATION, true);
    }
    $ua1 = 'Mozilla/5.0 (iPhone; CPU iPhone OS 13_2_3 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/13.0.3 Mobile/15E148 Safari/604.1';
    $ua = empty($_SERVER["HTTP_USER_AGENT"]) ? $ua1 : $_SERVER["HTTP_USER_AGENT"];
    curl_setopt($curl, CURLOPT_USERAGENT, $ua);
    // 兼容HTTPS
    if (FALSE !== stripos($url, 'https://')) {
        curl_setopt($curl, CURLOPT_SSL_VERIFYPEER, FALSE);
        curl_setopt($curl, CURLOPT_SSL_VERIFYHOST, FALSE);
        //ssl版本控制
        //curl_setopt($curl, CURLOPT_SSLVERSION, CURL_SSLVERSION_TLSv1);
        curl_setopt($curl, CURLOPT_SSLVERSION, true);
    }

    $header = array('Content-type: application/x-www-form-urlencoded;charset=UTF-8', 'X-Requested-With: XMLHttpRequest');
    $cookie and $header[] = "Cookie: $cookie";
    curl_setopt($curl, CURLOPT_HTTPHEADER, $header);

    if ($post) {
        // POST
        curl_setopt($curl, CURLOPT_POST, true);
        // 自动设置Referer
        curl_setopt($curl, CURLOPT_AUTOREFERER, true);
        curl_setopt($curl, CURLOPT_POSTFIELDS, $post);
    }

    if ($ms) {
        curl_setopt($curl, CURLOPT_NOSIGNAL, true); // 设置毫秒超时
        curl_setopt($curl, CURLOPT_TIMEOUT_MS, intval($timeout)); // 超时毫秒
    } else {
        curl_setopt($curl, CURLOPT_TIMEOUT, intval($timeout)); // 秒超时
    }
    //优先解析 IPv6 超时后IPv4
    //curl_setopt($curl, CURLOPT_IPRESOLVE, CURL_IPRESOLVE_V4);
    curl_setopt($curl, CURLOPT_ENCODING, 'gzip');
    // 返回执行结果
    $output = curl_exec($curl);
    // 有效URL，输出URL非URL页面内容 CURLOPT_RETURNTRANSFER 必须为false
    'GET' == $post and $output = curl_getinfo($curl, CURLINFO_EFFECTIVE_URL);
    curl_close($curl);
    return $output;
}

function save_image($img)
{
    $ch = curl_init();
    // 设定请求的RUL
    curl_setopt($ch, CURLOPT_URL, $img);
    // 设定返回信息中包含响应信息头 启用时会将头文件的信息作为数据流输出
    //curl_setopt($ch, CURLOPT_HEADER, false);
    //curl_setopt($ch, CURLOPT_USERAGENT, $_SERVER["HTTP_USER_AGENT"]);
    // true表示$html,false表示echo $html
    curl_setopt($ch, CURLOPT_RETURNTRANSFER, true);
    curl_setopt($ch, CURLOPT_CONNECTTIMEOUT, 10);
    curl_setopt($ch, CURLOPT_SSL_VERIFYHOST, false);
    curl_setopt($ch, CURLOPT_SSL_VERIFYPEER, false);
    //curl_setopt($ch, CURLOPT_BINARYTRANSFER, 1);
    //curl_setopt($ch, CURLOPT_FOLLOWLOCATION, 0);
    curl_setopt($ch, CURLOPT_ENCODING, 'gzip');
    $output = curl_exec($ch);
    curl_close($ch);
    return $output;
}

// 计算字串宽度:剧中对齐(字体大小/字串内容/字体链接/背景宽度/倍数)
function calculate_str_width($size, $str, $font, $width, $multiple = 2)
{
    $box = imagettfbbox($size, 0, $font, $str);
    return ($width - $box[4] - $box[6]) / $multiple;
}

// 搜索目录下的文件 比对文件后缀
function search_directory($path)
{
    if (is_dir($path)) {
        $paths = scandir($path);
        foreach ($paths as $val) {
            $sub_path = $path . '/' . $val;
            if ('.' == $val || '..' == $val) {
                continue;
            } else if (is_dir($sub_path)) {
                //echo '目录名:' . $val . '<br/>';
                search_directory($sub_path);
            } else {
                //echo ' 最底层文件: ' . $path . '/' . $val . ' <hr/>';
                $ext = strtolower(file_ext($sub_path));
                if (in_array($ext, array('php', 'asp', 'jsp', 'cgi', 'exe', 'dll'), TRUE)) {
                    echo '异常文件：' . $sub_path . ' <hr/>';
                }
            }
        }
    }
}

// 一维数组转字符串 $sign待签名字符串 $url为urlencode转码GET参数字符串
function array_to_string($arr, &$sign = '', &$url = '')
{
    if (count($arr) != count($arr, 1)) throw new Exception('Does not support multi-dimensional array to string');

    // 注销签名
    unset($arr['sign']);

    // 排序
    ksort($arr);
    reset($arr);

    // 转字符串做签名
    $url = '';
    $sign = '';
    foreach ($arr as $key => $val) {
        if (empty($val) || is_array($val)) continue;
        $url .= $key . '=' . urlencode($val) . '&';
        $sign .= $key . '=' . $val . '&';
    }
    $url = substr($url, 0, -1);
    $url = htmlspecialchars($url);
    $sign = substr($sign, 0, -1);
}

// 私钥生成签名
function rsa_create_sign($data, $key, $sign_type = 'RSA')
{
    if (!function_exists('openssl_sign')) throw new Exception('OpenSSL extension is not enabled');

    if (!defined('OPENSSL_ALGO_SHA256')) throw new Exception('Only versions above PHP 5.4.8 support SHA256');

    $key = wordwrap($key, 64, "\n", true);
    if (FALSE === $key) throw new Exception('Private Key Error');

    $key = "-----BEGIN RSA PRIVATE KEY-----\n$key\n-----END RSA PRIVATE KEY-----";

    if ('RSA2' == $sign_type) {
        openssl_sign($data, $sign, $key, OPENSSL_ALGO_SHA256);
    } else {
        openssl_sign($data, $sign, $key, OPENSSL_ALGO_SHA1);
    }

    // 加密
    return base64_encode($sign);
}

// 公钥验证签名
function rsa_verify_sign($data, $sign, $key, $sign_type = 'RSA')
{
    $key = wordwrap($key, 64, "\n", true);
    if (FALSE === $key) throw new Exception('Public Key Error');

    $key = "-----BEGIN PUBLIC KEY-----\n$key\n-----END PUBLIC KEY-----";

    // 签名正确返回1 签名不正确返回0 错误-1
    if ('RSA2' == $sign_type) {
        $result = openssl_verify($data, base64_decode($sign), $key, OPENSSL_ALGO_SHA256);
    } else {
        $result = openssl_verify($data, base64_decode($sign), $key, OPENSSL_ALGO_SHA1);
    }

    return $result === 1;
}

// Array to xml array('appid' => 'appid', 'code' => 'success')
function array_to_xml($arr)
{
    if (!is_array($arr) || empty($arr)) throw new Exception('Array Error');

    $xml = "<xml>";
    foreach ($arr as $key => $val) {
        if (is_numeric($val)) {
            $xml .= "<" . $key . ">" . $val . "</" . $key . ">";
        } else {
            $xml .= "<" . $key . "><![CDATA[" . $val . "]]></" . $key . ">";
        }
    }
    $xml .= "</xml>";
    return $xml;
}

// Xml to array
function xml_to_array($xml)
{
    if (!$xml) throw new Exception('XML error');

    $old = libxml_disable_entity_loader(true);

    // xml解析
    $result = (array)simplexml_load_string($xml, null, LIBXML_NOCDATA | LIBXML_COMPACT);
    // 恢复旧值
    if (FALSE === $old) libxml_disable_entity_loader(false);

    return $result;
}

// 逐行读取
function well_import($file)
{
    if ($handle = fopen($file, 'r')) {
        while (!feof($handle)) {
            yield trim(fgets($handle));
        }
        fclose($handle);
    }
}

// 计算总行数
function well_import_total($file, $key = 'well_import_total')
{
    static $cache = array();
    if (isset($cache[$key])) return $cache[$key];
    $count = cache_get($key);
    if (NULL === $count) {
        $count = 0;
        $globs = well_import($file);
        while ($globs->valid()) {
            ++$count;
            $globs->next(); // 指向下一个
        }
        $count and cache_set($key, $count, 300);
    }

    return $cache[$key] = $count;
}

$g_dir_file = FALSE;
function well_search_dir($path)
{
    global $g_dir_file;
    FALSE === $g_dir_file and $g_dir_file = array();
    if (is_dir($path)) {
        $paths = scandir($path);
        foreach ($paths as $val) {
            $sub_path = $path . '/' . $val;
            if ('.' == $val || '..' == $val) {
                continue;
            } else if (is_dir($sub_path)) {
                well_search_dir($sub_path);
            } else {
                $g_dir_file[] = $sub_path;
            }

        }
    }

    return $g_dir_file;
}




?><!DOCTYPE html><html lang="zh-cn"><head><meta http-equiv="Content-Type" content="text/html; charset=utf-8" /><meta http-equiv="X-UA-Compatible" content="IE=Edge,chrome=1" /><meta http-equiv="Cache-Control" content="no-transform" /><meta http-equiv="Cache-Control" content="no-siteapp" /><meta name="viewport"
          content="width=device-width, initial-scale=1.0, minimum-scale=1.0, maximum-scale=1.0, user-scalable=no"><meta name="referrer" content="origin-when-cross-origin"><meta name="applicable-device" content="pc,mobile" /><meta name="MobileOptimized" content="width" /><meta name="HandheldFriendly" content="true" /><meta name="renderer" content="webkit" /><meta name="keywords" content="大数据项目：职务分析（一）——数据获取" /><meta name="description" content="大数据项目：职务分析（一）——数据获取" /><title>大数据项目：职务分析（一）——数据获取-阿南达文事网
</title><link rel="shortcut icon" href="/view/img/favicon.ico" /><link rel="icon" sizes="32x32" href="/view/img/favicon.ico"><link rel="Bookmark" href="/view/img/favicon.ico" /><link rel="stylesheet" href="/view/template/mitiqin/css/style.css?2.3.0"><link rel="stylesheet" href="/view/template/mitiqin/css/iconfont.css?2.3.0"><script src="/view/template/mitiqin/js/jquery-2.2.4.min.js?2.3.0" type="text/javascript"></script></head><body><div id="header"><div class="header container"><div class="logo"><a href="/"  title="阿南达文事网"><img src="/view/template/mitiqin/img/logo.png" alt="阿南达文事网"></a></div><div id="monavber" class="nav" data-type="index" data-infoid><ul id="nav" class="navbar"><li class="navbar-item" data-active="fid-0"><a href="/">首页</a></li><li class="navbar-item" fid="1" data-active="fid-1"><a href="/wenke">文科汇编</a></li><li class="navbar-item" fid="4" data-active="fid-4"><a href="/IT">编程日记</a></li></ul></div><div id="mnav"><i class="iconfont icon-app"></i></div><div id="search"><i class="iconfont icon-search"></i></div><div class="search"><form id="form-search" action="/operate/search.html"><input type="text" name="keyword" placeholder="关键词" value=""/><button type="submit" value="搜索"><i class="iconfont icon-search"></i></button><input type="hidden" name="range" value="1"></form></div></div></div>
<div class="breadcrumb2 container-w"><div id="b" class="cl"><div class="fl"><i class="iconfont icon-home"></i><a href="/">首页</a>&nbsp;>&nbsp;
<a href="/IT">编程日记</a><a href="/IT/1731227755a1030586.html" title="首页">大数据项目：职务分析（一）——数据获取</a></div></div></div><div class="main container-w cl"><div class="mainl"><div class="art-post"><div class="title"><h1>大数据项目：职务分析（一）——数据获取</h1><div class="info"><span><a href="/IT"><i class="iconfont icon-crossborder-fill"></i>编程日记</a></span><span><i class="iconfont icon-browse"></i>63</span><span><i class="iconfont icon-message-fill"></i>0</span><span>
                        更新时间：2025-05-10 13:16:44</span></div></div><div class="art-content"><article class="single-post"><div class="entry"><p><p>
<h2>大数据项目：职务分析（一）——数据获取</h2>
<p>项目介绍&#xff1a;该项目适合学习的时候使用&#xff0c;因为项目比较小&#xff0c;主要目的对猎聘当中的各个岗位的数据的获取和简单的分析&#xff0c;从多个方面分析岗位之间的关系以及薪资的差异。</p> 
<p>采用的技术有&#xff1a;</p> 
<p>python爬虫&#xff1a;</p> 
<p>hadoop&#xff1a;hdfs存储数据</p> 
<p>hive on spark : 进行数据分析</p> 
<p>sqoop: 将分析的结果传输到关系型数据库当中</p> 
<p>superset&#xff1a;进行数据的可视化</p> 
<p></p> 
<p>首先是将数据从猎聘官网当中获取&#xff1a;</p> 
<p style="text-align:center;"></p> 
<p>爬取技术一栏当中的似是一个岗位对应的数据。</p> 
<p>先获得各个岗位的url&#xff0c;进行跳转&#xff0c;在每个网页当获取有用的信息&#xff1a;比如&#xff1a;岗位&#xff0c;地址&#xff0c;薪资&#xff0c;公司规模&#xff0c;要求掌握的技术&#xff0c;学历要求和经验要求&#xff0c;最后&#xff0c;对这一页的数据爬取完之后&#xff0c;进行跳转&#xff0c;通过find-element-by-xpath(),来锁定到<strong>下一页</strong>的链接上&#xff0c;跳转到下一页再进行数据的爬取&#xff0c;如此往复&#xff0c;从而&#xff0c;得到所有的想要的数据。 </p> 
<p></p> 
<p> 话不多说&#xff0c;代码实现为&#xff1a;</p> 
<pre><code class="language-python">from selenium import webdriver
from selenium.webdriver.chrome.webdriver import Options
from lxml import etree
import osdef share_brower():chrome_options &#61; Options()# chrome_options.add_argument(&#39;--headless&#39;)   # 来判断浏览器的前后台运行&#xff0c;有图形化可以更好的展现她的活动chrome_options.add_argument(&#39;--disable-gpu&#39;)path&#61;&#39;C:\Program Files (x86)\Google\Chrome\Application\chrome.exe&#39;chrome_options.binary_location &#61; pathbrower &#61; webdriver.Chrome(chrome_options&#61;chrome_options)return browerdef save(source, number, name1):tree &#61; etree.HTML(source)position &#61; tree.xpath(&#39;//ul/li//div[&#64;class&#61;&#34;job-title-box&#34;]/div[1]/text()&#39;)addr &#61; tree.xpath(&#39;//ul/li//div[&#64;class&#61;&#34;job-title-box&#34;]/div[2]/span[2]/text()&#39;)salary &#61; tree.xpath(&#39;//ul/li//div[&#64;class&#61;&#34;job-detail-header-box&#34;]/span/text()&#39;)company &#61; tree.xpath(&#39;//ul/li//div[&#64;class&#61;&#34;job-company-info-box&#34;]/span/text()&#39;)scale &#61; tree.xpath(&#39;//ul/li//div[&#64;class&#61;&#34;job-company-info-box&#34;]/div[&#64;class&#61;&#34;company-tags-box ellipsis-1&#34;]/span[last()]/text()&#39;)experience &#61; tree.xpath(&#39;//ul/li//div[&#64;class&#61;&#34;job-labels-box&#34;]/span[1]/text()&#39;)xueli &#61; tree.xpath(&#39;//ul/li//div[&#64;class&#61;&#34;job-labels-box&#34;]/span[2]/text()&#39;)keyword &#61; tree.xpath(&#39;//ul/li//div[&#64;class&#61;&#34;job-labels-box&#34;]/span/text()&#39;)mi &#61; min(len(position), len(addr), len(salary), len(company), len(scale), len(xueli), len(experience))with open(&#39;./date/&#39; &#43; name1.strip() &#43; &#34;/&#34; &#43; str(number) &#43; &#39;.csv&#39;, &#39;w&#39;, encoding&#61;&#39;utf-8&#39;) as fs:for l in range(mi):new &#61; position[l] &#43; &#39;,&#39; &#43; addr[l] &#43; &#39;,&#39; &#43; salary[l] &#43; &#39;,&#39; &#43; company[l] &#43; &#39;,&#39; &#43; scale[l]&#43;&#39;,&#39;&#43;experience[l]&#43;&#39;,&#39;&#43;xueli[l]&#43;&#39;\t\n&#39;fs.write(new)fs.close()with open(&#39;./keyword.txt&#39;, &#39;a&#39;, encoding&#61;&#39;utf-8&#39;) as fs:ne &#61; &#39;&#39;for i in keyword:ne &#61; ne &#43; i &#43; &#39; &#39;fs.write(ne)fs.close()base_url &#61; &#39;&#39;
brower &#61; share_brower()
brower.get(&#39;/&#39;)
brower.implicitly_wait(3)
page &#61; brower.page_source
tree &#61; etree.HTML(page)
name &#61; tree.xpath(&#39;//ul[&#64;class&#61;&#34;sidebar float-left&#34;]/li[1]//dd/a/text()&#39;)
url &#61; tree.xpath(&#39;//ul[&#64;class&#61;&#34;sidebar float-left&#34;]/li[1]//dd/a/&#64;href&#39;)
for i in range(len(name)):if not os.path.exists(&#39;./date/&#39;&#43;name[i]):os.mkdir(&#39;./date/&#39;&#43;name[i]) #创建文件夹brower.get(base_url&#43;url[i])brower.implicitly_wait(3)source &#61; brower.page_sourcenumber &#61; 1save(source, number, name[i])print(name[i])try:for j in range(9):element &#61; brower.find_element_by_xpath(&#39;//div[&#64;class&#61;&#34;list-pagination-box&#34;]//li[last()]/a&#39;)element.click()save(brower.page_source, number, name[i])number &#43;&#61; 1except RuntimeError:print(&#34;*&#34;*30&#43;&#34;有错误&#xff0c;但是可以执行的哦&#xff01;&#xff01;&#34;)continueelse:print(&#34;文件已经存在&#34;)os.rmdir(&#39;./date/&#39;&#43;name[i])continue
## //ul[&#64;class&#61;&#34;sidebar float-left&#34;]/li[1]//dd/a/text() 相关职业
# //ul[&#64;class&#61;&#34;sidebar float-left&#34;]/li[1]//dd/a/&#64;href  对应的连接 每个连接底下都有十个页面 、爬取当中的数据
# 数据的存放 总共有49个类别的技术岗位 分别放在49个问价夹底下&#xff0c;文件夹以对应的职业命名 底下十个文件&#xff0c;每个文件表示每一页的数据
# &#xff0c;文件的命名方式以1-10.csv ,保存的时候中间以逗号隔开&#xff0c;保存当当前的路径底下&#xff0c;然后爬取成功之后同意上传到大数据集
# 群的本地文件夹下面# //ul/li//div[&#64;class&#61;&#34;job-title-box&#34;]/div[1]/text()  职位
# //ul/li//div[&#64;class&#61;&#34;job-title-box&#34;]/div[2]/span[2]/text()  地址
# //ul/li//div[&#64;class&#61;&#34;job-detail-header-box&#34;]/span/text()   薪资
# //ul/li//div[&#64;class&#61;&#34;job-company-info-box&#34;]/span/text()   企业
# //ul/li//div[&#64;class&#61;&#34;job-company-info-box&#34;]/div[&#64;class&#61;&#34;company-tags-box ellipsis-1&#34;]
# /span[last()]/text() 公司规模
# //ul/li//div[&#64;class&#61;&#34;job-labels-box&#34;]/span[1]/text()   工作经验
# //ul/li//div[&#64;class&#61;&#34;job-labels-box&#34;]/span[2]/text()  招聘学历要求
# //ul/li//div[&#64;class&#61;&#34;job-labels-box&#34;]/span/text()   //用正则将数据的后序删除掉&#xff0c;或者在hadoop当中处理
# //div[&#64;class&#61;&#34;list-pagination-box&#34;]//li[last()] 下一页的标签 循环九次brower.quit()
# 最后退出</code></pre> 
<p> 最后结果为&#xff1a;</p> 
<p style="text-align:center;"></p> 
<p style="text-align:center;"> </p> 
<p style="text-align:center;"> </p> 
<p> 每一层和里面的数据保存形式&#xff0c;都如上所述&#xff0c;后序通过简单的mapreduce实现数据的处理&#xff0c;上传至hdfs当中&#xff0c;下期继续。。。。</p></div></article></div><div class="info"><div><span>本文发布于:2024-11-10，感谢您对本站的认可！</span></div><div><span>本文链接:</span><a href="http://anandasy.com/IT/1731227755a1030586.html" title="大数据项目：职务分析（一）——数据获取">http://anandasy.com/IT/1731227755a1030586.html</a></div><div><span>版权声明:本站内容均来自互联网，仅供演示用，请勿用于商业和其他非法用途。如果侵犯了您的权益请与我们联系，我们将在24小时内删除。</span></div></div><span class="tag"><i class="iconfont icon-discount" style="font-size:12px;font-weight:bold;opacity:.7;">本文标签：</i><a href="/tag/1027416.html" target="_blank">大数据项目职务分析（一）数据获取</a></span></div><div class="post_comments" id="comments"><div id="comt-respond" class="commentpost"><h4>发布评论
<span><a rel="nofollow" id="cancel-reply" href="#comment" style="display:none;"><small>取消回复</small></a></span></h4><form action="/comment/create/1030586.html?safe_token=VNw4sxArkYOIWtYbg9j_2BYxTpl2MrWDjJCVRFIUTWLxLZwm2tEyzxm69_2BkqtB0td5YukNYHO9T_2Bc3YoRw" method="post" name="saypl" id="frmSumbit"><input type="hidden" name="doctype" value="1" /><input type="hidden" name="quotepid" value="0" /><div id="comment-tools"><div class="tools_text"><textarea placeholder="请在这里留言..." name="message" id="txaArticle" class="text input-block-level comt-area" cols="50" rows="4" tabindex="5"></textarea></div></div><div class="psumbit"><input name="sumbit" type="submit" tabindex="6" value="发布" class="button" /></div></form></div><div class="commentlist"><div class="comment-tab"><div class="come-comt">评论列表<span id="comment_count">（有<span id="infocommentnumarea" style="color:#c81111">0</span>条评论）</span></div></div><ul class="diy-comment"></ul></div></div></div><div class="mainr">