Python爬虫应用 - PayPal职位抓取¶

# 解析网页
import requests
from bs4 import BeautifulSoup

# 表格操作
import numpy as np
import pandas as pd

# 通用
import re
import os
import unicodedata

# 访问URL，并取回返回结果
def url_request(url):
    header = {
        'User-Agent': 'Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/30.0.1599.101 Safari/537.36'}
    r = requests.get(url, headers=header)
    print('Connection status:', r.status_code, '\n')
    return r.text

# 找到页面中需要的元素，存储职位列表信息
def job_parser(html):
    header,desc,link = [[] for i in range(3)]
    soup = BeautifulSoup(html, 'html.parser')
    # 右键通过打开浏览器检查器，在元素tab中查看网页源码，可看到职位名称的类别名字为primary-text-color job-result-title，并且是一个a标签
    job_header = soup.find_all('a', attrs={'class': 'primary-text-color job-result-title'})
    # 元素查找方法同上
    job_link = soup.find_all('a', attrs={'class': 'primary-text-color job-result-title'}, href=True)

    header = [i.contents[0] for i in job_header]
    link = ['https://jobsearch.paypal-corp.com/'+i['href'] for i in job_link]

    # 将结果存起来
    return pd.DataFrame({'Title':header, 'Link':link})

# 创建一个存储结果的dataframe
df = pd.DataFrame(columns=['Title','Link'])
# 创建URL的模板，添加不同的页码就可匹配不同的页面
job_url_header = 'https://jobsearch.paypal-corp.com/en-US/search?facetcountry=cn&facetcity=shanghai&pagenumber='

# 遍历所有页面，并存储结果
for i in range(2):
  job_url = job_url_header + str(i+1)
  print('URL: {}'.format(job_url))
  job_html = url_request(job_url)
  # 将每个页面的结果存起来
  df = df.append(job_parser(job_html))

def get_jd(url):
  jd_html = url_request(url)
  soup = BeautifulSoup(jd_html, 'html.parser')
  jd_desc = soup.find('div', attrs={'class': 'jdp-job-description-card content-card'})
  # JD格式不一，此处仅做演示
  if jd_desc:
    if jd_desc.findAll('ul')[:]:
      desc = [i.text + '\n{}'.format(j.text) for i,j in zip(jd_desc.findAll('p')[:], jd_desc.findAll('ul')[:])]
    else:
      desc = [i.text  for i in jd_desc.findAll('p')[:]]

    return unicodedata.normalize('NFKD', '\n'.join(i for i in desc))

# 对之前存储的内容使用详情抓取函数，将详情保存下来。
df['JD'] = df['Link'].apply(get_jd)

# 打印结果
df.tail(2)