1.使用selenium爬取唯品会相关数据importrandomfromseleniumimportwebdriverfromselenium.webdriver.common.byimportByfromselenium.common.exceptionsimportNoSuchElementExceptionimportrandomimportpymongoimporttimeclassWph_shopping:def__init__(self):self.cookie{cookie: vip_first_visitor:1,vip_address:%257B%2522pid%2522%253A%2522105102%2522%252C%2522cid%2522%253A%2522105102102%2522%252C%2522pname%2522%253A%2522%255Cu8d35%255Cu5dde%255Cu7701%2522%252C%2522cname%2522%253A%2522%255Cu516d%255Cu76d8%255Cu6c34%255Cu5e02%2522%257D,vip_province:105102,vip_province_name:%E8%B4%B5%E5%B7%9E%E7%9C%81,vip_city_name:%E5%85%AD%E7%9B%98%E6%B0%B4%E5%B8%82,vip_city_code:105102102,vip_wh:VIP_CD,vip_ipver:31,mars_cid:1789807019466_c13216fd06575620f094b472c89d398e,mars_sid:e4d070bd42dd676ecb1b1962256f5222,mst_area_code:104104,mars_pid:0,visit_id:567B6DB8268305C5110FF6DD9508F77F,vipshop_passport_src:https%3A%2F%2Fwww.vip.com%2F%26st_name%3Dmst_tfs_punish_login,_jzqco:%7C%7C%7C%7C%7C1.1864491312.1789807026631.1789807026631.1789807026631.1789807026631.1789807026631.0.0.0.1.1,vip_sec_fp_vvid:MWZjZmM5ZDEtMDIzMS00MGEzLTllMTMtOTE2ODg1Zjk5Mzg1MTc4OTgwNzAyNjgyOECKgFw,v_s_fp_switch:1,PASSPORT_ACCESS_TOKEN:C72C7DBF25DDFD0E67CB23AA99042F01B9BD51A3,VipRUID:293126899,VipUID:4c44c970ee7ed3dcc8c47e7d3fdb82a8,VipRNAME:187*****047,VipLID:0%7C1789807057%7C2ba69b,VipDegree:E1,user_class:b,vip_sec_fp_vid:293126899,pc_fdc_area_id:105102102,pc_fdc_source_ip:1,is_default_area:1,VipUINFO:luc%3Ab%7Csuc%3Ab%7Cbct%3Ac_new%7Chct%3Ac_new%7Cbdts%3A0%7Cbcts%3A0%7Ckfts%3A0%7Cc10%3A0%7Crcabt%3A0%7Cp2%3A0%7Cp3%3A1%7Cp4%3A0%7Cp5%3A1%7Cul%3A3105,vpc_uinfo:fr249%3AB,vpc_fdc_city_code:105102102:105102102,vpc_common_usp:fr328:3105,fr249:B,fe_global_sync:1,VIP_QR_FIRST:1,vip_tracker_source_from:,sfl_d:0,tfs_c:1,vip_sec_fp_wtk:cwEAAzVqMZ4kYbCT6tFJ3ji6wQNI_1TrMvd8aGgbuWgfnZ6EtL3m7v6ZUvGs0_Lvmh_oKdPlGtqhj22RzI4uxDSb4QuBNC8,pg_session_no:10,tfs_fp_token:cwEAAzVqMZ4kYbCT6tFJ3ji6wQNI_1TrMvd8aGgbuWgfnZ6EtL3m7v6ZUvGs0_Lvmh_oKdPlGtqhj22RzI4uxDSb4QuBNC8,tfs_fp_timestamp:1789807917376,waitlist:%7B%22pollingId%22%3A%22D3E69CB7-63F1-421D-B2A8-6286CC099EC7%22%2C%22pollingStamp%22%3A1789807922082%7D}optionswebdriver.EdgeOptions()# add_argument 添加浏览器自动化参数括号里面的是关闭Chrome自动化受控特征options.add_argument(--disable-blink-featuresAutomationControlled)# 禁用自动化扩展去掉开发者扩展弹框options.add_experimental_option(useAutomationExtension,False)# 去掉开发者警告# 去掉顶部的“Chrome正在接受自动化软件控制”提示options.add_experimental_option(excludeSwitches,[enable-automation])# 不加载图片prefs{profile.managed_default_content_settings.images:2}options.add_experimental_option(prefs,prefs)# 加载浏览器self.driverwebdriver.Edge(optionsoptions)##放大窗口self.driver.maximize_window()self.clientpymongo.MongoClient(hostlocalhost,port27017)self.dbself.client[wph][shop]defopen_page(self):self.driver.get(https://category.vip.com/suggest.php?keyword%E7%AC%94%E8%AE%B0%E6%9C%ACff235%7C12%7C1%7C1)forkey,valueinself.cookie.items():cookie_dirt{name:key,value:value}self.driver.add_cookie(cookie_dirt)# 刷新数据self.driver.refresh()##休眠三秒时间time.sleep(3)##随机休眠时间time.sleep(random.randint(1,3))defpage_down(self):forxinrange(1,10):# js绝对滚动jsfdocument.documentElement.scrollTop{x*1000}self.driver.execute_script(js)time.sleep(random.randint(500,800)/1000)defget_data(self):# 执行滚动self.page_down()# 获取相关的数据div_listself.driver.find_elements(By.XPATH,//section[idJ_searchCatList]/div[classc-goods-item J-goods-item c-goods-item--auto-width])fordivindiv_list:moneydiv.find_element(By.XPATH,.//div[classc-goods-item__sale-price J-goods-item__sale-price]).text titlediv.find_element(By.XPATH,.//div[2]/div[2]).text img_urldiv.find_element(By.XPATH,.//div[classc-goods-item__img]/img)item{title:title,money:money,img_url:img_url.get_attribute(src)}self.save_data(item)print(数据保存成功)# 实现翻页self.next_page()defnext_page(self):try:buttonself.driver.find_element(By.XPATH,//a[idJ_nextPage_link])ifbutton:button.click()self.get_data()else:self.driver.quit()exceptNoSuchElementException:self.driver.quit()self.client.close()print(结束)defsave_data(self,item):self.db.insert_one(item)defmain(self):self.open_page()self.get_data()if__name____main__:wphWph_shopping()wph.main()2.使用platwright爬取考研网相关数据fromplaywright.sync_apiimportsync_playwrightimportpymongoclassKanyan:def__init__(self):self.urlhttps://www.handebook.com/web/#/1234/documentself.clientpymongo.MongoClient(hostlocalhost,port27017)self.dbself.client[kanyan][datas]defget_response(self,page):page.wait_for_selector(//div[classschool-list],statevisible)div_listpage.locator(//div[classcontent-document]).all()fordivindiv_list:item{}item[title]div.locator(//span[1]).inner_text()item[price]div.locator(//span[2]).inner_text()self.db.insert_one(item)print(保存数据成功)self.next_page(page)defnext_page(self,page):page.locator(//button[classbtn-next]).scroll_into_view_if_needed()next_page_buttonpage.locator(//button[classbtn-next])ifnext_page_button:next_page_button.click()self.get_response(page)else:print(没有数据了)defmain(self):withsync_playwright()aspw:driverpw.chromium.launch(headlessFalse)pagedriver.new_page()page.goto(self.url)self.get_response(page)if__name____main__:kanyanKanyan()kanyan.main()3.使用DrisionPage爬取网易社招相关数据.import pymongo from DrissionPage import ChromiumPage from DrissionPage.common import By import pymongo import time dbpymongo.MongoClient(localhost, 27017) clientdb[WXZP][datas] page ChromiumPage() page.listen.start(api/hr163/position/queryPage) page.get(https://hr.163.com/job-list.html?workType1langzh) time.sleep(5) for item in page.listen.steps(count10): datasitem.response.body[data][list][0] client.insert_one(datas) # for data in datas: # print(data[name]) print(datas) flex_next_button page.wait.eles_loaded(( By.XPATH, //*[idp-job-list]/div[2]/div[2]/div/div/ul/li[9]/a ), timeout3) if flex_next_button: page.ele((By.XPATH, //*[idp-job-list]/div[2]/div[2]/div/div/ul/li[last()-1]/a)).click() time.sleep(0.2) else: print(没有数据了) page.quit() Python相关的知识及使用
1.使用selenium爬取唯品会相关数据importrandomfromseleniumimportwebdriverfromselenium.webdriver.common.byimportByfromselenium.common.exceptionsimportNoSuchElementExceptionimportrandomimportpymongoimporttimeclassWph_shopping:def__init__(self):self.cookie{cookie: vip_first_visitor:1,vip_address:%257B%2522pid%2522%253A%2522105102%2522%252C%2522cid%2522%253A%2522105102102%2522%252C%2522pname%2522%253A%2522%255Cu8d35%255Cu5dde%255Cu7701%2522%252C%2522cname%2522%253A%2522%255Cu516d%255Cu76d8%255Cu6c34%255Cu5e02%2522%257D,vip_province:105102,vip_province_name:%E8%B4%B5%E5%B7%9E%E7%9C%81,vip_city_name:%E5%85%AD%E7%9B%98%E6%B0%B4%E5%B8%82,vip_city_code:105102102,vip_wh:VIP_CD,vip_ipver:31,mars_cid:1789807019466_c13216fd06575620f094b472c89d398e,mars_sid:e4d070bd42dd676ecb1b1962256f5222,mst_area_code:104104,mars_pid:0,visit_id:567B6DB8268305C5110FF6DD9508F77F,vipshop_passport_src:https%3A%2F%2Fwww.vip.com%2F%26st_name%3Dmst_tfs_punish_login,_jzqco:%7C%7C%7C%7C%7C1.1864491312.1789807026631.1789807026631.1789807026631.1789807026631.1789807026631.0.0.0.1.1,vip_sec_fp_vvid:MWZjZmM5ZDEtMDIzMS00MGEzLTllMTMtOTE2ODg1Zjk5Mzg1MTc4OTgwNzAyNjgyOECKgFw,v_s_fp_switch:1,PASSPORT_ACCESS_TOKEN:C72C7DBF25DDFD0E67CB23AA99042F01B9BD51A3,VipRUID:293126899,VipUID:4c44c970ee7ed3dcc8c47e7d3fdb82a8,VipRNAME:187*****047,VipLID:0%7C1789807057%7C2ba69b,VipDegree:E1,user_class:b,vip_sec_fp_vid:293126899,pc_fdc_area_id:105102102,pc_fdc_source_ip:1,is_default_area:1,VipUINFO:luc%3Ab%7Csuc%3Ab%7Cbct%3Ac_new%7Chct%3Ac_new%7Cbdts%3A0%7Cbcts%3A0%7Ckfts%3A0%7Cc10%3A0%7Crcabt%3A0%7Cp2%3A0%7Cp3%3A1%7Cp4%3A0%7Cp5%3A1%7Cul%3A3105,vpc_uinfo:fr249%3AB,vpc_fdc_city_code:105102102:105102102,vpc_common_usp:fr328:3105,fr249:B,fe_global_sync:1,VIP_QR_FIRST:1,vip_tracker_source_from:,sfl_d:0,tfs_c:1,vip_sec_fp_wtk:cwEAAzVqMZ4kYbCT6tFJ3ji6wQNI_1TrMvd8aGgbuWgfnZ6EtL3m7v6ZUvGs0_Lvmh_oKdPlGtqhj22RzI4uxDSb4QuBNC8,pg_session_no:10,tfs_fp_token:cwEAAzVqMZ4kYbCT6tFJ3ji6wQNI_1TrMvd8aGgbuWgfnZ6EtL3m7v6ZUvGs0_Lvmh_oKdPlGtqhj22RzI4uxDSb4QuBNC8,tfs_fp_timestamp:1789807917376,waitlist:%7B%22pollingId%22%3A%22D3E69CB7-63F1-421D-B2A8-6286CC099EC7%22%2C%22pollingStamp%22%3A1789807922082%7D}optionswebdriver.EdgeOptions()# add_argument 添加浏览器自动化参数括号里面的是关闭Chrome自动化受控特征options.add_argument(--disable-blink-featuresAutomationControlled)# 禁用自动化扩展去掉开发者扩展弹框options.add_experimental_option(useAutomationExtension,False)# 去掉开发者警告# 去掉顶部的“Chrome正在接受自动化软件控制”提示options.add_experimental_option(excludeSwitches,[enable-automation])# 不加载图片prefs{profile.managed_default_content_settings.images:2}options.add_experimental_option(prefs,prefs)# 加载浏览器self.driverwebdriver.Edge(optionsoptions)##放大窗口self.driver.maximize_window()self.clientpymongo.MongoClient(hostlocalhost,port27017)self.dbself.client[wph][shop]defopen_page(self):self.driver.get(https://category.vip.com/suggest.php?keyword%E7%AC%94%E8%AE%B0%E6%9C%ACff235%7C12%7C1%7C1)forkey,valueinself.cookie.items():cookie_dirt{name:key,value:value}self.driver.add_cookie(cookie_dirt)# 刷新数据self.driver.refresh()##休眠三秒时间time.sleep(3)##随机休眠时间time.sleep(random.randint(1,3))defpage_down(self):forxinrange(1,10):# js绝对滚动jsfdocument.documentElement.scrollTop{x*1000}self.driver.execute_script(js)time.sleep(random.randint(500,800)/1000)defget_data(self):# 执行滚动self.page_down()# 获取相关的数据div_listself.driver.find_elements(By.XPATH,//section[idJ_searchCatList]/div[classc-goods-item J-goods-item c-goods-item--auto-width])fordivindiv_list:moneydiv.find_element(By.XPATH,.//div[classc-goods-item__sale-price J-goods-item__sale-price]).text titlediv.find_element(By.XPATH,.//div[2]/div[2]).text img_urldiv.find_element(By.XPATH,.//div[classc-goods-item__img]/img)item{title:title,money:money,img_url:img_url.get_attribute(src)}self.save_data(item)print(数据保存成功)# 实现翻页self.next_page()defnext_page(self):try:buttonself.driver.find_element(By.XPATH,//a[idJ_nextPage_link])ifbutton:button.click()self.get_data()else:self.driver.quit()exceptNoSuchElementException:self.driver.quit()self.client.close()print(结束)defsave_data(self,item):self.db.insert_one(item)defmain(self):self.open_page()self.get_data()if__name____main__:wphWph_shopping()wph.main()2.使用platwright爬取考研网相关数据fromplaywright.sync_apiimportsync_playwrightimportpymongoclassKanyan:def__init__(self):self.urlhttps://www.handebook.com/web/#/1234/documentself.clientpymongo.MongoClient(hostlocalhost,port27017)self.dbself.client[kanyan][datas]defget_response(self,page):page.wait_for_selector(//div[classschool-list],statevisible)div_listpage.locator(//div[classcontent-document]).all()fordivindiv_list:item{}item[title]div.locator(//span[1]).inner_text()item[price]div.locator(//span[2]).inner_text()self.db.insert_one(item)print(保存数据成功)self.next_page(page)defnext_page(self,page):page.locator(//button[classbtn-next]).scroll_into_view_if_needed()next_page_buttonpage.locator(//button[classbtn-next])ifnext_page_button:next_page_button.click()self.get_response(page)else:print(没有数据了)defmain(self):withsync_playwright()aspw:driverpw.chromium.launch(headlessFalse)pagedriver.new_page()page.goto(self.url)self.get_response(page)if__name____main__:kanyanKanyan()kanyan.main()3.使用DrisionPage爬取网易社招相关数据.import pymongo from DrissionPage import ChromiumPage from DrissionPage.common import By import pymongo import time dbpymongo.MongoClient(localhost, 27017) clientdb[WXZP][datas] page ChromiumPage() page.listen.start(api/hr163/position/queryPage) page.get(https://hr.163.com/job-list.html?workType1langzh) time.sleep(5) for item in page.listen.steps(count10): datasitem.response.body[data][list][0] client.insert_one(datas) # for data in datas: # print(data[name]) print(datas) flex_next_button page.wait.eles_loaded(( By.XPATH, //*[idp-job-list]/div[2]/div[2]/div/div/ul/li[9]/a ), timeout3) if flex_next_button: page.ele((By.XPATH, //*[idp-job-list]/div[2]/div[2]/div/div/ul/li[last()-1]/a)).click() time.sleep(0.2) else: print(没有数据了) page.quit()