script-run.py

import time import requests, json from lxml import etree from retry import retry from datetime import datetime from def_pakege import get_time, to_xpath, save_content, save_date cookies = { 'Hm_lvt_a024fd6185174b7675b9ad94c7a5507b': '1722997495', 'HMACCOUNT': '575BFDB4CC7879E4', 'Hm_lpvt_a024fd6185174b7675b9ad94c7a5507b': '1723011614', } headers = { 'Accept': 'text/javascript, application/javascript, application/ecmascript, application/x-ecmascript, */*; q=0.01', 'Accept-Language': 'zh-CN,zh;q=0.9', 'Connection': 'keep-alive', 'Referer': 'https://sdxw.iqilu.com/w/mpdetail/402121.html?catid=402121', 'User-Agent': 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36', 'X-Requested-With': 'XMLHttpRequest', } @retry(delay=5) def sh_requests(params): response = requests.get('https://sdxw.iqilu.com/v1/app/article/common/json', params=params, cookies=cookies, headers=headers) return response.json() def shandiannews_run(i): p = 1 while True: params = { 'departmentid': i['id'], 'page': str(p), '_orgid_': '21', # 'cqcallback': 'jQuery112406744793210450484_1723011613268', '_': str(int(time.time() * 1000)), } print(params) data = sh_requests(params) data = data['data']['infos'] if data == []: return 0 for item in data: html = to_xpath(item['url'], headers=headers) if html=='cte' or html==None: continue content = html.xpath('//div[@class="art_content"]//text()') content = ''.join(content) if content != [] else '' if len(content) < 30: continue ans = { 'title': item['title'], 'url': item['url'], 'content': content, 'pub_dt': time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(item['publish_at_time'])), 'download_date': get_time(), 'type': item['catename'] } save_content(ans, 'shandiannews') time.sleep(0.3) p += 1 a = [{'name': '山东地质', 'id': '402571'}, {'name': '健康平阴', 'id': '402121'}, {'name': '菏泽市东明县', 'id': '401670'}, {'name': '菏泽市鄄城县', 'id': '401669'}, {'name': '菏泽市郓城县', 'id': '401668'}, {'name': '菏泽市巨野县', 'id': '401667'}, {'name': '菏泽市成武县', 'id': '401666'}, {'name': '菏泽市单县', 'id': '401665'}, {'name': '菏泽市曹县', 'id': '401664'}, {'name': '菏泽市定陶区', 'id': '401663'}, {'name': '菏泽市牡丹区', 'id': '401662'}, {'name': '滨州市博兴县', 'id': '401661'}, {'name': '滨州市惠民县', 'id': '401660'}, {'name': '滨州市阳信县', 'id': '401659'}, {'name': '济宁市嘉祥县', 'id': '401658'}, {'name': '滨州市无棣县', 'id': '401656'}, {'name': '滨州市邹平市', 'id': '401655'}, {'name': '滨州市沾化区', 'id': '401654'}, {'name': '滨州市滨城区', 'id': '401653'}, {'name': '聊城市高唐县', 'id': '401652'}, {'name': '聊城市东阿县', 'id': '401651'}, {'name': '聊城市阳谷县', 'id': '401650'}, {'name': '聊城市莘县', 'id': '401649'}, {'name': '聊城市冠县', 'id': '401648'}, {'name': '聊城市临清市', 'id': '401647'}, {'name': '聊城市茌平区', 'id': '401646'}, {'name': '聊城市东昌府区', 'id': '401645'}, {'name': '德州市德州运河经济开发区', 'id': '401642'}, {'name': '德州市德州经济技术开发区', 'id': '401641'}, {'name': '德州市武城县', 'id': '401640'}, {'name': '日照市经济技术开发区', 'id': '401639'}, {'name': '德州市齐河县', 'id': '401638'}, {'name': '日照市莒县', 'id': '401637'}, {'name': '德州市夏津县', 'id': '401636'}, {'name': '日照市五莲县', 'id': '401635'}, {'name': '日照市岚山区', 'id': '401634'}, {'name': '日照市东港区', 'id': '401633'}, {'name': '威海市乳山市', 'id': '401632'}, {'name': '威海市荣成市', 'id': '401631'}, {'name': '威海市文登区', 'id': '401630'}, {'name': '威海市环翠区', 'id': '401629'}, {'name': '德州市临邑县', 'id': '401628'}, {'name': '德州市平原县', 'id': '401627'}, {'name': '泰安市东平县', 'id': '401626'}, {'name': '济南市商河县', 'id': '401625'}, {'name': '济南市平阴县', 'id': '401624'}, {'name': '德州市庆云县', 'id': '401623'}, {'name': '泰安市宁阳县', 'id': '401622'}, {'name': '济南市钢城区', 'id': '401621'}, {'name': '德州市宁津县', 'id': '401620'}, {'name': '济南市莱芜区', 'id': '401619'}, {'name': '济南市济阳区', 'id': '401618'}, {'name': '泰安市肥城市', 'id': '401617'}, {'name': '德州市禹城市', 'id': '401616'}, {'name': '济南市章丘区', 'id': '401615'}, {'name': '泰安市新泰市', 'id': '401614'}, {'name': '泰安市岱岳区', 'id': '401613'}, {'name': '泰安市泰山区', 'id': '401612'}, {'name': '济南市长清区', 'id': '401610'}, {'name': '德州市乐陵市', 'id': '401609'}, {'name': '德州市陵城区', 'id': '401608'}, {'name': '济南市历城区', 'id': '401607'}, {'name': '德州市德城区', 'id': '401606'}, {'name': '临沂市临沭县', 'id': '401605'}, {'name': '济南市天桥区', 'id': '401604'}, {'name': '临沂市蒙阴县', 'id': '401603'}, {'name': '潍坊市昌乐县', 'id': '401602'}, {'name': '济南市槐荫区', 'id': '401601'}, {'name': '临沂市莒南县', 'id': '401600'}, {'name': '济宁市梁山县', 'id': '401599'}, {'name': '济宁市泗水县', 'id': '401598'}, {'name': '济宁市汶上县', 'id': '401597'}, {'name': '济南市市中区', 'id': '401596'}, {'name': '济宁市金乡县', 'id': '401595'}, {'name': '济南市历下区', 'id': '401594'}, {'name': '临沂市平邑县', 'id': '401593'}, {'name': '烟台市昆嵛山国家级自然保护区', 'id': '401591'}, {'name': '济宁市微山县', 'id': '401590'}, {'name': '烟台市长岛海洋生态文明综合试验区', 'id': '401589'}, {'name': '济宁市邹城市', 'id': '401588'}, {'name': '临沂市费县', 'id': '401587'}, {'name': '济宁市曲阜市', 'id': '401586'}, {'name': '烟台市高新技术产业开发区', 'id': '401585'}, {'name': '临沂市兰陵县', 'id': '401584'}, {'name': '烟台市经济技术开发区', 'id': '401582'}, {'name': '临沂市沂水县', 'id': '401581'}, {'name': '济宁市兖州区', 'id': '401580'}, {'name': '济宁市任城区', 'id': '401579'}, {'name': '烟台市海阳市', 'id': '401578'}, {'name': '临沂市郯城县', 'id': '401577'}, {'name': '烟台市栖霞市', 'id': '401576'}, {'name': '烟台市招远市', 'id': '401575'}, {'name': '临沂市沂南县', 'id': '401574'}, {'name': '潍坊市临朐县', 'id': '401573'}, {'name': '烟台市莱州市', 'id': '401572'}, {'name': '临沂市河东区', 'id': '401571'}, {'name': '烟台市莱阳市', 'id': '401570'}, {'name': '烟台市龙口市', 'id': '401569'}, {'name': '临沂市罗庄区', 'id': '401568'}, {'name': '潍坊市昌邑市', 'id': '401567'}, {'name': '临沂市兰山区', 'id': '401566'}, {'name': '烟台市蓬莱区', 'id': '401565'}, {'name': '潍坊市高密市', 'id': '401564'}, {'name': '烟台市牟平区', 'id': '401563'}, {'name': '潍坊市安丘市', 'id': '401562'}, {'name': '烟台市莱山区', 'id': '401561'}, {'name': '潍坊市寿光市', 'id': '401560'}, {'name': '烟台市福山区', 'id': '401559'}, {'name': '烟台市芝罘区', 'id': '401558'}, {'name': '潍坊市诸城市', 'id': '401557'}, {'name': '东营市利津县', 'id': '401556'}, {'name': '东营市广饶县', 'id': '401555'},