记录openlaw的反爬

网络爬虫

浏览数:300

2019-5-10

地址:http://openlaw.cn/login.jsp

需要登陆,找到Form data

 

根据提交数据可以知道需要找到_csrf和加密后的password,_csrf由服务器生成,在登陆页面

接下来找加密password的js代码,ctrl+F搜索password,可以看出password在加密后重新填入input再提交

找到加密函数KeyEncrypt

复制到snippets调试,调用了JSEncrypt,CryptoJs,继续搜索

 

全都复制加到snippets里执行

这样就获得了加密passwoed的js代码,接下来需要让这段代码在python里执行,可以用execjs和js2py,推荐用execjs

把snippets的代码复制出来另存为js,用execjs执行

要注意运行时环境,如果是node.js或Jscript都会报错:navigator is not define,nodejs和Jscript都不是浏览器,可以用PhantomJS。

#运行时环境设置
import execjs import execjs.runtime_names os.environ["EXECJS_RUNTIME"] = "PhantomJS" PhantomJS = execjs.get(execjs.runtime_names.PhantomJS) print(execjs.get().name)

js文件路径前要加r,加密密码结果如图

因为csfr和post data在两页,所以需要一起运行

#获取csfr
headers={ "Accept":"text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3", "Accept-Encoding":"gzip,deflate", "Accept-Language":"zh-CN,zh;q=0.9", "Cache-Control":"max-age=0", "Connection":"keep-alive", "Content-Type":"application/x-www-form-urlencoded", "Host":"openlaw.cn", "Upgrade-Insecure-Requests":"1", "User-Agent":"Mozilla/5.0(WindowsNT10.0;Win64;x64)AppleWebKit/537.36(KHTML,likeGecko)Chrome/73.0.3683.86Safari/537.36", } r=session.get("http://openlaw.cn/login.jsp?",headers=headers1) tree=etree.HTML(r.text) csrf=tree.xpath('//input[@name="_csrf"]/@value')[0]

#post form data post_url="http://openlaw.cn/login" with open(r"C:\Users\37576\Desktop\123.js",encoding="utf-8")as f: js=f.read() ctx = execjs.compile(js) pwd=ctx.call("keyEncrypt","XXXXXXXX")#密码 data={ "_csrf":csrf, "username":"#########",#账户 "password":pwd, "_spring_security_remember_me":"true", } res=session.post(post_url,data=data,headers=headers)

 tree=etree.HTML(res.text)
 lala=tree.xpath(‘//div[@class=”bbp-user-section”]//text()’)

登陆成功

 

接下来分析抓取页面

url="http://openlaw.cn/guidance/16444eba67afe97881fa6521d333ef10"
headers1={
"Accept":"text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3",
"Accept-Encoding":"gzip,deflate",
"Accept-Language":"zh-CN,zh;q=0.9",
"Connection":"keep-alive",
"Host":"openlaw.cn",
"Upgrade-Insecure-Requests":"1",
"User-Agent":"Mozilla/5.0(WindowsNT10.0;Win64;x64)AppleWebKit/537.36(KHTML,likeGecko)Chrome/73.0.3683.86Safari/537.36",
}
a=session.get(url,headers=headers1)

抓取结果全是乱码

用火狐浏览器抓包分析

第一页与刚才抓到的页面一样,第二页有数据,在第一页面下面

对比分析发现请求地址一样,但cookie不一样,参考资料了解到是js加密cookie,带新cookie的请求才能显示正确页面,所以需要找到添加cookie的js代码

查资料得知下面这种加密叫jjencode,jjencodeDemo地址:http://utf-8.jp/public/jjencode.html

],_$_:++$,$_$$:({}+"")[$],$$_$:($[$]+"")[$],_$$:++$,$$$_:(!""+"")[$],$__:++$,$_$:++$,$$__:({}+"")[$],$$_:++$,$$$:++$,$___:++$,$__$:++$};$.$_=($.$_=$+"")[$.$_$]+($._$=$.$_[$.__$])+($.$$=($.$+"")[$.__$])+((!$)+"")[$._$$]+($.__=$.$_[$.$$_])+($.$=(!""+"")[$.__$])+($._=(!""+"")[$._$_])+$.$_[$.$_$]+$.__+$._$+$.$;$.$$=$.$+(!""+"")[
#这种加密叫jsfunk,地址是:http://www.jsfuck.com/
+[]+!+[]+!+[]]+(!![]+[])[+!+[]]]+[])[!+[]+!+[]+!+[]]+(!![]+[][(![]+[])[+[]]+([![]]+[][[]])[+!+[]+[+[]]]+(![]+[])[!+[]+!+[]]+(!![]+[])[+[]]+(!![]+[])[!+[]+!+[]+!+[]]+(!![]+[])[+!+[]]])[+!+[]+[+[]]]+([][[]]+[])[+!+[]]+(![]+[])[!+[]+!+[]+!+[]]+(!![]+[])[+[]]+(!![]+[])[+!+[]]+([][[]]

下面是aaencode,地址是http://utf-8.jp/public/aaencode.html

゚ω゚ノ= /`m´)ノ ~┻━┻ //*´∇`*/ [‘_’]; o=(゚ー゚) =_=3; c=(゚Θ゚) =(゚ー゚)-(゚ー゚); (゚Д゚) =(゚Θ゚)= (o^_^o)/ (o^_^o);(゚Д゚)={゚Θ゚: ‘_’ ,゚ω゚ノ : ((゚ω゚ノ==3) +’_’) [゚Θ゚] ,゚ー゚ノ :(゚ω゚ノ+ ‘_’)[o^_^o -(゚Θ゚)] ,゚Д゚ノ:((゚ー゚==3) +’_’)[゚ー゚] }; (゚Д゚) [゚Θ゚] =((゚ω゚ノ==3) +’_’) [c^_^o];(゚Д゚) [‘c’] = ((゚Д゚)+’_’) [ (゚ー゚)+(゚ー゚)-(゚Θ゚) ];

加密原理百度

代码在可以执行获得内容或者找工具解密成明文

(function anonymous() {
    if (window.wp1) {
        document.cookie = 'OPEN_E4C0C08491=' + wp3;
        document.cookie = 'OPEN_ID=B32pJOwfO/AJZJ3ZGmnunQNTaUie84jnyHoNCq/onZQ='
    }
}
)

代码有添加cookie,第二行拉出来单独执行

有报错但得到了解析后的代码

页面上所以js代码如下:

var wp1 = 'xKIOt5jUZQcS8wurcX4lghwlET9K0IWBUsS3ZqmSamCBTAPg7Fl5sVr443dyK42p';
var wp2 = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/=";
var wp3 = "548545fe3ced4f3494934e537e304705";
var wp4 = "464acf618bf62df3e7e37e80c2c1a04d";
var wp5 = "0aa9566c4aa30883e34d0e1a530bf508";
var wp6 = "a8f49232d343e4b94ff7958088fa6734";
var wp7 = "82f778b45a9fce8589105c852174947b";
var wp8 = "3221d819c9b96d2631d0b5084c736f69";
if (window.wp1) {
        document.cookie = 'OPEN_E4C0C08491=' + wp3;
        document.cookie = 'OPEN_ID=B32pJOwfO/AJZJ3ZGmnunQNTaUie84jnyHoNCq/onZQ='
    }

var getGlobal = function() {
        if (typeof self !== '') {
            return self
        }
        ;if (typeof window !== '') {
            return window
        }
        ;if (typeof global !== '') {
            return global
        }
        ;throw new Error('unable to locate global object')
    };
function _process(s) {
        var result = s.substring(5, 7).concat('s').concat(s.substring(14, 15)).concat('v').concat(s.substring(9, 13)).concat('g').concat(s.substring(5)).concat('l').concat(s.substring(11, 19));
        return result.substr(0, 32);
    }
    ;var _switch = false;
    if ((getGlobal() === window)) {
        _switch = true
    } else {
        _switch = false
    };
document.cookie = 'SIGNIN_ID=3e2291010f36c9244c633aba4cdcdd42'; document.cookie = 'SIGNIN_UC=9e65d8fc646e49fb7a0f9f8237915bfb'; document.cookie = 'UNDEFINED=432e9111ecf89746f3046ca970814f41'; if (_switch) { document.cookie = 'OPEN_589D5869B5=' + _process(wp3); setTimeout("location.reload();", 100); }

有六条cookie修改完后通过location.reload()刷新页面,带着新cookie刷新后的页面能访问到有数据的页面

接下来需要把js代码拉出来执行获得cookie,但是有一点问题就是cookie数据:OPEN_E4C0C08491,OPEN_ID,SIGNIN_ID,UNDEFINED,OPEN_589D5869B5不知道从哪里来的,找了很久都没找到

没有这些数据就没办法写cookie,就此卡住。。。等水平提高再来解决

为了获取页面数据,下面用selenium解决该问题

#配置chorme无头浏览器
from selenium import webdriver from selenium.webdriver.chrome.options import Options chrome_options = Options() chrome_options.add_argument('--headless') driver = webdriver.Chrome(chrome_options=chrome_options,executable_path = 'D:\python\chromedriver.exe') from time import sleep
#登陆账号
driver.get("http://openlaw.cn/login.jsp?logout") driver.find_element_by_id("username").send_keys("XXXXXXXXX") driver.find_element_by_id("password").send_keys("XXXXXXXXX") sleep(2) driver.find_element_by_id("submit").click()
#获取页面数据
driver.get("http://openlaw.cn/guidance/167ce490ba94c3ed26baab5689fdd620") sleep(2) title=driver.find_element_by_xpath('//h1[@class="entry-title"]').text guid=driver.find_element_by_xpath('//div[@class="annotator-wrapper"]/p').text content=driver.find_element_by_xpath('//div[@class="annotator-wrapper"]/p[2]').text page_detail={"title":title,"content":guid+"\n"+content.replace("\u3000"," ")}

结果

小结

1.RSA学习地址:https://blog.csdn.net/u014044812/article/details/80782448

2.用代理或访问频率异常会频繁出现验证码

3.访问有资源的网页时服务器会验证cookie,如果不带有特定cookie访问,网页会转到空白页执行js代码添加cookie,然后302或reload带着新cookie重新请求出有数据的页面

4.常见的js混淆方式

5.还不熟练js调试用法,等水平提高继续破解该反爬

 

作者:神一样了