爬虫初学爬微博页面遇难

BSCEO · 发表于 2022-5-25 18:48

想要爬取微博的某个帖子的评论区，网页源码是会随着下拉不断的变化的，而且虽然在变化，但是却不是那种加载多出来，而是在原代码上变化的。
还有个疑惑就是为什么用requests库爬它的页面代码会有一些看不懂的乱码符号，    // óÃ»§éí·Y′æÔú￡¬3￠êÔ»Ö¸′óÃ»§éí·Y¡￡类似这种，有大佬解答吗？

<!DOCTYPE html>
<html>
<head>
<meta http-equiv="Content-type" content="text/html; charset=gb2312"/>
<title>Sina Visitor System</title>
</head>
<body>
<span id="message"></span>
<script type="text/javascript" src="/js/visitor/mini_original.js?v=20161116"></script>
<script type="text/javascript">
window.use_fp = "1" == "1"; // êÇ·ñ2é¼ˉéè±¸Ö¸ÎÆ¡￡
var url = url || {};
(function () {
      this.l = function (u, c) {
         try {
            var s = document.createElement("script");
            s.type = "text/javascript";
            s[document.all ? "onreadystatechange" : "onload"] = function () {

                  if (document.all && this.readyState != "loaded" && this.readyState != "complete") {
                     return
                  }
                  this[document.all ? "onreadystatechange" : "onload"] = null;
                  this.parentNode.removeChild(this);
                  if (c) {
                     c()
                  }
            };
            s.src = u;
            document.getElementsByTagName("head")[0].appendChild(s)
         } catch (e) {
         }
      };
}).call(url);

// á÷3ìèë¿ú¡￡
wload(function () {

      try {

         var need_restore = "1" == "1"; // êÇ·ñ×ß»Ö¸′éí·Yá÷3ì¡￡

         // èç1ûDèòa×ß»Ö¸′éí·Yá÷3ì￡¬3￠êÔ′ó cookie »ñè¡óÃ»§éí·Y¡￡
         if (!need_restore || !Store.CookieHelper.get("SRF")) {

            // èô»ñè¡ê§°ü×ß′′½¨·Ã¿íá÷3ì¡￡
            // á÷3ìÖ′DDê±¼ä1y3¤￡¨3¬1y 3s￡©￡¬ÔòèÏÎa3ö′í¡￡
            var error_timeout = window.setTimeout("error_back()", 5000);

            tid.get(function (tid, where, confidence) {
                  // è¡Ö¸ÎÆË3àûíê3é￡¬Çå3y3ö′í timeout ¡￡
                  window.clearTimeout(error_timeout);
                  incarnate(tid, where, confidence);
            });
         } else {
            // óÃ»§éí·Y′æÔú￡¬3￠êÔ»Ö¸′óÃ»§éí·Y¡￡
            restore();
         }
      } catch (e) {
         // 3ö′í¡￡
         error_back();
      }
});

// ¡°·μ»Ø¡± »Øμ÷oˉêy¡￡
var return_back = function (response) {

      if (response["retcode"] == 20000000) {
         back();
      } else {
         // 3ö′í¡￡
         error_back(response["msg"]);
      }
};

// ìø×a»Ø3õê¼μØÖ·¡￡
var back = function() {

      var url = "https://weibo.com/1878335471/LuCF3n3IL";
      if (url != "none") {
         window.location.href = url;
      }
};

// ¿çóò1ã2￥¡￡
var cross_domain = function (response) {

      var from = "weibo";
      var entry = "miniblog";
      if (response["retcode"] == 20000000) {

         var crossdomain_host = "login.sina.com.cn";
         if (crossdomain_host != "none") {

            var cross_domain_intr = window.location.protocol + "//" + crossdomain_host + "/visitor/visitor?a=crossdomain&cb=return_back&s=" +
                     encodeURIComponent(response["data"]["sub"]) + "&sp=" + encodeURIComponent(response["data"]["subp"]) + "&from=" + from + "&_rand=" + Math.random() + "&entry=" + entry;
            url.l(cross_domain_intr);
         } else {

            back();
         }
      } else {

         // 3ö′í¡￡
         error_back(response["msg"]);
      }
};

// ÎaóÃ»§¸3óè·Ã¿íéí·Y ¡￡
var incarnate = function (tid, where, conficence) {

      var gen_conf = "";
      var from = "weibo";
      var incarnate_intr = window.location.protocol + "//" + window.location.host + "/visitor/visitor?a=incarnate&t=" +
            encodeURIComponent(tid) + "&w=" + encodeURIComponent(where) + "&c=" + encodeURIComponent(conficence) +
            "&gc=" + encodeURIComponent(gen_conf) + "&cb=cross_domain&from=" + from + "&_rand=" + Math.random();
      url.l(incarnate_intr);
};

// »Ö¸′óÃ»§¶aê§μÄéí·Y¡￡
var restore = function () {

      var from = "weibo";
      var restore_intr = window.location.protocol + "//" + window.location.host +
            "/visitor/visitor?a=restore&cb=restore_back&from=" + from + "&_rand=" + Math.random();

      url.l(restore_intr);
};

// ¿çóò»Ö¸′¶aê§μÄéí·Y¡￡
var restore_back = function (response) {

      // éí·Y»Ö¸′3é1|×ß1ã2￥á÷3ì￡¬·ñÔò×ß′′½¨·Ã¿íá÷3ì¡￡
      if (response["retcode"] == 20000000) {

         var url = "https://weibo.com/1878335471/LuCF3n3IL";
         var alt = response["data"]["alt"];
         var savestate = response["data"]["savestate"];
         if (alt != "") {
            requrl = (url == "none") ? "" : "&url=" + encodeURIComponent(url);
            var params = "entry=sso&alt=" + encodeURIComponent(alt) + "&returntype=META" +
                  "&gateway=1&savestate=" + encodeURIComponent(savestate) + requrl;
            window.location.href = "https://login.sina.com.cn/sso/login.php?" + params;
         } else {

            cross_domain(response);
         }
      } else if(response['retcode'] == 50111261 && isInIframe()) {
         //do nothing
      } else {

         tid.get(function (tid, where, confidence) {
            incarnate(tid, where, confidence);
         });
      }
};

// 3ö′íÇé¿ö·μ»ØμÇÂ¼ò3¡￡
var error_back = function (msg) {

      var url = "https://weibo.com/1878335471/LuCF3n3IL";
      var clientType = "pc";
      if (url != "none") {

         if (url.indexOf("ssovie4c55=0") === -1) {
            url += (((url.indexOf("?") === -1) ? "?" : "&") + "ssovie4c55=0");
         }
         if (clientType == "mobile") {
            window.location.href = "https://passport.weibo.cn/signin/login?r="+url;
         } else{
            window.location.href = "https://weibo.com/login.php";
         }
      } else {

         if(document.getElementById("message")) {
            document.getElementById("message").innerHTML = "Error occurred" + (msg ? (": " + msg) : ".");
         }
      }
};

var isInIframe = function () {
      try {
         return window.self !== window.top;
      } catch (e) {
         return true;
      }
};

</script>
</body>
</html>

李玉风我爱你 · 发表于 2022-5-25 20:10

编码是gb2312 requests解码的时候是否选择正确？

lgsp_Jim · 发表于 2022-5-25 20:19

微博有反爬机制，你那个网页源码是微博的警告页面！

GitHubList · 发表于 2022-5-26 00:11

有反爬机制的网站,用selenium
虽然效率低,但是简单上手快

BSCEO · 发表于 2022-5-26 09:15

GitHubList 发表于 2022-5-26 00:11
有反爬机制的网站,用selenium
虽然效率低,但是简单上手快

好的好的

帐号		自动登录	找回密码
密码			注册[Register]

[求助] 爬虫初学爬微博页面遇难

浏览过的版块