splash-bot-crawler/crawler.js

/*
 * Mobifier, the gallery bot
 * Copyright (C) 2021 Daniele Bonini
 * 
 * This program is free software: you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published by
 * the Free Software Foundation, either version 3 of the License, or
 * (at your option) any later version.
 * 
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 * 
 * You should have received a copy of the GNU General Public License
 * along with this program.  If not, see <https://www.gnu.org/licenses/>.
 *  
 */    

var recNum = 0;
var nImgProcessed = 0;
var output = 1; 
var verbose = 0; 
var host = "mbfy.it";
var targetDomain = "example.com";
var galTitle = "EXAMPLE";
var landingPage = "Example.com";

var crawlres = [];
var logoFound = false;

function loadDoc() {

    recNum++;

    bConnectionOK = false;

    if (verbose===0 && recNum===1) {

      document.write("Copyrights (C) 2021 Daniele Bonini. GNU General Public License ver3<br>");
      document.write("<br>");
      document.write("This is a bit of doc:<br>");
      document.write("- out: std (for standard output) or json<br>");
      document.write("- v: 0 (for standard verbosity) or 1 (for quite output)<br>");
      document.write("<br>");
      document.write("The service is running and trying to connect to the target server.<br>");
      document.write("The connection to the target web server can eventually fail.<br>");
      document.write("Cause could be found among the following reasons:<br>");
      document.write("- Internet connection lost.<br>");
      document.write("- DNS failure.<br>");
      document.write("- Target server is down.<br>");
      document.write("- Access-Control-Allow-Origin header missing on the target server.<br>");
      document.write("<br>");

    }

    var xhttp = new XMLHttpRequest();
    if (host == "mbfier.com") {
      xhttp.open("GET", "http://"+targetDomain+".mbfier.com/upload.php?url=http://"+targetDomain, true);  
    } else {
      xhttp.open("GET", "http://"+targetDomain+".mbfy.it/upload.php?url=http://"+targetDomain, true);
    }
    xhttp.send();
    xhttp.onreadystatechange = function() {

      if (this.readyState == 4 && this.status == 200) {
  
        bConnectionOK = true;

        if (verbose===0) {
          document.write("Connection succeeded.<br><br>")
        }
        
        try {
          document.getElementById("grausi").value = this.responseText;  
        }
        catch (error) {
          //console.error(error);
          if (!document.getElementById("grausi")) {
            document.write("<input id='grausi' type='hidden' value=''>");
            document.getElementById("grausi").value = this.responseText;
          }  
        } 
        finally {
          document.getElementById("grausi").addEventListener("change", launchParsing(), true);  
        }
        
      } else {

        if ((this.readyState == 4 && this.status == 0) && (!bConnectionOK)) {
          if (verbose===0) {
            document.write("Final result:<br>");
            document.write("Connection problems detected.<br><br>");
          }
        } 
      }

    }
}  

function writeHeader() {
  //TITLE BAR    
  if ((output == 1 && verbose == 1) && (nImgProcessed==0 && recNum==1)) {
    divTitle = "<div style='position:relative;margin-top:8px;top:0px;left:0px;width:100%'>";
    document.write(divTitle);
    divTitle = "<div style='height:50px;text-align:center;opacity:1.0;width:350px;clear:both;margin:auto;vertical-align:middle;border:0px solid green;background: rgba(3, 169, 245, 0.7);color:#FFFFFF;'><span style='position:relative;top:-15px;font-size:25px;font-weight:900;'><br>"+galTitle+"<br><br></span></div></div>";
    document.write(divTitle);
  }  
  // END TITLE
}

function crawlImages() {
  var bTitle = false;
  var found = "||";
  var z = 0;
  var str = document.getElementById('grausi').value;
  str = str.replace("/\n/"," ");
  //var regexp = /<img.*?src="[^*?"<>|]+".*?>/gi;
  //var regexp = /("|')([^*?"<>|]+\.(png|gif|jpg|jpeg))("|')/gi;
  //var regexp = /<img\n?.*src=("|')?([^:*?"<>|]+\.(png|gif|jpg|jpeg))("|')?.*?\n?\/?>/gis;
  //var regexp = /<img.*\n?.*src=("|')?(?!(cid:|data:))([^*?"<>|]+\.(png|gif|jpg|jpeg))("|')?.*?\n?[^*?"<>|]*\/?>/gi;
  var regexp = /<img\n?.*?\n?\/?>/gis;
  var x = str.match(regexp);
  if (x) {
    for (i=0;i<x.length;i++) {
      str2 = x[i]; //.toLowerCase();
      str3 = str2.toLowerCase();
      if (str2.length <= 4) {
        continue;
      }
      if ((str2.indexOf("blank.gif")>-1 || str2.indexOf("pixel.gif")>-1 || str2.indexOf("pxl.gif")>-1 || str2.indexOf("pix.gif")>-1) && str2.indexOf("data-src")==-1) {
        continue;
      }
      if (str2.indexOf(" src=\"data:")>-1 || str2.indexOf(" src='data:")>-1 || str2.indexOf(" src=data:")>-1) {
        str2 = str2.replace(" src=", " zzz=");
        str2 = str2.replace(" data-src=", " src=");
      }
      if (str3.indexOf("credit-card")>-1 || str3.indexOf("loyalty")>-1) {
        continue;
      }
      oo = str2.indexOf(" src=");
      ii = str2.indexOf(" data-src=");
      jj = str2.indexOf(".jpg");
      yy = str2.indexOf(".jpeg");
      kk = str2.indexOf(".png");
      ww = str2.indexOf(".gif");
      if (((oo==-1) && (oo<ii)) && ii>-1 && ((jj>ii) || (yy>ii) || (kk>ii) || (ww>ii))) {
        //alert(str2);
        //var regexp3 = /<img\n?.*data-src=("|')([^*?"<>|]+\.(png|gif|jpg|jpeg))("|').*?\n?\/?>/gis;
        //var zz = str2.match(regexp3);
        //if (zz[0]) {
          //str3 = zz[0];
          str2 = str2.replace(" src=", " zzz=");
          str2 = str2.replace(" data-src=", " src=");
          //str2 = str3;
        //}
      }  
      str2 = str2.trim();
      if (str2 !== "") {
        newurl = str2; 
        //var regexp2 = /([\w\.-]+\.[a-z\.]{2,8})/gi;
        //var regexp2 = /([\w\.-]+\.(png|jpg|jpeg))/gi;
        //var regexp2 = /src="([\w\:\/\.-]+\.(png|jpg|jpeg))"/gi;
        //var regexp2 = /<img.*?src="[^*?"<>|]+".*?>/gi;
        //var regexp2 = /("|')([^*?"<>|]+\.(png|gif|jpg|jpeg))("|')/gi;
        //var regexp2 = /<img\n?.*src=("|')([^*?"<>|]+\.(png|gif|jpg|jpeg))("|').*?>/gi;
        var regexp2 = /<img\n?.*?\n?\/?>/gis;
        
        var y = newurl.match(regexp2);

        if (y[0]) {
	        if (found.indexOf("|" + y[0] + "|") == -1) {
            crawlres[z] = y[0];
            z++;
	          found = found + "|" + y[0] + "|";
	        }
        }
      }
    }
  }

  /*
   * DUMP
   */
  dump = false;
  if (dump || (verbose == 0) || ((output==2) && (verbose == 1))) {
    for (i=0;i<crawlres.length;i++) {
      str = crawlres[i];                        
      str = str.replace("<", "&lt;");
      str = str.replace(">", "&gt;");
      crawlres[i] = str;
    }
  }
  
  if (output === 1) {

    if (verbose === 0) {
      document.write("Image list:<br>");
    }

    /*
     * PATH FIX
     */
    for (i=0;i<crawlres.length;i++) {
      str = crawlres[i];                        
      ipos = str.indexOf("src=//");
      if (ipos>-1) {
        str = str.replace("src=//", "src=http://");
        crawlres[i] = str;
      }
      ipos = str.indexOf("src='//");
      if (ipos>-1) {
        str = str.replace("src='//", "src='http://");
        crawlres[i] = str;
      }
      ipos = str.indexOf("src=\"//");
      if (ipos>-1) {
        str = str.replace("src=\"//", "src=\"http://");
        crawlres[i] = str;
      }      
      ipos = str.indexOf("src=/");
      if (ipos>-1) {
        str = str.replace("src=/", "src=http://"+targetDomain+"/");
        crawlres[i] = str;
      }
      ipos = str.indexOf("src='/");
      if (ipos>-1) {
        str = str.replace("src='/", "src='http://"+targetDomain+"/");
        crawlres[i] = str;
      }
      ipos = str.indexOf("src=\"/");
      if (ipos>-1) {
        str = str.replace("src=\"/", "src=\"http://"+targetDomain+"/");
        crawlres[i] = str;
      }      
      ipos = str.indexOf("src=http");
      if (ipos>-1) {
        continue;
      }
      ipos = str.indexOf("src='http");
      if (ipos>-1) {
        continue;
      }
      ipos = str.indexOf("src=\"http");
      if (ipos>-1) {
        continue;
      }
      str = str.replace("src=", "src=http://"+targetDomain+"/");
      crawlres[i] = str;
      str = str.replace("src='", "src='http://"+targetDomain+"/");
      crawlres[i] = str;
      str = str.replace("src=\"", "src=\"http://"+targetDomain+"/");
      crawlres[i] = str;
    }
     
     
    /* 
     * SEARCH FOR LOGO
     */
        var newcrawlres = [];
        var found = -1;
        for (i=0;i<crawlres.length;i++) {
          str = crawlres[i];                        
          if (str.indexOf("\"logo\"")>-1 || str.indexOf("'logo'")>-1 || str.indexOf("logo.png")>-1 || str.indexOf("logo.gif")>-1 || str.indexOf("logo.jpg")>-1 || str.indexOf("logo.jpeg")>-1) {
            found=i;
            //alert("found="+found);
            break;
          }
        }

        if (found>-1) {
          newcrawlres[0] = crawlres[found];
          //alert(newcrawlres[0]);
          var j=1;
          for (i=0;i<found;i++) {
            newcrawlres[j] = crawlres[i];              
            j++;
          }                
          for (i=found+1;i<crawlres.length;i++) {
            newcrawlres[j] = crawlres[i];              
            j++;
          }                
          crawlres = newcrawlres;
          logoFound = true;
          //alert(newcrawlres[0]);
        } 
    // END SEARCH 
    
    for (i=0;i<crawlres.length;i++) {
      document.write(crawlres[i]);
      if (verbose === 0) {
        document.write("<br>");
      }      
    }
    
  } else {

    document.write(JSON.stringify(crawlres));

  }

}

function postCrawling() {
	var colImages = document.images;
  var j = 0;
  var logoHeight = "300px";
	for(var i = 0; i < colImages.length; i++) {
    var oriWidth = colImages[i].width;
    var oriHeight = colImages[i].height;
    var d = oriWidth / oriHeight;
    if((parseInt(colImages[i].width) < 200) || (j>4) || colImages[i].style.visibility == "hidden" || colImages[i].style.display == "none") {
       colImages[i].style.display = "none";
       colImages[i].style.visibility = "hidden";
    } else {
       colImages[i].style.width = "100%"; 

       newWidth = colImages[i].width;
       newHeight = parseInt(newWidth / d); 

       colImages[i].style.maxWidth = "";
       colImages[i].style.float = "";
       colImages[i].style.height = "auto"; //newHeight + "px";
       colImages[i].style.position = "";
       colImages[i].style.top = "";
       colImages[i].style.left = "";
       colImages[i].style.margin = "";
       colImages[i].style.marginTop = "";
       colImages[i].style.marginBottom = "";
       colImages[i].style.marginLeft = "";
       colImages[i].style.marginRight = "";
       colImages[i].style.padding = "";
       colImages[i].style.paddingTop = "";
       colImages[i].style.paddingBottom = "";
       colImages[i].style.paddingLeft = "";
       colImages[i].style.paddingRight = "";
       colImages[i].style.border = "";

       //for transp pictures
       colImages[i].style.backgroundColor = "lightgray";

       if (i==0) {
         logoHeight = newHeight + "px";
       }
       j++;
    }
	}
  return j;
}

function writeFooter() {
  if (output == 1 && verbose == 1) {
    str = "<div style='text-align:center;font-size:9px'>Trademarks and brands are property of their respective owners.<br>.<br></div><div style='background-color:#03a9f5;height:105px;text-align:center;vetical-align:middle'><br><br><a href='/gr.php?to="+landingPage+"' target='_blank' style='text-decoration:none;font-weight:900;font-size:3.2vw;color:#FFFFFF;'>Go to "+landingPage+"</a></div>";
    str_redirect = "<script>setTimeout(function(){window.open('/gr.php?to="+landingPage+"','_self')},12000);</script>";
    document.write(str);
    document.write(str_redirect);
  }  
}

window.addEventListener("load", function() {
  if (!document.getElementById("grausi")) {
    document.write("<input id='grausi' type='hidden' value=''>");
  }  
  loadDoc();
}, true);  


function checkHttpResponse() {
  ret = false;
  str = document.getElementById("grausi").value;
  if (str.indexOf("<img")>-1) {
    ret = true;
  }
  return ret;
}

function launchParsing() {

  writeHeader();
  
  if (checkHttpResponse()) {
    
    crawlImages();
    setTimeout(nImgProcessed=postCrawling(), 1500);
    
    if (nImgProcessed==0 && recNum<3 && verbose!=0 && output!=2) {
      //document.body.innerHtml = "";
      loadDoc();
      return;
    }
    
  } else {
    if (recNum<3 && verbose!=0 && output!=2) {
      loadDoc();
    }
  }
  
  setTimeout(writeFooter(), 1700);

}