danwin1210/onion-link-list
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Use encoding in all curl requests

Browse Source
This commit is contained in:
Daniel Winzen
2020-10-25 19:00:01 +01:00
parent d45ee056b4
commit 095685aa77
6 changed files with 3 additions and 14 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
cron/tests.php
View File

@ -20,7 +20,6 @@ foreach($onions as $onion){
curl_setopt($ch, CURLOPT_HEADER, true);
curl_setopt($ch, CURLOPT_FOLLOWLOCATION, true);
curl_setopt($ch, CURLOPT_MAXREDIRS, 2);
curl_setopt($ch, CURLOPT_ENCODING, '');
curl_setopt($ch, CURLOPT_HTTPHEADER, ["Host: $onion[address].onion", 'User-Agent: Mozilla/5.0 (Windows NT 10.0; rv:68.0) Gecko/20100101 Firefox/68.0', 'Accept: text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8', 'Accept-Language: en-US,en;q=0.5', 'Accept-Encoding: gzip, deflate', 'Connection: keep-alive', 'Upgrade-Insecure-Requests: 1']);
curl_setopt($ch, CURLOPT_URL, "http://$onion[address].onion/");
curl_multi_add_handle($mh, $ch);
@ -79,7 +78,6 @@ foreach($curl_handles as $handle){
curl_setopt($ch, CURLOPT_HEADER, true);
curl_setopt($ch, CURLOPT_FOLLOWLOCATION, true);
curl_setopt($ch, CURLOPT_MAXREDIRS, 2);
curl_setopt($ch, CURLOPT_ENCODING, '');
curl_setopt($ch, CURLOPT_HTTPHEADER, ["Host: $onion[address].onion", 'User-Agent: Mozilla/5.0 (Windows NT 10.0; rv:68.0) Gecko/20100101 Firefox/68.0', 'Accept: text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8', 'Accept-Language: en-US,en;q=0.5', 'Accept-Encoding: gzip, deflate', 'Connection: keep-alive', 'Upgrade-Insecure-Requests: 1']);
curl_setopt($ch, CURLOPT_URL, "http://$onion[address].onion/".$matches[1]);
$content=curl_exec($ch);
@ -90,7 +88,6 @@ foreach($curl_handles as $handle){
curl_setopt($ch, CURLOPT_HEADER, true);
curl_setopt($ch, CURLOPT_FOLLOWLOCATION, true);
curl_setopt($ch, CURLOPT_MAXREDIRS, 2);
curl_setopt($ch, CURLOPT_ENCODING, '');
curl_setopt($ch, CURLOPT_HTTPHEADER, ["Host: $onion[address].onion", 'User-Agent: Mozilla/5.0 (Windows NT 10.0; rv:68.0) Gecko/20100101 Firefox/68.0', 'Accept: text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8', 'Accept-Language: en-US,en;q=0.5', 'Accept-Encoding: gzip, deflate', 'Connection: keep-alive', 'Upgrade-Insecure-Requests: 1']);
curl_setopt($ch, CURLOPT_URL, $matches[1]);
$content=curl_exec($ch);
@ -109,7 +106,6 @@ foreach($curl_handles as $handle){
curl_setopt($ch, CURLOPT_HEADER, true);
curl_setopt($ch, CURLOPT_FOLLOWLOCATION, true);
curl_setopt($ch, CURLOPT_MAXREDIRS, 2);
curl_setopt($ch, CURLOPT_ENCODING, '');
curl_setopt($ch, CURLOPT_HTTPHEADER, ["Host: $onion[address].onion", 'User-Agent: Mozilla/5.0 (Windows NT 10.0; rv:68.0) Gecko/20100101 Firefox/68.0', 'Accept: text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8', 'Accept-Language: en-US,en;q=0.5', 'Accept-Encoding: gzip, deflate', 'Connection: keep-alive', 'Upgrade-Insecure-Requests: 1']);
curl_setopt($ch, CURLOPT_URL, $link_to_check);
$content=curl_exec($ch);

8
cron/update.php
View File

@ -48,13 +48,7 @@ function check_links(array &$onions, CurlHandle &$ch, string $link_to_check, boo
if($scan_children && empty($scanned_onions[$md5])){
$scanned_onions[$md5]=$link;
$ch_child=curl_init();
curl_setopt($ch_child, CURLOPT_RETURNTRANSFER, true);
curl_setopt($ch_child, CURLOPT_USERAGENT, USERAGENT);
curl_setopt($ch_child, CURLOPT_PROXY, PROXY);
curl_setopt($ch_child, CURLOPT_PROXYTYPE, CURLPROXY_SOCKS5_HOSTNAME);
curl_setopt($ch_child, CURLOPT_CONNECTTIMEOUT, 25);
curl_setopt($ch_child, CURLOPT_TIMEOUT, 30);
curl_setopt($ch_child, CURLOPT_ENCODING, '');
set_curl_options($ch_child);
curl_setopt($ch_child, CURLOPT_URL, "http://$link.onion");
curl_multi_add_handle($mh, $ch_child);
$curl_handles []= $ch_child;