60 * 30) continue; if ($parts[3] !== $search_url) continue; $url = $parts[2]; $query_id = $parts[1]; break; } fclose($cache); } } if ($url === null) { $timestamp = time(); $s = curl_init($search_url); curl_setopt($s, CURLOPT_RETURNTRANSFER, true); $html = curl_exec($s); preg_match_all('/id="control_([^"]*)" value="([^"]*)"/', $html, $matches, PREG_SET_ORDER); $control = []; foreach ($matches as $m) { $control[$m[1]] = $m[2]; } $query_id = $control['iniqueid']; $server_count = $control['countserver']; $lines = []; for ($i = 0; $i < 30; $i++) { sleep(1); $s = curl_init("https://www.easy-cert.com/_includes/search/result_items.php"); curl_setopt($s, CURLOPT_RETURNTRANSFER, true); curl_setopt($s, CURLOPT_HTTPHEADER, [ 'Referer: https://www.easy-cert.com/htm/suchergebnis.htm', 'X-Requested-With: XMLHttpRequest', 'X-USE: 1dfd3nsdv234njsdf923masddj123n12l31lg28gsdf2k34', ]); curl_setopt($s, CURLOPT_POSTFIELDS, "uniqueid=$query_id&countServer=$server_count"); $json = json_decode(curl_exec($s)); if ($json->{'count'} >= $server_count) { $url = "https://www.easy-cert.com/" . str_replace('../', '', $json->{'filename'}); break; } } if ($url !== null) file_put_contents('.cache.csv', "$timestamp;$query_id;$url;$search_url\n", FILE_APPEND); } if ($url === null) { header("Status: 500"); header("Content-Length: 0"); exit; } $sed = [ 's/^.*"searchresults":\[//', 's/],.*$//', 's/},{/},\n {/g', '/^$/d', 's/^/ /', 's/}$/},/', ]; header("Content-Type: application/json; charset=UTF-8"); echo '{"searchUrl":' . json_encode($search_url) . ',"queryId":' . json_encode($query_id) . ',"rawFileUrl":' . json_encode($url) . ',"timestamp":' . json_encode(gmdate('Y-m-d\TH:i:s\Z', $timestamp)) . ',"limit":' . json_encode($limit) . ',"offset":' . json_encode($offset) . ",\"data\":[\n"; $fd_spec = [ 0 => ["pipe", "r"], // stdin 1 => ["pipe", "w"], // stdout 2 => ["pipe", "w"], // stderr 3 => ["pipe", "w"], // line count ]; $offset++; $process = proc_open( ['bash', '-c', "curl -s -N " . escapeshellarg($url) . " | " . // fetch data (silent and unbuffered) "xxd -p | " . // convert to hex stream "sed 's/7d2c7b/7d2c0a7b/g' | " . // roughly break up into lines (s/},{/},\n{/g) "xxd -r -p | " . // convert to text stream "sed '" . implode(';', $sed) . "' | " . // apply sed commands "tee >(wc -l 1>&3) | " . // copy stdout into wc and write result into fd 3 "tail -n +$offset | " . // apply offset ($limit !== null ? " head -n $limit | " : "") . // optionally apply limit "sed '\$s/.$//'"], // remove last comma of last line $fd_spec, $pipes ); fclose($pipes[0]); while (($line = fgets($pipes[1])) !== false) { echo $line; } fclose($pipes[1]); $stderr = stream_get_contents($pipes[2]); fclose($pipes[2]); $count = intval(trim(stream_get_contents($pipes[3]))); fclose($pipes[3]); $return_value = proc_close($process); echo '],"totalCount":' . json_encode($count) . "}\n";