diff --git a/www/organic/.php/pdf.php b/www/organic/.php/pdf.php new file mode 100644 index 0000000..c12874f --- /dev/null +++ b/www/organic/.php/pdf.php @@ -0,0 +1,229 @@ += $from; $i--) { + $el = $array[$i]; + if (sizeof($postalCode) > 0) { + if (sizeof($address) === 0) $el = rtrim($el, ", \n\r\t\v\0"); + if (strlen($el) === 0) continue; + array_unshift($address, $el); + } else if (preg_match("/^[A-Z0-9.\-]{3,},?$/", $el)) { + array_unshift($postalCode, trim($el, ", \n\r\t\v\0")); + } else { + array_unshift($city, $el); + } + } + return [implode(' ', $address), implode(' ', $postalCode), implode(' ', $city)]; +} + +function parse_pdf(string $filename, bool $loadUrl = false): array | null { + if (!file_exists($filename)) + return null; + if (exec("pdftotext -raw " . escapeshellarg($filename) . " -", $text) === false) + return null; + + $text = implode("\n", $text); + exec("pdfsig " . escapeshellarg($filename), $sig); + $sig = implode("\n", $sig); + + if ($loadUrl) { + $prefix = "/tmp/pdfimages-" . uniqid(); + exec("zbarimg -q --raw $(pdfimages -print-filenames " . escapeshellarg($filename) . " $prefix) | uniq; rm -rf $prefix-*.*", $qrCodes); + } else { + $qrCodes = null; + } + + $r = preg_match('@([a-z]{2}) (https://webgate\.ec\.europa\.eu/tracesnt/directory/publication/organic-operator/(.*?)\.pdf) (\d+) / (\d+)@', $text, $matches); + if ($r === 1) { + // TRACES certificate + + $data = []; + $parts = preg_split('@\n(I+\.\d+) ([^\n]*)@', $text, -1, PREG_SPLIT_DELIM_CAPTURE); + $status = str_replace("\n", '', $parts[0]); + for ($i = 3; $i < sizeof($parts); $i += 3) { + $data[$parts[$i - 2]] = trim($parts[$i]); + } + + $lang = $matches[1]; + $splitAddr = [ + 'de' => 'Adresse', + 'en' => 'Address', + ][$lang]; + $splitCountry = [ + 'de' => 'Land', + 'en' => 'Country', + ][$lang]; + $statusMap = [ + 'de' => [ + 'AUSGESTELLT' => 'issued', + ], + 'en' => [ + 'ISSUED' => 'issued', + ] + ][$lang]; + $activityMap = [ + 'de' => [ + 'Aufbereitung' => 'preparation', + 'Ausfuhr' => 'export', + 'Einfuhr' => 'import', + 'Lagerung' => 'storing', + 'Produktion' => 'production', + 'Vertrieb' => 'distribution', + 'Vertrieb/Inverkehrbringen' => 'distribution_placing_on_the_market', + ], + 'en' => [ + 'Distribution' => 'distribution', + 'Distribution/Placing on the market' => 'distribution_placing_on_the_market', + 'Export' => 'export', + 'Import' => 'import', + 'Preparation' => 'preparation', + 'Production' => 'production', + 'Storing' => 'storing', + ], + ][$lang]; + + $certUrl = $matches[2]; + $certId = $matches[3]; + $authorityId = explode('.', $certId)[0]; + $operatorId = explode('.', $certId)[1]; + $operator = preg_split('@\s+@', trim($data['I.3'])); + $p1 = array_search($splitAddr, $operator); + $p2 = array_search($splitCountry, $operator); + $operatorName = trim(implode(' ', array_filter($operator, fn($k,$i) => $i > 0 && $i < $p1, ARRAY_FILTER_USE_BOTH)), ', '); + [$opAddr, $opPostal, $opCity] = get_address($operator, $p1 + 1, $p2 - 1); + + $authority = preg_split('@\s+@', trim($data['I.4'])); + $until = array_search("($authorityId)", $authority); + $p1 = array_search($splitAddr, $authority); + $p2 = array_search($splitCountry, $authority); + $authorityName = implode(' ', array_filter($authority, fn($k,$i) => $i > 0 && $i < $p1 - 1 && ($i !== $p1 - 2 || !str_starts_with($k, '(')), ARRAY_FILTER_USE_BOTH)); + [$aAddr, $aPostal, $aCity] = get_address($authority, $p1 + 1, $p2 - 1); + + $activities = []; + foreach (explode("\n", $data['I.5']) as $a) { + $activities[] = $activityMap[trim($a, '• ')]; + } + + preg_match_all('/\([a-g]\)/', $data['I.6'], $matches, PREG_SET_ORDER); + $products = []; + foreach ($matches as $m) { + $products[] = $m[0]; + } + + preg_match_all('@\d+/\d+/\d+@', $data['I.8'], $matches, PREG_SET_ORDER); + $valid1 = implode('-', array_reverse(explode('/', $matches[0][0]))); + $valid2 = implode('-', array_reverse(explode('/', $matches[1][0]))); + + $sigs = []; + foreach (array_slice(explode("\nSignature #", $sig), 1) as $s) { + $sData = []; + $sData2 = []; + preg_match_all('/\n {2}- (([^:\n]*): )?([^\n]*)/', $s, $matches, PREG_SET_ORDER); + foreach ($matches as $m) { + if (strlen($m[2]) === 0) { + $sData2[] = $m[3]; + } else { + $sData[$m[2]] = $m[3]; + } + } + $sigs[] = [ + 'signerCommonName' => $sData['Signer Certificate Common Name'], + 'valid' => $sData['Signature Validation'] === 'Signature is Valid.', + 'trusted' => $sData['Certificate Validation'] === 'Certificate is Trusted.', + 'totalDocument' => in_array('Total document signed', $sData2), + 'timestamp' => gmdate('Y-m-d\TH:i:s\Z', strtotime($sData['Signing Time'])), + 'type' => $sData['Signature Type'], + 'hashAlgorithm' => $sData['Signing Hash Algorithm'], + 'signerDistinguishedName' => $sData['Signer full Distinguished Name'], + 'fieldName' => $sData['Signature Field Name'], + ]; + } + + return [ + 'type' => 'traces', + 'lang' => $lang, + 'id' => $certId, + 'status' => $statusMap[$status], + 'operator' => [ + 'id' => $operatorId, + 'groupOfOperators' => !str_starts_with($data['I.2'], '☑'), + 'name' => $operatorName, + 'address' => $opAddr, + 'postalCode' => $opPostal, + 'city' => $opCity, + 'countryCode' => $operator[sizeof($operator) - 1], + ], + 'authority' => [ + 'id' => $authorityId, + 'name' => $authorityName, + 'address' => $aAddr, + 'postalCode' => $aPostal, + 'city' => $aCity, + 'countryCode' => $authority[sizeof($authority) - 1], + ], + 'activities' => $activities, + 'productCategories' => $products, + 'validFrom' => $valid1, + 'validTo' => $valid2, + 'url' => $certUrl, + 'digitalSignatures' => $sigs, + 'qrCodes' => $qrCodes, + ]; + } + + $isLacon = str_contains($text, "\nLACON GmbH\n") && (str_contains($text, "\nAnlage zum Zertifikat\n")); + if (preg_match('/AT-BIO-[0-9]{3}/', $text, $matches) === 1 || $isLacon) { + $authorityId = $matches[0] ?? ($isLacon ? 'AT-BIO-402' : null); + $certId = null; + $certNr = null; + if (preg_match("/$authorityId\.040-[0-9]{7}\.[0-9]{4}\.[0-9]{3}/", $text, $matches) === 1) { + $certId = $matches[0]; + } + if (preg_match_all("/\b[0-9]+([._-])[0-9]+\g{-1}[0-9]+\b/", $text, $matches, PREG_SET_ORDER) !== false) { + foreach ($matches as $m) { + if (strlen($m[0]) > 10 && !str_ends_with($certId, $m[0])) + $certNr = $m[0]; + } + } + + if ($certId === null && $qrCodes !== null && sizeof($qrCodes) > 0 && str_starts_with($qrCodes[0], 'www.easy-cert.com/')) { + $res = exec("curl -L " . escapeshellarg($qrCodes[0]) . " | grep -A 100 '' | sed 's/<[^>]*>//g;s/\\s\\+/ /g;s/^ //g;s/ $//g;s/\\n//m' | grep 'Certificate number' -A 1"); + if ($res !== false && preg_match("/$authorityId\.040-[0-9]{7}\.[0-9]{4}\.[0-9]{3}/", $res, $matches) === 1) { + $certId = $matches[0]; + } + } + + $operator = ['lfbisNr' => null]; + if ($authorityId === 'AT-BIO-301') { + // TODO + } else if ($authorityId === 'AT-BIO-302') { + if (preg_match("/^lfbis[^0-9]*([0-9]{6,7})$/im", $text, $matches) === 1) + $operator['lfbisNr'] = str_pad($matches[1], 7, '0', STR_PAD_LEFT); + } else if ($authorityId === 'AT-BIO-401') { + // TODO + } else if ($authorityId === 'AT-BIO-402') { + if (preg_match("/^Betriebsnummer ([0-9]{6,7})$/im", $text, $matches) === 1) + $operator['lfbisNr'] = str_pad($matches[1], 7, '0', STR_PAD_LEFT); + } else if ($authorityId === 'AT-BIO-902') { + if (preg_match("/ lfbis:([0-9]{6,7})$/im", $text, $matches) === 1) + $operator['lfbisNr'] = str_pad($matches[1], 7, '0', STR_PAD_LEFT); + } else if ($authorityId === 'AT-BIO-903') { + // TODO + } + + return [ + 'type' => $authorityId, + 'lang' => 'de', + 'id' => $certId, + 'nr' => $certNr, + 'operator' => $operator, + 'authority' => ['id' => $authorityId], + 'qrCodes' => $qrCodes, + ]; + } + + return ['type' => 'unknown']; +} diff --git a/www/organic/certificates/.cache.csv b/www/organic/certificates/.cache.csv new file mode 100644 index 0000000..e69de29 diff --git a/www/organic/certificates/index.php b/www/organic/certificates/index.php new file mode 100644 index 0000000..ac7d2af --- /dev/null +++ b/www/organic/certificates/index.php @@ -0,0 +1,44 @@ + $cert, + 'appendix' => $appendix, +]); diff --git a/www/organic/external/bioc/operators.php b/www/organic/external/bioc/operators.php index 328eadd..5e7264c 100644 --- a/www/organic/external/bioc/operators.php +++ b/www/organic/external/bioc/operators.php @@ -121,7 +121,8 @@ if ($country === null) { $data = []; -$url = "https://www.bioc.info/search/producerSearchQuery?search[name]=" . urlencode($name) . "&search[citycode]=" . urlencode($postalCode) . "&search[operatorId]=" . urlencode($idNr) . "&search[country]=$country"; +$origin = "https://www.bioc.info/search/producersearchresult?producerSearch%5Bcountry%5D={$country}&producerSearch%5Bcitycode%5D=" . urlencode($postalCode) . "&producerSearch%5Bname%5D=" . urlencode($name) . "&producerSearch%5BoperatorId%5D=" . urlencode($idNr); +$url = "https://www.bioc.info/search/producerSearchQuery?search[country]={$country}&search[citycode]=" . urlencode($postalCode) . "&search[name]=" . urlencode($name) . "&search[operatorId]=" . urlencode($idNr); $m = curl_multi_init(); $requests = []; foreach ($source_groups as $group) { @@ -134,6 +135,7 @@ foreach ($source_groups as $group) { $running = 1; do { curl_multi_exec($m, $running); + if ($running) usleep(250_000); } while ($running); foreach ($requests as $r) { curl_multi_remove_handle($m, $r); @@ -141,7 +143,7 @@ foreach ($requests as $r) { curl_multi_close($m); foreach ($requests as $s) { - if (($json = curl_multi_getcontent($s)) === false) { + if (($json = curl_multi_getcontent($s)) === null) { header('Status: 500'); header('Content-Length: 34'); echo "{\"error\":\"internal server error\"}\n"; @@ -153,10 +155,8 @@ foreach ($requests as $s) { } } -header('Content-Type: application/json; charset=UTF-8'); - $first = true; -echo "{\"data\":[\n"; +echo "{\"originUrl\":\"$origin\",\"data\":[\n"; foreach ($data as $id => $row) { if (!$first) echo ",\n"; $row = explode(" None: args = parser.parse_args() s = requests.Session() + s.verify = False while True: try: r = s.get(f'{URL}?menu_sid=5002') diff --git a/www/organic/external/bioqs/.operators.py b/www/organic/external/bioqs/.operators.py index 96429cb..f5ff43a 100755 --- a/www/organic/external/bioqs/.operators.py +++ b/www/organic/external/bioqs/.operators.py @@ -33,29 +33,35 @@ def main() -> None: query = {'PartnerCertSearchForm:pcs_' + q.split('=', 1)[0]: urllib.parse.unquote(q.split('=', 1)[-1]) for q in args.query.split('&')} s = requests.Session() - while True: - try: - r = s.get(f'{URL}?menu_sid=5002') - uri = ACTION_RE.findall(r.text)[0] - break - except IndexError: - pass - hidden = {m[1]: m[2] for m in HIDDEN_RE.finditer(r.text)} + s.verify = False + try: + while True: + try: + r = s.get(f'{URL}?menu_sid=5002') + uri = ACTION_RE.findall(r.text)[0] + break + except IndexError: + pass + hidden = {m[1]: m[2] for m in HIDDEN_RE.finditer(r.text)} - r = s.post(f'{BASE_URL}{uri}', data={ - **query, - 'PartnerCertSearchForm:button_search': 'Suche starten...', - 'PartnerCertSearchForm_SUBMIT': '1', - 'javax.faces.ViewState': hidden['javax.faces.ViewState'], - }) + r = s.post(f'{BASE_URL}{uri}', data={ + **query, + 'PartnerCertSearchForm:button_search': 'Suche starten...', + 'PartnerCertSearchForm_SUBMIT': '1', + 'javax.faces.ViewState': hidden['javax.faces.ViewState'], + }) + + result_table = r.text[r.text.find('') + 8] + uncollapsed_rows = [tuple(remove_tags(m[1]) + for m in TD_RE.finditer(row[0])) + for row in UNCOLLAPSED_ROW_RE.finditer(result_table)] + collapsed_rows = [[tuple(remove_tags((ATTACHMENT_RE.search(m[1]) or m)[1]) for m in TD_RE.finditer(row[1])) + for row in ROW_RE.finditer(tbl[0])] + for tbl in COLLAPSED_ROW_RE.finditer(result_table)] + except: + print('[]') + return - result_table = r.text[r.text.find('') + 8] - uncollapsed_rows = [tuple(remove_tags(m[1]) - for m in TD_RE.finditer(row[0])) - for row in UNCOLLAPSED_ROW_RE.finditer(result_table)] - collapsed_rows = [[tuple(remove_tags((ATTACHMENT_RE.search(m[1]) or m)[1]) for m in TD_RE.finditer(row[1])) - for row in ROW_RE.finditer(tbl[0])] - for tbl in COLLAPSED_ROW_RE.finditer(result_table)] print('[') first = True for row, tbl in zip(uncollapsed_rows, collapsed_rows): diff --git a/www/organic/external/combined/operators.php b/www/organic/external/combined/operators.php new file mode 100644 index 0000000..02fdc2d --- /dev/null +++ b/www/organic/external/combined/operators.php @@ -0,0 +1,141 @@ + $op]; +foreach ($data[0] as $op) { + if (($m = array_find($operators, fn($o) => isset($o['easyCert']) && $o['easyCert']['id'] === $op['id'])) !== null) { + $m['bioc'] = $op; + } else if (isset($op['details']) && ($m = array_find($operators, fn($o) => isset($o['easyCert']) && $o['easyCert']['idNr'] === $op['details']['idNr'])) !== null) { + $m['bioc'] = $op; + } else if (isset($op['details']) && ($m = array_find($operators, fn($o) => isset($o['bioc']['details']) && isset($op['details']) && $o['bioc']['name'] === $op['name'] && $o['bioc']['details']['address'] === $op['details']['address'])) !== null) { + $m['bioc'] = $op; + } else { + $operators[] = ['bioc' => $op]; + } +} +foreach ($data[1] as $op) { + if (false) { + + } else { + $operators[] = ['bioqs' => $op]; + } +} +foreach ($data[3] as $op) $operators[] = ['lkv' => $op]; + +// for (const op of easyCert.data) operators.push({easyCert: op}); +// for (const op of bioc.data) { +// let m; +// if ((m = operators.find(o => o.easyCert?.id === op.id))) { +// m.bioc = op; +// } else if ((m = operators.find(o => o.easyCert?.idNr === op.details.idNr))) { +// m.bioc = op; +// } else if ((m = operators.find(o => o.bioc?.name === op.name && o.bioc?.details.address === op.details.address))) { +// m.bioc2 = op; +// } else { +// operators.push({bioc: op}); +// } +// } +// for (const op of bioqs.data) { +// let m; +// if (op.lfbisNr && (m = operators.find(o => o.easyCert?.idNr === op.lfbisNr))) { +// m.bioqs = op; +// } else if (op.lfbisNr && (m = operators.find(o => o.bioc?.details.idNr === op.lfbisNr))) { +// m.bioqs = op; +// } else if (op.idNr && (m = operators.find(o => o.bioc?.details.idNr === op.idNr))) { +// m.bioqs = op; +// } else if ((m = operators.find(o => (o.easyCert?.name ?? o.bioc?.name) === op.name && (o.easyCert?.details.address ?? o.bioc?.details.address) === op.address))) { +// m.bioqs = op; +// } else { +// operators.push({bioqs: op}); +// } +// } +// for (const op of lkv.data) operators.push({lkv: op}); +// +// for (const op of operators) { +// op.name = op.easyCert?.name ?? op.bioc?.name ?? op.bioqs?.name ?? op.lkv?.name ?? null; +// op.address = op.easyCert?.details.address ?? op.bioc?.details.address ?? op.bioqs?.address ?? op.lkv?.address ?? null; +// op.postalCode = op.easyCert?.postalCode ?? op.bioc?.postalCode ?? op.bioqs?.postalCode ?? op.lkv?.postalCode ?? null; +// op.city = op.easyCert?.city ?? op.bioc?.city ?? op.bioqs?.city ?? op.lkv?.city ?? null; +// op.countryCode = op.easyCert?.countryCode ?? op.bioc?.details.countryCode ?? 'AT'; +// op.lfbisNr = op.bioqs?.lfbisNr ?? null; +// const certNrs = [...new Set([].concat( +// op.easyCert?.details.certificates.map(c => c.nr) ?? [], +// op.bioc?.details.certificates.map(c => c.nr) ?? [], +// op.bioqs?.certificates.map(c => c.nr) ?? [], +// op.lkv?.certificates.map(c => c.nr) ?? [] +// ))]; +// op.ooId = certNrs +// .map(nr => nr.match(/[A-Z]{2}-.{3}-[0-9]{3}\.[0-9]{3}-[0-9]{7}\.[0-9]{4}\.[0-9]{3}/)) +// .filter(nr => nr !== null) +// .map(nr => nr[0].split('.')[1])[0] ?? null; +// } +// +// operators.sort((a, b) => a.name.localeCompare(b.name)); + +$first = true; +echo "{\"data\":[\n"; +foreach ($operators as $op) { + if (!$first) echo ",\n"; + echo " "; + echo jenc($op); + $first = false; +} +echo "\n]}\n"; diff --git a/www/organic/external/lkv/operators.php b/www/organic/external/lkv/operators.php index a3bb088..6a2bd79 100644 --- a/www/organic/external/lkv/operators.php +++ b/www/organic/external/lkv/operators.php @@ -1,6 +1,6 @@ + + + Bio Zertifikate + + + +

Bio Zertifikate

+
+ + +
+ + diff --git a/www/organic/index.php b/www/organic/index.php index 6293b9b..656e558 100644 --- a/www/organic/index.php +++ b/www/organic/index.php @@ -36,13 +36,13 @@ $AUTHORITIES = [ ]; header('Access-Control-Allow-Origin: *'); +header('Content-Type: application/json; charset=UTF-8'); function jenc($data): string { return json_encode($data, JSON_UNESCAPED_SLASHES | JSON_UNESCAPED_UNICODE); } if ($_SERVER['REQUEST_METHOD'] !== 'GET' && $_SERVER['REQUEST_METHOD'] !== 'HEAD') { - header('Content-Type: application/json; charset=UTF-8'); header('Status: 405'); header('Content-Length: 56'); header('Allow: GET, HEAD'); @@ -58,44 +58,7 @@ if ($info !== '/' && str_ends_with($info, '/')) { } $parts = explode('/', $info); -if (str_starts_with($info, '/certificates/')) { - $id = $parts[2]; - if (str_ends_with($id, '.txt')) { - $id = substr($id, 0, -4); - } - if (str_contains($id, '/') || !file_exists("certificates/$id.pdf")) { - header('Content-Type: application/json; charset=UTF-8'); - header('Status: 404'); - header('Content-Length: 22'); - echo "{\"error\":\"not found\"}\n"; - exit; - } - if (str_ends_with($parts[2], '.txt')) { - $mode = '-layout'; - if (isset($_GET['raw']) && strtolower($_GET['raw']) === 'true') { - $mode = '-raw'; - } - header('Content-Type: text/plain; charset=UTF-8'); - system("pdftotext $mode 'certificates/$id.pdf' -"); - exit; - } - - if (str_ends_with($id, '.appendix')) { - header('Status: 303'); - header('Location: ' . substr($id, 0, -9));; - exit; - } - - $cert = shell_exec("pdftotext -raw 'certificates/$id.pdf' -"); - $appendix = shell_exec("pdftotext -raw 'certificates/$id.appendix.pdf' -"); - - $p1 = strpos($cert, "\nI.3 "); - $p2 = strpos($cert, "\nI.4 "); - echo substr($cert, $p1 + 5, $p2 - $p1 - 5); - - exit; -} else if ($info === '/authorities') { - header('Content-Type: application/json; charset=UTF-8'); +if ($info === '/authorities') { echo "{\"data\":[\n"; $first = true; foreach ($AUTHORITIES as $auth) { @@ -107,7 +70,6 @@ if (str_starts_with($info, '/certificates/')) { exit; } else if (str_starts_with($info, '/authorities/')) { $code = $parts[2]; - header('Content-Type: application/json; charset=UTF-8'); if (array_key_exists($code, $AUTHORITIES)) { echo jenc($AUTHORITIES[$code]); echo "\n"; @@ -117,17 +79,6 @@ if (str_starts_with($info, '/certificates/')) { echo "{\"error\":\"not found\"}\n"; } exit; -} else if ($info === '/operators') { - header('Content-Type: application/json; charset=UTF-8'); - header('Status: 501'); - header('Content-Length: 27'); - echo "{\"error\":\"not implemented\"}\n"; - exit; -} else if (str_starts_with($info, '/operators/')) { - $ooid = $parts[2]; - header('Content-Type: text/plain; charset=UTF-8'); - echo "Organic Operator Id: $ooid\n"; - exit; } header('Status: 404'); diff --git a/www/organic/operators/.ids/.gitkeep b/www/organic/operators/.ids/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/www/organic/operators/index.php b/www/organic/operators/index.php new file mode 100644 index 0000000..5d289f9 --- /dev/null +++ b/www/organic/operators/index.php @@ -0,0 +1,346 @@ + $v) { + if ($v !== null) + $data[$k] = $v; + } + fseek($file, 0); + ftruncate($file, 0); + foreach ($data as $k => $v) { + fwrite($file, "$k:$v\n"); + } + fclose($file); + return !$existed; +} + +function find_local(string $anyId): string | null { + $found = []; + foreach (scandir('.ids/') as $file) { + if (!str_ends_with($file, '.txt')) continue; + $data = file_get_contents(".ids/$file"); + if (str_contains($data, "\n$anyId\n")) + $found[] = substr($file, 0, strlen($file) - 4); + } + return sizeof($found) === 1 ? $found[0] : null; +} + +function update_cache(string $country, string $postalCode): void { + $anyNew = false; + $traces = jget("https://webgate.ec.europa.eu/tracesnt/directory/publication/organic-operator/for/query?countryCode=$country&operatorPostalCode=$postalCode&sort=issuedOn"); + foreach ($traces as $op) { + // traces + $ooId = $op['operatorIdentifier']; + $name = $op['operator']['name']; + $address = $op['operator']['address']['street']['value']; + $cityPostalCode = $op['operator']['address']['cityReference']['postalCode']; + $cityName = $op['operator']['address']['cityReference']['name']; + $new = update_local($ooId, ['plz' => $postalCode, 'name' => "$name|$address|$cityPostalCode $cityName"]); + $anyNew = $anyNew || $new; + } + if (!$anyNew) return; + + $m = curl_multi_init(); + $requests = []; + foreach (['easy-cert', 'bioc', 'bioqs', 'lkv'] as $api) { + $s = curl_init("https://elwig.at/organic/external/$api/operators?country=$country&postalCode=$postalCode"); + $requests[] = $s; + curl_setopt($s, CURLOPT_RETURNTRANSFER, true); + curl_multi_add_handle($m, $s); + } + + $running = 1; + do { + curl_multi_exec($m, $running); + if ($running) usleep(250_000); + } while ($running); + foreach ($requests as $r) { + curl_multi_remove_handle($m, $r); + } + curl_multi_close($m); + $data = []; + foreach ($requests as $s) { + if (($json = curl_multi_getcontent($s)) === null) { + header('Status: 500'); + header('Content-Length: 34'); + echo "{\"error\":\"internal server error\"}\n"; + exit; + } + $data[] = json_decode($json, true)['data']; + } + + foreach ($data[0] as $op) { + // easy-cert + $ooId = find_local("easy-cert:$op[db]:$op[id]"); + $lfbisNr = null; + if ($ooId === null && ($opDetails = jget($op['url'])) !== null) { + foreach ($opDetails['certificates'] as $cert) { + if (preg_match("/\.([0-9]{3}-[0-9]{7})\./", $cert['nr'], $matches) === 1) { + $ooId = $matches[1]; + } + } + if ($ooId === null || $lfbisNr === null) { + foreach ($opDetails['certificates'] as $cert) { + if ($cert['pdfUrl'] === null) continue; + $pdfDetails = pdf_fetch($cert['pdfUrl']); + if (isset($pdfDetails['operator']['lfbisNr'])) $lfbisNr = $pdfDetails['operator']['lfbisNr']; + if (!isset($pdfDetails['id'])) continue; + if (preg_match("/\.([0-9]{3}-[0-9]{7})\./", $pdfDetails['id'], $matches) === 1) { + $ooId = $matches[1]; + } + } + } + } + if ($ooId === null) + continue; + update_local($ooId, ['plz' => $postalCode, 'lfbis' => $lfbisNr, 'easy-cert' => "$op[db]:$op[id]"]); + } + foreach ($data[1] as $op) { + // bioc + $ooId = find_local("bioc:$op[id]"); + if ($ooId === null && ($opDetails = jget($op['url'])) !== null) { + foreach ($opDetails['certificates'] as $cert) { + if (preg_match("/\.([0-9]{3}-[0-9]{7})\./", $cert['nr'], $matches) === 1) { + $ooId = $matches[1]; + } + } + if ($ooId === null) { + foreach ($opDetails['certificates'] as $cert) { + if ($cert['pdfUrl'] === null) continue; + $pdfDetails = pdf_fetch($cert['pdfUrl']); + if (isset($pdfDetails['operator']['lfbisNr'])) $op['lfbisNr'] = $pdfDetails['operator']['lfbisNr']; + if (!isset($pdfDetails['id'])) continue; + if (preg_match("/\.([0-9]{3}-[0-9]{7})\./", $pdfDetails['id'], $matches) === 1) { + $ooId = $matches[1]; + } + } + } + } + if ($ooId === null) + continue; + update_local($ooId, ['plz' => $postalCode, 'bioc' => "$op[id]"]); + } + foreach ($data[2] as $op) { + // bioqs + $ooId = find_local("bioqs:$op[id]"); + $lfbisNr = null; + if ($ooId === null || $lfbisNr === null) { + foreach ($op['certificates'] as $cert) { + if (preg_match("/\.([0-9]{3}-[0-9]{7})\./", $cert['nr'], $matches) === 1) { + $ooId = $matches[1]; + } + } + if ($ooId === null || $lfbisNr === null) { + foreach ($op['certificates'] as $cert) { + if ($cert['pdfUrl'] === null) continue; + $pdfDetails = pdf_fetch($cert['pdfUrl']); + if (isset($pdfDetails['operator']['lfbisNr'])) $lfbisNr = $pdfDetails['operator']['lfbisNr']; + if (!isset($pdfDetails['id'])) continue; + if (preg_match("/\.([0-9]{3}-[0-9]{7})\./", $pdfDetails['id'], $matches) === 1) { + $ooId = $matches[1]; + } + } + } + } + if ($ooId === null) + continue; + update_local($ooId, ['plz' => $postalCode, 'lfbis' => $lfbisNr, 'bioqs' => "$op[id]"]); + } + foreach ($data[3] as $op) { + // lkv + $ooId = find_local("lkv:$op[id]"); + if ($ooId === null) + continue; + update_local($ooId, ['plz' => $postalCode, 'lfbis' => $op['lfbisNr'], 'lkv' => "$op[id]"]); + update_local($ooId, ['plz' => $postalCode, 'lfbis' => $op['lfbisNr'], 'lkv' => "$op[id]"]); + } +} + +$info = substr($_SERVER['PATH_INFO'], 1); +if ($info === '') { + // search + $limit = $_GET['limit'] ?? null; + $offset = intval($_GET['offset'] ?? "0"); + if ($limit === '') $limit = null; + if ($limit !== null) $limit = intval($limit); + + $country = $_GET['country'] ?? null; + $postalCode = $_GET['postalCode'] ?? null; + if ($country === null || $postalCode === null) { + header('Status: 400'); + header('Content-Length: 109'); + echo "{\"error\":\"bad request\",\"message\":\"The 'country' and 'postalCode' request URL query parameters are required\"}\n"; + exit; + } + + update_cache($country, $postalCode); + + echo "{\"data\":["; + if ($country === 'AT') { + $first = true; + foreach (scandir('.ids/') as $file) { + if (!str_ends_with($file, '.txt') || !str_starts_with($file, '040-')) continue; + $id = substr($file, 0, strlen($file) - 4); + $data = file_get_contents(".ids/$file"); + if (str_starts_with($data, "plz:$postalCode\n")) { + preg_match("/^name:(.*?)\|(.*?)\|([0-9]{4}) (.*?)$/m", $data, $name); + preg_match("/^lfbis:(.*?)$/m", $data, $lfbis); + if (!$first) echo ","; + echo "\n "; + echo jenc([ + 'id' => $id, + 'lfbisNr' => $lfbis[1], + 'name' => $name[1], + 'address' => $name[2], + 'postalCode' => $name[3], + 'city' => $name[4], + 'country' => $country, + 'url' => "https://elwig.at/organic/operators/$id", + ]); + $first = false; + } + } + } + echo "\n]}\n"; + + exit; +} + +if (preg_match("/^((easy-cert:[a-z]+|bioqs|bioc|lkv):[0-9a-zA-Z-_]+|lfbis:[0-9]+)$/", $info) === 1) { + $found = find_local($info); + if ($found !== null) { + header('Status: 303'); + header('Location: /organic/operators/' . $found); + header('Content-Length: 0'); + exit; + } + header('Status: 404'); + header('Content-Length: 22'); + echo "{\"error\":\"not found\"}\n"; + exit; +} else if (!file_exists(".ids/$info.txt")) { + header('Status: 404'); + header('Content-Length: 22'); + echo "{\"error\":\"not found\"}\n"; + exit; +} + +$data = file_get_contents(".ids/$info.txt"); +preg_match("/^name:(.*?)\|(.*?)\|([0-9]{4}) (.*?)$/m", $data, $name); +preg_match("/^lfbis:(.*?)$/m", $data, $lfbis); +preg_match("/^easy-cert:(.*?)$/m", $data, $easyCert); +preg_match("/^bioqs:(.*?)$/m", $data, $bioQs); +preg_match("/^bioc:(.*?)$/m", $data, $bioC); + +if (sizeof($easyCert) > 1 && ($opDetails = jget("https://elwig.at/organic/external/easy-cert/operators/$easyCert[1]")) !== null) { + foreach ($opDetails['certificates'] as $cert) { + if ($cert['pdfUrl'] === null) continue; + pdf_fetch($cert['pdfUrl']); + } +} +if (sizeof($bioQs) > 1 && ($opDetails = jget("https://elwig.at/organic/external/bioqs/operators/$bioQs[1]")) !== null) { + foreach ($opDetails['certificates'] as $cert) { + if ($cert['pdfUrl'] === null) continue; + pdf_fetch($cert['pdfUrl']); + } +} +if (sizeof($bioC) > 1 && ($opDetails = jget("https://elwig.at/organic/external/bioc/operators/$bioC[1]")) !== null) { + foreach ($opDetails['certificates'] as $cert) { + if ($cert['pdfUrl'] === null) continue; + pdf_fetch($cert['pdfUrl']); + } +} + +$certs = []; +$certIds = []; +foreach (scandir('../certificates/') as $file) { + if (!str_contains($file, ".$info.")) + continue; + $certId = substr($file, 0, 31); + if (in_array($certId, $certIds)) + continue; + $certIds[] = $certId; + $cert = parse_pdf("../certificates/$certId.pdf"); + $appendix = parse_pdf("../certificates/$certId.appendix.pdf"); + if ($cert === null && $appendix === null) + continue; + + $parts = explode('.', $file); + $certs[] = [ + 'id' => $certId, + 'nr' => $appendix['nr'] ?? null, + 'validFrom' => $cert['validFrom'] ?? null, + 'validTo' => $cert['validTo'] ?? null, + 'authority' => ['code' => $parts[0]], + 'operator' => ['id' => $parts[1]], + 'url' => "https://elwig.at/organic/certificates/$certId", + 'tracesPdfUrl' => $cert !== null ? "https://elwig.at/organic/certificates/$certId.pdf" : null, + 'appendixPdfUrl' => $appendix !== null ? "https://elwig.at/organic/certificates/$certId.appendix.pdf" : null, + ]; +} + +$details = []; +if (preg_match("/^easy-cert:(.*?)$/m", $data, $easyCert) === 1) + $details[] = "https://elwig.at/organic/external/easy-cert/operators/$easyCert[1]"; +if (preg_match("/^bioc:(.*?)$/m", $data, $bioc) === 1) + $details[] = "https://elwig.at/organic/external/bioc/operators/$bioc[1]"; +if (preg_match("/^bioqs:(.*?)$/m", $data, $bioqs) === 1) + $details[] = "https://elwig.at/organic/external/bioqs/operators/$bioqs[1]"; +if (preg_match("/^lkv:(.*?)$/m", $data, $lkv) === 1) + $details[] = "https://elwig.at/organic/external/lkv/operators/$lkv[1]"; +echo jenc([ + 'id' => $info, + 'lfbisNr' => $lfbis[1], + 'name' => $name[1], + 'address' => $name[2], + 'postalCode' => $name[3], + 'city' => $name[4], + 'country' => 'AT', + 'url' => "https://elwig.at/organic/operators/$info", + 'tracesUrl' => "https://webgate.ec.europa.eu/tracesnt/directory/publication/organic-operator/index#!?query=$info", + 'originUrls' => $details, + 'certificates' => $certs, +]); diff --git a/www/organic/pdf.php b/www/organic/pdf.php index 9e1b9f8..b0bd4cc 100644 --- a/www/organic/pdf.php +++ b/www/organic/pdf.php @@ -1,5 +1,7 @@ = $from; $i--) { - $el = $array[$i]; - if (sizeof($postalCode) > 0) { - if (sizeof($address) === 0) $el = rtrim($el, ", \n\r\t\v\0"); - if (strlen($el) === 0) continue; - array_unshift($address, $el); - } else if (preg_match("/^[A-Z0-9.\-]{3,},?$/", $el)) { - array_unshift($postalCode, trim($el, ", \n\r\t\v\0")); - } else { - array_unshift($city, $el); - } - } - return [implode(' ', $address), implode(' ', $postalCode), implode(' ', $city)]; -} - function jenc($data): string { return json_encode($data, JSON_UNESCAPED_SLASHES | JSON_UNESCAPED_UNICODE); } @@ -47,7 +30,9 @@ if (!$file || !$headerfile) { header('Content-Length: 0'); exit; } -$filename = stream_get_meta_data($file)['uri']; +$cacheFile = null; +$tmpfilename = stream_get_meta_data($file)['uri']; +$filename = $tmpfilename; $headerfilename = stream_get_meta_data($headerfile)['uri']; if ($_SERVER['REQUEST_METHOD'] === 'POST') { $stdin = fopen("php://input", "rb"); @@ -58,13 +43,30 @@ if ($_SERVER['REQUEST_METHOD'] === 'POST') { } fclose($stdin); } else { - if (exec("curl -s -D " . escapeshellarg($headerfilename) . " -o " . escapeshellarg($filename) . " " . escapeshellarg($url)) === false) { - header('Status: 500'); - header('Content-Length: 0'); - exit; + if ($cache = fopen('certificates/.cache.csv', 'r')) { + while (($line = fgets($cache)) !== false) { + $parts = explode(';', trim($line), 3); + $timestamp = intval($parts[0]); + if ($parts[2] !== $url) continue; + $cacheFile = 'certificates/' . $parts[1] . '.pdf'; + break; + } + fclose($cache); + } + + if ($cacheFile !== null) { + $filename = $cacheFile; + } else { + if (exec("curl -sk -D " . escapeshellarg($headerfilename) . " -o " . escapeshellarg($filename) . " " . escapeshellarg($url)) === false) { + header('Status: 500'); + header('Content-Length: 0'); + exit; + } } } +$timestamp = time(); + if ($format === 'text') { header('Content-Type: text/plain; charset=UTF-8'); passthru("pdftotext -raw " . escapeshellarg($filename) . " -"); @@ -73,176 +75,34 @@ if ($format === 'text') { passthru("pdfsig " . escapeshellarg($filename)); } else if ($format === 'json') { header('Content-Type: application/json; charset=UTF-8'); - if (exec("pdftotext -raw " . escapeshellarg($filename) . " -", $text) === false) { + + if (($cert = parse_pdf($filename, true)) === null) { header('Status: 500'); header('Content-Length: 0'); exit; } - $text = implode("\n", $text); - exec("pdfsig " . escapeshellarg($filename), $sig); - $sig = implode("\n", $sig); + $certType = $cert['type']; + $certId = $cert['id']; - $r = preg_match('@([a-z]{2}) (https://webgate\.ec\.europa\.eu/tracesnt/directory/publication/organic-operator/(.*?)\.pdf) (\d+) / (\d+)@', $text, $matches); - if ($r === 1) { - // TRACES certificate - - $data = []; - $parts = preg_split('@\n(I+\.\d+) ([^\n]*)@', $text, -1, PREG_SPLIT_DELIM_CAPTURE); - $status = str_replace("\n", '', $parts[0]); - for ($i = 3; $i < sizeof($parts); $i += 3) { - $data[$parts[$i - 2]] = trim($parts[$i]); + if ($certType === 'traces') { + if ($certId !== null && $url !== null && $cacheFile === null) { + copy($filename, "certificates/$certId.pdf"); + file_put_contents('certificates/.cache.csv', "$timestamp;$certId;$url\n", FILE_APPEND); } - - $lang = $matches[1]; - $splitAddr = [ - 'de' => 'Adresse', - 'en' => 'Address', - ][$lang]; - $splitCountry = [ - 'de' => 'Land', - 'en' => 'Country', - ][$lang]; - $statusMap = [ - 'de' => [ - 'AUSGESTELLT' => 'issued', - ], - 'en' => [ - 'ISSUED' => 'issued', - ] - ][$lang]; - $activityMap = [ - 'de' => [ - 'Aufbereitung' => 'preparation', - 'Ausfuhr' => 'export', - 'Einfuhr' => 'import', - 'Lagerung' => 'storing', - 'Produktion' => 'production', - 'Vertrieb' => 'distribution', - 'Vertrieb/Inverkehrbringen' => 'distribution_placing_on_the_market', - ], - 'en' => [ - 'Distribution' => 'distribution', - 'Distribution/Placing on the market' => 'distribution_placing_on_the_market', - 'Export' => 'export', - 'Import' => 'import', - 'Preparation' => 'preparation', - 'Production' => 'production', - 'Storing' => 'storing', - ], - ][$lang]; - - $certUrl = $matches[2]; - $certId = $matches[3]; - $authorityId = explode('.', $certId)[0]; - $operatorId = explode('.', $certId)[1]; - $operator = preg_split('@\s+@', trim($data['I.3'])); - $p1 = array_search($splitAddr, $operator); - $p2 = array_search($splitCountry, $operator); - $operatorName = trim(implode(' ', array_filter($operator, fn($k,$i) => $i > 0 && $i < $p1, ARRAY_FILTER_USE_BOTH)), ', '); - [$opAddr, $opPostal, $opCity] = get_address($operator, $p1 + 1, $p2 - 1); - - $authority = preg_split('@\s+@', trim($data['I.4'])); - $until = array_search("($authorityId)", $authority); - $p1 = array_search($splitAddr, $authority); - $p2 = array_search($splitCountry, $authority); - $authorityName = implode(' ', array_filter($authority, fn($k,$i) => $i > 0 && $i < $p1 - 1 && ($i !== $p1 - 2 || !str_starts_with($k, '(')), ARRAY_FILTER_USE_BOTH)); - [$aAddr, $aPostal, $aCity] = get_address($authority, $p1 + 1, $p2 - 1); - - $activities = []; - foreach (explode("\n", $data['I.5']) as $a) { - $activities[] = $activityMap[trim($a, '• ')]; + } else if ($certType !== 'unknown') { + if ($certId !== null && $url !== null && $cacheFile === null) { + copy($filename, "certificates/$certId.appendix.pdf"); + file_put_contents('certificates/.cache.csv', "$timestamp;$certId.appendix;$url\n", FILE_APPEND); } - - preg_match_all('/\([a-g]\)/', $data['I.6'], $matches, PREG_SET_ORDER); - $products = []; - foreach ($matches as $m) { - $products[] = $m[0]; - } - - preg_match_all('@\d+/\d+/\d+@', $data['I.8'], $matches, PREG_SET_ORDER); - $valid1 = implode('-', array_reverse(explode('/', $matches[0][0]))); - $valid2 = implode('-', array_reverse(explode('/', $matches[1][0]))); - - $sigs = []; - foreach (array_slice(explode("\nSignature #", $sig), 1) as $s) { - $sData = []; - $sData2 = []; - preg_match_all('/\n {2}- (([^:\n]*): )?([^\n]*)/', $s, $matches, PREG_SET_ORDER); - foreach ($matches as $m) { - if (strlen($m[2]) === 0) { - $sData2[] = $m[3]; - } else { - $sData[$m[2]] = $m[3]; - } - } - $sigs[] = [ - 'signerCommonName' => $sData['Signer Certificate Common Name'], - 'valid' => $sData['Signature Validation'] === 'Signature is Valid.', - 'trusted' => $sData['Certificate Validation'] === 'Certificate is Trusted.', - 'totalDocument' => in_array('Total document signed', $sData2), - 'timestamp' => gmdate('Y-m-d\TH:i:s\Z', strtotime($sData['Signing Time'])), - 'type' => $sData['Signature Type'], - 'hashAlgorithm' => $sData['Signing Hash Algorithm'], - 'signerDistinguishedName' => $sData['Signer full Distinguished Name'], - 'fieldName' => $sData['Signature Field Name'], - ]; - } - - echo "{\"type\":\"traces\",\"lang\":\"$lang\",\"id\":\"$certId\",\"status\":\"$statusMap[$status]\""; - echo ",\n \"operator\":{\"id\":" . jenc($operatorId). - ',"groupOfOperators":' . jenc(!str_starts_with($data['I.2'], '☑')) . - ',"name":' . jenc($operatorName) . - ',"address":' . jenc($opAddr) . - ',"postalCode":' . jenc($opPostal) . - ',"city":' . jenc($opCity) . - ',"countryCode":' . jenc($operator[sizeof($operator) - 1]) . - "},\n \"authority\":{\"id\":" . jenc($authorityId) . - ',"name":' . jenc($authorityName) . - ',"address":' . jenc($aAddr) . - ',"postalCode":' . jenc($aPostal) . - ',"city":' . jenc($aCity) . - ',"countryCode":' . jenc($authority[sizeof($authority) - 1]) . - "},\n \"activities\":" . jenc($activities) . - ",\n \"productCategories\":" . jenc($products) . - ",\n \"validFrom\":" . jenc($valid1) . - ',"validTo":' . jenc($valid2) . - ",\n \"url\":\"$certUrl\"" . - ",\n \"digitalSignatures\":" . jenc($sigs) . - "\n}\n"; - exit; } - if (preg_match('/AT-BIO-[0-9]{3}/', $text, $matches) === 1) { - $authorityId = $matches[0]; - $certId = null; - $certNr = null; - if (preg_match("/$authorityId\.040-[0-9]{7}\.[0-9]{4}\.[0-9]{3}/", $text, $matches) === 1) - $certId = $matches[0]; - if (preg_match_all("/\b[0-9]+([._-])[0-9]+\g{-1}[0-9]+\b/", $text, $matches, PREG_SET_ORDER) !== false) { - foreach ($matches as $m) { - if (strlen($m[0]) > 10 && !str_ends_with($certId, $m[0])) - $certNr = $m[0]; - } - } - $operator = ['lfbisNr' => null]; - if ($authorityId === 'AT-BIO-301') { - // TODO - } else if ($authorityId === 'AT-BIO-302') { - if (preg_match("/^lfbis[^0-9]*([0-9]{7})$/im", $text, $matches) === 1) - $operator['lfbisNr'] = $matches[1]; - } else if ($authorityId === 'AT-BIO-401') { - // TODO - } else if ($authorityId === 'AT-BIO-402') { - // TODO - } else if ($authorityId === 'AT-BIO-903') { - // TODO - } - echo "{\"type\":\"$authorityId\",\"lang\":\"de\",\"id\":" . jenc($certId) . ",\"nr\":" . jenc($certNr); - echo ",\n \"operator\":" . jenc($operator) . ",\n \"authority\":{\"id\":" . jenc($authorityId) . "}}\n"; - exit; - } - - echo "{\"type\":\"unknown\"}\n"; + echo jenc($cert); + exit; +} else if ($cacheFile === $filename) { + header('Content-Type: application/pdf'); + header('Content-Length: ' . filesize($filename)); + header('Content-Disposition: inline; filename="' . basename($filename) . '"'); + readfile($filename); } else { $headers = []; $status_code = null;