-
Notifications
You must be signed in to change notification settings - Fork 2
/
crawl.php
47 lines (32 loc) · 1.33 KB
/
crawl.php
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
<?php
require __DIR__ . '/vendor/autoload.php';
include './datum.php';
include "./sitemap-generator.php";
$BASE_URL = 'https://micovid-data.s3.us-east-2.amazonaws.com/';
// $DOWNLOAD_URL = 'http://micovid-data.s3-website.us-east-2.amazonaws.com/';
$xml = simplexml_load_file($BASE_URL) or die("Error: Cannot create object");
$final_data = array();
$fileList = array();
foreach ($xml->children() as $files) {
if ($files->Key) {
if (preg_match('/kent\/\d{4}\-\d{2}\-\d{2}/', $files->Key[0])) {
$fileList[] = $files->Key[0];
}
}
}
system('aws s3 sync s3://micovid-data/kent/ kent');
foreach ($fileList as $file) {
$data = file_get_contents($file, false);
$json_data = json_decode($data, true);
$datum = new Datum($json_data["submitted"], $json_data["negative"], $json_data["positive"], $json_data["pending"], $json_data["deaths"], $json_data["created_at"]);
$final_data[$json_data["created_at"]] = $datum;
}
$fh = fopen('/var/www/html/data.json', 'w') or die("Error opening output file");
fwrite($fh, json_encode($final_data, JSON_UNESCAPED_UNICODE));
fclose($fh);
Print("Generating Sitemap\n");
// Create an object of the generator class passing the config file
$smg = new SitemapGenerator(include("sitemap-config.php"));
// Run the generator
$smg->GenerateSitemap();
print("\nCrawl finished\n");