3 namespace Rapsys\AirBundle\Command
;
5 use Doctrine\Bundle\DoctrineBundle\Command\DoctrineCommand
;
6 use Doctrine\Persistence\ManagerRegistry
;
7 use Symfony\Component\Console\Input\InputInterface
;
8 use Symfony\Component\Console\Output\OutputInterface
;
9 use Symfony\Component\Filesystem\Exception\IOException
;
10 use Symfony\Component\Filesystem\Filesystem
;
12 use Rapsys\AirBundle\Entity\Session
;
14 class WeatherCommand
extends DoctrineCommand
{
15 //Set failure constant
18 ///Set success constant
23 //Mostly useless in fact
25 //Required to simplify simplexml transition
27 //Required to avoid xml errors
28 'quote-nbsp' => false,
29 //Required to fix code
33 ///Set accuweather uris
34 private $accuweather = [
37 75001 => 'https://www.accuweather.com/en/fr/paris-01-louvre/75001/hourly-weather-forecast/179142_pc?day=',
38 75004 => 'https://www.accuweather.com/en/fr/paris-04-hotel-de-ville/75004/hourly-weather-forecast/179145_pc?day=',
39 75005 => 'https://www.accuweather.com/en/fr/paris-05-pantheon/75005/hourly-weather-forecast/179146_pc?day=',
40 75006 => 'https://www.accuweather.com/fr/fr/paris-06-luxembourg/75006/hourly-weather-forecast/179147_pc?day=',
41 75007 => 'https://www.accuweather.com/en/fr/paris-07-palais-bourbon/75007/hourly-weather-forecast/179148_pc?day=',
42 75009 => 'https://www.accuweather.com/en/fr/paris-09-opera/75009/hourly-weather-forecast/179150_pc?day=',
43 75010 => 'https://www.accuweather.com/en/fr/paris-10-entrepot/75010/hourly-weather-forecast/179151_pc?day=',
44 75012 => 'https://www.accuweather.com/en/fr/paris-12-reuilly/75012/hourly-weather-forecast/179153_pc?day=',
45 75013 => 'https://www.accuweather.com/en/fr/paris-13-gobelins/75013/hourly-weather-forecast/179154_pc?day=',
46 75015 => 'https://www.accuweather.com/en/fr/paris-15-vaugirard/75015/hourly-weather-forecast/179156_pc?day=',
47 75019 => 'https://www.accuweather.com/en/fr/paris-19-buttes-chaumont/75019/hourly-weather-forecast/179160_pc?day=',
48 75116 => 'https://www.accuweather.com/en/fr/paris-16-passy/75116/hourly-weather-forecast/179246_pc?day='
52 75001 => 'https://www.accuweather.com/en/fr/paris-01-louvre/75001/daily-weather-forecast/179142_pc',
53 75004 => 'https://www.accuweather.com/en/fr/paris-04-hotel-de-ville/75004/daily-weather-forecast/179145_pc',
54 75005 => 'https://www.accuweather.com/en/fr/paris-05-pantheon/75005/daily-weather-forecast/179146_pc',
55 75006 => 'https://www.accuweather.com/fr/fr/paris-06-luxembourg/75006/daily-weather-forecast/179147_pc',
56 75007 => 'https://www.accuweather.com/en/fr/paris-07-palais-bourbon/75007/daily-weather-forecast/179148_pc',
57 75009 => 'https://www.accuweather.com/en/fr/paris-09-opera/75009/daily-weather-forecast/179150_pc',
58 75010 => 'https://www.accuweather.com/en/fr/paris-10-entrepot/75010/daily-weather-forecast/179151_pc',
59 75012 => 'https://www.accuweather.com/en/fr/paris-12-reuilly/75012/daily-weather-forecast/179153_pc',
60 75013 => 'https://www.accuweather.com/en/fr/paris-13-gobelins/75013/daily-weather-forecast/179154_pc',
61 75015 => 'https://www.accuweather.com/en/fr/paris-15-vaugirard/75015/daily-weather-forecast/179156_pc',
62 75019 => 'https://www.accuweather.com/en/fr/paris-19-buttes-chaumont/75019/daily-weather-forecast/179160_pc',
63 75116 => 'https://www.accuweather.com/en/fr/paris-16-passy/75116/daily-weather-forecast/179246_pc'
70 ///Set manager registry
76 ///Weather command constructor
77 public function __construct(ManagerRegistry
$doctrine, Filesystem
$filesystem) {
78 parent
::__construct($doctrine);
81 $this->doctrine
= $doctrine;
84 $this->filesystem
= $filesystem;
87 ///Configure attribute command
88 protected function configure() {
92 ->setName('rapsysair:weather')
93 //Set description shown with bin/console list
94 ->setDescription('Updates session rain and temperature fields')
95 //Set description shown with bin/console --help airlibre:attribute
96 ->setHelp('This command updates session rain and temperature fields in next three days')
97 //Add daily and hourly aliases
98 ->setAliases(['rapsysair:weather:daily', 'rapsysair:weather:hourly']);
101 ///Process the attribution
102 protected function execute(InputInterface
$input, OutputInterface
$output) {
104 $kernel = $this->getApplication()->getKernel();
107 $tmpdir = $kernel->getContainer()->getParameter('kernel.project_dir').'/var/cache/weather';
110 //XXX: worst case scenario we have 3 files per zipcode plus daily
111 if (!is_dir($tmpdir)) {
114 $this->filesystem
->mkdir($tmpdir, 0775);
115 } catch (IOException
$exception) {
117 echo 'Create dir '.$exception->getPath().' failed'."\n";
130 //Init zipcodes array
136 //Process hourly accuweather
137 if (($command = $input->getFirstArgument()) == 'rapsysair:weather:hourly' || $command == 'rapsysair:weather') {
138 //Fetch hourly sessions to attribute
139 $types['hourly'] = $this->doctrine
->getRepository(Session
::class)->findAllPendingHourlyWeather();
141 //Iterate on each session
142 foreach($types['hourly'] as $sessionId => $session) {
144 $zipcode = $session->getLocation()->getZipcode();
147 $start = $session->getStart();
150 $day = $start->diff((new \
DateTime('now'))->setTime(0, 0, 0))->d +
1;
152 //Check if zipcode is set
153 if (!isset($zipcodes[$zipcode])) {
154 $zipcodes[$zipcode] = [];
157 //Check if zipcode date is set
158 if (!isset($zipcodes[$zipcode][$day])) {
159 $zipcodes[$zipcode][$day] = [ $sessionId => $sessionId ];
161 $zipcodes[$zipcode][$day][$sessionId] = $sessionId;
165 $stop = $session->getStop();
168 $day = $stop->diff((new \
DateTime('now'))->setTime(0, 0, 0))->d +
1;
170 //Check if zipcode date is set
171 if (!isset($zipcodes[$zipcode][$day])) {
172 $zipcodes[$zipcode][$day] = [ $sessionId => $sessionId ];
174 $zipcodes[$zipcode][$day][$sessionId] = $sessionId;
179 //Process daily accuweather
180 if ($command == 'rapsysair:weather:daily' || $command == 'rapsysair:weather') {
181 //Fetch daily sessions to attribute
182 $types['daily'] = $this->doctrine
->getRepository(Session
::class)->findAllPendingDailyWeather();
184 //Iterate on each session
185 foreach($types['daily'] as $sessionId => $session) {
187 $zipcode = $session->getLocation()->getZipcode();
190 $start = $session->getStart();
195 //Check if zipcode is set
196 if (!isset($zipcodes[$zipcode])) {
197 $zipcodes[$zipcode] = [];
200 //Check if zipcode date is set
201 if (!isset($zipcodes[$zipcode][$day])) {
202 $zipcodes[$zipcode][$day] = [ $sessionId => $sessionId ];
204 $zipcodes[$zipcode][$day][$sessionId] = $sessionId;
215 //Iterate on zipcodes
216 foreach($zipcodes as $zipcode => $days) {
218 foreach($days as $day => $null) {
219 //Try to load content from cache
220 if (!is_file($file = $tmpdir.'/'.$zipcode.'.'.$day.'.html') || stat($file)['ctime'] <= (time() - ($day == 'daily' ? 4 : 2)*3600) || ($content = file_get_contents($file)) === false) {
221 //Prevent timing detection
222 //XXX: from 0.1 to 5 seconds
223 usleep(rand(1,50) * 100000);
226 //TODO: for daily we may load data for requested quarter of the day
227 $content = $this->curl_get($day == 'daily' ? $this->accuweather
['daily'][$zipcode] : $this->accuweather
['hourly'][$zipcode].$day);
230 if (file_put_contents($tmpdir.'/'.$zipcode.'.'.$day.'.html', $content) === false) {
232 echo 'Write to '.$tmpdir.'/'.$zipcode.'.'.$day.'.html failed'."\n";
240 $tidy->parseString($content, $this->config
, 'utf8');
243 //XXX: don't care about theses errors, tidy is here to fix...
244 #if (!empty($tidy->errorBuffer)) {
245 # var_dump($tidy->errorBuffer);
246 # die('Tidy errors');
250 //XXX: trash all xmlns= broken tags
251 $sx = new \
SimpleXMLElement(str_replace(['xmlns=', 'xlink:href='], ['xns=', 'href='], $tidy));
254 if ($day == 'daily') {
255 //Iterate on each link containing data
256 foreach($sx->xpath('//a[contains(@class,"daily-forecast-card")]') as $node) {
258 $dsm = trim($node->div
[0]->h2
[0]->span
[1]);
261 $temperature = str_replace('Ā°', '', $node->div
[0]->div
[0]->span
[0]);
264 $rainrisk = trim(str_replace('%', '', $node->div
[1]))/100;
267 $data[$zipcode][$dsm]['daily'] = [
268 'temperature' => $temperature,
269 'rainrisk' => $rainrisk
274 //Iterate on each div containing data
275 #(string)$sx->xpath('//div[@class="hourly-card-nfl"]')[0]->attributes()->value
276 #/html/body/div[1]/div[5]/div[1]/div[1]/div[1]/div[1]/div[1]/div/h2/span[1]
277 foreach($sx->xpath('//div[@data-shared="false"]') as $node) {
279 $hour = trim(str_replace(' h', '', $node->div
[0]->div
[0]->div
[0]->div
[0]->div
[0]->h2
[0]));
281 //Compute dsm from day (1=d,2=d+1,3=d+2)
282 $dsm = (new \
DateTime('+'.($day - 1).' day'))->format('d/m');
285 $temperature = str_replace('Ā°', '', $node->div
[0]->div
[0]->div
[0]->div
[0]->div
[1]);
288 $realfeel = trim(str_replace(['RealFeelĀ®', 'Ā°'], '', $node->div
[0]->div
[0]->div
[0]->div
[1]->div
[0]->div
[0]->div
[0]));
291 $rainrisk = floatval(str_replace('%', '', trim($node->div
[0]->div
[0]->div
[0]->div
[2]->div
[0]))/100);
293 //Set rainfall to 0 (mm)
296 //Iterate on each entry
297 //TODO: wind and other infos are present in $node->div[1]->div[0]->div[1]->div[0]->p
298 foreach($node->div
[1]->div
[0]->div
[1]->div
[0]->p
as $p) {
299 //Lookup for rain entry if present
300 if (in_array(trim($p), ['Rain', 'Pluie'])) {
302 $rainfall = floatval(str_replace(' mm', '', $p->span
[0]));
307 $data[$zipcode][$dsm][$hour] = [
308 'temperature' => $temperature,
309 'realfeel' => $realfeel,
310 'rainrisk' => $rainrisk,
311 'rainfall' => $rainfall
322 foreach($types as $type => $sessions) {
323 //Iterate on each type
324 foreach($sessions as $sessionId => $session) {
326 $zipcode = $session->getLocation()->getZipcode();
329 $start = $session->getStart();
332 if ($type == 'daily') {
334 $period = [ $start ];
338 $stop = $session->getStop();
341 $period = new \
DatePeriod(
344 //Iterate on each hour
345 new \
DateInterval('PT1H'),
346 //End with begin + length
356 'realfeelmin' => null,
357 'realfeelmax' => null,
359 'temperaturemin' => null,
360 'temperaturemax' => null
363 //Iterate on the period
364 foreach($period as $time) {
366 $dsm = $time->format('d/m');
369 $hour = $type=='daily'?$type:$time->format('H');
371 //Check data availability
372 //XXX: sometimes startup delay causes weather data to be unavailable for session first hour
373 if (!isset($data[$zipcode][$dsm][$hour])) {
374 //Skip unavailable data
379 $info = $data[$zipcode][$dsm][$hour];
381 //Check if rainrisk is higher
382 if ($meteo['rainrisk'] === null || $info['rainrisk'] > $meteo['rainrisk']) {
383 //Set highest rain risk
384 $meteo['rainrisk'] = floatval($info['rainrisk']);
387 //Check if rainfall is set
388 if (isset($info['rainfall'])) {
390 $meteo['rainfall'] +
= floatval($info['rainfall']);
394 $meteo['temperature'][$hour] = $info['temperature'];
397 if ($type != 'daily') {
398 //Check min temperature
399 if ($meteo['temperaturemin'] === null || $info['temperature'] < $meteo['temperaturemin']) {
400 $meteo['temperaturemin'] = floatval($info['temperature']);
403 //Check max temperature
404 if ($meteo['temperaturemax'] === null || $info['temperature'] > $meteo['temperaturemax']) {
405 $meteo['temperaturemax'] = floatval($info['temperature']);
409 //Check if realfeel is set
410 if (isset($info['realfeel'])) {
412 $meteo['realfeel'][$hour] = $info['realfeel'];
415 if ($meteo['realfeelmin'] === null || $info['realfeel'] < $meteo['realfeelmin']) {
416 $meteo['realfeelmin'] = floatval($info['realfeel']);
420 if ($meteo['realfeelmax'] === null || $info['realfeel'] > $meteo['realfeelmax']) {
421 $meteo['realfeelmax'] = floatval($info['realfeel']);
426 //Check if rainfall is set and differ
427 if ($session->getRainfall() !== $meteo['rainfall']) {
429 $session->setRainfall($meteo['rainfall']);
432 //Check if rainrisk differ
433 if ($session->getRainrisk() !== $meteo['rainrisk']) {
435 $session->setRainrisk($meteo['rainrisk']);
438 //Check realfeel array
439 if ($meteo['realfeel'] !== []) {
441 $realfeel = floatval(round(array_sum($meteo['realfeel'])/count($meteo['realfeel']),1));
443 //Check if realfeel differ
444 if ($session->getRealfeel() !== $realfeel) {
445 //Set average realfeel
446 $session->setRealfeel($realfeel);
449 //Check if realfeelmin differ
450 if ($session->getRealfeelmin() !== $meteo['realfeelmin']) {
452 $session->setRealfeelmin($meteo['realfeelmin']);
455 //Check if realfeelmax differ
456 if ($session->getRealfeelmax() !== $meteo['realfeelmax']) {
458 $session->setRealfeelmax($meteo['realfeelmax']);
462 //Check temperature array
463 if ($meteo['temperature'] !== []) {
464 //Compute temperature
465 $temperature = floatval(round(array_sum($meteo['temperature'])/count($meteo['temperature']),1));
467 //Check if temperature differ
468 if ($session->getTemperature() !== $temperature) {
469 //Set average temperature
470 $session->setTemperature($temperature);
473 //Check if temperaturemin differ
474 if ($session->getTemperaturemin() !== $meteo['temperaturemin']) {
476 $session->setTemperaturemin($meteo['temperaturemin']);
479 //Check if temperaturemax differ
480 if ($session->getTemperaturemax() !== $meteo['temperaturemax']) {
482 $session->setTemperaturemax($meteo['temperaturemax']);
488 //Flush to get the ids
489 $this->doctrine
->getManager()->flush();
495 return self
::SUCCESS
;
501 * @return bool|void Return success or exit
503 function curl_init() {
505 if (($this->ch
= curl_init()) === false) {
507 echo 'Curl init failed: '.curl_error($this->ch
)."\n";
518 CURLOPT_HTTP_VERSION
=> CURL_HTTP_VERSION_2_0
,
520 CURLOPT_HTTPHEADER
=> [
521 //XXX: it seems that you can disable akamai fucking protection with Pragma: akamai-x-cache-off
522 //XXX: see https://support.globaldots.com/hc/en-us/articles/115003996705-Akamai-Pragma-Headers-overview
523 #'Pragma: akamai-x-cache-off',
524 //XXX: working curl command
525 #curl --http2 --cookie file.jar --cookie-jar file.jar -v -i -k -H 'Accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9' -H 'Accept-Language: en-GB,en;q=0.9' -H 'Cache-Control: no-cache' -H 'Connection: keep-alive' -H 'Host: www.accuweather.com' -H 'Pragma: no-cache' -H 'Upgrade-Insecure-Requests: 1' -H 'User-Agent: Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/86.0.4240.198 Safari/537.36' 'https://www.accuweather.com/'
527 'Accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9',
528 //Set accept language
529 'Accept-Language: en-GB,en;q=0.9',
531 'Cache-Control: no-cache',
532 //Keep connection alive
533 'Connection: keep-alive',
536 //Force secure requests
537 'Upgrade-Insecure-Requests: 1',
539 'User-Agent: Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/86.0.4240.198 Safari/537.36',
540 //Force akamai cookie
541 //XXX: seems to come from http request
545 CURLOPT_COOKIEFILE
=> '',
546 //Disable location following
547 CURLOPT_FOLLOWLOCATION
=> false,
549 #CURLOPT_URL => $url = 'https://www.accuweather.com/',
551 CURLOPT_HEADER
=> true,
553 CURLOPT_RETURNTRANSFER
=> true,
556 CURLINFO_HEADER_OUT
=> true
561 echo 'Curl setopt array failed: '.curl_error($this->ch
)."\n";
573 * @return string|void Return url content or exit
575 function curl_get($url) {
577 if (curl_setopt($this->ch
, CURLOPT_URL
, $url) === false) {
579 echo 'Setopt for '.$url.' failed: '.curl_error($this->ch
)."\n";
582 curl_close($this->ch
);
588 //Check return status
589 if (($response = curl_exec($this->ch
)) === false) {
591 echo 'Get for '.$url.' failed: '.curl_error($this->ch
)."\n";
593 //Display sent headers
594 var_dump(curl_getinfo($this->ch
, CURLINFO_HEADER_OUT
));
600 curl_close($this->ch
);
607 if (empty($hs = curl_getinfo($this->ch
, CURLINFO_HEADER_SIZE
))) {
609 echo 'Getinfo for '.$url.' failed: '.curl_error($this->ch
)."\n";
611 //Display sent headers
612 var_dump(curl_getinfo($this->ch
, CURLINFO_HEADER_OUT
));
618 curl_close($this->ch
);
625 if (empty($header = substr($response, 0, $hs))) {
627 echo 'Header for '.$url.' empty: '.curl_error($this->ch
)."\n";
629 //Display sent headers
630 var_dump(curl_getinfo($this->ch
, CURLINFO_HEADER_OUT
));
636 curl_close($this->ch
);
642 //Check request success
643 if (strlen($header) <= 10 || substr($header, 0, 10) !== 'HTTP/2 200') {
645 echo 'Status for '.$url.' failed: '.curl_error($this->ch
)."\n";
647 //Display sent headers
648 var_dump(curl_getinfo($this->ch
, CURLINFO_HEADER_OUT
));
654 curl_close($this->ch
);
661 return substr($response, $hs);
667 * @return bool Return success or failure
669 function curl_close() {
670 return curl_close($this->ch
);