Add 75010 uris

[airbundle] / Command / WeatherCommand.php

diff --git a/Command/WeatherCommand.php b/Command/WeatherCommand.php
index 00f922cf113544855fc988a52c8113d70c053a7b..3b6c88f22dad0bb289c08213db5eed9c7dec3beb 100644 (file)
--- a/Command/WeatherCommand.php
+++ b/Command/WeatherCommand.php
@@ -3,10 +3,12 @@
 namespace Rapsys\AirBundle\Command;
 
 use Doctrine\Bundle\DoctrineBundle\Command\DoctrineCommand;
 namespace Rapsys\AirBundle\Command;
 
 use Doctrine\Bundle\DoctrineBundle\Command\DoctrineCommand;

+use Doctrine\Persistence\ManagerRegistry;

 use Symfony\Component\Console\Input\InputInterface;
 use Symfony\Component\Console\Output\OutputInterface;
 use Symfony\Component\Console\Input\InputInterface;
 use Symfony\Component\Console\Output\OutputInterface;

-use Symfony\Component\Filesystem\Exception\IOExceptionInterface;
+use Symfony\Component\Filesystem\Exception\IOException;

 use Symfony\Component\Filesystem\Filesystem;
 use Symfony\Component\Filesystem\Filesystem;

+

 use Rapsys\AirBundle\Entity\Session;
 
 class WeatherCommand extends DoctrineCommand {
 use Rapsys\AirBundle\Entity\Session;
 
 class WeatherCommand extends DoctrineCommand {


@@ -33,9 +35,14 @@ class WeatherCommand extends DoctrineCommand {
                //Hourly uri
                'hourly' => [
                        75001 => 'https://www.accuweather.com/en/fr/paris-01-louvre/75001/hourly-weather-forecast/179142_pc?day=',
                //Hourly uri
                'hourly' => [
                        75001 => 'https://www.accuweather.com/en/fr/paris-01-louvre/75001/hourly-weather-forecast/179142_pc?day=',

+                       75004 => 'https://www.accuweather.com/en/fr/paris-04-hotel-de-ville/75004/hourly-weather-forecast/179145_pc?day=',

                        75005 => 'https://www.accuweather.com/en/fr/paris-05-pantheon/75005/hourly-weather-forecast/179146_pc?day=',
                        75005 => 'https://www.accuweather.com/en/fr/paris-05-pantheon/75005/hourly-weather-forecast/179146_pc?day=',

+                       75006 => 'https://www.accuweather.com/fr/fr/paris-06-luxembourg/75006/hourly-weather-forecast/179147_pc?day=',

                        75007 => 'https://www.accuweather.com/en/fr/paris-07-palais-bourbon/75007/hourly-weather-forecast/179148_pc?day=',
                        75009 => 'https://www.accuweather.com/en/fr/paris-09-opera/75009/hourly-weather-forecast/179150_pc?day=',
                        75007 => 'https://www.accuweather.com/en/fr/paris-07-palais-bourbon/75007/hourly-weather-forecast/179148_pc?day=',
                        75009 => 'https://www.accuweather.com/en/fr/paris-09-opera/75009/hourly-weather-forecast/179150_pc?day=',

+                       75010 => 'https://www.accuweather.com/en/fr/paris-10-entrepot/75010/hourly-weather-forecast/179151_pc?day=',
+                       75012 => 'https://www.accuweather.com/en/fr/paris-12-reuilly/75012/hourly-weather-forecast/179153_pc?day=',
+                       75013 => 'https://www.accuweather.com/en/fr/paris-13-gobelins/75013/hourly-weather-forecast/179154_pc?day=',

                        75015 => 'https://www.accuweather.com/en/fr/paris-15-vaugirard/75015/hourly-weather-forecast/179156_pc?day=',
                        75019 => 'https://www.accuweather.com/en/fr/paris-19-buttes-chaumont/75019/hourly-weather-forecast/179160_pc?day=',
                        75116 => 'https://www.accuweather.com/en/fr/paris-16-passy/75116/hourly-weather-forecast/179246_pc?day='
                        75015 => 'https://www.accuweather.com/en/fr/paris-15-vaugirard/75015/hourly-weather-forecast/179156_pc?day=',
                        75019 => 'https://www.accuweather.com/en/fr/paris-19-buttes-chaumont/75019/hourly-weather-forecast/179160_pc?day=',
                        75116 => 'https://www.accuweather.com/en/fr/paris-16-passy/75116/hourly-weather-forecast/179246_pc?day='


@@ -43,9 +50,14 @@ class WeatherCommand extends DoctrineCommand {
                //Daily uri
                'daily' => [
                        75001 => 'https://www.accuweather.com/en/fr/paris-01-louvre/75001/daily-weather-forecast/179142_pc',
                //Daily uri
                'daily' => [
                        75001 => 'https://www.accuweather.com/en/fr/paris-01-louvre/75001/daily-weather-forecast/179142_pc',

+                       75004 => 'https://www.accuweather.com/en/fr/paris-04-hotel-de-ville/75004/daily-weather-forecast/179145_pc',

                        75005 => 'https://www.accuweather.com/en/fr/paris-05-pantheon/75005/daily-weather-forecast/179146_pc',
                        75005 => 'https://www.accuweather.com/en/fr/paris-05-pantheon/75005/daily-weather-forecast/179146_pc',

+                       75006 => 'https://www.accuweather.com/fr/fr/paris-06-luxembourg/75006/daily-weather-forecast/179147_pc',

                        75007 => 'https://www.accuweather.com/en/fr/paris-07-palais-bourbon/75007/daily-weather-forecast/179148_pc',
                        75009 => 'https://www.accuweather.com/en/fr/paris-09-opera/75009/daily-weather-forecast/179150_pc',
                        75007 => 'https://www.accuweather.com/en/fr/paris-07-palais-bourbon/75007/daily-weather-forecast/179148_pc',
                        75009 => 'https://www.accuweather.com/en/fr/paris-09-opera/75009/daily-weather-forecast/179150_pc',

+                       75010 => 'https://www.accuweather.com/en/fr/paris-10-entrepot/75010/daily-weather-forecast/179151_pc',
+                       75012 => 'https://www.accuweather.com/en/fr/paris-12-reuilly/75012/daily-weather-forecast/179153_pc',
+                       75013 => 'https://www.accuweather.com/en/fr/paris-13-gobelins/75013/daily-weather-forecast/179154_pc',

                        75015 => 'https://www.accuweather.com/en/fr/paris-15-vaugirard/75015/daily-weather-forecast/179156_pc',
                        75019 => 'https://www.accuweather.com/en/fr/paris-19-buttes-chaumont/75019/daily-weather-forecast/179160_pc',
                        75116 => 'https://www.accuweather.com/en/fr/paris-16-passy/75116/daily-weather-forecast/179246_pc'
                        75015 => 'https://www.accuweather.com/en/fr/paris-15-vaugirard/75015/daily-weather-forecast/179156_pc',
                        75019 => 'https://www.accuweather.com/en/fr/paris-19-buttes-chaumont/75019/daily-weather-forecast/179160_pc',
                        75116 => 'https://www.accuweather.com/en/fr/paris-16-passy/75116/daily-weather-forecast/179246_pc'


@@ -55,6 +67,23 @@ class WeatherCommand extends DoctrineCommand {
        ///Set curl handler
        private $ch = null;
 
        ///Set curl handler
        private $ch = null;
 

+       ///Set manager registry
+       private $doctrine;
+
+       ///Set filesystem
+       private $filesystem;
+
+       ///Weather command constructor
+       public function __construct(ManagerRegistry $doctrine, Filesystem $filesystem) {
+               parent::__construct($doctrine);
+
+               //Set entity manager
+               $this->doctrine = $doctrine;
+
+               //Set filesystem
+               $this->filesystem = $filesystem;
+       }
+

        ///Configure attribute command
        protected function configure() {
                //Configure the class
        ///Configure attribute command
        protected function configure() {
                //Configure the class


@@ -71,11 +100,29 @@ class WeatherCommand extends DoctrineCommand {
 
        ///Process the attribution
        protected function execute(InputInterface $input, OutputInterface $output) {
 
        ///Process the attribution
        protected function execute(InputInterface $input, OutputInterface $output) {

-               //Fetch doctrine
-               $doctrine = $this->getDoctrine();
+               //Kernel object
+               $kernel = $this->getApplication()->getKernel();
+
+               //Tmp directory
+               $tmpdir = $kernel->getContainer()->getParameter('kernel.project_dir').'/var/cache/weather';
+
+               //Set tmpdir
+               //XXX: worst case scenario we have 3 files per zipcode plus daily
+               if (!is_dir($tmpdir)) {
+                       try {
+                               //Create dir
+                               $this->filesystem->mkdir($tmpdir, 0775);
+                       } catch (IOException $exception) {
+                               //Display error
+                               echo 'Create dir '.$exception->getPath().' failed'."\n";
+
+                               //Exit with failure
+                               exit(self::FAILURE);
+                       }
+               }

 
 

-               //Get manager
-               $manager = $doctrine->getManager();
+               //Cleanup kernel
+               unset($kernel);

 
                //Tidy object
                $tidy = new \tidy();
 
                //Tidy object
                $tidy = new \tidy();


@@ -89,7 +136,7 @@ class WeatherCommand extends DoctrineCommand {
                //Process hourly accuweather
                if (($command = $input->getFirstArgument()) == 'rapsysair:weather:hourly' || $command == 'rapsysair:weather') {
                        //Fetch hourly sessions to attribute
                //Process hourly accuweather
                if (($command = $input->getFirstArgument()) == 'rapsysair:weather:hourly' || $command == 'rapsysair:weather') {
                        //Fetch hourly sessions to attribute

-                       $types['hourly'] = $doctrine->getRepository(Session::class)->findAllPendingHourlyWeather();
+                       $types['hourly'] = $this->doctrine->getRepository(Session::class)->findAllPendingHourlyWeather();

 
                        //Iterate on each session
                        foreach($types['hourly'] as $sessionId => $session) {
 
                        //Iterate on each session
                        foreach($types['hourly'] as $sessionId => $session) {


@@ -132,7 +179,7 @@ class WeatherCommand extends DoctrineCommand {
                //Process daily accuweather
                if ($command == 'rapsysair:weather:daily' || $command == 'rapsysair:weather') {
                        //Fetch daily sessions to attribute
                //Process daily accuweather
                if ($command == 'rapsysair:weather:daily' || $command == 'rapsysair:weather') {
                        //Fetch daily sessions to attribute

-                       $types['daily'] = $doctrine->getRepository(Session::class)->findAllPendingDailyWeather();
+                       $types['daily'] = $this->doctrine->getRepository(Session::class)->findAllPendingDailyWeather();

 
                        //Iterate on each session
                        foreach($types['daily'] as $sessionId => $session) {
 
                        //Iterate on each session
                        foreach($types['daily'] as $sessionId => $session) {


@@ -159,24 +206,6 @@ class WeatherCommand extends DoctrineCommand {
                        }
                }
 
                        }
                }
 

-               //Get filesystem
-               $filesystem = new Filesystem();
-
-               //Set tmpdir
-               //XXX: worst case scenario we have 3 files per zipcode
-               if (!is_dir($tmpdir = sys_get_temp_dir().'/accuweather')) {
-                       try {
-                               //Create dir
-                           $filesystem->mkdir($tmpdir, 0775);
-                       } catch (IOExceptionInterface $exception) {
-                               //Display error
-                               echo 'Create dir '.$exception->getPath().' failed'."\n";
-
-                               //Exit with failure
-                               exit(self::FAILURE);
-                       }
-               }
-

                //Init curl
                $this->curl_init();
 
                //Init curl
                $this->curl_init();
 


@@ -224,7 +253,7 @@ class WeatherCommand extends DoctrineCommand {
                                //Process daily
                                if ($day == 'daily') {
                                        //Iterate on each link containing data
                                //Process daily
                                if ($day == 'daily') {
                                        //Iterate on each link containing data

-                                       foreach($sx->xpath('//a[@class="daily-forecast-card"]') as $node) {
+                                       foreach($sx->xpath('//a[contains(@class,"daily-forecast-card")]') as $node) {

                                                //Get date
                                                $dsm = trim($node->div[0]->h2[0]->span[1]);
 
                                                //Get date
                                                $dsm = trim($node->div[0]->h2[0]->span[1]);
 


@@ -232,7 +261,7 @@ class WeatherCommand extends DoctrineCommand {
                                                $temperature = str_replace('°', '', $node->div[0]->div[0]->span[0]);
 
                                                //Get rainrisk
                                                $temperature = str_replace('°', '', $node->div[0]->div[0]->span[0]);
 
                                                //Get rainrisk

-                                               $rainrisk = str_replace('%', '', trim($node->div[2]))/100;
+                                               $rainrisk = trim(str_replace('%', '', $node->div[1]))/100;

 
                                                //Store data
                                                $data[$zipcode][$dsm]['daily'] = [
 
                                                //Store data
                                                $data[$zipcode][$dsm]['daily'] = [


@@ -247,28 +276,31 @@ class WeatherCommand extends DoctrineCommand {
                                        #/html/body/div[1]/div[5]/div[1]/div[1]/div[1]/div[1]/div[1]/div/h2/span[1]
                                        foreach($sx->xpath('//div[@data-shared="false"]') as $node) {
                                                //Get hour
                                        #/html/body/div[1]/div[5]/div[1]/div[1]/div[1]/div[1]/div[1]/div/h2/span[1]
                                        foreach($sx->xpath('//div[@data-shared="false"]') as $node) {
                                                //Get hour

-                                               $hour = trim($node->div[0]->div[0]->h2[0]->span[0]);
+                                               $hour = trim(str_replace(' h', '', $node->div[0]->div[0]->div[0]->div[0]->div[0]->h2[0]));

 
 

-                                               //Get dsm
-                                               $dsm = trim($node->div[0]->div[0]->h2[0]->span[1]);
+                                               //Compute dsm from day (1=d,2=d+1,3=d+2)
+                                               $dsm = (new \DateTime('+'.($day - 1).' day'))->format('d/m');

 
                                                //Get temperature
 
                                                //Get temperature

-                                               $temperature = str_replace('°', '', $node->div[0]->div[0]->div[0]);
+                                               $temperature = str_replace('°', '', $node->div[0]->div[0]->div[0]->div[0]->div[1]);

 
                                                //Get realfeel
 
                                                //Get realfeel

-                                               $realfeel = str_replace(['RealFeel® ', '°'], '', trim($node->div[0]->div[0]->span[0]));
+                                               $realfeel = trim(str_replace(['RealFeel®', '°'], '', $node->div[0]->div[0]->div[0]->div[1]->div[0]->div[0]->div[0]));

 
                                                //Get rainrisk
 
                                                //Get rainrisk

-                                               $rainrisk = str_replace('%', '', trim($node->div[0]->div[0]->div[1]))/100;
-
-                                               //Label is Rain when we have a rainfall
-                                               if (($pluviolabel = trim($node->div[1]->div[0]->div[0]->div[1]->p[1])) == 'Rain') {
-                                                       //Get rainfall
-                                                       $rainfall = str_replace(' mm', '', $node->div[1]->div[0]->div[0]->div[1]->p[1]->span[0]);
-                                               //Cloud Cover, no rainfall
-                                               } else {
-                                                       //Set rainfall to 0 (mm)
-                                                       $rainfall = 0;
+                                               $rainrisk = floatval(str_replace('%', '', trim($node->div[0]->div[0]->div[0]->div[2]->div[0]))/100);
+
+                                               //Set rainfall to 0 (mm)
+                                               $rainfall = 0;
+
+                                               //Iterate on each entry
+                                               //TODO: wind and other infos are present in $node->div[1]->div[0]->div[1]->div[0]->p
+                                               foreach($node->div[1]->div[0]->div[1]->div[0]->p as $p) {
+                                                       //Lookup for rain entry if present
+                                                       if (in_array(trim($p), ['Rain', 'Pluie'])) {
+                                                               //Get rainfall
+                                                               $rainfall = floatval(str_replace(' mm', '', $p->span[0]));
+                                                       }

                                                }
 
                                                //Store data
                                                }
 
                                                //Store data


@@ -337,11 +369,11 @@ class WeatherCommand extends DoctrineCommand {
                                        $hour = $type=='daily'?$type:$time->format('H');
 
                                        //Check data availability
                                        $hour = $type=='daily'?$type:$time->format('H');
 
                                        //Check data availability

-                                       //XXX: should never happen
-                                       #if (!isset($data[$zipcode][$dsm][$hour])) {
-                                       #       //Skip unavailable data
-                                       #       continue;
-                                       #}
+                                       //XXX: sometimes startup delay causes weather data to be unavailable for session first hour
+                                       if (!isset($data[$zipcode][$dsm][$hour])) {
+                                               //Skip unavailable data
+                                               continue;
+                                       }

 
                                        //Set info alias
                                        $info = $data[$zipcode][$dsm][$hour];
 
                                        //Set info alias
                                        $info = $data[$zipcode][$dsm][$hour];


@@ -411,7 +443,6 @@ class WeatherCommand extends DoctrineCommand {
                                        //Check if realfeel differ
                                        if ($session->getRealfeel() !== $realfeel) {
                                                //Set average realfeel
                                        //Check if realfeel differ
                                        if ($session->getRealfeel() !== $realfeel) {
                                                //Set average realfeel

-                                               #$meteo['realfeel'] = array_sum($meteo['realfeel'])/count($meteo['realfeel']);

                                                $session->setRealfeel($realfeel);
                                        }
 
                                                $session->setRealfeel($realfeel);
                                        }
 


@@ -436,7 +467,6 @@ class WeatherCommand extends DoctrineCommand {
                                        //Check if temperature differ
                                        if ($session->getTemperature() !== $temperature) {
                                                //Set average temperature
                                        //Check if temperature differ
                                        if ($session->getTemperature() !== $temperature) {
                                                //Set average temperature

-                                               #$meteo['temperature'] = array_sum($meteo['temperature'])/count($meteo['temperature']);

                                                $session->setTemperature($temperature);
                                        }
 
                                                $session->setTemperature($temperature);
                                        }
 


@@ -456,7 +486,7 @@ class WeatherCommand extends DoctrineCommand {
                }
 
                //Flush to get the ids
                }
 
                //Flush to get the ids

-               $manager->flush();
+               $this->doctrine->getManager()->flush();

 
                //Close curl handler
                $this->curl_close();
 
                //Close curl handler
                $this->curl_close();