From 592005065f366821edc91771bc0ceb7aac583099 Mon Sep 17 00:00:00 2001 From: Robbie Mackay Date: Wed, 5 Aug 2015 12:51:11 +1200 Subject: [PATCH] Add CSV imports via CLI Summary: - Add CSV imports via CLI - Add Import Usecase - Fix typo in Console\Command - Use camelCase for console command actions - Better validation errors Refs T272 Test Plan: - `./bin/ushahidi import list` - to see possible formats. - `./bin/ushahidi import run -f ../primary-schools-2007-top50.csv -m ../primary-schools-map.json --values ../primary-schools-values.json` Where map.json is a json file of source-destination mappings like ``` { "[Id]" : "[values][school-id][0]", "[Name_of_School]": "[title]" } ``` and values.json is a json file of fixed values like ``` { "status":"published", "form":2 } ``` Reviewers: will, lkamau, jasonmule Subscribers: jasonmule, will, rjmackay, lkamau Maniphest Tasks: T272 Differential Revision: https://phabricator.ushahidi.com/D874 --- .../classes/Ushahidi/Console/Dataprovider.php | 6 +- application/classes/Ushahidi/Core.php | 4 + .../classes/Ushahidi/FileReader/CSV.php | 48 +++ application/classes/Ushahidi/Repository.php | 3 +- .../Transformer/MappingTransformer.php | 47 +++ .../Ushahidi/Validator/Post/Create.php | 2 +- .../Validator/Post/ValueValidator.php | 2 + application/messages/post.php | 16 +- composer.json | 5 +- composer.lock | 301 ++++++++++++++++-- src/Console/Authorizer/ConsoleAuthorizer.php | 35 ++ src/Console/Command.php | 4 +- src/Console/Import.php | 143 +++++++++ src/Core/Exception/ValidatorException.php | 2 + src/Core/Tool/AuthorizerTrait.php | 13 + src/Core/Tool/FileReader.php | 27 ++ src/Core/Tool/MappingTransformer.php | 22 ++ src/Core/Tool/Transformer.php | 19 ++ src/Core/Traits/DataTransformer.php | 6 +- src/Core/Usecase/ImportRepository.php | 19 ++ src/Core/Usecase/ImportUsecase.php | 162 ++++++++++ src/Init.php | 17 +- 22 files changed, 858 insertions(+), 45 deletions(-) create mode 100644 application/classes/Ushahidi/FileReader/CSV.php create mode 100644 application/classes/Ushahidi/Transformer/MappingTransformer.php create mode 100644 src/Console/Authorizer/ConsoleAuthorizer.php create mode 100644 src/Console/Import.php create mode 100644 src/Core/Tool/FileReader.php create mode 100644 src/Core/Tool/MappingTransformer.php create mode 100644 src/Core/Tool/Transformer.php create mode 100644 src/Core/Usecase/ImportRepository.php create mode 100644 src/Core/Usecase/ImportUsecase.php diff --git a/application/classes/Ushahidi/Console/Dataprovider.php b/application/classes/Ushahidi/Console/Dataprovider.php index cd2d1d02b7..fcf6334ec8 100644 --- a/application/classes/Ushahidi/Console/Dataprovider.php +++ b/application/classes/Ushahidi/Console/Dataprovider.php @@ -50,7 +50,7 @@ protected function get_providers(InputInterface $input, OutputInterface $output return $providers; } - protected function execute_list(InputInterface $input, OutputInterface $output) + protected function executeList(InputInterface $input, OutputInterface $output) { $providers = $this->get_providers($input, $output); @@ -66,7 +66,7 @@ protected function execute_list(InputInterface $input, OutputInterface $output) return $list; } - protected function execute_incoming(InputInterface $input, OutputInterface $output) + protected function executeIncoming(InputInterface $input, OutputInterface $output) { $providers = $this->get_providers($input, $output); $limit = $input->getOption('limit'); @@ -84,7 +84,7 @@ protected function execute_incoming(InputInterface $input, OutputInterface $outp return $totals; } - protected function execute_outgoing(InputInterface $input, OutputInterface $output) + protected function executeOutgoing(InputInterface $input, OutputInterface $output) { $providers = $this->get_providers($input, $output); $limit = $input->getOption('limit'); diff --git a/application/classes/Ushahidi/Core.php b/application/classes/Ushahidi/Core.php index 72eca9d2a1..54c8c6c72c 100644 --- a/application/classes/Ushahidi/Core.php +++ b/application/classes/Ushahidi/Core.php @@ -129,6 +129,7 @@ public static function init() $di->params['Ushahidi\Factory\ValidatorFactory']['map']['posts'] = [ 'create' => $di->lazyNew('Ushahidi_Validator_Post_Create'), 'update' => $di->lazyNew('Ushahidi_Validator_Post_Create'), + 'import' => $di->lazyNew('Ushahidi_Validator_Post_Create'), ]; $di->params['Ushahidi\Factory\ValidatorFactory']['map']['tags'] = [ 'create' => $di->lazyNew('Ushahidi_Validator_Tag_Create'), @@ -434,6 +435,9 @@ public static function init() 'repo' => $di->lazyGet('repository.post') ]; + $di->set('transformer.mapping', $di->lazyNew('Ushahidi_Transformer_MappingTransformer')); + $di->set('filereader.csv', $di->lazyNew('Ushahidi_FileReader_CSV')); + /** * 1. Load the plugins */ diff --git a/application/classes/Ushahidi/FileReader/CSV.php b/application/classes/Ushahidi/FileReader/CSV.php new file mode 100644 index 0000000000..7a56f2ff1e --- /dev/null +++ b/application/classes/Ushahidi/FileReader/CSV.php @@ -0,0 +1,48 @@ + + * @package Ushahidi\Application + * @copyright 2014 Ushahidi + * @license https://www.gnu.org/licenses/agpl-3.0.html GNU Affero General Public License Version 3 (AGPL3) + */ + +use \ArrayIterator; +use League\Csv\Reader; +use Ushahidi\Core\Tool\FileReader; + +class Ushahidi_FileReader_CSV implements FileReader +{ + + protected $limit; + public function setLimit($limit) { + $this->limit = $limit; + } + + protected $offset; + public function setOffset($offset) { + $this->offset = $offset; + } + + public function process($filename) { + // @todo inject factory function to get Reader + $reader = Reader::createFromFileObject(new SplFileObject($filename)); + + // Filter out empty rows + $nbColumns = count($reader->fetchOne()); + $reader->addFilter(function($row) use ($nbColumns) { + return count($row) == $nbColumns; + }); + + if ($this->offset) { + $reader->setOffset($this->offset); + } + if ($this->limit) { + $reader->setLimit($this->limit); + } + + return new ArrayIterator($reader->fetchAssoc()); + } +} diff --git a/application/classes/Ushahidi/Repository.php b/application/classes/Ushahidi/Repository.php index b384ab2e79..bba21322c3 100644 --- a/application/classes/Ushahidi/Repository.php +++ b/application/classes/Ushahidi/Repository.php @@ -19,7 +19,8 @@ abstract class Ushahidi_Repository implements Usecase\ReadRepository, Usecase\UpdateRepository, Usecase\DeleteRepository, - Usecase\SearchRepository + Usecase\SearchRepository, + Usecase\ImportRepository { use CollectionLoader; diff --git a/application/classes/Ushahidi/Transformer/MappingTransformer.php b/application/classes/Ushahidi/Transformer/MappingTransformer.php new file mode 100644 index 0000000000..478ac730ff --- /dev/null +++ b/application/classes/Ushahidi/Transformer/MappingTransformer.php @@ -0,0 +1,47 @@ + + * @package Ushahidi\Application + * @copyright 2014 Ushahidi + * @license https://www.gnu.org/licenses/agpl-3.0.html GNU Affero General Public License Version 3 (AGPL3) + */ + +use Ushahidi\Core\Tool\MappingTransformer; +use Ddeboer\DataImport\Step\MappingStep; + +class Ushahidi_Transformer_MappingTransformer implements MappingTransformer +{ + protected $map; + // MappingTransformer + public function setMap(Array $map) + { + $this->map = new MappingStep($map); + } + + protected $fixedValues; + // MappingTransformer + public function setFixedValues(Array $fixedValues) + { + $this->fixedValues = $fixedValues; + } + + // Tranformer + public function interact(Array $data) + { + $this->map->process($data); + + $data = array_merge($data, $this->fixedValues); + + return $data; + } +} diff --git a/application/classes/Ushahidi/Validator/Post/Create.php b/application/classes/Ushahidi/Validator/Post/Create.php index dda457609e..b4b94f9a83 100644 --- a/application/classes/Ushahidi/Validator/Post/Create.php +++ b/application/classes/Ushahidi/Validator/Post/Create.php @@ -192,7 +192,7 @@ public function checkValues(Validation $validation, $attributes, $data) } elseif ($error = $validator->check($values)) { - $validation->error('values', $error, [$key]); + $validation->error('values', $error, [$key, $values]); } } } diff --git a/application/classes/Ushahidi/Validator/Post/ValueValidator.php b/application/classes/Ushahidi/Validator/Post/ValueValidator.php index 06e19a9d52..ef15278818 100644 --- a/application/classes/Ushahidi/Validator/Post/ValueValidator.php +++ b/application/classes/Ushahidi/Validator/Post/ValueValidator.php @@ -12,6 +12,8 @@ // Note: this doesn't actually implement Ushahidi\Core\Tool\Validator abstract class Ushahidi_Validator_Post_ValueValidator { + protected $default_error_source = 'post'; + protected $config; public function setConfig(Array $config = null) diff --git a/application/messages/post.php b/application/messages/post.php index a44d52b741..5a1924da9b 100644 --- a/application/messages/post.php +++ b/application/messages/post.php @@ -17,5 +17,19 @@ 'exists' => 'The role you are publishing to ":value" does not exist' ], 'stageDoesNotExist' => 'Stage ":param1" does not exist', - 'stageRequired' => 'Stage ":param1" is required before publishing' + 'stageRequired' => 'Stage ":param1" is required before publishing', + + 'values' => [ + 'date' => ':field.:param1 must be a date, Given: :param2', + 'decimal' => ':field.:param1 must be a decimal with :param2 places, Given: :param2', + 'digit' => ':field.:param1 must be a digit, Given: :param2', + 'email' => ':field.:param1 must be an email address, Given: :param2', + 'exists' => ':field.:param1 must be a valid post id, Post id: :param2', + 'max_length' => ':field.:param1 must not exceed :param2 characters long, Given: :param2', + 'invalidForm' => ':field.:param1 has the wrong post type, Post id: :param2', + 'numeric' => ':field.:param1 must be numeric, Given: :param2', + 'scalar' => ':field.:param1 must be scalar, Given: :param2', + 'point' => ':field.:param1 must be an array of lat and lon', + 'url' => ':field.:param1 must be a url, Given: :param2', + ] ]; diff --git a/composer.json b/composer.json index 2ce31de087..67baa72607 100644 --- a/composer.json +++ b/composer.json @@ -72,7 +72,10 @@ "ext-gd": "*", "ext-json": "*", "ext-mbstring": "*", - "ext-mcrypt": "*" + "ext-mcrypt": "*", + "league/csv": "~7.1@dev", + "ddeboer/data-import": "dev-master", + "symfony/property-access": "~2.5" }, "require-dev": { "behat/behat": "~2.5.2", diff --git a/composer.lock b/composer.lock index 417f6125f3..31f5769ea2 100644 --- a/composer.lock +++ b/composer.lock @@ -1,10 +1,10 @@ { "_readme": [ "This file locks the dependencies of your project to a known state", - "Read more about it at http://getcomposer.org/doc/01-basic-usage.md#composer-lock-the-lock-file", + "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#composer-lock-the-lock-file", "This file is @generated automatically" ], - "hash": "ebc9f73a34a93852daa4c241ecb180bd", + "hash": "170b1bfc696949538cc8e79ff9220436", "packages": [ { "name": "abraham/twitteroauth", @@ -165,7 +165,7 @@ }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/composer/installers/zipball/f8c20b427de1cfe7a28a015c1640ce4e4eef1e33", + "url": "https://api.github.com/repos/composer/installers/zipball/e420b539e8d7b38b7c6f3f99dccc0386bd3dfe41", "reference": "f8c20b427de1cfe7a28a015c1640ce4e4eef1e33", "shasum": "" }, @@ -250,6 +250,78 @@ ], "time": "2015-03-26 16:11:30" }, + { + "name": "ddeboer/data-import", + "version": "dev-master", + "source": { + "type": "git", + "url": "https://github.com/ddeboer/data-import.git", + "reference": "043baa8a6c82412972eae7b156774906b6a577a6" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/ddeboer/data-import/zipball/043baa8a6c82412972eae7b156774906b6a577a6", + "reference": "043baa8a6c82412972eae7b156774906b6a577a6", + "shasum": "" + }, + "require": { + "php": ">=5.4.0", + "psr/log": "~1.0" + }, + "require-dev": { + "doctrine/dbal": "~2.4", + "doctrine/orm": "~2.4", + "ext-iconv": "*", + "ext-mbstring": "*", + "ext-sqlite3": "*", + "henrikbjorn/phpspec-code-coverage": "~1.0", + "phpoffice/phpexcel": "*", + "phpspec/phpspec": "~2.1", + "symfony/console": "~2.5.0", + "symfony/property-access": "~2.5", + "symfony/validator": "~2.3.0" + }, + "suggest": { + "doctrine/dbal": "If you want to use the DbalReader", + "ext-iconv": "For the CharsetValueConverter", + "ext-mbstring": "For the CharsetValueConverter", + "phpoffice/phpexcel": "If you want to use the ExcelReader", + "symfony/console": "If you want to use the ConsoleProgressWriter", + "symfony/property-access": "If you want to use the ObjectConverter", + "symfony/validator": "to use the ValidatorFilter" + }, + "type": "library", + "autoload": { + "psr-4": { + "Ddeboer\\DataImport\\": "src/", + "Ddeboer\\DataImport\\Tests\\": "tests/" + } + }, + "notification-url": "https://packagist.org/downloads/", + "license": [ + "MIT" + ], + "authors": [ + { + "name": "The community", + "homepage": "https://github.com/ddeboer/data-import/graphs/contributors" + }, + { + "name": "David de Boer", + "email": "david@ddeboer.nl" + } + ], + "description": "Import data from, and export data to, a range of file formats and media", + "keywords": [ + "csv", + "data", + "doctrine", + "excel", + "export", + "import" + ], + "time": "2015-05-30 08:46:44" + }, { "name": "ircmaxell/password-compat", "version": "1.0.x-dev", @@ -302,7 +374,7 @@ }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/kohana/cache/zipball/e001903db55c2af92737da2f3cf957e94282417c", + "url": "https://api.github.com/repos/kohana/cache/zipball/8bcf01225f7b32fc7fbf6929826ff9791831849f", "reference": "e001903db55c2af92737da2f3cf957e94282417c", "shasum": "" }, @@ -359,7 +431,7 @@ }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/kohana/core/zipball/37740b1dce1d7aa276d21bae6b7161ddaef244d6", + "url": "https://api.github.com/repos/kohana/core/zipball/b158eaeffbd6fb6d4cbd8e52d8ec3b3ee2145d63", "reference": "37740b1dce1d7aa276d21bae6b7161ddaef244d6", "shasum": "" }, @@ -584,6 +656,63 @@ ], "time": "2014-10-10 14:38:27" }, + { + "name": "league/csv", + "version": "dev-master", + "source": { + "type": "git", + "url": "https://github.com/thephpleague/csv.git", + "reference": "2ee1760c262c41986f6371775907fc9e8603fd26" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/thephpleague/csv/zipball/2ee1760c262c41986f6371775907fc9e8603fd26", + "reference": "2ee1760c262c41986f6371775907fc9e8603fd26", + "shasum": "" + }, + "require": { + "ext-mbstring": "*", + "php": ">=5.4.0" + }, + "require-dev": { + "phpunit/phpunit": "~4.0", + "scrutinizer/ocular": "~1.1" + }, + "type": "library", + "extra": { + "branch-alias": { + "dev-master": "7.1-dev" + } + }, + "autoload": { + "psr-4": { + "League\\Csv\\": "src" + } + }, + "notification-url": "https://packagist.org/downloads/", + "license": [ + "MIT" + ], + "authors": [ + { + "name": "Ignace Nyamagana Butera", + "email": "nyamsprod@gmail.com", + "homepage": "https://github.com/nyamsprod/", + "role": "Developer" + } + ], + "description": "Csv data manipulation made easy in PHP", + "homepage": "http://csv.thephpleague.com", + "keywords": [ + "csv", + "export", + "filter", + "import", + "read", + "write" + ], + "time": "2015-06-10 11:12:37" + }, { "name": "league/flysystem", "version": "0.4.5", @@ -774,6 +903,52 @@ ], "time": "2014-08-21 16:41:35" }, + { + "name": "psr/log", + "version": "dev-master", + "source": { + "type": "git", + "url": "https://github.com/php-fig/log.git", + "reference": "9e45edca52cc9c954680072c93e621f8b71fab26" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/php-fig/log/zipball/9e45edca52cc9c954680072c93e621f8b71fab26", + "reference": "9e45edca52cc9c954680072c93e621f8b71fab26", + "shasum": "" + }, + "require": { + "php": ">=5.3.0" + }, + "type": "library", + "extra": { + "branch-alias": { + "dev-master": "1.0.x-dev" + } + }, + "autoload": { + "psr-4": { + "Psr\\Log\\": "Psr/Log/" + } + }, + "notification-url": "https://packagist.org/downloads/", + "license": [ + "MIT" + ], + "authors": [ + { + "name": "PHP-FIG", + "homepage": "http://www.php-fig.org/" + } + ], + "description": "Common interface for logging libraries", + "keywords": [ + "log", + "psr", + "psr-3" + ], + "time": "2015-06-02 13:48:41" + }, { "name": "robmorgan/phinx", "version": "v0.4.3", @@ -955,7 +1130,7 @@ }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/symfony/Config/zipball/1d865a833a85c535bff72832fc41940178dc5716", + "url": "https://api.github.com/repos/symfony/Config/zipball/b5063937fab6fdfb7bacc00bc8c0cd7ee0c50070", "reference": "1d865a833a85c535bff72832fc41940178dc5716", "shasum": "" }, @@ -1006,7 +1181,7 @@ }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/symfony/Console/zipball/656a3f550462635f00cf5f1ed7aabd2b1f1bc299", + "url": "https://api.github.com/repos/symfony/Console/zipball/0e5e18ae09d3f5c06367759be940e9ed3f568359", "reference": "656a3f550462635f00cf5f1ed7aabd2b1f1bc299", "shasum": "" }, @@ -1064,7 +1239,7 @@ }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/symfony/Filesystem/zipball/0d0790beeb3bdb997a6cb2d6f4e53f607337efe5", + "url": "https://api.github.com/repos/symfony/Filesystem/zipball/2d7b2ddaf3f548f4292df49a99d19c853d43f0b8", "reference": "0d0790beeb3bdb997a6cb2d6f4e53f607337efe5", "shasum": "" }, @@ -1103,6 +1278,66 @@ "homepage": "http://symfony.com", "time": "2015-03-22 16:57:18" }, + { + "name": "symfony/property-access", + "version": "2.8.x-dev", + "source": { + "type": "git", + "url": "https://github.com/symfony/PropertyAccess.git", + "reference": "5b1635c98f34b3fa258213e47213e949a4ccb4c0" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/symfony/PropertyAccess/zipball/5b1635c98f34b3fa258213e47213e949a4ccb4c0", + "reference": "5b1635c98f34b3fa258213e47213e949a4ccb4c0", + "shasum": "" + }, + "require": { + "php": ">=5.3.9" + }, + "require-dev": { + "symfony/phpunit-bridge": "~2.7|~3.0.0" + }, + "type": "library", + "extra": { + "branch-alias": { + "dev-master": "2.8-dev" + } + }, + "autoload": { + "psr-4": { + "Symfony\\Component\\PropertyAccess\\": "" + } + }, + "notification-url": "https://packagist.org/downloads/", + "license": [ + "MIT" + ], + "authors": [ + { + "name": "Fabien Potencier", + "email": "fabien@symfony.com" + }, + { + "name": "Symfony Community", + "homepage": "https://symfony.com/contributors" + } + ], + "description": "Symfony PropertyAccess Component", + "homepage": "https://symfony.com", + "keywords": [ + "access", + "array", + "extraction", + "index", + "injection", + "object", + "property", + "property path", + "reflection" + ], + "time": "2015-07-16 12:22:14" + }, { "name": "symfony/yaml", "version": "2.6.x-dev", @@ -1114,7 +1349,7 @@ }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/symfony/Yaml/zipball/6059e5c76321906186043c909a65f08aa7ad4397", + "url": "https://api.github.com/repos/symfony/Yaml/zipball/c044d1744b8e91aaaa0d9bac683ab87ec7cbf359", "reference": "6059e5c76321906186043c909a65f08aa7ad4397", "shasum": "" }, @@ -1391,7 +1626,7 @@ }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/Behat/Behat/zipball/51d9a2fb1c802a681c2717e48b4b8d184595bd18", + "url": "https://api.github.com/repos/Behat/Behat/zipball/794f294f373acbfc5f8952adfb11471a93a2a864", "reference": "51d9a2fb1c802a681c2717e48b4b8d184595bd18", "shasum": "" }, @@ -1514,7 +1749,7 @@ }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/minkphp/Mink/zipball/1ccdea1f492e118c32e06caebe5eb3c8e4bdcc17", + "url": "https://api.github.com/repos/minkphp/Mink/zipball/d95155ed2bc5ae940d78be286727a3b712c2513e", "reference": "1ccdea1f492e118c32e06caebe5eb3c8e4bdcc17", "shasum": "" }, @@ -1674,7 +1909,7 @@ }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/minkphp/MinkGoutteDriver/zipball/63a593c25a449292787c7418f08b6181c27271f6", + "url": "https://api.github.com/repos/minkphp/MinkGoutteDriver/zipball/cc5ce119b5a8e06662f634b35967aff0b0c7dfdd", "reference": "63a593c25a449292787c7418f08b6181c27271f6", "shasum": "" }, @@ -1726,7 +1961,7 @@ }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/doctrine/instantiator/zipball/3d9669e597439e8d205baf315efb757038fb4dea", + "url": "https://api.github.com/repos/doctrine/instantiator/zipball/8e884e78f9f0eb1329e445619e04456e64d8051d", "reference": "3d9669e597439e8d205baf315efb757038fb4dea", "shasum": "" }, @@ -1775,12 +2010,12 @@ "version": "1.0.x-dev", "source": { "type": "git", - "url": "https://github.com/fabpot/Goutte.git", + "url": "https://github.com/FriendsOfPHP/Goutte.git", "reference": "794b196e76bdd37b5155cdecbad311f0a3b07625" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/fabpot/Goutte/zipball/794b196e76bdd37b5155cdecbad311f0a3b07625", + "url": "https://api.github.com/repos/FriendsOfPHP/Goutte/zipball/794b196e76bdd37b5155cdecbad311f0a3b07625", "reference": "794b196e76bdd37b5155cdecbad311f0a3b07625", "shasum": "" }, @@ -1836,7 +2071,7 @@ }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/guzzle/guzzle3/zipball/0645b70d953bc1c067bbc8d5bc53194706b628d9", + "url": "https://api.github.com/repos/guzzle/guzzle3/zipball/b3f5050cb6270c7a728a0b74ac2de50a262b3e02", "reference": "0645b70d953bc1c067bbc8d5bc53194706b628d9", "shasum": "" }, @@ -1931,7 +2166,7 @@ }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/heroku/heroku-buildpack-php/zipball/8da6b92aad89b2e7254f5eb2ad39ffac315b2282", + "url": "https://api.github.com/repos/heroku/heroku-buildpack-php/zipball/044a289943c6aab0e23547f21b3e9d6a8d89f1e1", "reference": "8da6b92aad89b2e7254f5eb2ad39ffac315b2282", "shasum": "" }, @@ -1975,7 +2210,7 @@ }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/kohana/unittest/zipball/5cc6000d8757d361eb7808f4aa4f3423b3582931", + "url": "https://api.github.com/repos/kohana/unittest/zipball/a95fd65e53395f4e0af939180803d5c482a8a18d", "reference": "5cc6000d8757d361eb7808f4aa4f3423b3582931", "shasum": "" }, @@ -2192,7 +2427,7 @@ }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/phpspec/prophecy/zipball/8724cd239f8ef4c046f55a3b18b4d91cc7f3e4c5", + "url": "https://api.github.com/repos/phpspec/prophecy/zipball/5700f75b23b0dd3495c0f495fe33a5e6717ee160", "reference": "8724cd239f8ef4c046f55a3b18b4d91cc7f3e4c5", "shasum": "" }, @@ -2371,7 +2606,7 @@ }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/sebastianbergmann/php-file-iterator/zipball/a923bb15680d0089e2316f7a4af8f437046e96bb", + "url": "https://api.github.com/repos/sebastianbergmann/php-file-iterator/zipball/6150bf2c35d3fc379e50c7602b75caceaa39dbf0", "reference": "a923bb15680d0089e2316f7a4af8f437046e96bb", "shasum": "" }, @@ -2686,7 +2921,7 @@ }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/sebastianbergmann/comparator/zipball/1dd8869519a225f7f2b9eb663e225298fade819e", + "url": "https://api.github.com/repos/sebastianbergmann/comparator/zipball/937efb279bd37a375bcadf584dec0726f84dbf22", "reference": "1dd8869519a225f7f2b9eb663e225298fade819e", "shasum": "" }, @@ -2750,7 +2985,7 @@ }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/sebastianbergmann/diff/zipball/863df9687835c62aa423a22412d26fa2ebde3fd3", + "url": "https://api.github.com/repos/sebastianbergmann/diff/zipball/6899b3e33bfbd386d88b5eea5f65f563e8793051", "reference": "863df9687835c62aa423a22412d26fa2ebde3fd3", "shasum": "" }, @@ -2802,7 +3037,7 @@ }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/sebastianbergmann/exporter/zipball/84839970d05254c73cde183a721c7af13aede943", + "url": "https://api.github.com/repos/sebastianbergmann/exporter/zipball/66fb20c9e1b3617651c3f66be7a1747479d96ba5", "reference": "84839970d05254c73cde183a721c7af13aede943", "shasum": "" }, @@ -2868,7 +3103,7 @@ }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/sebastianbergmann/recursion-context/zipball/3989662bbb30a29d20d9faa04a846af79b276252", + "url": "https://api.github.com/repos/sebastianbergmann/recursion-context/zipball/994d4a811bafe801fb06dccbee797863ba2792ba", "reference": "3989662bbb30a29d20d9faa04a846af79b276252", "shasum": "" }, @@ -2997,7 +3232,7 @@ }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/symfony/BrowserKit/zipball/c135dd2a06216cfcbb00863770c3df075450992e", + "url": "https://api.github.com/repos/symfony/BrowserKit/zipball/176905d3d74c2f99e6ab70f4f5a89460532495ae", "reference": "c135dd2a06216cfcbb00863770c3df075450992e", "shasum": "" }, @@ -3053,7 +3288,7 @@ }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/symfony/CssSelector/zipball/4af0110c47b5905944c3688a73e9452077e68a26", + "url": "https://api.github.com/repos/symfony/CssSelector/zipball/0b5c07b516226b7dd32afbbc82fe547a469c5092", "reference": "4af0110c47b5905944c3688a73e9452077e68a26", "shasum": "" }, @@ -3107,7 +3342,7 @@ }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/symfony/DependencyInjection/zipball/b9008df7404a3cb7983ad04e88d607260aeaaad9", + "url": "https://api.github.com/repos/symfony/DependencyInjection/zipball/319f1b373888b72d7ba50253adedd75214bd01d2", "reference": "b9008df7404a3cb7983ad04e88d607260aeaaad9", "shasum": "" }, @@ -3168,7 +3403,7 @@ }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/symfony/DomCrawler/zipball/a4d52bbd4775d6891735c973cf39e38b27e0e677", + "url": "https://api.github.com/repos/symfony/DomCrawler/zipball/9dabece63182e95c42b06967a0d929a5df78bc35", "reference": "a4d52bbd4775d6891735c973cf39e38b27e0e677", "shasum": "" }, @@ -3222,7 +3457,7 @@ }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/symfony/EventDispatcher/zipball/186349c2966529804e38685f671e64746dde220b", + "url": "https://api.github.com/repos/symfony/EventDispatcher/zipball/9310b5f9a87ec2ea75d20fec0b0017c77c66dac3", "reference": "186349c2966529804e38685f671e64746dde220b", "shasum": "" }, @@ -3281,7 +3516,7 @@ }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/symfony/Finder/zipball/492de776d92afc774f78cb3d47635b5da6ad2507", + "url": "https://api.github.com/repos/symfony/Finder/zipball/ae0f363277485094edc04c9f3cbe595b183b78e4", "reference": "492de776d92afc774f78cb3d47635b5da6ad2507", "shasum": "" }, @@ -3331,7 +3566,7 @@ }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/symfony/Process/zipball/6028c8268fa2afe7e500f9652709270607b535a1", + "url": "https://api.github.com/repos/symfony/Process/zipball/48aeb0e48600321c272955132d7606ab0a49adb3", "reference": "6028c8268fa2afe7e500f9652709270607b535a1", "shasum": "" }, @@ -3381,7 +3616,7 @@ }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/symfony/Translation/zipball/bd939f05cdaca128f4ddbae1b447d6f0203b60af", + "url": "https://api.github.com/repos/symfony/Translation/zipball/d84291215b5892834dd8ca8ee52f9cbdb8274904", "reference": "bd939f05cdaca128f4ddbae1b447d6f0203b60af", "shasum": "" }, @@ -3445,6 +3680,8 @@ "wouter/acl": 20, "vlucas/phpdotenv": 20, "zeelot/kohana-media": 20, + "league/csv": 20, + "ddeboer/data-import": 20, "kohana/unittest": 20, "heroku/heroku-buildpack-php": 20 }, diff --git a/src/Console/Authorizer/ConsoleAuthorizer.php b/src/Console/Authorizer/ConsoleAuthorizer.php new file mode 100644 index 0000000000..3e317758af --- /dev/null +++ b/src/Console/Authorizer/ConsoleAuthorizer.php @@ -0,0 +1,35 @@ + + * @package Ushahidi\Application + * @copyright 2014 Ushahidi + * @license https://www.gnu.org/licenses/agpl-3.0.html GNU Affero General Public License Version 3 (AGPL3) + */ + +namespace Ushahidi\Console\Authorizer; + +use Ushahidi\Core\Entity; +use Ushahidi\Core\Tool\Authorizer; +use Ushahidi\Core\Traits\PrivAccess; +use Ushahidi\Core\Traits\UserContext; + +// The `ConsoleAuthorizer` class is responsible for access checks for console tasks +class ConsoleAuthorizer implements Authorizer +{ + // The access checks are run under the context of a specific user + // @todo refactor to avoid including this. CLI doesn't have a user context + use UserContext; + + // It uses `PrivAccess` to provide the `getAllowedPrivs` method. + use PrivAccess; + + /* Authorizer */ + public function isAllowed(Entity $entity, $privilege) + { + // All console requests are authorized + return true; + } +} diff --git a/src/Console/Command.php b/src/Console/Command.php index b163577d21..1cebb4ca8d 100644 --- a/src/Console/Command.php +++ b/src/Console/Command.php @@ -27,7 +27,7 @@ protected function execute(InputInterface $input, OutputInterface $output) { // Enforce a default action of list. $action = $input->getArgument('action') ?: 'list'; - $execute = 'execute_' . $action; + $execute = 'execute' . ucfirst($action); // Reroute to the specific action. $response = $this->$execute($input, $output); @@ -36,7 +36,7 @@ protected function execute(InputInterface $input, OutputInterface $output) // Display arrays as tables. $table = $this->getHelperSet()->get('table'); - $key = array_keys($response); + $keys = array_keys($response); if (is_array(current($response))) { // Assume that an array of arrays is a result list. diff --git a/src/Console/Import.php b/src/Console/Import.php new file mode 100644 index 0000000000..e173bca193 --- /dev/null +++ b/src/Console/Import.php @@ -0,0 +1,143 @@ + + * @package Ushahidi\Console + * @copyright 2014 Ushahidi + * @license https://www.gnu.org/licenses/agpl-3.0.html GNU Affero General Public License Version 3 (AGPL3) + */ + +namespace Ushahidi\Console; + +use SplFileObject; +use Symfony\Component\Console\Input\InputArgument; +use Symfony\Component\Console\Input\InputInterface; +use Symfony\Component\Console\Input\InputOption; +use Symfony\Component\Console\Input\ArrayInput; +use Symfony\Component\Console\Output\OutputInterface; +use Symfony\Component\Console\Logger\ConsoleLogger; +use Psr\Log\LoggerInterface; + +use League\Csv\Reader; +use Ushahidi\Core\Tool\MappingTransformer; +use Ushahidi\Core\Usecase\ImportUsecase; + +class Import extends Command +{ + + /** + * Data Importers + * @var [Ushahidi\DataImport\Importer, ..] + */ + protected $importers; + + /** + * Set Data Importers + * @param [Ushahidi\DataImport\Importer, ..] $importers + */ + public function setImporters(Array $importers) + { + $this->importers = $importers; + } + + protected function configure() + { + $this + ->setName('import') + ->setDescription('Import data') + ->addArgument('action', InputArgument::OPTIONAL, 'list, run', 'list') + // @todo provide allowed types list + ->addOption('type', ['t'], InputOption::VALUE_OPTIONAL, 'import source type', 'csv') + ->addOption('file', ['f'], InputOption::VALUE_REQUIRED, 'Source filename') + ->addOption('map', ['m'], InputOption::VALUE_REQUIRED, 'Source-Destination field mapping') + ->addOption('values', [], InputOption::VALUE_REQUIRED, 'Static field values') + ->addOption('limit', [], InputOption::VALUE_OPTIONAL, 'Number of records to import') + ->addOption('offset', [], InputOption::VALUE_OPTIONAL, 'Offset to start importing from') + ; + } + + protected function executeList(InputInterface $input, OutputInterface $output) + { + return [ + [ + 'Supported Types' => 'CSV' + ] + ]; + } + + protected function executeRun(InputInterface $input, OutputInterface $output) + { + // Get the filename + $filename = $input->getOption('file'); + + // Load mapping and pass to transformer + $map = file_get_contents($input->getOption('map')); + $this->transformer->setMap(json_decode($map, true)); + + // Load fixed values and pass to transformer + $values = file_get_contents($input->getOption('values')); + $this->transformer->setFixedValues(json_decode($values, true)); + + // Get CSV reader + $reader = $this->getReader($input->getOption('type')); + + // Set limit.. + if ($limit = $input->getOption('limit')) { + $reader->setLimit($limit); + } + // .. and offset + if ($offset = $input->getOption('offset')) { + $reader->setOffset($offset); + } + + // Get the traversable results + $payload = $reader->process($filename); + + // Get the usecase and pass in authorizer, payload and transformer + $this->usecase + ->setPayload($payload) + ->setTransformer($this->transformer); + + // Execute the import + return $this->usecase->interact(); + } + + /** + * Map of readers + * @var [FileReader, ...] + */ + protected $readerMap = []; + + public function setReaderMap($map) + { + $this->readerMap = $map; + } + + protected function getReader($type) + { + return $this->readerMap[$type](); + } + + /** + * @var Ushahidi\Core\Tool\MappingTransformer + */ + protected $transformer; + + public function setTransformer(MappingTransformer $transformer) + { + $this->transformer = $transformer; + } + + /** + * @var Ushahidi\Core\Usecase\ImportUsecase + * @todo support multiple entity types + */ + protected $usecase; + + public function setImportUsecase(ImportUsecase $usecase) + { + $this->usecase = $usecase; + } +} diff --git a/src/Core/Exception/ValidatorException.php b/src/Core/Exception/ValidatorException.php index 1064e0d9a2..c59aeb91fb 100644 --- a/src/Core/Exception/ValidatorException.php +++ b/src/Core/Exception/ValidatorException.php @@ -17,6 +17,8 @@ class ValidatorException extends \InvalidArgumentException public function __construct($message, Array $errors, Exception $previous = null) { + $message = $message . ":\n" . implode("\n", $errors); + parent::__construct($message, 0, $previous); $this->setErrors($errors); } diff --git a/src/Core/Tool/AuthorizerTrait.php b/src/Core/Tool/AuthorizerTrait.php index 53743f6249..900d99fcca 100644 --- a/src/Core/Tool/AuthorizerTrait.php +++ b/src/Core/Tool/AuthorizerTrait.php @@ -119,6 +119,19 @@ protected function verifyCreateAuth(Entity $entity) $this->verifyAuth($entity, 'create'); } + /** + * Verifies the current user is allowed import access on $entity + * + * @param Entity $entity + * @param Data $input + * @return void + * @throws AuthorizerException + */ + protected function verifyImportAuth(Entity $entity) + { + $this->verifyAuth($entity, 'import'); + } + /** * Get all allowed privs on an Entity * @param Entity $entity diff --git a/src/Core/Tool/FileReader.php b/src/Core/Tool/FileReader.php new file mode 100644 index 0000000000..b207cd243f --- /dev/null +++ b/src/Core/Tool/FileReader.php @@ -0,0 +1,27 @@ + + * @package Ushahidi\Platform + * @copyright 2014 Ushahidi + * @license https://www.gnu.org/licenses/agpl-3.0.html GNU Affero General Public License Version 3 (AGPL3) + */ + +namespace Ushahidi\Core\Tool; + +interface FileReader +{ + /** + * Read a file and return a Traversable object + * + * @param String $filename + * @return Traversable + */ + public function process($filename); + + public function setOffset($offset); + + public function setLimit($limit); +} diff --git a/src/Core/Tool/MappingTransformer.php b/src/Core/Tool/MappingTransformer.php new file mode 100644 index 0000000000..aefea4aef8 --- /dev/null +++ b/src/Core/Tool/MappingTransformer.php @@ -0,0 +1,22 @@ + + * @package Ushahidi\Platform + * @copyright 2014 Ushahidi + * @license https://www.gnu.org/licenses/agpl-3.0.html GNU Affero General Public License Version 3 (AGPL3) + */ + +namespace Ushahidi\Core\Tool; + +interface MappingTransformer extends Transformer +{ + public function setMap(Array $map); + public function setFixedValues(Array $fixedValues); +} diff --git a/src/Core/Tool/Transformer.php b/src/Core/Tool/Transformer.php new file mode 100644 index 0000000000..33e2d1ae52 --- /dev/null +++ b/src/Core/Tool/Transformer.php @@ -0,0 +1,19 @@ + + * @package Ushahidi\Platform + * @copyright 2014 Ushahidi + * @license https://www.gnu.org/licenses/agpl-3.0.html GNU Affero General Public License Version 3 (AGPL3) + */ + +namespace Ushahidi\Core\Tool; + +interface Transformer +{ + public function interact(Array $data); +} diff --git a/src/Core/Traits/DataTransformer.php b/src/Core/Traits/DataTransformer.php index 917a503a93..ef0dfa16aa 100644 --- a/src/Core/Traits/DataTransformer.php +++ b/src/Core/Traits/DataTransformer.php @@ -3,8 +3,10 @@ /** * Ushahidi Data Transformer Trait * - * Gives objects a new `transform($data, $definition)` method, which can be - * used to ensure data type consistency. + * Gives objects new `transform($data)` and `getDefinition()` methods, + * which can be used to ensure data type consistency. + * + * @todo rename to differentiate from Transformer tools * * @author Ushahidi Team * @package Ushahidi\Platform diff --git a/src/Core/Usecase/ImportRepository.php b/src/Core/Usecase/ImportRepository.php new file mode 100644 index 0000000000..570b4f6ce1 --- /dev/null +++ b/src/Core/Usecase/ImportRepository.php @@ -0,0 +1,19 @@ + + * @package Ushahidi\Platform + * @copyright 2014 Ushahidi + * @license https://www.gnu.org/licenses/agpl-3.0.html GNU Affero General Public License Version 3 (AGPL3) + */ + +namespace Ushahidi\Core\Usecase; + +use Ushahidi\Core\Entity; + +interface ImportRepository extends CreateRepository, UpdateRepository +{ + +} diff --git a/src/Core/Usecase/ImportUsecase.php b/src/Core/Usecase/ImportUsecase.php new file mode 100644 index 0000000000..38400572c5 --- /dev/null +++ b/src/Core/Usecase/ImportUsecase.php @@ -0,0 +1,162 @@ + + * @package Ushahidi\Platform + * @copyright 2014 Ushahidi + * @license https://www.gnu.org/licenses/agpl-3.0.html GNU Affero General Public License Version 3 (AGPL3) + */ + +namespace Ushahidi\Core\Usecase; + +use Traversable; +use Ushahidi\Core\Entity; +use Ushahidi\Core\Usecase; +use Ushahidi\Core\Tool\AuthorizerTrait; +use Ushahidi\Core\Tool\FormatterTrait; +use Ushahidi\Core\Tool\ValidatorTrait; +use Ushahidi\Core\Tool\Transformer; +use Ushahidi\Core\Traits\ModifyRecords; + +class ImportUsecase implements Usecase +{ + // Uses several traits to assign tools. Each of these traits provides a + // setter method for the tool. For example, the AuthorizerTrait provides + // a `setAuthorizer` method which only accepts `Authorizer` instances. + use AuthorizerTrait, + FormatterTrait, + ValidatorTrait; + + /** + * @var ImportRepository + */ + protected $repo; + + /** + * Inject a repository that can create entities. + * + * @param $repo ImportRepository + * @return $this + */ + public function setRepository(ImportRepository $repo) + { + $this->repo = $repo; + return $this; + } + + /** + * @var Traversable + */ + protected $payload; + + /** + * Inject a repository that can create entities. + * + * @todo setPayload doesn't match signature for other usecases + * + * @param $repo Iterator + * @return $this + */ + public function setPayload(Traversable $payload) + { + $this->payload = $payload; + return $this; + } + + /** + * @var Transformer + */ + protected $transformer; + + /** + * Inject a repository that can create entities. + * + * @param $repo Iterator + * @return $this + */ + public function setTransformer(Transformer $transformer) + { + $this->transformer = $transformer; + return $this; + } + + // Usecase + public function isWrite() + { + return true; + } + + // Usecase + public function isSearch() + { + return false; + } + + // Usecase + public function interact() + { + // Start count of records processed, and errors + $processed = $errors = 0; + + // Fetch an empty entity.. + $entity = $this->getEntity(); + + // ... verify that the entity can be created by the current user + $this->verifyImportAuth($entity); + + // Fetch a record + foreach ($this->payload as $index => $record) { + + // ... transform record + $entity = $this->transform($record); + + // ... verify that the entity can be created by the current user + $this->verifyCreateAuth($entity); + + // ... verify that the entity is in a valid state + $this->verifyValid($entity); + + // ... persist the new entity + $id = $this->repo->create($entity); + + $processed++; + } + + // ... and return the formatted entity + return [ + 'processed' => $processed, + 'errors' => $errors + ]; + } + + // ValidatorTrait + protected function verifyValid(Entity $entity) + { + if (!$this->validator->check($entity->asArray())) { + $this->validatorError($entity); + } + } + + /** + * Get an empty entity + * + * @return Entity + */ + protected function getEntity() + { + return $this->repo->getEntity(); + } + + /** + * [transform description] + * @return [type] [description] + */ + protected function transform($record) + { + $record = $this->transformer->interact($record); + + return $this->repo->getEntity()->setState($record); + } +} diff --git a/src/Init.php b/src/Init.php index fcb842366f..f6f0caf2ab 100644 --- a/src/Init.php +++ b/src/Init.php @@ -66,6 +66,18 @@ function feature($name) // Any command can be registered with the console app. $di->params['Ushahidi\Console\Application']['injectCommands'] = []; +$di->setter['Ushahidi\Console\Application']['injectCommands'][] = $di->lazyNew('Ushahidi\Console\Import'); + +// Set up Import command +$di->setter['Ushahidi\Console\Import']['setReaderMap'] = []; +$di->setter['Ushahidi\Console\Import']['setReaderMap']['csv'] = $di->lazyGet('filereader.csv'); +$di->setter['Ushahidi\Console\Import']['setTransformer'] = $di->lazyGet('transformer.mapping'); +$di->setter['Ushahidi\Console\Import']['setImportUsecase'] = $di->lazy(function () use ($di) { + return service('factory.usecase') + ->get('posts', 'import') + // Override authorizer for console + ->setAuthorizer($di->get('authorizer.console')); +}); // Validators are used to parse **and** verify input data used for write operations. $di->set('factory.validator', $di->lazyNew('Ushahidi\Factory\ValidatorFactory')); @@ -211,7 +223,8 @@ function feature($name) 'update' => $di->lazyNew('Ushahidi\Core\Usecase\Post\UpdatePost'), 'delete' => $di->lazyNew('Ushahidi\Core\Usecase\Post\DeletePost'), 'search' => $di->lazyNew('Ushahidi\Core\Usecase\Post\SearchPost'), - 'stats' => $di->lazyNew('Ushahidi\Core\Usecase\Post\StatsPost') + 'stats' => $di->lazyNew('Ushahidi\Core\Usecase\Post\StatsPost'), + 'import' => $di->lazyNew('Ushahidi\Core\Usecase\ImportUsecase') ]; // Add custom usecases for sets_posts @@ -276,4 +289,4 @@ function feature($name) 'post_repo' => $di->lazyGet('repository.post'), ]; - +$di->set('authorizer.console', $di->lazyNew('Ushahidi\Console\Authorizer\ConsoleAuthorizer'));