From 2dcb49475b29e0faec1a6a8b76aecaa5860e0367 Mon Sep 17 00:00:00 2001 From: Fang Jin Date: Wed, 23 Jul 2014 18:27:13 -0400 Subject: [PATCH] init draft --- .gitignore | 4 + .travis.yml | 14 + composer.json | 23 ++ phpunit.xml | 18 ++ public/.gitkeep | 0 src/QPlot/Importer/Importer.php | 267 ++++++++++++++++++ .../Importer/ImporterServiceProvider.php | 49 ++++ src/config/.gitkeep | 0 src/config/config.php | 123 ++++++++ src/controllers/.gitkeep | 0 src/lang/.gitkeep | 0 src/migrations/.gitkeep | 0 src/views/.gitkeep | 0 tests/.gitkeep | 0 14 files changed, 498 insertions(+) create mode 100644 .gitignore create mode 100644 .travis.yml create mode 100644 composer.json create mode 100644 phpunit.xml create mode 100644 public/.gitkeep create mode 100644 src/QPlot/Importer/Importer.php create mode 100644 src/QPlot/Importer/ImporterServiceProvider.php create mode 100644 src/config/.gitkeep create mode 100644 src/config/config.php create mode 100644 src/controllers/.gitkeep create mode 100644 src/lang/.gitkeep create mode 100644 src/migrations/.gitkeep create mode 100644 src/views/.gitkeep create mode 100644 tests/.gitkeep diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..5826402 --- /dev/null +++ b/.gitignore @@ -0,0 +1,4 @@ +/vendor +composer.phar +composer.lock +.DS_Store diff --git a/.travis.yml b/.travis.yml new file mode 100644 index 0000000..aa14ee5 --- /dev/null +++ b/.travis.yml @@ -0,0 +1,14 @@ +language: php + +php: + - 5.3 + - 5.4 + - 5.5 + - 5.6 + - hhvm + +before_script: + - composer self-update + - composer install --prefer-source --no-interaction --dev + +script: phpunit diff --git a/composer.json b/composer.json new file mode 100644 index 0000000..8bc1adc --- /dev/null +++ b/composer.json @@ -0,0 +1,23 @@ +{ + "name": "qplot/importer", + "description": "", + "authors": [ + { + "name": "Fang Jin", + "email": "windmaomao@gmail.com" + } + ], + "require": { + "php": ">=5.4.0", + "illuminate/support": "4.2.*" + }, + "autoload": { + "classmap": [ + "src/migrations" + ], + "psr-0": { + "QPlot\\Importer\\": "src/" + } + }, + "minimum-stability": "stable" +} diff --git a/phpunit.xml b/phpunit.xml new file mode 100644 index 0000000..3347b75 --- /dev/null +++ b/phpunit.xml @@ -0,0 +1,18 @@ + + + + + ./tests/ + + + diff --git a/public/.gitkeep b/public/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/src/QPlot/Importer/Importer.php b/src/QPlot/Importer/Importer.php new file mode 100644 index 0000000..4496264 --- /dev/null +++ b/src/QPlot/Importer/Importer.php @@ -0,0 +1,267 @@ +app = $app; + $this->messages = new MessageBag(); + } + + /** + * Import + */ + public function import($callback = '') { + $config = $this->app['config']; + + // check path exists + $this->path = base_path() . '/' . $config->get('importer::config.path'); + if (!file_exists($this->path)) { + $this->messages->add('error', 'import path does not exist'); + return; + } + + // check database connection + $db = $this->app['config']->get('importer::config.database'); + try { + $this->db = $this->app['db']->connection($db); + } catch (Exception $e) { + $this->messages->add('error', 'database connection error for ' . $db); + return; + } + + // assign callback + $this->beforeSaveCallback = $callback; + if (!is_callable($this->beforeSaveCallback)) { + $this->messages->add('error', 'callback function is not callable.'); + return; + } + + // process all imports + $this->data = []; + $imports = $config->get('importer::config.imports'); + foreach($imports as $name => &$import) { + $import['name'] = $name; + $this->process($import); + } + } + + /** + * + */ + public function getMessages() { + return $this->messages; + } + + /** + * Process each import file + */ + protected function process($import) { + $changeme = $this->app['hash']->make('changeme'); + + // Preset variables + $mapping = $import['mapping']; + $unique_field = $import['unique']; + $Table = str_plural($import['model']); // etc. users + $Model = ucfirst($import['model']); // etc. User + $Rules = $import['rules']; + $data = []; + + // Load file + $filename = $this->path . '/' . $import['file']; + + if (!file_exists($filename)) { + $this->messages->add('error', 'File missing: ' . $filename); + } + ini_set("auto_detect_line_endings", true); + $content = file($filename, FILE_IGNORE_NEW_LINES); + + // Load csv and fetch header + $rows = array_map('str_getcsv', $content); + $headers = array_shift($rows); // header required + + // Check unique col + if ($unique_field) { + if (!isset($mapping[$unique_field])) { + $this->messages->add('error', 'Unique Column Def missing: ' . $unique_field); + return; + } + $unique_col = $mapping[$unique_field]; + + // Check existing rows + $unique_mapping_index = array_search($unique_col, $headers); + if (!$unique_mapping_index) { + $this->messages->add('error', 'Unique Column missing: ' . $unique_field); + return; + } + $unique_cols = array_pluck($rows, $unique_mapping_index); + $query = call_user_func( + $Model . '::whereIn', + $unique_field, $unique_cols + ); + $data['unique'] = $query->lists($unique_field); + } else { + $data['unique'] = []; + } + + // Check mapping and prepare + foreach($mapping as $field => $map) { + if (!is_array($map)) { + if (!in_array($map, $headers)) { + $this->messages->add('error', 'Column missing: ' . $map); + return; + } + } else { + // for reference row + if ($map['type'] == 'reference') { + $ref_mapping_index = array_search($map['column'], $headers); + if (!$ref_mapping_index) { + $this->messages->add('error', 'Unique Column missing: ' . $map['column']); + return; + } + $ref_cols = array_pluck($rows, $ref_mapping_index); + $RefModel = ucfirst($map['model']); + $query = call_user_func( + $RefModel . '::whereIn', + $map['foreign_field'], $ref_cols + ); + $result_fields = [$map['foreign_ref']]; + $result_fields = array_merge($result_fields, [$map['foreign_field']]); + $result_fields = array_merge($result_fields, $map['foreign_data']); + $results = $query->get($result_fields); +// $data[$field] = $query->lists($map['foreign_ref'], $map['foreign_field']); + $ref_data = []; + foreach($results as $result) { + $key = $result[$map['foreign_field']]; + $ref_data[$key] = $result->toArray(); + } + $data[$field] = $ref_data; + } + } + } + + // Import + $importing = []; + $records = []; + $i = 0; + foreach($rows as $item) { + $i++; + + // assemble row + $row = array_combine($headers, $item); + + // remove non-unique + if ($unique_field) { + $unique_value = $row[$unique_col]; + // skip existing one + if (in_array($unique_value, $data['unique']) || in_array($unique_value, $importing)) { + continue; + } + } else { + $unique_value = $i; + } + + // assemble record + $record = []; + foreach($mapping as $field => $map) { + if (!is_array($map)) { + $record[$field] = $row[$map]; + } else { + switch($map['type']) { + case 'constant': + $record[$field] = $map['value']; + break; + case 'reference': + $ref_data = &$data[$field]; + $value = $row[$map['column']]; + if (isset($ref_data[$value])) + { + $record[$field] = $ref_data[$value][$map['foreign_ref']]; + } else + { + $this->messages->add('error', 'Reference missing: ' . $unique_field); + } + break; + } + } + } + + // call custom function + call_user_func_array($this->beforeSaveCallback, array($import, &$record, &$data)); + + // validate record + $validator = $this->app['validator']; + $validation = $validator->make($record, $Rules); + if ($validation->fails()) { + foreach($validation->messages()->all() as $msg) { + $this->messages->add('error', "[$Model] " . $unique_value . ': ' . $msg); + } + } else { + $importing[] = $unique_value; + $records[] = $record; + } + } +// Debugbar::info($records); + + if ($records) { + try { + $this->db->table($Table)->insert($records); + } catch(Exception $e) { + $this->messages->add('error', $e->getMessage()); + return; + } + } + + $this->messages->add('info', count($importing) . ' ' . $import['name'] . ' has been imported'); + return; + } + +} \ No newline at end of file diff --git a/src/QPlot/Importer/ImporterServiceProvider.php b/src/QPlot/Importer/ImporterServiceProvider.php new file mode 100644 index 0000000..8ad01d1 --- /dev/null +++ b/src/QPlot/Importer/ImporterServiceProvider.php @@ -0,0 +1,49 @@ +package('qplot/importer'); + $this->app['config']->package('qplot/importer', $this->guessPackagePath() . '/config'); + + } + + /** + * Register the service provider. + * + * @return void + */ + public function register() + { + $this->app['importer'] = $this->app->share(function ($app) + { + return new Importer(); + }); + } + + /** + * Get the services provided by the provider. + * + * @return array + */ + public function provides() + { + return array('importer'); + } + +} diff --git a/src/config/.gitkeep b/src/config/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/src/config/config.php b/src/config/config.php new file mode 100644 index 0000000..b256d2b --- /dev/null +++ b/src/config/config.php @@ -0,0 +1,123 @@ + 'public/files', + + /* + |-------------------------------------------------------------------------- + | Database connection + |-------------------------------------------------------------------------- + | + | Which database connection + | + */ + + 'database' => 'mysql', + + /* + |-------------------------------------------------------------------------- + | Import settings + |-------------------------------------------------------------------------- + | + | Import settings + | + */ + 'imports' => array( + + /* + |-------------------------------------------------------------------------- + | Import name + |-------------------------------------------------------------------------- + | + | Import code name + | + */ + 'customers' => [ + 'file' => 'customers.csv', + 'model' => 'user', + 'unique' => 'fullname', + 'mapping' => [ + 'fullname' => 'Full Name', + 'email' => 'E-mail', + 'created_at' => 'Signup Date', + ], + 'rules' => [ + 'fullname' => 'required', + 'email' => 'required|email|unique:users', + 'username' => 'required', + ] + ], + 'kids' => [ + 'file' => 'appointments.csv', + 'model' => 'kid', + 'unique' => 'fullname', + 'mapping' => [ + 'fullname' => 'Kid Name', + 'parent_id' => [ + 'column' => 'Full Name', + 'type' => 'reference', + 'model' => 'user', + 'foreign_ref' => 'id', + 'foreign_field' => 'fullname', + 'foreign_data' => [] + ] + ], + 'rules' => [] + ], + 'services' => [ + 'file' => 'appointments.csv', + 'model' => 'service', + 'unique' => 'title', + 'mapping' => [ + 'title' => 'Service', + 'duration' => 'Hours', + 'calendar_id' => [ + 'type' => 'constant', + 'value' => 1, + ] + ], + 'rules' => [] + ], + 'appointments' => [ + 'file' => 'appointments.csv', + 'model' => 'appointment', + 'unique' => '', + 'mapping' => [ + 'kid_id' => [ + 'column' => 'Kid Name', + 'type' => 'reference', + 'model' => 'kid', + 'foreign_ref' => 'id', + 'foreign_field' => 'fullname', + 'foreign_data' => [] + ], + 'service_id' => [ + 'column' => 'Service', + 'type' => 'reference', + 'model' => 'service', + 'foreign_ref' => 'id', + 'foreign_field' => 'title', + 'foreign_data' => ['duration'] + ], + 'date' => 'Date', + 'from' => 'Time', + 'to' => 'Service' + ], + 'rules' => [] + ] + + ), + +); diff --git a/src/controllers/.gitkeep b/src/controllers/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/src/lang/.gitkeep b/src/lang/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/src/migrations/.gitkeep b/src/migrations/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/src/views/.gitkeep b/src/views/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/tests/.gitkeep b/tests/.gitkeep new file mode 100644 index 0000000..e69de29