Merge pull request #295 from tdt/development

Pull request for v5.6.0

Author: coreation

.gitignore

@@ -15,3 +15,4 @@ composer.phar
 app/config/workbench.php
 public/packages
 workbench
+node_modules/

README.md

@@ -6,12 +6,12 @@
 [![Latest Stable Version](https://poser.pugx.org/tdt/core/version.png)](https://packagist.org/packages/tdt/core)
 [![Build Status](https://travis-ci.org/tdt/core.png?branch=development)](https://travis-ci.org/tdt/core) [![Dependency Status](https://www.versioneye.com/php/tdt:core/badge.png)](https://www.versioneye.com/php/tdt:core)
 
-The DataTank core is the framework in which the main application of The DataTank is built. The DataTank aims at publishing data to URI's in web readable formats. This means that you provide a nice JSON, XML, PHP, ... serialization on a certain URI from which the data resides somewhere in a CSV, XLS, XML, JSON, SHP, ... file.
+The DataTank core is the framework in which the main application of The DataTank is built. The DataTank is a web application that publishes data to URIs in web readable formats. This means that you provide a nice JSON, XML, PHP, ... serialization on a certain URI of data that resides somewhere in a CSV, XLS, XML, JSON, SHP, ... file or any other machine readable data container. In short, it provides an instant REST API on top of any machine readable data.
 
 # Read more
 
 If you want to read more about The DataTank project visit our [website](http://thedatatank.com) or take a look at our [documentation](http://docs.thedatatank.com).
 
-The DataTank is free software (AGPL, © 2011,2012 iRail NPO, 2012 OKFN Belgium) to create an API for non-local/dynamic data in no time.
+The DataTank is free software (AGPL, © 2011,2012 iRail NPO, 2012 Open Knowledge Belgium) to create an API for data in no time.
 
-Any questions? Add a support issue.
+Any questions? Add a support issue!

app/Tdt/Core/BaseController.php

@@ -4,6 +4,7 @@
 
 /**
  * The base controller
+ *
  * @copyright (C) 2011, 2014 by OKFN Belgium vzw/asbl
  * @license AGPLv3
  * @author Michiel Vancoillie <[email protected]>

app/Tdt/Core/DataControllers/CSVController.php

@@ -32,7 +32,6 @@ public function __construct(TabularColumnsRepositoryInterface $tabular_columns,
 
     public function readData($source_definition, $rest_parameters = array())
     {
-
         list($limit, $offset) = Pager::calculateLimitAndOffset();
 
         // Disregard the paging when rest parameters are given
@@ -41,6 +40,9 @@ public function readData($source_definition, $rest_parameters = array())
             $offset = 0;
         }
 
+        // Get the current column configuration from the CSV file
+        $parsed_columns = self::parseColumns($source_definition);
+
         // Check the given URI
         if (!empty($source_definition['uri'])) {
             $uri = $source_definition['uri'];
@@ -56,6 +58,34 @@ public function readData($source_definition, $rest_parameters = array())
         // Get CSV columns
         $columns = $this->tabular_columns->getColumns($source_definition['id'], 'CsvDefinition');
 
+        // Check if they match with the freshly parsed columns
+        if (count($parsed_columns) != count($columns)) {
+
+            // Save the new config
+            $this->tabular_columns->deleteBulk($source_definition['id'], 'CsvDefinition');
+            $this->tabular_columns->storeBulk($source_definition['id'], 'CsvDefinition', $columns);
+
+        } else {
+            foreach ($parsed_columns as $parsed_column) {
+
+                $column = array_shift($columns);
+
+                foreach ($parsed_column as $key => $val) {
+                    if ($val != $column[$key]) {
+
+                        // Save the new config
+                        $this->tabular_columns->deleteBulk($source_definition['id'], 'CsvDefinition');
+                        $this->tabular_columns->storeBulk($source_definition['id'], 'CsvDefinition', $columns);
+
+                        break;
+                    }
+                }
+            }
+        }
+
+        // In any case (changed column configuration or not) we can set the columns to the parsed ones
+        $columns = $parsed_columns;
+
         // Get the geo properties
         $geo_properties = $this->geo_properties->getGeoProperties($source_definition['id'], 'CsvDefinition');
 
@@ -102,11 +132,10 @@ public function readData($source_definition, $rest_parameters = array())
 
                 if ($total_rows >= $start_row) {
 
-                    $num = count($data);
-
                     // Create the values array, containing the (aliased) name of the column
                     // to the value of a the row which $data represents
                     $values = $this->createValues($columns, $data);
+
                     if ($offset <= $hits && $offset + $limit > $hits) {
 
                         $obj = new \stdClass();
@@ -128,6 +157,7 @@ public function readData($source_definition, $rest_parameters = array())
                     }
                     $hits++;
                 }
+
                 $total_rows++;
             }
             fclose($handle);

app/Tdt/Core/DataControllers/JSONController.php

@@ -16,6 +16,10 @@
  */
 class JSONController extends ADataController
 {
+    public static function getParameters()
+    {
+        return [];
+    }
 
     public function readData($source_definition, $rest_parameters = array())
     {

app/Tdt/Core/DataControllers/JSONLDController.php

@@ -16,6 +16,10 @@
  */
 class JSONLDController extends ADataController
 {
+    public static function getParameters()
+    {
+        return [];
+    }
 
     public function readData($source_definition, $rest_parameters = array())
     {

app/Tdt/Core/DataControllers/MYSQLController.php

@@ -0,0 +1,136 @@
+<?php
+
+namespace Tdt\Core\DataControllers;
+
+use Tdt\Core\Datasets\Data;
+use Tdt\Core\Pager;
+use Tdt\Core\Repositories\Interfaces\TabularColumnsRepositoryInterface;
+use Tdt\Core\Repositories\Interfaces\GeoPropertyRepositoryInterface;
+use Illuminate\Database\QueryException;
+
+/**
+ * MySQL controller
+ *
+ * @copyright (C) 2011, 2014 by OKFN Belgium vzw/asbl
+ * @license AGPLv3
+ * @author Jan Vansteenlandt <[email protected]>
+ */
+class MYSQLController extends ADataController
+{
+    private $columnsRepo;
+
+    public function __construct(TabularColumnsRepositoryInterface $columnsRepo, GeoPropertyRepositoryInterface $geoRepo)
+    {
+        $this->columnsRepo = $columnsRepo;
+        $this->geoRepo = $geoRepo;
+    }
+
+    public function readData($source_definition, $rest_parameters = array())
+    {
+        //Check if the query is already paged.
+        $limitInQuery = false;
+
+        if (stripos($source_definition['query'], 'limit')) {
+            $limitInQuery = true;
+        } else {
+            list($limit, $offset) = Pager::calculateLimitAndOffset();
+        }
+
+        // Disregard the paging when rest parameters are given
+        if (!empty($rest_parameters)) {
+            $limit = 500;
+            $offset = 0;
+        }
+
+        // Get the columns from the repository
+        $columns = $this->columnsRepo->getColumns($source_definition['id'], 'MysqlDefinition');
+
+        // Get the geo properties
+        $geo_properties = $this->geoRepo->getGeoProperties($source_definition['id'], 'MysqlDefinition');
+
+        $geo = array();
+
+        foreach ($geo_properties as $geo_prop) {
+            $geo[$geo_prop['property']] = $geo_prop['path'];
+        }
+
+        if (!$columns) {
+            // 500 error because this shouldn't happen in normal conditions
+            // Columns are parsed upon adding a CSV resource and are always present
+            \App::abort(500, "Cannot find the columns of the MySQL table file, this might be due to a corrupted database or a broken configuration.");
+        }
+
+        // Create aliases for the columns
+        $aliases = $this->columnsRepo->getColumnAliases($source_definition['id'], 'MysqlDefinition');
+        $pk = null;
+
+        foreach ($columns as $column) {
+
+            if (!empty($column['is_pk'])) {
+                $pk = $column['column_name_alias'];
+            }
+        }
+
+        // Connect to the database
+        $db_config = array(
+            'driver'    => 'mysql',
+            'host'      => $source_definition['host'],
+            'database'  => $source_definition['database'],
+            'username'  => $source_definition['username'],
+            'password'  => $source_definition['password'],
+            'charset'   => 'utf8',
+            'collation' => $source_definition['collation'],
+        );
+
+        // Configure a connection
+        \Config::set('database.connections.mysqltmp', $db_config);
+
+        // Make a database connection
+        $db = \DB::connection('mysqltmp');
+
+        try {
+
+            $query = $source_definition['query'];
+
+            // Get the total amount of records for the query for pagination
+            preg_match("/select.*?(from.*)/msi", $query, $matches);
+
+            if (empty($matches[1])) {
+                \App::abort(400, 'Failed to make a count statement, make sure the SQL query is valid.');
+            }
+
+            $count_query = 'select count(*) as count ' . $matches[1];
+
+            $count_result = $db->select($count_query);
+
+            $total_rows = $count_result[0]->count;
+
+            if (!$limitInQuery) {
+
+                if (!empty($limit)) {
+                    $query .= ' limit ' . $limit;
+                }
+
+                if (!empty($offset)) {
+                    $query .= ' offset ' . $offset;
+                }
+            }
+
+            $result = $db->select($query);
+
+        } catch (QueryException $ex) {
+            \App::abort(400, "A bad query has been made, make sure all passed statements are SQL friendly. The error message was: " . $ex->getMessage());
+        }
+
+        // Get the paging headers
+        $paging = Pager::calculatePagingHeaders($limit, $offset, $total_rows);
+
+        $data_result = new Data();
+        $data_result->data = $result;
+        $data_result->paging = $paging;
+        $data_result->geo = $geo;
+        $data_result->preferred_formats = $this->getPreferredFormats();
+
+        return $data_result;
+    }
+}

app/Tdt/Core/DataControllers/RDFController.php

@@ -14,6 +14,10 @@
  */
 class RDFController extends ADataController
 {
+    public static function getParameters()
+    {
+        return [];
+    }
 
     public function readData($source_definition, $rest_parameters = array())
     {