allow to execute sql vommands from a file; refactor

Author: Gaetano Giunta

TODO.md

@@ -1,7 +1,8 @@
 - worker: improve sql execution cmd:
-  + allow it to take sql snippet from file
   + allow it to pick a set of desired servers
   + disallow execution of commands that are part of the db client instead of being sent to the server, such as eg. 'use db'
+    - ok for mysql? (to be tested), missing for psql
+  + examine in detail the differences between running a command vs a file (eg. transaction usage)
 
 - worker: improve profile of 'user' account (esp: add APP_ENV and APP_DEBUG env vars)
 

WHATSNEW.md

@@ -6,8 +6,9 @@ Version 0.3 (unreleased)
 - Improved the `sql:execute` command:
     - measure time and memory taken for each db
     - allow to print output in json/yaml/php format
- 
- 
+    - allow to execute sql commands stored in a file besides specifying them as cli option
+
+
 Version 0.2
 -----------
 

app/src/API/Interfaces/ForkedSqlExecutor.php renamed to app/src/API/Interfaces/ForkedCommandExecutor.php

@@ -4,11 +4,11 @@
 
 use Symfony\Component\Process\Process;
 
-interface ForkedSqlExecutor
+interface ForkedCommandExecutor
 {
     /**
      * @param string $sql
      * @return Process
      */
-    public function getProcess($sql);
+    public function getExecuteCommandProcess($sql);
 }

app/src/API/Interfaces/ForkedFileExecutor.php

@@ -0,0 +1,14 @@
+<?php
+
+namespace Db3v4l\API\Interfaces;
+
+use Symfony\Component\Process\Process;
+
+interface ForkedFileExecutor
+{
+    /**
+     * @param string $filename
+     * @return Process
+     */
+    public function getExecuteFileProcess($filename);
+}

app/src/Command/SqlExecute.php

@@ -38,7 +38,8 @@ protected function configure()
     {
         $this
             ->setDescription('Executes an SQL command in parallel on all configured database servers')
-            ->addOption('sql', null, InputOption::VALUE_REQUIRED, 'The command to execute')
+            ->addOption('sql', null, InputOption::VALUE_REQUIRED, 'The sql command(s) string to execute')
+            ->addOption('file', null, InputOption::VALUE_REQUIRED, 'A file with sql commands to execute')
             ->addOption('output-type', null, InputOption::VALUE_REQUIRED, 'The format for the output: json, php, text or yml', 'text')
             ->addOption('timeout', null, InputOption::VALUE_REQUIRED, 'The maximum time to wait for execution (secs)', 600)
             ->addOption('max-parallel', null, InputOption::VALUE_REQUIRED, 'The maximum number of processes to run in parallel', 16)
@@ -65,13 +66,17 @@ protected function execute(InputInterface $input, OutputInterface $output)
 
         $dbList = $this->dbManager->listDatabases();
         $sql = $input->getOption('sql');
+        $file = $input->getOption('file');
         $timeout = $input->getOption('timeout');
         $maxParallel = $input->getOption('max-parallel');
         $dontForceSigchildEnabled = $input->getOption('dont-force-enabled-sigchild');
         $format = $input->getOption('output-type');
 
-        if ($sql === '') {
-            throw new \Exception("Please provide an sql command/snippted to be executed");
+        if ($sql == null && $file == null) {
+            throw new \Exception("Please provide an sql command/file to be executed");
+        }
+        if ($sql != null && $file != null) {
+            throw new \Exception("Please provide either an sql command or file to be executed, not both");
         }
 
         // On Debian, which we use by default, SF has troubles understanding that php was compiled with --enable-sigchild
@@ -82,14 +87,27 @@ protected function execute(InputInterface $input, OutputInterface $output)
             Process::forceSigchildEnabled(true);
         }
 
+        if ($format === 'text') {
+            $this->writeln('<info>Preparing commands...</info>', OutputInterface::VERBOSITY_VERBOSE);
+        }
+
         /** @var Process[] $processes */
         $processes = [];
         $executors = [];
         foreach ($dbList as $dbName) {
             $dbConnectionSpec = $this->dbManager->getDatabaseConnectionSpecification($dbName);
 
             $executor = $this->executorFactory->createForkedExecutor($dbConnectionSpec);
-            $process = $executor->getProcess($sql);
+
+            if ($sql != null) {
+                $process = $executor->getExecuteCommandProcess($sql);
+            } else {
+                $process = $executor->getExecuteFileProcess($file);
+            }
+
+            if ($format === 'text') {
+                $this->writeln('Command line: ' . $process->getCommandLine(), OutputInterface::VERBOSITY_VERBOSE);
+            }
 
             $process->setTimeout($timeout);
 

app/src/Service/SqlExecutor/Forked/Doctrine.php

@@ -2,16 +2,16 @@
 
 namespace Db3v4l\Service\SqlExecutor\Forked;
 
-use Db3v4l\API\Interfaces\ForkedSqlExecutor;
+use Db3v4l\API\Interfaces\ForkedCommandExecutor;
 use Db3v4l\Util\Process;
 
-class Doctrine extends ForkedExecutor implements ForkedSqlExecutor
+class Doctrine extends ForkedExecutor implements ForkedCommandExecutor
 {
     /**
      * @param string $sql
      * @return Process
      */
-    public function getProcess($sql)
+    public function getExecuteCommandProcess($sql)
     {
         throw new \RuntimeException('TO BE IMPLEMENTED');
     }

app/src/Service/SqlExecutor/Forked/NativeClient.php

@@ -2,17 +2,37 @@
 
 namespace Db3v4l\Service\SqlExecutor\Forked;
 
-use Db3v4l\API\Interfaces\ForkedSqlExecutor;
+use Db3v4l\API\Interfaces\ForkedCommandExecutor;
+use Db3v4l\API\Interfaces\ForkedFileExecutor;
 use Db3v4l\Util\Process;
 
-class NativeClient extends ForkedExecutor implements ForkedSqlExecutor
+class NativeClient extends ForkedExecutor implements ForkedCommandExecutor, ForkedFileExecutor
 {
     /**
      * @param string $sql
      * @return Process
+     */
+    public function getExecuteCommandProcess($sql)
+    {
+        return $this->getProcess($sql);
+    }
+
+    /**
+     * @param string $filename
+     * @return Process
+     */
+    public function getExecuteFileProcess($filename)
+    {
+        return $this->getProcess($filename, true);
+    }
+
+    /**
+     * @param string $sqlOrFilename
+     * @param bool $isFile
+     * @return Process
      * @todo allow to inject location of db clients via setter/constructor
      */
-    public function getProcess($sql)
+    public function getProcess($sqlOrFilename, $isFile = false)
     {
         $clientType = $this->getDbClientFromDriver($this->databaseConfiguration['driver']);
 
@@ -25,9 +45,11 @@ public function getProcess($sql)
                     '--user=' . $this->databaseConfiguration['user'],
                     '-p' . $this->databaseConfiguration['password'],
                     '--binary-mode', // 'It also disables all mysql commands except charset and delimiter in non-interactive mode (for input piped to mysql or loaded using the source command)'
-                    '--execute=' . $sql,
                     // $dbname
                 ];
+                if (!$isFile) {
+                    $options[] = '--execute=' . $sqlOrFilename;
+                }
                 $env = [
                     // problematic when wrapping the process in a call to `time`...
                     //'MYSQL_PWD' => $this->databaseConfiguration['password'],
@@ -41,9 +63,11 @@ public function getProcess($sql)
                     //'--host=' . $this->databaseConfiguration['host'],
                     //'--port=' . $this->databaseConfiguration['port'] ?? '5432',
                     //'--username=' . $this->databaseConfiguration['user'],
-                    '--command=' . $sql,
                     //'--dbname=' . $dbname
                 ];
+                if (!$isFile) {
+                    $options[] = '--command=' . $sqlOrFilename;
+                }
                 $env = [
                     // problematic when wrapping the process in a call to `time`...
                     //'PGPASSWORD' => $this->databaseConfiguration['password'],
@@ -53,7 +77,14 @@ public function getProcess($sql)
                 throw new \OutOfBoundsException("Unsupported db client '$clientType'");
         }
 
-        return new Process($this->buildCommandLine($command, $options), null, $env);
+        $commandLine = $this->buildCommandLine($command, $options);
+
+        /// @todo for psql this is probably better done via --file
+        if ($isFile) {
+            $commandLine .= ' < ' . escapeshellarg($sqlOrFilename);
+        }
+
+        return new Process($commandLine, null, $env);
     }
 
     /**

app/src/Service/SqlExecutor/Forked/TimedExecutor.php

@@ -2,28 +2,44 @@
 
 namespace Db3v4l\Service\SqlExecutor\Forked;
 
-use Db3v4l\API\Interfaces\ForkedSqlExecutor;
+use Db3v4l\API\Interfaces\ForkedCommandExecutor;
+use Db3v4l\API\Interfaces\ForkedFileExecutor;
 use Db3v4l\API\Interfaces\TimedExecutor as TimedExecutorInterface;
 
-class TimedExecutor implements ForkedSqlExecutor, TimedExecutorInterface
+class TimedExecutor implements ForkedCommandExecutor, ForkedFileExecutor, TimedExecutorInterface
 {
-    /** @var ForkedSqlExecutor */
+    /** @var ForkedCommandExecutor */
     protected $wrappedExecutor;
     protected $timingFile;
 
-    public function __construct(ForkedSqlExecutor $wrappedExecutor)
+    protected $timeCmd = '/usr/bin/time';
+
+    public function __construct(ForkedCommandExecutor $wrappedExecutor)
     {
         $this->wrappedExecutor = $wrappedExecutor;
     }
 
-    public function getProcess($sql)
+    public function getExecuteCommandProcess($sql)
+    {
+        $process = $this->wrappedExecutor->getExecuteCommandProcess($sql);
+
+        // wrap in a `time` call
+        $this->timingFile = tempnam(sys_get_temp_dir(), 'db3val_');
+        $process->setCommandLine(
+            $this->timeCmd . ' ' . escapeshellarg('--output=' . $this->timingFile) . ' ' . escapeshellarg('--format=%M %e') . ' '
+            . $process->getCommandLine());
+
+        return $process;
+    }
+
+    public function getExecuteFileProcess($sql)
     {
-        $process = $this->wrappedExecutor->getProcess($sql);
+        $process = $this->wrappedExecutor->getExecuteFileProcess($sql);
 
         // wrap in a `time` call
         $this->timingFile = tempnam(sys_get_temp_dir(), 'db3val_');
         $process->setCommandLine(
-            'time ' . escapeshellarg('--output=' . $this->timingFile) . ' ' . escapeshellarg('--format=%M %e') . ' '
+            $this->timeCmd . ' ' . escapeshellarg('--output=' . $this->timingFile) . ' ' . escapeshellarg('--format=%M %e') . ' '
             . $process->getCommandLine());
 
         return $process;

app/src/Service/SqlExecutorFactory.php

@@ -2,7 +2,8 @@
 
 namespace Db3v4l\Service;
 
-use Db3v4l\API\Interfaces\ForkedSqlExecutor;
+use Db3v4l\API\Interfaces\ForkedCommandExecutor;
+use Db3v4l\API\Interfaces\ForkedFileExecutor;
 use Db3v4l\Service\SqlExecutor\Forked\NativeClient;
 use Db3v4l\Service\SqlExecutor\Forked\Doctrine;
 use Db3v4l\Service\SqlExecutor\Forked\TimedExecutor;
@@ -12,8 +13,8 @@ class SqlExecutorFactory
     /**
      * @param array $databaseConnectionConfiguration
      * @param string $executionStrategy
-     * @bool $timed
-     * @return ForkedSqlExecutor
+     * @param bool $timed
+     * @return ForkedCommandExecutor|ForkedFileExecutor
      * @throws \OutOfBoundsException
      */
     public function createForkedExecutor($databaseConnectionConfiguration, $executionStrategy = 'NativeClient', $timed = true)