diff --git a/DependencyInjection/FOSOAuthServerExtension.php b/DependencyInjection/FOSOAuthServerExtension.php
index 602de0aa..f46396e1 100644
--- a/DependencyInjection/FOSOAuthServerExtension.php
+++ b/DependencyInjection/FOSOAuthServerExtension.php
@@ -90,16 +90,16 @@ public function load(array $configs, ContainerBuilder $container)
 
         if (!empty($config['authorize'])) {
             $this->loadAuthorize($config['authorize'], $container, $loader);
-        }
 
-        // Authorize form factory definition
-        // TODO: Go back to xml configuration when bumping the requirement to Symfony >=2.6
-        $authorizeFormDefinition = $container->getDefinition('fos_oauth_server.authorize.form');
-        if (method_exists($authorizeFormDefinition, 'setFactory')) {
-            $authorizeFormDefinition->setFactory(array(new Reference('form.factory'), 'createNamed'));
-        } else {
-            $authorizeFormDefinition->setFactoryService('form.factory');
-            $authorizeFormDefinition->setFactoryMethod('createNamed');
+            // Authorize form factory definition
+            // TODO: Go back to xml configuration when bumping the requirement to Symfony >=2.6
+            $authorizeFormDefinition = $container->getDefinition('fos_oauth_server.authorize.form');
+            if (method_exists($authorizeFormDefinition, 'setFactory')) {
+                $authorizeFormDefinition->setFactory(array(new Reference('form.factory'), 'createNamed'));
+            } else {
+                $authorizeFormDefinition->setFactoryService('form.factory');
+                $authorizeFormDefinition->setFactoryMethod('createNamed');
+            }
         }
     }