Feature/multiple identifiers

config/module.config.php

@@ -175,7 +175,7 @@
                 'adapter' => 'uri',
             ],
             'resource' => [
-                'label' => 'Omeka resource (by ID)', // @translate
+                'label' => 'Omeka resource', // @translate
                 'adapter' => 'resource',
             ],
         ],
@@ -192,7 +192,10 @@
             'csv_import_multivalue_separator' => ',',
             'csv_import_rows_by_batch' => 20,
             'csv_import_global_language' => '',
-            'csv_import_identifier_property' => '',
+            'csv_import_identifier_properties' => [
+                'o:id',
+                'dcterms:identifier',
+            ],
             'csv_import_automap_check_names_alone' => false,
         ],
     ],

src/Controller/IndexController.php

@@ -321,15 +321,18 @@ protected function cleanArgs(array $post)
             }
         }
 
-        // Check the identifier property.
-        if (array_key_exists('identifier_property', $args)) {
-            $identifierProperty = $args['identifier_property'];
-            if (empty($identifierProperty) && $identifierProperty !== 'internal_id') {
-                $properties = $api->search('properties', ['term' => $identifierProperty])->getContent();
-                if (empty($properties)) {
-                    $args['identifier_property'] = null;
+        // Check the identifier properties.
+        if (array_key_exists('identifier_properties', $args)) {
+            $identifierProperties = $args['identifier_properties'] ? $args['identifier_properties'] : [];
+            foreach ($identifierProperties as $key => $identifierProperty) {
+                if ($identifierProperty !== 'o:id') {
+                    $property = $api->searchOne('properties', ['term' => $identifierProperty])->getContent();
+                    if (empty($property)) {
+                        unset($args['identifier_properties'][$key]);
+                    }
                 }
             }
+            $args['identifier_properties'] = array_values($args['identifier_properties']);
         }
 
         if (!array_key_exists('column-multivalue', $post)) {

src/Form/MappingForm.php

@@ -265,6 +265,28 @@ public function init()
                 ],
             ]);
 
+            $basicSettingsFieldset->add([
+                'name' => 'identifier_properties',
+                'type' => PropertySelect::class,
+                'options' => [
+                    'label' => 'Resource identifier properties', // @translate
+                    'info' => 'Use these properties, generally "Internal id" or "dcterms:identifier", to identify the existing resources to link or to get. In all cases, it is strongly recommended to add one or more unique identifiers to all your resources.', // @translate
+                    'empty_option' => 'Select below', // @translate
+                    'prepend_value_options' => [
+                        'o:id' => 'Internal ID', // @translate
+                    ],
+                    'term_as_value' => true,
+                ],
+                'attributes' => [
+                    'multiple' => true,
+                    'value' => $userSettings->get(
+                        'csv_import_identifier_properties',
+                        $default['csv_import_identifier_properties']),
+                    'class' => 'chosen-select',
+                    'data-placeholder' => 'Select a property', // @translate
+                ],
+            ]);
+
             $this->add([
                 'type' => 'fieldset',
                 'name' => 'advanced-settings',
@@ -319,27 +341,6 @@ public function init()
                 ]);
             }
 
-            $advancedSettingsFieldset->add([
-                'name' => 'identifier_property',
-                'type' => PropertySelect::class,
-                'options' => [
-                    'label' => 'Resource identifier property', // @translate
-                    'info' => 'Use this property, generally "dcterms:identifier", to identify the existing resources, so it will be possible to update them. One column of the file must map the selected property. In all cases, it is strongly recommended to add one ore more unique identifiers to all your resources.', // @translate
-                    'empty_option' => 'Select below', // @translate
-                    'prepend_value_options' => [
-                        'internal_id' => 'Internal ID', // @translate
-                    ],
-                    'term_as_value' => true,
-                ],
-                'attributes' => [
-                    'value' => $userSettings->get(
-                        'csv_import_identifier_property',
-                        $default['csv_import_identifier_property']),
-                    'class' => 'action-option chosen-select',
-                    'data-placeholder' => 'Select a property', // @translate
-                ],
-            ]);
-
             $advancedSettingsFieldset->add([
                 'name' => 'action_unidentified',
                 'type' => 'radio',
@@ -411,7 +412,7 @@ public function init()
                 'required' => false,
             ]);
             $advancedSettingsInputFilter->add([
-                'name' => 'identifier_property',
+                'name' => 'identifier_properties',
                 'required' => false,
             ]);
             $advancedSettingsInputFilter->add([

src/Job/Import.php

@@ -5,7 +5,7 @@
 use CSVImport\Mvc\Controller\Plugin\FindResourcesFromIdentifiers;
 use CSVImport\Source\SourceInterface;
 use finfo;
-use Omeka\Api\Manager;
+use Omeka\Mvc\Controller\Plugin\Api;
 use Omeka\Job\AbstractJob;
 use Omeka\Stdlib\Message;
 use Zend\Log\Logger;
@@ -28,7 +28,7 @@ class Import extends AbstractJob
     protected $rowsByBatch = 20;
 
     /**
-     * @var Manager
+     * @var Api
      */
     protected $api;
 
@@ -83,9 +83,9 @@ class Import extends AbstractJob
     protected $identifiers;
 
     /**
-     * @var string|int
+     * @var array
      */
-    protected $identifierPropertyId;
+    protected $identifierProperties;
 
     /**
      * @var bool
@@ -101,7 +101,7 @@ public function perform()
     {
         ini_set('auto_detect_line_endings', true);
         $services = $this->getServiceLocator();
-        $this->api = $services->get('Omeka\ApiManager');
+        $this->api = $services->get('ControllerPluginManager')->get('api');
         $this->logger = $services->get('Omeka\Logger');
         $this->findResourcesFromIdentifiers = $services->get('ControllerPluginManager')
             ->get('findResourcesFromIdentifiers');
@@ -154,28 +154,32 @@ public function perform()
             return $this->endJob();
         }
 
-        // The main identifier property may be used as term or as id in some
-        // places, so prepare it one time only.
-        if (empty($args['identifier_property']) || $args['identifier_property'] === 'internal_id') {
-            $this->identifierPropertyId = $args['identifier_property'];
-        } elseif (is_numeric($args['identifier_property'])) {
-            $this->identifierPropertyId = (int) $args['identifier_property'];
-        } else {
-            $result = $this->api
-                ->search('properties', ['term' => $args['identifier_property']])->getContent();
-            $this->identifierPropertyId = $result ? $result[0]->id() : null;
+        // The main identifier properties may be used as term or as id in some
+        // places, so prepare them one time only.
+        $this->identifierProperties = [];
+        foreach ($args['identifier_properties'] as $identifierProperty) {
+            if ($identifierProperty === 'o:id') {
+                $this->identifierProperties[] = 'o:id';
+            } elseif (is_numeric($identifierProperty)) {
+                $this->identifierProperties[] = (int) $identifierProperty;
+            } else {
+                $result = $this->api
+                    ->searchOne('properties', ['term' => $identifierProperty])->getContent();
+                if ($result) {
+                    $this->identifierProperties[] = $result->id();
+                }
+            }
+        }
+        if (!$this->identifierProperties) {
+            $this->identifierProperties = ['o:id'];
         }
 
         if (!empty($args['rows_by_batch'])) {
             $this->rowsByBatch = (int) $args['rows_by_batch'];
         }
 
-        // The core allows batch processes only for creation and deletion.
-        if (!in_array($args['action'], [self::ACTION_CREATE, self::ACTION_DELETE, self::ACTION_SKIP])
-            // It allows to identify resources too, so to use a new resource
-            // from a previous row.
-            || ($args['action'] === self::ACTION_CREATE && $this->resourceType === 'resources')
-        ) {
+        // The core allows batch processes only for deletion.
+        if (!in_array($args['action'], [self::ACTION_DELETE, self::ACTION_SKIP])) {
             $this->rowsByBatch = 1;
         }
 
@@ -262,7 +266,7 @@ protected function processBatchData(array $data)
             case self::ACTION_REPLACE:
                 $findResourcesFromIdentifiers = $this->findResourcesFromIdentifiers;
                 $identifiers = $this->extractIdentifiers($data);
-                $ids = $findResourcesFromIdentifiers($identifiers, $this->identifierPropertyId, $this->resourceType);
+                $ids = $findResourcesFromIdentifiers($identifiers, $this->identifierProperties, $this->resourceType);
                 $ids = $this->assocIdentifierKeysAndIds($identifiers, $ids);
                 $idsToProcess = array_filter($ids);
                 $idsRemaining = array_diff_key($ids, $idsToProcess);
@@ -298,7 +302,7 @@ protected function processBatchData(array $data)
             case self::ACTION_DELETE:
                 $findResourcesFromIdentifiers = $this->findResourcesFromIdentifiers;
                 $identifiers = $this->extractIdentifiers($data);
-                $ids = $findResourcesFromIdentifiers($identifiers, $this->identifierPropertyId, $this->resourceType);
+                $ids = $findResourcesFromIdentifiers($identifiers, $this->identifierProperties, $this->resourceType);
                 $idsToProcess = array_filter($ids);
                 $idsRemaining = array_diff_key($ids, $idsToProcess);
 
@@ -525,11 +529,11 @@ protected function identifyMedias(array $data, array $ids)
                 if (empty($media['o:source']) || empty($media['o:ingester'])) {
                     continue;
                 }
-                $identifierProperties = [];
-                $identifierProperties['o:ingester'] = $media['o:ingester'];
-                $identifierProperties['o:item']['o:id'] = $ids[$key];
+                $identifier = [];
+                $identifier['o:ingester'] = $media['o:ingester'];
+                $identifier['o:item']['o:id'] = $ids[$key];
                 $resourceId = $findResourceFromIdentifier(
-                    $media['o:source'], $identifierProperties, 'media');
+                    $media['o:source'], [$identifier], 'media');
                 if ($resourceId) {
                     $media['o:id'] = $resourceId;
                 }
@@ -712,7 +716,7 @@ protected function filterDataWithoutIdentifier(array $data, array $identifiers)
     protected function idsForLog($ids, $hasIdentifierKeys = false)
     {
         switch ($this->args['identifier_property']) {
-            case 'internal_id':
+            case 'o:id':
                 // Nothing to do.
                 break;
             default:
@@ -804,11 +808,11 @@ protected function updateRevise($resourceType, $id, $data, $action)
      */
     protected function removeEmptyData(array $data)
     {
-        // Data are updated in place.
-        foreach ($data as $name => &$metadata) {
+        foreach ($data as $name => $metadata) {
             switch ($name) {
                 case 'o:resource_template':
                 case 'o:resource_class':
+                case 'o:thumbnail':
                 case 'o:owner':
                 case 'o:item':
                     if (empty($metadata) || empty($metadata['o:id'])) {
@@ -827,6 +831,7 @@ protected function removeEmptyData(array $data)
                 case 'o:ingester':
                 case 'o:source':
                 case 'ingest_filename':
+                case 'o:size':
                     unset($data[$name]);
                     break;
                 case 'o:is_public':
@@ -835,12 +840,12 @@ protected function removeEmptyData(array $data)
                         unset($data[$name]);
                     }
                     break;
+                // Properties.
                 default:
-                    if (is_array($metadata)) {
-                        if (empty($metadata)) {
-                            unset($data[$name]);
-                        }
+                    if (is_array($metadata) && empty($metadata)) {
+                        unset($data[$name]);
                     }
+                    break;
             }
         }
         return $data;
@@ -951,7 +956,7 @@ protected function extractPropertyValuesFromResource($resourceJson)
     }
 
     /**
-     * Deduplicate data ids for collections of items set, items, media....
+     * Deduplicate data ids for collections of items set, items, media…
      *
      * @param array $data
      * @return array
@@ -981,17 +986,40 @@ protected function deduplicatePropertyValues($values)
     {
         // Base to normalize data in order to deduplicate them in one pass.
         $base = [];
-        $base['literal'] = ['property_id' => 0, 'type' => 'literal', '@language' => '', '@value' => ''];
-        $base['resource'] = ['property_id' => 0, 'type' => 'resource', 'value_resource_id' => 0];
-        $base['url'] = ['property_id' => 0, 'type' => 'url', '@id' => 0, 'o:label' => ''];
+
+        $base['literal'] = ['is_public' => true, 'property_id' => 0, 'type' => 'literal', '@language' => null, '@value' => ''];
+        $base['resource'] = ['is_public' => true, 'property_id' => 0, 'type' => 'resource', 'value_resource_id' => 0];
+        $base['uri'] = ['is_public' => true, 'o:label' => null, 'property_id' => 0, 'type' => 'uri', '@id' => ''];
         foreach ($values as $key => $value) {
             $values[$key] = array_values(
                 // Deduplicate values.
-                array_map('unserialize', array_unique(array_map('serialize',
+                array_map('unserialize', array_unique(array_map(
+                    'serialize',
                     // Normalize values.
                     array_map(function ($v) use ($base) {
-                        return array_replace($base[$v['type']], array_intersect_key($v, $base[$v['type']]));
-            }, $value)))));
+                        // Data types "resource" and "uri" have "@id" (in json).
+                        $mainType = array_key_exists('value_resource_id', $v)
+                            ? 'resource'
+                            : (array_key_exists('@id', $v) ? 'uri' : 'literal');
+                        // Keep order and meaning keys.
+                        $r = array_replace($base[$mainType], array_intersect_key($v, $base[$mainType]));
+                        $r['is_public'] = (bool) $r['is_public'];
+                        switch ($mainType) {
+                            case 'literal':
+                                if (empty($r['@language'])) {
+                                    $r['@language'] = null;
+                                }
+                                break;
+                            case 'uri':
+                                if (empty($r['o:label'])) {
+                                    $r['o:label'] = null;
+                                }
+                                break;
+                        }
+                        return $r;
+                    }, $value)
+                )))
+            );
         }
         return $values;
     }
@@ -1071,7 +1099,7 @@ protected function checkOptions()
 
         // Specific check when a identifier is required.
         elseif (!in_array($args['action'], [self::ACTION_CREATE, self::ACTION_SKIP])) {
-            if (empty($args['identifier_property'])) {
+            if (empty($args['identifier_properties'])) {
                 $this->hasErr = true;
                 $this->logger->err(new Message('The action "%s" requires a resource identifier property.', // @translate
                     $args['action']));

src/Mapping/AbstractMapping.php

@@ -57,7 +57,7 @@ public function init(array $args, ServiceLocatorInterface $serviceLocator)
         $this->args = $args;
         $this->serviceLocator = $serviceLocator;
         $this->logger = $serviceLocator->get('Omeka\Logger');
-        $this->api = $serviceLocator->get('Omeka\ApiManager');
+        $this->api = $serviceLocator->get('ControllerPluginManager')->get('api');
     }
 
     public function getServiceLocator()

src/Mapping/AbstractResourceMapping.php

@@ -241,7 +241,7 @@ protected function processCellMedia($index, array $values)
             // Check params to avoid useless search and improve speed.
             $action = $this->args['action'];
             $identifier = reset($values);
-            $identifierProperty = $this->map['item'][$index] ?: 'internal_id';
+            $identifierProperty = $this->map['item'][$index] ?: 'o:id';
             $resourceType = 'items';
 
             if (empty($identifier)) {
@@ -268,7 +268,7 @@ protected function processCellMedia($index, array $values)
         }
     }
 
-    protected function findResource($identifier, $identifierProperty = 'internal_id')
+    protected function findResource($identifier, $identifierProperty = 'o:id')
     {
         $resourceType = $this->args['resource_type'];
         $findResourceFromIdentifier = $this->findResourceFromIdentifier;