diff --git a/indexing/src/main/java/com/google/enterprise/cloudsearch/sdk/indexing/StructuredData.java b/indexing/src/main/java/com/google/enterprise/cloudsearch/sdk/indexing/StructuredData.java index 98038c4..d0e8889 100644 --- a/indexing/src/main/java/com/google/enterprise/cloudsearch/sdk/indexing/StructuredData.java +++ b/indexing/src/main/java/com/google/enterprise/cloudsearch/sdk/indexing/StructuredData.java @@ -38,6 +38,7 @@ import com.google.api.services.cloudsearch.v1.model.StructuredDataObject; import com.google.api.services.cloudsearch.v1.model.TextValues; import com.google.api.services.cloudsearch.v1.model.TimestampValues; +import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Converter; import com.google.common.base.Strings; import com.google.common.collect.ImmutableList; @@ -117,6 +118,8 @@ public class StructuredData { public static final String DATETIME_PATTERNS = "structuredData.dateTimePatterns"; public static final String LOCAL_SCHEMA = "structuredData.localSchema"; + public static final String IGNORE_CONVERSION_ERRORS = "structuredData.ignoreConversionErrors"; + private static final String DATETIME_PATTERNS_DELIMITER = ";"; private static final ImmutableList DEFAULT_DATETIME_PARSERS = ImmutableList.of( @@ -131,6 +134,8 @@ public class StructuredData { private static final AtomicBoolean initialized = new AtomicBoolean(); private static final List dateTimeParsers = new ArrayList<>(); + private static final AtomicBoolean ignoreConversionErrors = new AtomicBoolean(); + /** A map from object definition names to instances of this class. */ private static final Map structuredDataMapping = new HashMap(); @@ -178,6 +183,8 @@ public static synchronized void initFromConfiguration(IndexingService indexingSe throw patternErrors; } + ignoreConversionErrors.set(Configuration.getBoolean(IGNORE_CONVERSION_ERRORS, Boolean.FALSE).get()); + Schema schema; String localSchemaPath = Configuration.getString(LOCAL_SCHEMA, "").get(); if (!localSchemaPath.isEmpty()) { @@ -265,6 +272,15 @@ public static boolean hasObjectDefinition(String objectType) { return structuredDataMapping.containsKey(objectType); } + /** + * Same as setting structuredData.ignoreConversionErrors in configuration + * @param ignore + */ + @VisibleForTesting + public static void setIgnoreConversionErrors(boolean ignore) { + ignoreConversionErrors.set(ignore); + } + /** * Generate a {@link StructuredDataObject} for the given object type using the input values. * @@ -356,15 +372,40 @@ private NamedProperty getProperty(String propertyName, Collection values return null; } if (!isRepeated) { - return propertyBuilder.getNamedProperty( - propertyName, Collections.singletonList(valueConverter.convert(nonNullValues.get(0)))); + try { + return propertyBuilder.getNamedProperty( + propertyName, Collections.singletonList(valueConverter.convert(nonNullValues.get(0)))); + } catch (IllegalArgumentException e) { + if (ignoreConversionErrors.get()) { + logger.log(Level.FINEST, "Ignoring conversion error: {0}", e.getMessage()); + return null; + } + throw e; + } } else { + List nonNullConvertedValues = nonNullValues + .stream() + .map(v -> convert(valueConverter, v)) + .filter(Objects::nonNull) + .collect(Collectors.toList()); + if (nonNullConvertedValues.isEmpty()) { + return null; + } return propertyBuilder.getNamedProperty( propertyName, - nonNullValues - .stream() - .map(v -> valueConverter.convert(v)) - .collect(Collectors.toList())); + nonNullConvertedValues); + } + } + + private T convert(Converter converter, Object v) { + try { + return converter.convert(v); + } catch (IllegalArgumentException e) { + if (ignoreConversionErrors.get()) { + logger.log(Level.FINEST, "Ignoring conversion error: {0}", e.getMessage()); + return null; + } + throw e; } } } diff --git a/indexing/src/test/java/com/google/enterprise/cloudsearch/sdk/indexing/StructuredDataTest.java b/indexing/src/test/java/com/google/enterprise/cloudsearch/sdk/indexing/StructuredDataTest.java index 5ef8bd9..91de7a8 100644 --- a/indexing/src/test/java/com/google/enterprise/cloudsearch/sdk/indexing/StructuredDataTest.java +++ b/indexing/src/test/java/com/google/enterprise/cloudsearch/sdk/indexing/StructuredDataTest.java @@ -796,6 +796,28 @@ public void testDateTime_unparsedCharacters() throws IOException { converter.convert("2018-08-08T15:48:17.000-07:00 and so on"); } + @Test + public void testIgnoreConversionErrors() throws IOException { + setupSchema(); + StructuredData.setIgnoreConversionErrors(true); + StructuredDataObject expected = + new StructuredDataObject() + .setProperties( + Arrays.asList( + new NamedProperty() + .setName("dateProperty") + .setDateValues(new DateValues().setValues( + Arrays.asList(new Date().setDay(1).setMonth(1).setYear(2019)))) + )); + + Multimap values = ArrayListMultimap.create(); + values.put("dateProperty", " - - "); + values.put("dateProperty", "2019-01-01"); + values.put("dateProperty", "bad-date"); + assertEquals(expected, StructuredData.getStructuredData("myObject", values)); + } + + private void setupSchema() { Schema schema = new Schema(); schema.setObjectDefinitions(