Restrict term length to never exceed Lucene's fixed term length

Change-Id: Icc3be552e95ca15967b544168e0c3be4d533d00f

Author: Akron

Changes

@@ -3,6 +3,8 @@
     - [feature] Make VC cache location customizable (margaretha)
     - [bugfix] Improve handling of C2 #IN frames serialization
       (diewald)
+    - [bugfix] Restrict term length to never exceed Lucene
+      boundaries (diewald)
 
 0.62.2 2024-02-20
     - [feature] Support MMap directory parameters directly

src/main/java/de/ids_mannheim/korap/index/MultiTerm.java

@@ -56,6 +56,8 @@ public class MultiTerm implements Comparable<MultiTerm> {
 
     private static short i, l;
 
+    private static final int MAX_TERM_LENGTH = 1024;
+    
     // This advices the java compiler to ignore all loggings
     public static final boolean DEBUG = false;
     private final Logger log = LoggerFactory
@@ -122,6 +124,11 @@ public String getTerm () {
      * @return The {@link MultIterm} object for chaining.
      */
     public MultiTerm setTerm (String term) {
+        if (term.length() > (MAX_TERM_LENGTH - 2)) {
+            term = term.substring(0, MAX_TERM_LENGTH - 2);
+            log.warn("Term %s... exceeds %i cahracters - cutted", term, MAX_TERM_LENGTH);
+        }
+
         this.term = term;
         return this;
     };
@@ -468,7 +475,7 @@ private void _fromString (String term) throws CorpusDataException {
                                 + termSurface[0]);
             };
         };
-        this.term = _unescape(stringOffset[0]);
+        this.setTerm(_unescape(stringOffset[0]));
     };
 
 

src/test/java/de/ids_mannheim/korap/TestIndexer.java

@@ -24,6 +24,7 @@ public class TestIndexer {
     private final ByteArrayOutputStream outputStream = new ByteArrayOutputStream();
     private String info = "usage: Krill indexer";
     private File outputDirectory = new File("test-index");
+    private File outputDirectory2 = new File("test-index2");
 
     @Test
     public void testArguments () throws IOException {
@@ -91,6 +92,17 @@ public void testMissingInput () throws IOException {
         assertEquals(true, outputStream.toString().startsWith(info));
     }
 
+    @Test
+    public void testUnicodeProblem () throws IOException {
+        Indexer.main(new String[] {
+                "-c", "src/test/resources/krill.properties",
+                "-i", "src/test/resources/bug",
+                "-o", "test-index2"
+            });
+        logger.info(outputStream.toString());
+        assertEquals(outputStream.toString(), "Added 1 file.\n");
+    }
+
     @Before
     public void setOutputStream () {
         System.setOut(new PrintStream(outputStream));
@@ -107,6 +119,11 @@ public void cleanOutputDirectory () {
         if (outputDirectory.exists()) {
             logger.debug("Output directory exists");
             deleteFile(outputDirectory);
+            deleteFile(outputDirectory2);
+        }
+        if (outputDirectory2.exists()) {
+            logger.debug("Output directory 2 exists");
+            deleteFile(outputDirectory2);
         }
     }