Release LevelDB 1.14

Fix issues 200, 201

Also,
* Fix link to bigtable paper in docs.
* New sstables will have the file extension .ldb. .sst files will
continue to be recognized.
* When building for iOS, use xcrun to execute the compiler. This may
affect issue 177.

Author: davidsgrogan

AUTHORS

@@ -9,3 +9,4 @@ Sanjay Ghemawat <[email protected]>
 
 # Partial list of contributors:
 Kevin Regan <[email protected]>
+Johan Bilien <[email protected]>

Makefile

@@ -44,6 +44,7 @@ TESTS = \
 	filename_test \
 	filter_block_test \
 	issue178_test \
+	issue200_test \
 	log_test \
 	memenv_test \
 	skiplist_test \
@@ -71,7 +72,7 @@ SHARED = $(SHARED1)
 else
 # Update db.h if you change these.
 SHARED_MAJOR = 1
-SHARED_MINOR = 13
+SHARED_MINOR = 14
 SHARED1 = libleveldb.$(PLATFORM_SHARED_EXT)
 SHARED2 = $(SHARED1).$(SHARED_MAJOR)
 SHARED3 = $(SHARED1).$(SHARED_MAJOR).$(SHARED_MINOR)
@@ -154,6 +155,9 @@ filter_block_test: table/filter_block_test.o $(LIBOBJECTS) $(TESTHARNESS)
 issue178_test: issues/issue178_test.o $(LIBOBJECTS) $(TESTHARNESS)
 	$(CXX) $(LDFLAGS) issues/issue178_test.o $(LIBOBJECTS) $(TESTHARNESS) -o $@ $(LIBS)
 
+issue200_test: issues/issue200_test.o $(LIBOBJECTS) $(TESTHARNESS)
+	$(CXX) $(LDFLAGS) issues/issue200_test.o $(LIBOBJECTS) $(TESTHARNESS) -o $@ $(LIBS)
+
 log_test: db/log_test.o $(LIBOBJECTS) $(TESTHARNESS)
 	$(CXX) $(LDFLAGS) db/log_test.o $(LIBOBJECTS) $(TESTHARNESS) -o $@ $(LIBS)
 
@@ -191,14 +195,14 @@ IOSVERSION=$(shell defaults read $(PLATFORMSROOT)/iPhoneOS.platform/version CFBu
 	mkdir -p ios-x86/$(dir $@)
 	$(CXX) $(CXXFLAGS) -isysroot $(SIMULATORROOT)/SDKs/iPhoneSimulator$(IOSVERSION).sdk -arch i686 -c $< -o ios-x86/$@
 	mkdir -p ios-arm/$(dir $@)
-	$(DEVICEROOT)/usr/bin/$(CXX) $(CXXFLAGS) -isysroot $(DEVICEROOT)/SDKs/iPhoneOS$(IOSVERSION).sdk -arch armv6 -arch armv7 -c $< -o ios-arm/$@
+	xcrun -sdk iphoneos $(CXX) $(CXXFLAGS) -isysroot $(DEVICEROOT)/SDKs/iPhoneOS$(IOSVERSION).sdk -arch armv6 -arch armv7 -c $< -o ios-arm/$@
 	lipo ios-x86/$@ ios-arm/$@ -create -output $@
 
 .c.o:
 	mkdir -p ios-x86/$(dir $@)
 	$(CC) $(CFLAGS) -isysroot $(SIMULATORROOT)/SDKs/iPhoneSimulator$(IOSVERSION).sdk -arch i686 -c $< -o ios-x86/$@
 	mkdir -p ios-arm/$(dir $@)
-	$(DEVICEROOT)/usr/bin/$(CC) $(CFLAGS) -isysroot $(DEVICEROOT)/SDKs/iPhoneOS$(IOSVERSION).sdk -arch armv6 -arch armv7 -c $< -o ios-arm/$@
+	xcrun -sdk iphoneos $(CC) $(CFLAGS) -isysroot $(DEVICEROOT)/SDKs/iPhoneOS$(IOSVERSION).sdk -arch armv6 -arch armv7 -c $< -o ios-arm/$@
 	lipo ios-x86/$@ ios-arm/$@ -create -output $@
 
 else

db/db_iter.cc

@@ -161,12 +161,13 @@ void DBIter::Next() {
       saved_key_.clear();
       return;
     }
+    // saved_key_ already contains the key to skip past.
+  } else {
+    // Store in saved_key_ the current key so we skip it below.
+    SaveKey(ExtractUserKey(iter_->key()), &saved_key_);
   }
 
-  // Temporarily use saved_key_ as storage for key to skip.
-  std::string* skip = &saved_key_;
-  SaveKey(ExtractUserKey(iter_->key()), skip);
-  FindNextUserEntry(true, skip);
+  FindNextUserEntry(true, &saved_key_);
 }
 
 void DBIter::FindNextUserEntry(bool skipping, std::string* skip) {

db/db_test.cc

@@ -147,7 +147,7 @@ class SpecialEnv : public EnvWrapper {
 
     Status s = target()->NewWritableFile(f, r);
     if (s.ok()) {
-      if (strstr(f.c_str(), ".sst") != NULL) {
+      if (strstr(f.c_str(), ".ldb") != NULL) {
         *r = new SSTableFile(this, *r);
       } else if (strstr(f.c_str(), "MANIFEST") != NULL) {
         *r = new ManifestFile(this, *r);
@@ -484,6 +484,24 @@ class DBTest {
     }
     return false;
   }
+
+  // Returns number of files renamed.
+  int RenameLDBToSST() {
+    std::vector<std::string> filenames;
+    ASSERT_OK(env_->GetChildren(dbname_, &filenames));
+    uint64_t number;
+    FileType type;
+    int files_renamed = 0;
+    for (size_t i = 0; i < filenames.size(); i++) {
+      if (ParseFileName(filenames[i], &number, &type) && type == kTableFile) {
+        const std::string from = TableFileName(dbname_, number);
+        const std::string to = SSTTableFileName(dbname_, number);
+        ASSERT_OK(env_->RenameFile(from, to));
+        files_renamed++;
+      }
+    }
+    return files_renamed;
+  }
 };
 
 TEST(DBTest, Empty) {
@@ -1632,6 +1650,22 @@ TEST(DBTest, MissingSSTFile) {
       << s.ToString();
 }
 
+TEST(DBTest, StillReadSST) {
+  ASSERT_OK(Put("foo", "bar"));
+  ASSERT_EQ("bar", Get("foo"));
+
+  // Dump the memtable to disk.
+  dbfull()->TEST_CompactMemTable();
+  ASSERT_EQ("bar", Get("foo"));
+  Close();
+  ASSERT_GT(RenameLDBToSST(), 0);
+  Options options = CurrentOptions();
+  options.paranoid_checks = true;
+  Status s = TryReopen(&options);
+  ASSERT_TRUE(s.ok());
+  ASSERT_EQ("bar", Get("foo"));
+}
+
 TEST(DBTest, FilesDeletedAfterCompaction) {
   ASSERT_OK(Put("foo", "v2"));
   Compact("a", "z");

db/filename.cc

@@ -30,6 +30,11 @@ std::string LogFileName(const std::string& name, uint64_t number) {
 }
 
 std::string TableFileName(const std::string& name, uint64_t number) {
+  assert(number > 0);
+  return MakeFileName(name, number, "ldb");
+}
+
+std::string SSTTableFileName(const std::string& name, uint64_t number) {
   assert(number > 0);
   return MakeFileName(name, number, "sst");
 }
@@ -71,7 +76,7 @@ std::string OldInfoLogFileName(const std::string& dbname) {
 //    dbname/LOG
 //    dbname/LOG.old
 //    dbname/MANIFEST-[0-9]+
-//    dbname/[0-9]+.(log|sst)
+//    dbname/[0-9]+.(log|sst|ldb)
 bool ParseFileName(const std::string& fname,
                    uint64_t* number,
                    FileType* type) {
@@ -106,7 +111,7 @@ bool ParseFileName(const std::string& fname,
     Slice suffix = rest;
     if (suffix == Slice(".log")) {
       *type = kLogFile;
-    } else if (suffix == Slice(".sst")) {
+    } else if (suffix == Slice(".sst") || suffix == Slice(".ldb")) {
       *type = kTableFile;
     } else if (suffix == Slice(".dbtmp")) {
       *type = kTempFile;

db/filename.h

@@ -37,6 +37,11 @@ extern std::string LogFileName(const std::string& dbname, uint64_t number);
 // "dbname".
 extern std::string TableFileName(const std::string& dbname, uint64_t number);
 
+// Return the legacy file name for an sstable with the specified number
+// in the db named by "dbname". The result will be prefixed with
+// "dbname".
+extern std::string SSTTableFileName(const std::string& dbname, uint64_t number);
+
 // Return the name of the descriptor file for the db named by
 // "dbname" and the specified incarnation number.  The result will be
 // prefixed with "dbname".

db/filename_test.cc

@@ -27,6 +27,7 @@ TEST(FileNameTest, Parse) {
     { "100.log",            100,   kLogFile },
     { "0.log",              0,     kLogFile },
     { "0.sst",              0,     kTableFile },
+    { "0.ldb",              0,     kTableFile },
     { "CURRENT",            0,     kCurrentFile },
     { "LOCK",               0,     kDBLockFile },
     { "MANIFEST-2",         2,     kDescriptorFile },

db/repair.cc

@@ -263,6 +263,12 @@ class Repairer {
     std::string fname = TableFileName(dbname_, t->meta.number);
     int counter = 0;
     Status status = env_->GetFileSize(fname, &t->meta.file_size);
+    if (!status.ok()) {
+      fname = SSTTableFileName(dbname_, t->meta.number);
+      Status s2 = env_->GetFileSize(fname, &t->meta.file_size);
+      if (s2.ok())
+        status = Status::OK();
+    }
     if (status.ok()) {
       Iterator* iter = table_cache_->NewIterator(
           ReadOptions(), t->meta.number, t->meta.file_size);
@@ -293,6 +299,8 @@ class Repairer {
       }
       delete iter;
     }
+    // If there was trouble opening an .sst file this will report that the .ldb
+    // file was not found, which is kind of lame but shouldn't happen often.
     Log(options_.info_log, "Table #%llu: %d entries %s",
         (unsigned long long) t->meta.number,
         counter,

db/table_cache.cc

@@ -54,6 +54,12 @@ Status TableCache::FindTable(uint64_t file_number, uint64_t file_size,
     RandomAccessFile* file = NULL;
     Table* table = NULL;
     s = env_->NewRandomAccessFile(fname, &file);
+    if (!s.ok()) {
+      std::string old_fname = SSTTableFileName(dbname_, file_number);
+      if (env_->NewRandomAccessFile(old_fname, &file).ok()) {
+        s = Status::OK();
+      }
+    }
     if (s.ok()) {
       s = Table::Open(*options_, file, file_size, &table);
     }

doc/impl.html

@@ -11,7 +11,7 @@ <h1>Files</h1>
 
 The implementation of leveldb is similar in spirit to the
 representation of a single
-<a href="http://labs.google.com/papers/bigtable.html">
+<a href="http://research.google.com/archive/bigtable.html">
 Bigtable tablet (section 5.3)</a>.
 However the organization of the files that make up the representation
 is somewhat different and is explained below.

include/leveldb/db.h

@@ -14,7 +14,7 @@ namespace leveldb {
 
 // Update Makefile if you change these
 static const int kMajorVersion = 1;
-static const int kMinorVersion = 13;
+static const int kMinorVersion = 14;
 
 struct Options;
 struct ReadOptions;

issues/issue200_test.cc

@@ -0,0 +1,59 @@
+// Copyright (c) 2013 The LevelDB Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file. See the AUTHORS file for names of contributors.
+
+// Test for issue 200: when iterator switches direction from backward
+// to forward, the current key can be yielded unexpectedly if a new
+// mutation has been added just before the current key.
+
+#include "leveldb/db.h"
+#include "util/testharness.h"
+
+namespace leveldb {
+
+class Issue200 { };
+
+TEST(Issue200, Test) {
+  // Get rid of any state from an old run.
+  std::string dbpath = test::TmpDir() + "/leveldb_issue200_test";
+  DestroyDB(dbpath, Options());
+
+  DB *db;
+  Options options;
+  options.create_if_missing = true;
+  ASSERT_OK(DB::Open(options, dbpath, &db));
+
+  WriteOptions write_options;
+  ASSERT_OK(db->Put(write_options, "1", "b"));
+  ASSERT_OK(db->Put(write_options, "2", "c"));
+  ASSERT_OK(db->Put(write_options, "3", "d"));
+  ASSERT_OK(db->Put(write_options, "4", "e"));
+  ASSERT_OK(db->Put(write_options, "5", "f"));
+
+  ReadOptions read_options;
+  Iterator *iter = db->NewIterator(read_options);
+
+  // Add an element that should not be reflected in the iterator.
+  ASSERT_OK(db->Put(write_options, "25", "cd"));
+
+  iter->Seek("5");
+  ASSERT_EQ(iter->key().ToString(), "5");
+  iter->Prev();
+  ASSERT_EQ(iter->key().ToString(), "4");
+  iter->Prev();
+  ASSERT_EQ(iter->key().ToString(), "3");
+  iter->Next();
+  ASSERT_EQ(iter->key().ToString(), "4");
+  iter->Next();
+  ASSERT_EQ(iter->key().ToString(), "5");
+
+  delete iter;
+  delete db;
+  DestroyDB(dbpath, options);
+}
+
+}  // namespace leveldb
+
+int main(int argc, char** argv) {
+  return leveldb::test::RunAllTests();
+}

util/arena.cc

@@ -40,7 +40,7 @@ char* Arena::AllocateFallback(size_t bytes) {
 }
 
 char* Arena::AllocateAligned(size_t bytes) {
-  const int align = sizeof(void*);    // We'll align to pointer size
+  const int align = (sizeof(void*) > 8) ? sizeof(void*) : 8;
   assert((align & (align-1)) == 0);   // Pointer size should be a power of 2
   size_t current_mod = reinterpret_cast<uintptr_t>(alloc_ptr_) & (align-1);
   size_t slop = (current_mod == 0 ? 0 : align - current_mod);