Merge pull request #92 from twitter/detokenizer_bug_fix

bug fixed

src/main/scala/com/twitter/penguin/korean/tokenizer/KoreanDetokenizer.scala

@@ -43,7 +43,7 @@ object KoreanDetokenizer {
   private def collapseTokens(tokenized: Seq[KoreanToken]): List[String] = {
     val (output, isPrefix) = tokenized.foldLeft((List[String](), false)) {
       case ((output: List[String], isPrefix: Boolean), token: KoreanToken) =>
-        if (isPrefix || SuffixPos.contains(token.pos)) {
+        if (output.nonEmpty && (isPrefix || SuffixPos.contains(token.pos))) {
           val attached = output.lastOption.getOrElse("") + token.text
           (output.init :+ attached, false)
         } else if (PrefixPos.contains(token.pos)) {

src/test/scala/com/twitter/penguin/korean/tokenizer/KoreanDetokenizerTest.scala

@@ -38,4 +38,31 @@ class KoreanDetokenizerTest extends TestBase {
           === "뭐 완벽하진 않지만 그럭저럭 쓸 만하군..."
     )
   }
+
+  test("detokenizer should correctly detokenize the edge cases") {
+    assert(
+      detokenize(List(""))
+          === ""
+    )
+
+    assert(
+      detokenize(List())
+          === ""
+    )
+
+    assert(
+      detokenize(List("완벽"))
+          === "완벽"
+    )
+
+    assert(
+      detokenize(List("이"))
+          === "이"
+    )
+
+    assert(
+      detokenize(List("이", "제품을", "사용하겠습니다"))
+          === "이 제품을 사용하겠습니다"
+    )
+  }
 }