diff --git a/src/Token/Builder.php b/src/Token/Builder.php
index 583ba86e6..e34dcaf28 100644
--- a/src/Token/Builder.php
+++ b/src/Token/Builder.php
@@ -9,7 +9,6 @@
 
 namespace Lcobucci\JWT\Token;
 
-use BadMethodCallException;
 use Lcobucci\Jose\Parsing;
 use Lcobucci\JWT\Builder as BuilderInterface;
 use Lcobucci\JWT\Signer;
@@ -59,13 +58,12 @@ public function __construct(Parsing\Encoder $encoder)
     public function canOnlyBeUsedBy(string $audience, bool $addHeader = false): BuilderInterface
     {
         $audiences = $this->claims['aud'] ?? [];
-        $audiences[] = $audience;
 
-        return $this->setRegisteredClaim(
-            'aud',
-            array_values(array_map('strval', $audiences)),
-            $addHeader
-        );
+        if (!in_array($audience, $audiences)) {
+            $audiences[] = $audience;
+        }
+
+        return $this->setRegisteredClaim('aud', $audiences, $addHeader);
     }
 
     /**
diff --git a/test/unit/Token/BuilderTest.php b/test/unit/Token/BuilderTest.php
index e851f8a25..70ed3a195 100644
--- a/test/unit/Token/BuilderTest.php
+++ b/test/unit/Token/BuilderTest.php
@@ -80,6 +80,25 @@ public function canOnlyBeUsedByMustAppendToTheAudClaim()
         self::assertAttributeEquals(['aud' => ['test', 'test2']], 'claims', $builder);
     }
 
+    /**
+     * @test
+     *
+     * @uses \Lcobucci\JWT\Token\Builder::__construct
+     * @uses \Lcobucci\JWT\Token\Builder::with
+     *
+     * @covers \Lcobucci\JWT\Token\Builder::canOnlyBeUsedBy
+     * @covers \Lcobucci\JWT\Token\Builder::setRegisteredClaim
+     */
+    public function canOnlyBeUsedByShouldPreventDuplicatedEntries()
+    {
+        $builder = $this->createBuilder();
+        $builder->canOnlyBeUsedBy('test');
+        $builder->canOnlyBeUsedBy('test');
+
+        self::assertAttributeEquals(['alg' => 'none', 'typ' => 'JWT'], 'headers', $builder);
+        self::assertAttributeEquals(['aud' => ['test']], 'claims', $builder);
+    }
+
     /**
      * @test
      *