Skip to content

Commit

Permalink
Implement attachment captioning
Browse files Browse the repository at this point in the history 
Starting with Client-Server API v1.10 [0], the `body` field in messages of type `m.image`,
`m.audio`, `m.video` and `m.file` can be used as the caption of the attachment. This is fact the way
that Nheko rends captions on images, for example.

This commit introduces a field in the `UploadHandle`s awaiting upload on the timeline's `InputBar`
which holds a caption taken from the input text area. The decision is as follows:
 - If text bar is empty or full of blanks, send all media with no caption
 - If the text is an incomplete command, fail
 - If there are no pending uploads, proceed as done previously (if there is no command recognized
   send the text, or try and execute the command and if it fails send the text)
 - If there are pending uploads, only accept uploads if nothing resembling a command name is in the
   text area. That text becomes the caption for all pending media. Otherwise, try and execute the
   command, and, if it fails, send it as text.

While this workflow for captioning so far is a bit jank, it is the least effort implementation.

Links:
[0]: https://spec.matrix.org/v1.10/client-server-api/#mimage

Signed-off-by: lymkwi <[email protected]>
  • Loading branch information
@Lymkwi
Lymkwi committed Apr 30, 2024
1 parent f465a5b commit ffee5a4
Show file tree
Hide file tree
Showing 2 changed files with 45 additions and 8 deletions.
46 changes: 38 additions & 8 deletions src/timeline/InputBar.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -394,6 +394,9 @@ InputBar::send()
{
QInputMethod *im = QGuiApplication::inputMethod();
im->commit();
// If the input from the UI is only blanks or no text, this trigger should
// be used to confirm media upload. If that is not the case however, but
// but there are pending uploads, we fall into one of the cases seen later.
if (text().trimmed().isEmpty()) {
acceptUploads();
return;
Expand All @@ -407,8 +410,18 @@ InputBar::send()
updateTextContentProperties(text());
if (containsIncompleteCommand_)
return;
if (commandName.isEmpty() || !command(commandName, args))
message(text());
if (unconfirmedUploads.empty()) {
if (commandName.isEmpty() || !command(commandName, args)) {
message(text());
}
} else {
if (commandName.isEmpty()) {
// This is a set of uploads with text
acceptUploadsWithCaption(text());
} else if (!command(commandName, args)) {
message(text());
}
}

if (!wasEdit) {
history_.push_front(QLatin1String(""));
Expand Down Expand Up @@ -708,6 +721,7 @@ void
InputBar::image(const QString &filename,
const std::optional<mtx::crypto::EncryptedFile> &file,
const QString &url,
const std::optional<QString> &caption,
const QString &mime,
uint64_t dsize,
const QSize &dimensions,
Expand All @@ -721,7 +735,8 @@ InputBar::image(const QString &filename,
image.info.mimetype = mime.toStdString();
image.info.size = dsize;
image.info.blurhash = blurhash.toStdString();
image.body = filename.toStdString();
// Depending on the input bar's situation, retrieve the text
image.body = caption.has_value() ? caption.value().toStdString() : filename.toStdString();
image.info.h = dimensions.height();
image.info.w = dimensions.width();

Expand Down Expand Up @@ -752,13 +767,14 @@ void
InputBar::file(const QString &filename,
const std::optional<mtx::crypto::EncryptedFile> &encryptedFile,
const QString &url,
const std::optional<QString> &caption,
const QString &mime,
uint64_t dsize)
{
mtx::events::msg::File file;
file.info.mimetype = mime.toStdString();
file.info.size = dsize;
file.body = filename.toStdString();
file.body = caption.has_value() ? caption.value().toStdString() : filename.toStdString();

if (encryptedFile)
file.file = encryptedFile;
Expand All @@ -775,14 +791,15 @@ void
InputBar::audio(const QString &filename,
const std::optional<mtx::crypto::EncryptedFile> &file,
const QString &url,
const std::optional<QString> &caption,
const QString &mime,
uint64_t dsize,
uint64_t duration)
{
mtx::events::msg::Audio audio;
audio.info.mimetype = mime.toStdString();
audio.info.size = dsize;
audio.body = filename.toStdString();
audio.body = caption.has_value() ? caption.value().toStdString() : filename.toStdString();
audio.url = url.toStdString();

if (duration > 0)
Expand All @@ -803,6 +820,7 @@ void
InputBar::video(const QString &filename,
const std::optional<mtx::crypto::EncryptedFile> &file,
const QString &url,
const std::optional<QString> &caption,
const QString &mime,
uint64_t dsize,
uint64_t duration,
Expand All @@ -817,7 +835,7 @@ InputBar::video(const QString &filename,
video.info.mimetype = mime.toStdString();
video.info.size = dsize;
video.info.blurhash = blurhash.toStdString();
video.body = filename.toStdString();
video.body = caption.has_value() ? caption.value().toStdString() : filename.toStdString();

if (duration > 0)
video.info.duration = duration;
Expand Down Expand Up @@ -1282,10 +1300,12 @@ InputBar::finalizeUpload(MediaUpload *upload, const QString &url)
auto mimeClass = upload->mimeClass();
auto size = upload->size();
auto encryptedFile = upload->encryptedFile_();
auto caption = upload->caption();
if (mimeClass == u"image")
image(filename,
encryptedFile,
url,
caption,
mime,
size,
upload->dimensions(),
Expand All @@ -1295,11 +1315,12 @@ InputBar::finalizeUpload(MediaUpload *upload, const QString &url)
upload->thumbnailImg().size(),
upload->blurhash());
else if (mimeClass == u"audio")
audio(filename, encryptedFile, url, mime, size, upload->duration());
audio(filename, encryptedFile, url, caption, mime, size, upload->duration());
else if (mimeClass == u"video")
video(filename,
encryptedFile,
url,
caption,
mime,
size,
upload->duration(),
Expand All @@ -1310,7 +1331,7 @@ InputBar::finalizeUpload(MediaUpload *upload, const QString &url)
upload->thumbnailImg().size(),
upload->blurhash());
else
file(filename, encryptedFile, url, mime, size);
file(filename, encryptedFile, url, caption, mime, size);

removeRunUpload(upload);
}
Expand Down Expand Up @@ -1405,6 +1426,15 @@ InputBar::acceptUploads()
}
}

void
InputBar::acceptUploadsWithCaption(QString caption)
{
for (UploadHandle &upload : unconfirmedUploads) {
upload->caption_ = std::optional(caption);
}
acceptUploads();
}

void
InputBar::declineUploads()
{
Expand Down
7 changes: 7 additions & 0 deletions src/timeline/InputBar.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -86,6 +86,7 @@ class MediaUpload final : public QObject
return MediaType::File;
}
[[nodiscard]] QString url() const { return url_; }
[[nodiscard]] std::optional<QString> caption() const { return caption_; }
[[nodiscard]] QString mimetype() const { return mimetype_; }
[[nodiscard]] QString mimeClass() const { return mimeClass_; }
[[nodiscard]] QString filename() const { return originalFilename_; }
Expand Down Expand Up @@ -143,6 +144,7 @@ private slots:
QString blurhash_;
QString thumbnailUrl_;
QString url_;
std::optional<QString> caption_;
std::optional<mtx::crypto::EncryptedFile> encryptedFile, thumbnailEncryptedFile;

QImage thumbnail_;
Expand Down Expand Up @@ -236,6 +238,7 @@ public slots:
void sticker(QStringList descriptor);

void acceptUploads();
void acceptUploadsWithCaption(QString);
void declineUploads();

private slots:
Expand Down Expand Up @@ -264,6 +267,7 @@ private slots:
void image(const QString &filename,
const std::optional<mtx::crypto::EncryptedFile> &file,
const QString &url,
const std::optional<QString> &caption,
const QString &mime,
uint64_t dsize,
const QSize &dimensions,
Expand All @@ -275,17 +279,20 @@ private slots:
void file(const QString &filename,
const std::optional<mtx::crypto::EncryptedFile> &encryptedFile,
const QString &url,
const std::optional<QString> &caption,
const QString &mime,
uint64_t dsize);
void audio(const QString &filename,
const std::optional<mtx::crypto::EncryptedFile> &file,
const QString &url,
const std::optional<QString> &caption,
const QString &mime,
uint64_t dsize,
uint64_t duration);
void video(const QString &filename,
const std::optional<mtx::crypto::EncryptedFile> &file,
const QString &url,
const std::optional<QString> &caption,
const QString &mime,
uint64_t dsize,
uint64_t duration,
Expand Down

0 comments on commit ffee5a4

Please sign in to comment.