adasauce
/
nheko
forked from mirror/nheko
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Readd pagination and fix redactions

Browse Source
master
Nicolas Werner 5 years ago
parent 9479fcde08
commit 9ae7d0dce3
7 changed files with 256 additions and 123 deletions
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Show Stats Download Patch File Download Diff File
  1. 6
      resources/qml/delegates/MessageDelegate.qml
  2. 177
      src/Cache.cpp
  3. 15
      src/Cache_p.h
  4. 56
      src/ChatPage.cpp
  5. 55
      src/timeline/EventStore.cpp
  6. 17
      src/timeline/EventStore.h
  7. 53
      src/timeline/TimelineModel.cpp

6
resources/qml/delegates/MessageDelegate.qml
Unescape View File

@ -66,6 +66,12 @@ Item {
text: qsTr("redacted")
}
}
DelegateChoice {
roleValue: MtxEvent.Redaction
Pill {
text: qsTr("redacted")
}
}
DelegateChoice {
roleValue: MtxEvent.Encryption
Pill {

177
src/Cache.cpp
Unescape View File

@ -1241,8 +1241,25 @@ Cache::getTimelineMentions()
return notifs;
}
std::string
Cache::previousBatchToken(const std::string &room_id)
{
auto txn = lmdb::txn::begin(env_, nullptr);
auto orderDb = getEventOrderDb(txn, room_id);
auto cursor = lmdb::cursor::open(txn, orderDb);
lmdb::val indexVal, val;
if (!cursor.get(indexVal, val, MDB_FIRST)) {
return "";
}
auto j = json::parse(std::string_view(val.data(), val.size()));
return j.value("prev_batch", "");
}
Cache::Messages
Cache::getTimelineMessages(lmdb::txn &txn, const std::string &room_id, int64_t index, bool forward)
Cache::getTimelineMessages(lmdb::txn &txn, const std::string &room_id, uint64_t index, bool forward)
{
// TODO(nico): Limit the messages returned by this maybe?
auto orderDb = getOrderToMessageDb(txn, room_id);
@ -1253,16 +1270,16 @@ Cache::getTimelineMessages(lmdb::txn &txn, const std::string &room_id, int64_t i
lmdb::val indexVal, event_id;
auto cursor = lmdb::cursor::open(txn, orderDb);
if (index == std::numeric_limits<int64_t>::max()) {
if (index == std::numeric_limits<uint64_t>::max()) {
if (cursor.get(indexVal, event_id, forward ? MDB_FIRST : MDB_LAST)) {
index = *indexVal.data<int64_t>();
index = *indexVal.data<uint64_t>();
} else {
messages.end_of_cache = true;
return messages;
}
} else {
if (cursor.get(indexVal, event_id, MDB_SET)) {
index = *indexVal.data<int64_t>();
index = *indexVal.data<uint64_t>();
} else {
messages.end_of_cache = true;
return messages;
@ -1296,7 +1313,7 @@ Cache::getTimelineMessages(lmdb::txn &txn, const std::string &room_id, int64_t i
cursor.close();
// std::reverse(timeline.events.begin(), timeline.events.end());
messages.next_index = *indexVal.data<int64_t>();
messages.next_index = *indexVal.data<uint64_t>();
messages.end_of_cache = !ret;
return messages;
@ -1402,16 +1419,16 @@ Cache::getTimelineRange(const std::string &room_id)
}
TimelineRange range{};
range.last = *indexVal.data<int64_t>();
range.last = *indexVal.data<uint64_t>();
if (!cursor.get(indexVal, val, MDB_FIRST)) {
return {};
}
range.first = *indexVal.data<int64_t>();
range.first = *indexVal.data<uint64_t>();
return range;
}
std::optional<int64_t>
std::optional<uint64_t>
Cache::getTimelineIndex(const std::string &room_id, std::string_view event_id)
{
auto txn = lmdb::txn::begin(env_, nullptr, MDB_RDONLY);
@ -1424,11 +1441,11 @@ Cache::getTimelineIndex(const std::string &room_id, std::string_view event_id)
return {};
}
return *val.data<int64_t>();
return *val.data<uint64_t>();
}
std::optional<std::string>
Cache::getTimelineEventId(const std::string &room_id, int64_t index)
Cache::getTimelineEventId(const std::string &room_id, uint64_t index)
{
auto txn = lmdb::txn::begin(env_, nullptr, MDB_RDONLY);
auto orderDb = getOrderToMessageDb(txn, room_id);
@ -2074,6 +2091,9 @@ Cache::saveTimelineMessages(lmdb::txn &txn,
const std::string &room_id,
const mtx::responses::Timeline &res)
{
if (res.events.empty())
return;
auto eventsDb = getEventsDb(txn, room_id);
auto relationsDb = getRelationsDb(txn, room_id);
@ -2090,16 +2110,16 @@ Cache::saveTimelineMessages(lmdb::txn &txn,
using namespace mtx::events::state;
lmdb::val indexVal, val;
int64_t index = 0;
auto cursor = lmdb::cursor::open(txn, orderDb);
uint64_t index = std::numeric_limits<uint64_t>::max() / 2;
auto cursor = lmdb::cursor::open(txn, orderDb);
if (cursor.get(indexVal, val, MDB_LAST)) {
index = *indexVal.data<int64_t>();
}
int64_t msgIndex = 0;
auto msgCursor = lmdb::cursor::open(txn, order2msgDb);
uint64_t msgIndex = std::numeric_limits<uint64_t>::max() / 2;
auto msgCursor = lmdb::cursor::open(txn, order2msgDb);
if (msgCursor.get(indexVal, val, MDB_LAST)) {
msgIndex = *indexVal.data<int64_t>();
msgIndex = *indexVal.data<uint64_t>();
}
bool first = true;
@ -2111,39 +2131,19 @@ Cache::saveTimelineMessages(lmdb::txn &txn,
continue;
lmdb::val ev{};
bool success =
lmdb::dbi_get(txn, eventsDb, lmdb::val(redaction->redacts), ev);
if (!success)
continue;
mtx::events::collections::TimelineEvent te;
try {
mtx::events::collections::from_json(
json::parse(std::string_view(ev.data(), ev.size())), te);
} catch (std::exception &e) {
nhlog::db()->error("Failed to parse message from cache {}",
e.what());
continue;
lmdb::dbi_put(
txn, eventsDb, lmdb::val(redaction->redacts), lmdb::val(event.dump()));
lmdb::dbi_put(
txn, eventsDb, lmdb::val(redaction->event_id), lmdb::val(event.dump()));
lmdb::val oldIndex{};
if (lmdb::dbi_get(
txn, msg2orderDb, lmdb::val(redaction->redacts), oldIndex)) {
lmdb::dbi_put(
txn, order2msgDb, oldIndex, lmdb::val(redaction->event_id));
lmdb::dbi_put(
txn, msg2orderDb, lmdb::val(redaction->event_id), oldIndex);
}
auto redactedEvent = std::visit(
[](const auto &ev) -> mtx::events::RoomEvent<mtx::events::msg::Redacted> {
mtx::events::RoomEvent<mtx::events::msg::Redacted> replacement =
{};
replacement.event_id = ev.event_id;
replacement.room_id = ev.room_id;
replacement.sender = ev.sender;
replacement.origin_server_ts = ev.origin_server_ts;
replacement.type = ev.type;
return replacement;
},
te.data);
lmdb::dbi_put(txn,
eventsDb,
lmdb::val(redaction->redacts),
lmdb::val(json(redactedEvent).dump()));
} else {
std::string event_id_val = event["event_id"].get<std::string>();
lmdb::val event_id = event_id_val;
@ -2193,6 +2193,83 @@ Cache::saveTimelineMessages(lmdb::txn &txn,
}
}
uint64_t
Cache::saveOldMessages(const std::string &room_id, const mtx::responses::Messages &res)
{
auto txn = lmdb::txn::begin(env_);
auto eventsDb = getEventsDb(txn, room_id);
auto relationsDb = getRelationsDb(txn, room_id);
auto orderDb = getEventOrderDb(txn, room_id);
auto msg2orderDb = getMessageToOrderDb(txn, room_id);
auto order2msgDb = getOrderToMessageDb(txn, room_id);
lmdb::val indexVal, val;
uint64_t index = std::numeric_limits<uint64_t>::max() / 2;
auto cursor = lmdb::cursor::open(txn, orderDb);
if (cursor.get(indexVal, val, MDB_FIRST)) {
index = *indexVal.data<uint64_t>();
}
uint64_t msgIndex = std::numeric_limits<uint64_t>::max() / 2;
auto msgCursor = lmdb::cursor::open(txn, order2msgDb);
if (msgCursor.get(indexVal, val, MDB_FIRST)) {
msgIndex = *indexVal.data<uint64_t>();
}
if (res.chunk.empty())
return index;
std::string event_id_val;
for (const auto &e : res.chunk) {
auto event = mtx::accessors::serialize_event(e);
event_id_val = event["event_id"].get<std::string>();
lmdb::val event_id = event_id_val;
lmdb::dbi_put(txn, eventsDb, event_id, lmdb::val(event.dump()));
--index;
json orderEntry = json::object();
orderEntry["event_id"] = event_id_val;
nhlog::db()->debug("saving '{}'", orderEntry.dump());
lmdb::dbi_put(
txn, orderDb, lmdb::val(&index, sizeof(index)), lmdb::val(orderEntry.dump()));
// TODO(Nico): Allow blacklisting more event types in UI
if (event["type"] != "m.reaction" && event["type"] != "m.dummy") {
--msgIndex;
lmdb::dbi_put(
txn, order2msgDb, lmdb::val(&msgIndex, sizeof(msgIndex)), event_id);
lmdb::dbi_put(
txn, msg2orderDb, event_id, lmdb::val(&msgIndex, sizeof(msgIndex)));
}
if (event.contains("content") && event["content"].contains("m.relates_to")) {
auto temp = event["content"]["m.relates_to"];
std::string relates_to = temp.contains("m.in_reply_to")
? temp["m.in_reply_to"]["event_id"]
: temp["event_id"];
if (!relates_to.empty())
lmdb::dbi_put(txn, relationsDb, lmdb::val(relates_to), event_id);
}
}
json orderEntry = json::object();
orderEntry["event_id"] = event_id_val;
orderEntry["prev_batch"] = res.end;
lmdb::cursor_put(
cursor.handle(), lmdb::val(&index, sizeof(index)), lmdb::val(orderEntry.dump()));
nhlog::db()->debug("saving '{}'", orderEntry.dump());
txn.commit();
return msgIndex;
}
mtx::responses::Notifications
Cache::getTimelineMentionsForRoom(lmdb::txn &txn, const std::string &room_id)
{
@ -2337,14 +2414,14 @@ Cache::deleteOldMessages()
auto eventsDb = getEventsDb(txn, room_id);
auto cursor = lmdb::cursor::open(txn, orderDb);
int64_t first, last;
uint64_t first, last;
if (cursor.get(indexVal, val, MDB_LAST)) {
last = *indexVal.data<int64_t>();
last = *indexVal.data<uint64_t>();
} else {
continue;
}
if (cursor.get(indexVal, val, MDB_FIRST)) {
first = *indexVal.data<int64_t>();
first = *indexVal.data<uint64_t>();
} else {
continue;
}

15
src/Cache_p.h
Unescape View File

@ -179,8 +179,8 @@ public:
};
Messages getTimelineMessages(lmdb::txn &txn,
const std::string &room_id,
int64_t index = std::numeric_limits<int64_t>::max(),
bool forward = false);
uint64_t index = std::numeric_limits<uint64_t>::max(),
bool forward = false);
std::optional<mtx::events::collections::TimelineEvent> getEvent(
const std::string &room_id,
@ -190,12 +190,15 @@ public:
const mtx::events::collections::TimelineEvent &event);
struct TimelineRange
{
int64_t first, last;
uint64_t first, last;
};
std::optional<TimelineRange> getTimelineRange(const std::string &room_id);
std::optional<int64_t> getTimelineIndex(const std::string &room_id,
std::string_view event_id);
std::optional<std::string> getTimelineEventId(const std::string &room_id, int64_t index);
std::optional<uint64_t> getTimelineIndex(const std::string &room_id,
std::string_view event_id);
std::optional<std::string> getTimelineEventId(const std::string &room_id, uint64_t index);
std::string previousBatchToken(const std::string &room_id);
uint64_t saveOldMessages(const std::string &room_id, const mtx::responses::Messages &res);
//! Remove old unused data.
void deleteOldMessages();

56
src/ChatPage.cpp
Unescape View File

@ -795,43 +795,39 @@ ChatPage::loadStateFromCache()
nhlog::db()->info("restoring state from cache");
QtConcurrent::run([this]() {
try {
cache::restoreSessions();
olm::client()->load(cache::restoreOlmAccount(), STORAGE_SECRET_KEY);
try {
cache::restoreSessions();
olm::client()->load(cache::restoreOlmAccount(), STORAGE_SECRET_KEY);
cache::populateMembers();
cache::populateMembers();
emit initializeEmptyViews(cache::roomMessages());
emit initializeRoomList(cache::roomInfo());
emit initializeMentions(cache::getTimelineMentions());
emit syncTags(cache::roomInfo().toStdMap());
emit initializeEmptyViews(cache::roomMessages());
emit initializeRoomList(cache::roomInfo());
emit initializeMentions(cache::getTimelineMentions());
emit syncTags(cache::roomInfo().toStdMap());
cache::calculateRoomReadStatus();
cache::calculateRoomReadStatus();
} catch (const mtx::crypto::olm_exception &e) {
nhlog::crypto()->critical("failed to restore olm account: {}", e.what());
emit dropToLoginPageCb(
tr("Failed to restore OLM account. Please login again."));
return;
} catch (const lmdb::error &e) {
nhlog::db()->critical("failed to restore cache: {}", e.what());
emit dropToLoginPageCb(
tr("Failed to restore save data. Please login again."));
return;
} catch (const json::exception &e) {
nhlog::db()->critical("failed to parse cache data: {}", e.what());
return;
}
} catch (const mtx::crypto::olm_exception &e) {
nhlog::crypto()->critical("failed to restore olm account: {}", e.what());
emit dropToLoginPageCb(tr("Failed to restore OLM account. Please login again."));
return;
} catch (const lmdb::error &e) {
nhlog::db()->critical("failed to restore cache: {}", e.what());
emit dropToLoginPageCb(tr("Failed to restore save data. Please login again."));
return;
} catch (const json::exception &e) {
nhlog::db()->critical("failed to parse cache data: {}", e.what());
return;
}
nhlog::crypto()->info("ed25519 : {}", olm::client()->identity_keys().ed25519);
nhlog::crypto()->info("curve25519: {}", olm::client()->identity_keys().curve25519);
nhlog::crypto()->info("ed25519 : {}", olm::client()->identity_keys().ed25519);
nhlog::crypto()->info("curve25519: {}", olm::client()->identity_keys().curve25519);
getProfileInfo();
getProfileInfo();
// Start receiving events.
emit trySyncCb();
});
// Start receiving events.
emit trySyncCb();
}
void

55
src/timeline/EventStore.cpp
Unescape View File

@ -34,12 +34,31 @@ EventStore::EventStore(std::string room_id, QObject *)
cache::client()->storeEvent(room_id_, id, {timeline});
if (!relatedTo.empty()) {
auto idx = idToIndex(id);
auto idx = idToIndex(relatedTo);
if (idx)
emit dataChanged(*idx, *idx);
}
},
Qt::QueuedConnection);
connect(
this,
&EventStore::oldMessagesRetrieved,
this,
[this](const mtx::responses::Messages &res) {
//
uint64_t newFirst = cache::client()->saveOldMessages(room_id_, res);
if (newFirst == first)
fetchMore();
else {
emit beginInsertRows(toExternalIdx(newFirst),
toExternalIdx(this->first - 1));
this->first = newFirst;
emit endInsertRows();
emit fetchedMore();
}
},
Qt::QueuedConnection);
}
void
@ -49,8 +68,16 @@ EventStore::handleSync(const mtx::responses::Timeline &events)
nhlog::db()->warn("{} called from a different thread!", __func__);
auto range = cache::client()->getTimelineRange(room_id_);
if (!range)
return;
if (events.limited) {
emit beginResetModel();
this->last = range->last;
this->first = range->first;
emit endResetModel();
if (range && range->last > this->last) {
} else if (range->last > this->last) {
emit beginInsertRows(toExternalIdx(this->last + 1), toExternalIdx(range->last));
this->last = range->last;
emit endInsertRows();
@ -290,3 +317,27 @@ EventStore::event(std::string_view id, std::string_view related_to, bool decrypt
return event_ptr;
}
void
EventStore::fetchMore()
{
mtx::http::MessagesOpts opts;
opts.room_id = room_id_;
opts.from = cache::client()->previousBatchToken(room_id_);
nhlog::ui()->debug("Paginating room {}, token {}", opts.room_id, opts.from);
http::client()->messages(
opts, [this, opts](const mtx::responses::Messages &res, mtx::http::RequestErr err) {
if (err) {
nhlog::net()->error("failed to call /messages ({}): {} - {} - {}",
opts.room_id,
mtx::errors::to_string(err->matrix_error.errcode),
err->matrix_error.error,
err->parse_error);
return;
}
emit oldMessagesRetrieved(std::move(res));
});
}

17
src/timeline/EventStore.h
Unescape View File

@ -8,6 +8,7 @@
#include <qhashfunctions.h>
#include <mtx/events/collections.hpp>
#include <mtx/responses/messages.hpp>
#include <mtx/responses/sync.hpp>
class EventStore : public QObject
@ -20,7 +21,7 @@ public:
struct Index
{
std::string room;
int64_t idx;
uint64_t idx;
friend uint qHash(const Index &i, uint seed = 0) noexcept
{
@ -66,12 +67,12 @@ public:
int size() const
{
return last != std::numeric_limits<int64_t>::max()
return last != std::numeric_limits<uint64_t>::max()
? static_cast<int>(last - first) + 1
: 0;
}
int toExternalIdx(int64_t idx) const { return static_cast<int>(idx - first); }
int64_t toInternalIdx(int idx) const { return first + idx; }
int toExternalIdx(uint64_t idx) const { return static_cast<int>(idx - first); }
uint64_t toInternalIdx(int idx) const { return first + idx; }
std::optional<int> idToIndex(std::string_view id) const;
std::optional<std::string> indexToId(int idx) const;
@ -79,11 +80,15 @@ public:
signals:
void beginInsertRows(int from, int to);
void endInsertRows();
void beginResetModel();
void endResetModel();
void dataChanged(int from, int to);
void newEncryptedImage(mtx::crypto::EncryptedFile encryptionInfo);
void eventFetched(std::string id,
std::string relatedTo,
mtx::events::collections::TimelineEvents timeline);
void oldMessagesRetrieved(const mtx::responses::Messages &);
void fetchedMore();
private:
mtx::events::collections::TimelineEvents *decryptEvent(
@ -92,8 +97,8 @@ private:
std::string room_id_;
int64_t first = std::numeric_limits<int64_t>::max(),
last = std::numeric_limits<int64_t>::max();
uint64_t first = std::numeric_limits<uint64_t>::max(),
last = std::numeric_limits<uint64_t>::max();
static QCache<IdIndex, mtx::events::collections::TimelineEvents> decryptedEvents_;
static QCache<Index, mtx::events::collections::TimelineEvents> events_;

53
src/timeline/TimelineModel.cpp
Unescape View File

@ -229,20 +229,33 @@ TimelineModel::TimelineModel(TimelineViewManager *manager, QString room_id, QObj
&EventStore::dataChanged,
this,
[this](int from, int to) {
emit dataChanged(index(events.size() - to, 0), index(events.size() - from, 0));
nhlog::ui()->debug(
"data changed {} to {}", events.size() - to - 1, events.size() - from - 1);
emit dataChanged(index(events.size() - to - 1, 0),
index(events.size() - from - 1, 0));
},
Qt::QueuedConnection);
connect(&events, &EventStore::beginInsertRows, this, [this](int from, int to) {
nhlog::ui()->info("begin insert from {} to {}",
events.size() - to + (to - from),
events.size() - from + (to - from));
beginInsertRows(QModelIndex(),
events.size() - to + (to - from),
events.size() - from + (to - from));
int first = events.size() - to;
int last = events.size() - from;
if (from >= events.size()) {
int batch_size = to - from;
first += batch_size;
last += batch_size;
} else {
first -= 1;
last -= 1;
}
nhlog::ui()->debug("begin insert from {} to {}", first, last);
beginInsertRows(QModelIndex(), first, last);
});
connect(&events, &EventStore::endInsertRows, this, [this]() { endInsertRows(); });
connect(&events, &EventStore::beginResetModel, this, [this]() { beginResetModel(); });
connect(&events, &EventStore::endResetModel, this, [this]() { endResetModel(); });
connect(&events, &EventStore::newEncryptedImage, this, &TimelineModel::newEncryptedImage);
connect(
&events, &EventStore::fetchedMore, this, [this]() { setPaginationInProgress(false); });
}
QHash<int, QByteArray>
@ -512,8 +525,9 @@ TimelineModel::canFetchMore(const QModelIndex &) const
{
if (!events.size())
return true;
if (!std::holds_alternative<mtx::events::StateEvent<mtx::events::state::Create>>(
*events.event(0)))
if (auto first = events.event(0);
first &&
!std::holds_alternative<mtx::events::StateEvent<mtx::events::state::Create>>(*first))
return true;
else
@ -540,27 +554,8 @@ TimelineModel::fetchMore(const QModelIndex &)
}
setPaginationInProgress(true);
mtx::http::MessagesOpts opts;
opts.room_id = room_id_.toStdString();
opts.from = prev_batch_token_.toStdString();
nhlog::ui()->debug("Paginating room {}", opts.room_id);
http::client()->messages(
opts, [this, opts](const mtx::responses::Messages &res, mtx::http::RequestErr err) {
if (err) {
nhlog::net()->error("failed to call /messages ({}): {} - {} - {}",
opts.room_id,
mtx::errors::to_string(err->matrix_error.errcode),
err->matrix_error.error,
err->parse_error);
setPaginationInProgress(false);
return;
}
emit oldMessagesRetrieved(std::move(res));
setPaginationInProgress(false);
});
events.fetchMore();
}
void

Write Preview
Loading…
Cancel
Save