commit 3181e4e96e9e952b92b79ed07653fe6b6a46c322
parent 367e1ce289cea6a3251b7350a6707c700bd8a544
Author: BlackDex <black.dex@gmail.com>
Date: Wed, 11 Jan 2023 20:23:53 +0100
Optimize CipherSyncData for very large vaults
As mentioned in #3111, using a very very large vault causes some issues.
Mainly because of a SQLite limit, but, it could also cause issue on
MariaDB/MySQL or PostgreSQL. It also uses a lot of memory, and memory
allocations.
This PR solves this by removing the need of all the cipher_uuid's just
to gather the correct attachments.
It will use the user_uuid and org_uuid's to get all attachments linked
to both, weither the user has access to them or not. This isn't an
issue, since the matching is done per cipher and the attachment data is
only returned if there is a matching cipher to where the user has access to.
I also modified some code to be able to use `::with_capacity(n)` where
possible. This prevents re-allocations if the `Vec` increases size,
which will happen a lot if there are a lot of ciphers.
According to my tests measuring the time it takes to sync, it seems to
have lowered the duration a bit more.
Fixes #3111
Diffstat:
4 files changed, 35 insertions(+), 26 deletions(-)
diff --git a/src/api/core/ciphers.rs b/src/api/core/ciphers.rs
@@ -104,16 +104,17 @@ async fn sync(data: SyncData, headers: Headers, mut conn: DbConn) -> Json<Value>
// Get all ciphers which are visible by the user
let ciphers = Cipher::find_by_user_visible(&headers.user.uuid, &mut conn).await;
- let cipher_sync_data = CipherSyncData::new(&headers.user.uuid, &ciphers, CipherSyncType::User, &mut conn).await;
+ let cipher_sync_data = CipherSyncData::new(&headers.user.uuid, CipherSyncType::User, &mut conn).await;
// Lets generate the ciphers_json using all the gathered info
- let mut ciphers_json = Vec::new();
+ let mut ciphers_json = Vec::with_capacity(ciphers.len());
for c in ciphers {
ciphers_json.push(c.to_json(&headers.host, &headers.user.uuid, Some(&cipher_sync_data), &mut conn).await);
}
- let mut collections_json = Vec::new();
- for c in Collection::find_by_user_uuid(headers.user.uuid.clone(), &mut conn).await {
+ let collections = Collection::find_by_user_uuid(headers.user.uuid.clone(), &mut conn).await;
+ let mut collections_json = Vec::with_capacity(collections.len());
+ for c in collections {
collections_json.push(c.to_json_details(&headers.user.uuid, Some(&cipher_sync_data), &mut conn).await);
}
@@ -148,9 +149,9 @@ async fn sync(data: SyncData, headers: Headers, mut conn: DbConn) -> Json<Value>
#[get("/ciphers")]
async fn get_ciphers(headers: Headers, mut conn: DbConn) -> Json<Value> {
let ciphers = Cipher::find_by_user_visible(&headers.user.uuid, &mut conn).await;
- let cipher_sync_data = CipherSyncData::new(&headers.user.uuid, &ciphers, CipherSyncType::User, &mut conn).await;
+ let cipher_sync_data = CipherSyncData::new(&headers.user.uuid, CipherSyncType::User, &mut conn).await;
- let mut ciphers_json = Vec::new();
+ let mut ciphers_json = Vec::with_capacity(ciphers.len());
for c in ciphers {
ciphers_json.push(c.to_json(&headers.host, &headers.user.uuid, Some(&cipher_sync_data), &mut conn).await);
}
@@ -1721,12 +1722,9 @@ pub enum CipherSyncType {
}
impl CipherSyncData {
- pub async fn new(user_uuid: &str, ciphers: &[Cipher], sync_type: CipherSyncType, conn: &mut DbConn) -> Self {
- // Generate a list of Cipher UUID's to be used during a query filter with an eq_any.
- let cipher_uuids = ciphers.iter().map(|c| c.uuid.clone()).collect();
-
- let mut cipher_folders: HashMap<String, String> = HashMap::new();
- let mut cipher_favorites: HashSet<String> = HashSet::new();
+ pub async fn new(user_uuid: &str, sync_type: CipherSyncType, conn: &mut DbConn) -> Self {
+ let cipher_folders: HashMap<String, String>;
+ let cipher_favorites: HashSet<String>;
match sync_type {
// User Sync supports Folders and Favorits
CipherSyncType::User => {
@@ -1738,18 +1736,25 @@ impl CipherSyncData {
}
// Organization Sync does not support Folders and Favorits.
// If these are set, it will cause issues in the web-vault.
- CipherSyncType::Organization => {}
+ CipherSyncType::Organization => {
+ cipher_folders = HashMap::with_capacity(0);
+ cipher_favorites = HashSet::with_capacity(0);
+ }
}
// Generate a list of Cipher UUID's containing a Vec with one or more Attachment records
- let mut cipher_attachments: HashMap<String, Vec<Attachment>> = HashMap::new();
- for attachment in Attachment::find_all_by_ciphers(&cipher_uuids, conn).await {
+ let user_org_uuids = UserOrganization::get_org_uuid_by_user(user_uuid, conn).await;
+ let attachments = Attachment::find_all_by_user_and_orgs(user_uuid, &user_org_uuids, conn).await;
+ let mut cipher_attachments: HashMap<String, Vec<Attachment>> = HashMap::with_capacity(attachments.len());
+ for attachment in attachments {
cipher_attachments.entry(attachment.cipher_uuid.clone()).or_default().push(attachment);
}
// Generate a HashMap with the Cipher UUID as key and one or more Collection UUID's
- let mut cipher_collections: HashMap<String, Vec<String>> = HashMap::new();
- for (cipher, collection) in Cipher::get_collections_with_cipher_by_user(user_uuid.to_string(), conn).await {
+ let user_cipher_collections = Cipher::get_collections_with_cipher_by_user(user_uuid.to_string(), conn).await;
+ let mut cipher_collections: HashMap<String, Vec<String>> =
+ HashMap::with_capacity(user_cipher_collections.len());
+ for (cipher, collection) in user_cipher_collections {
cipher_collections.entry(cipher).or_default().push(collection);
}
@@ -1768,14 +1773,14 @@ impl CipherSyncData {
.collect();
// Generate a HashMap with the collections_uuid as key and the CollectionGroup record
- let user_collections_groups = CollectionGroup::find_by_user(user_uuid, conn)
+ let user_collections_groups: HashMap<String, CollectionGroup> = CollectionGroup::find_by_user(user_uuid, conn)
.await
.into_iter()
.map(|collection_group| (collection_group.collections_uuid.clone(), collection_group))
.collect();
// Get all organizations that the user has full access to via group assignement
- let user_group_full_access_for_organizations =
+ let user_group_full_access_for_organizations: HashSet<String> =
Group::gather_user_organizations_full_access(user_uuid, conn).await.into_iter().collect();
Self {
diff --git a/src/api/core/emergency_access.rs b/src/api/core/emergency_access.rs
@@ -584,10 +584,9 @@ async fn view_emergency_access(emer_id: String, headers: Headers, mut conn: DbCo
}
let ciphers = Cipher::find_owned_by_user(&emergency_access.grantor_uuid, &mut conn).await;
- let cipher_sync_data =
- CipherSyncData::new(&emergency_access.grantor_uuid, &ciphers, CipherSyncType::User, &mut conn).await;
+ let cipher_sync_data = CipherSyncData::new(&emergency_access.grantor_uuid, CipherSyncType::User, &mut conn).await;
- let mut ciphers_json = Vec::new();
+ let mut ciphers_json = Vec::with_capacity(ciphers.len());
for c in ciphers {
ciphers_json
.push(c.to_json(&headers.host, &emergency_access.grantor_uuid, Some(&cipher_sync_data), &mut conn).await);
diff --git a/src/api/core/organizations.rs b/src/api/core/organizations.rs
@@ -617,9 +617,9 @@ async fn get_org_details(data: OrgIdData, headers: Headers, mut conn: DbConn) ->
async fn _get_org_details(org_id: &str, host: &str, user_uuid: &str, conn: &mut DbConn) -> Value {
let ciphers = Cipher::find_by_org(org_id, conn).await;
- let cipher_sync_data = CipherSyncData::new(user_uuid, &ciphers, CipherSyncType::Organization, conn).await;
+ let cipher_sync_data = CipherSyncData::new(user_uuid, CipherSyncType::Organization, conn).await;
- let mut ciphers_json = Vec::new();
+ let mut ciphers_json = Vec::with_capacity(ciphers.len());
for c in ciphers {
ciphers_json.push(c.to_json(host, user_uuid, Some(&cipher_sync_data), conn).await);
}
diff --git a/src/db/models/attachment.rs b/src/db/models/attachment.rs
@@ -187,10 +187,15 @@ impl Attachment {
}}
}
- pub async fn find_all_by_ciphers(cipher_uuids: &Vec<String>, conn: &mut DbConn) -> Vec<Self> {
+ // This will return all attachments linked to the user or org
+ // There is no filtering done here if the user actually has access!
+ // It is used to speed up the sync process, and the matching is done in a different part.
+ pub async fn find_all_by_user_and_orgs(user_uuid: &str, org_uuids: &Vec<String>, conn: &mut DbConn) -> Vec<Self> {
db_run! { conn: {
attachments::table
- .filter(attachments::cipher_uuid.eq_any(cipher_uuids))
+ .left_join(ciphers::table.on(ciphers::uuid.eq(attachments::cipher_uuid)))
+ .filter(ciphers::user_uuid.eq(user_uuid))
+ .or_filter(ciphers::organization_uuid.eq_any(org_uuids))
.select(attachments::all_columns)
.load::<AttachmentDb>(conn)
.expect("Error loading attachments")
