refactor server to allow pluggable db and tracing (#1036)

* refactor server to allow pluggable db and tracing

* clean up

* fix descriptions

* remove dependencies

15 files changed, 565 insertions, 0 deletions

diff --git a/atuin-server-postgres/Cargo.toml b/atuin-server-postgres/Cargo.toml
new file mode 100644
index 00000000..18864f6c
--- /dev/null
+++ b/atuin-server-postgres/Cargo.toml
@@ -0,0 +1,21 @@
+[package]
+name = "atuin-server-postgres"
+edition = "2018"
+description = "server postgres database library for atuin"
+
+version = { workspace = true }
+authors = { workspace = true }
+license = { workspace = true }
+homepage = { workspace = true }
+repository = { workspace = true }
+
+[dependencies]
+atuin-common = { path = "../atuin-common", version = "15.0.0" }
+atuin-server-database = { path = "../atuin-server-database", version = "15.0.0" }
+
+tracing = "0.1"
+chrono = { workspace = true }
+serde = { workspace = true }
+sqlx = { workspace = true }
+async-trait = { workspace = true }
+futures-util = "0.3"
diff --git a/atuin-server-postgres/migrations/20210425153745_create_history.sql b/atuin-server-postgres/migrations/20210425153745_create_history.sql
new file mode 100644
index 00000000..2c2d17b0
--- /dev/null
+++ b/atuin-server-postgres/migrations/20210425153745_create_history.sql
@@ -0,0 +1,11 @@
+create table history (
+	id bigserial primary key,
+	client_id text not null unique, -- the client-generated ID
+	user_id bigserial not null,     -- allow multiple users
+	hostname text not null,         -- a unique identifier from the client (can be hashed, random, whatever)
+	timestamp timestamp not null,   -- one of the few non-encrypted metadatas
+
+	data varchar(8192) not null,    -- store the actual history data, encrypted. I don't wanna know!
+
+	created_at timestamp not null default current_timestamp
+);
diff --git a/atuin-server-postgres/migrations/20210425153757_create_users.sql b/atuin-server-postgres/migrations/20210425153757_create_users.sql
new file mode 100644
index 00000000..a25dcced
--- /dev/null
+++ b/atuin-server-postgres/migrations/20210425153757_create_users.sql
@@ -0,0 +1,10 @@
+create table users (
+	id bigserial primary key,               -- also store our own ID
+	username varchar(32) not null unique,   -- being able to contact users is useful
+	email varchar(128) not null unique,     -- being able to contact users is useful
+	password varchar(128) not null unique
+);
+
+-- the prior index is case sensitive :(
+CREATE UNIQUE INDEX email_unique_idx on users (LOWER(email));
+CREATE UNIQUE INDEX username_unique_idx on users (LOWER(username));
diff --git a/atuin-server-postgres/migrations/20210425153800_create_sessions.sql b/atuin-server-postgres/migrations/20210425153800_create_sessions.sql
new file mode 100644
index 00000000..c2fb6559
--- /dev/null
+++ b/atuin-server-postgres/migrations/20210425153800_create_sessions.sql
@@ -0,0 +1,6 @@
+-- Add migration script here
+create table sessions (
+	id bigserial primary key,
+	user_id bigserial,
+	token varchar(128) unique not null
+);
diff --git a/atuin-server-postgres/migrations/20220419082412_add_count_trigger.sql b/atuin-server-postgres/migrations/20220419082412_add_count_trigger.sql
new file mode 100644
index 00000000..dd1afa88
--- /dev/null
+++ b/atuin-server-postgres/migrations/20220419082412_add_count_trigger.sql
@@ -0,0 +1,51 @@
+-- Prior to this, the count endpoint was super naive and just ran COUNT(1). 
+-- This is slow asf. Now that we have an amount of actual traffic, 
+-- stop doing that!
+-- This basically maintains a count, so we can read ONE row, instead of ALL the
+-- rows. Much better.
+-- Future optimisation could use some sort of cache so we don't even need to hit
+-- postgres at all.
+
+create table total_history_count_user(
+	id bigserial primary key, 
+	user_id bigserial,
+	total integer -- try and avoid using keywords - hence total, not count
+);
+
+create or replace function user_history_count()
+returns trigger as 
+$func$
+begin
+	if (TG_OP='INSERT') then
+		update total_history_count_user set total = total + 1 where user_id = new.user_id;
+
+		if not found then
+			insert into total_history_count_user(user_id, total) 
+			values (
+				new.user_id, 
+				(select count(1) from history where user_id = new.user_id)
+			);
+		end if;
+		
+	elsif (TG_OP='DELETE') then
+		update total_history_count_user set total = total - 1 where user_id = new.user_id;
+
+		if not found then
+			insert into total_history_count_user(user_id, total) 
+			values (
+				new.user_id, 
+				(select count(1) from history where user_id = new.user_id)
+			);
+		end if;
+	end if;
+
+	return NEW; -- this is actually ignored for an after trigger, but oh well
+end;
+$func$
+language plpgsql volatile -- pldfplplpflh
+cost 100; -- default value
+
+create trigger tg_user_history_count 
+	after insert or delete on history 
+	for each row 
+	execute procedure user_history_count();
diff --git a/atuin-server-postgres/migrations/20220421073605_fix_count_trigger_delete.sql b/atuin-server-postgres/migrations/20220421073605_fix_count_trigger_delete.sql
new file mode 100644
index 00000000..6198f300
--- /dev/null
+++ b/atuin-server-postgres/migrations/20220421073605_fix_count_trigger_delete.sql
@@ -0,0 +1,35 @@
+-- the old version of this function used NEW in the delete part when it should
+-- use OLD
+
+create or replace function user_history_count()
+returns trigger as 
+$func$
+begin
+	if (TG_OP='INSERT') then
+		update total_history_count_user set total = total + 1 where user_id = new.user_id;
+
+		if not found then
+			insert into total_history_count_user(user_id, total) 
+			values (
+				new.user_id, 
+				(select count(1) from history where user_id = new.user_id)
+			);
+		end if;
+		
+	elsif (TG_OP='DELETE') then
+		update total_history_count_user set total = total - 1 where user_id = old.user_id;
+
+		if not found then
+			insert into total_history_count_user(user_id, total) 
+			values (
+				old.user_id, 
+				(select count(1) from history where user_id = old.user_id)
+			);
+		end if;
+	end if;
+
+	return NEW; -- this is actually ignored for an after trigger, but oh well
+end;
+$func$
+language plpgsql volatile -- pldfplplpflh
+cost 100; -- default value
diff --git a/atuin-server-postgres/migrations/20220421174016_larger-commands.sql b/atuin-server-postgres/migrations/20220421174016_larger-commands.sql
new file mode 100644
index 00000000..0ac43433
--- /dev/null
+++ b/atuin-server-postgres/migrations/20220421174016_larger-commands.sql
@@ -0,0 +1,3 @@
+-- Make it 4x larger. Most commands are less than this, but as it's base64
+-- SOME are more than 8192. Should be enough for now.
+ALTER TABLE history ALTER COLUMN data TYPE varchar(32768);
diff --git a/atuin-server-postgres/migrations/20220426172813_user-created-at.sql b/atuin-server-postgres/migrations/20220426172813_user-created-at.sql
new file mode 100644
index 00000000..a9138194
--- /dev/null
+++ b/atuin-server-postgres/migrations/20220426172813_user-created-at.sql
@@ -0,0 +1 @@
+alter table users add column created_at timestamp not null default now();
diff --git a/atuin-server-postgres/migrations/20220505082442_create-events.sql b/atuin-server-postgres/migrations/20220505082442_create-events.sql
new file mode 100644
index 00000000..57e16ec7
--- /dev/null
+++ b/atuin-server-postgres/migrations/20220505082442_create-events.sql
@@ -0,0 +1,14 @@
+create type event_type as enum ('create', 'delete');
+ 
+create table events (
+	id bigserial primary key,
+	client_id text not null unique, -- the client-generated ID
+	user_id bigserial not null,     -- allow multiple users
+	hostname text not null,         -- a unique identifier from the client (can be hashed, random, whatever)
+	timestamp timestamp not null,   -- one of the few non-encrypted metadatas
+
+	event_type event_type,
+	data text not null,    -- store the actual history data, encrypted. I don't wanna know!
+
+	created_at timestamp not null default current_timestamp
+);
diff --git a/atuin-server-postgres/migrations/20220610074049_history-length.sql b/atuin-server-postgres/migrations/20220610074049_history-length.sql
new file mode 100644
index 00000000..b1c23016
--- /dev/null
+++ b/atuin-server-postgres/migrations/20220610074049_history-length.sql
@@ -0,0 +1,2 @@
+-- Add migration script here
+alter table history alter column data type text;
diff --git a/atuin-server-postgres/migrations/20230315220537_drop-events.sql b/atuin-server-postgres/migrations/20230315220537_drop-events.sql
new file mode 100644
index 00000000..fe3cae17
--- /dev/null
+++ b/atuin-server-postgres/migrations/20230315220537_drop-events.sql
@@ -0,0 +1,2 @@
+-- Add migration script here
+drop table events;
diff --git a/atuin-server-postgres/migrations/20230315224203_create-deleted.sql b/atuin-server-postgres/migrations/20230315224203_create-deleted.sql
new file mode 100644
index 00000000..9a9e6263
--- /dev/null
+++ b/atuin-server-postgres/migrations/20230315224203_create-deleted.sql
@@ -0,0 +1,5 @@
+-- Add migration script here
+alter table history add column if not exists deleted_at timestamp;
+
+-- queries will all be selecting the ids of history for a user, that has been deleted
+create index if not exists history_deleted_index on history(client_id, user_id, deleted_at);
diff --git a/atuin-server-postgres/migrations/20230515221038_trigger-delete-only.sql b/atuin-server-postgres/migrations/20230515221038_trigger-delete-only.sql
new file mode 100644
index 00000000..3d0bba52
--- /dev/null
+++ b/atuin-server-postgres/migrations/20230515221038_trigger-delete-only.sql
@@ -0,0 +1,30 @@
+-- We do not need to run the trigger on deletes, as the only time we are deleting history is when the user
+-- has already been deleted
+-- This actually slows down deleting all the history a good bit!
+
+create or replace function user_history_count()
+returns trigger as 
+$func$
+begin
+	if (TG_OP='INSERT') then
+		update total_history_count_user set total = total + 1 where user_id = new.user_id;
+
+		if not found then
+			insert into total_history_count_user(user_id, total) 
+			values (
+				new.user_id, 
+				(select count(1) from history where user_id = new.user_id)
+			);
+		end if;
+	end if;
+
+	return NEW; -- this is actually ignored for an after trigger, but oh well
+end;
+$func$
+language plpgsql volatile -- pldfplplpflh
+cost 100; -- default value
+
+create or replace trigger tg_user_history_count 
+	after insert on history 
+	for each row 
+	execute procedure user_history_count();
diff --git a/atuin-server-postgres/src/lib.rs b/atuin-server-postgres/src/lib.rs
new file mode 100644
index 00000000..0dc51daf
--- /dev/null
+++ b/atuin-server-postgres/src/lib.rs
@@ -0,0 +1,332 @@
+use async_trait::async_trait;
+use atuin_server_database::models::{History, NewHistory, NewSession, NewUser, Session, User};
+use atuin_server_database::{Database, DbError, DbResult};
+use futures_util::TryStreamExt;
+use serde::{Deserialize, Serialize};
+use sqlx::postgres::PgPoolOptions;
+
+use sqlx::Row;
+
+use tracing::instrument;
+use wrappers::{DbHistory, DbSession, DbUser};
+
+mod wrappers;
+
+#[derive(Clone)]
+pub struct Postgres {
+    pool: sqlx::Pool<sqlx::postgres::Postgres>,
+}
+
+#[derive(Clone, Debug, Deserialize, Serialize)]
+pub struct PostgresSettings {
+    pub db_uri: String,
+}
+
+fn fix_error(error: sqlx::Error) -> DbError {
+    match error {
+        sqlx::Error::RowNotFound => DbError::NotFound,
+        error => DbError::Other(error.into()),
+    }
+}
+
+#[async_trait]
+impl Database for Postgres {
+    type Settings = PostgresSettings;
+    async fn new(settings: &PostgresSettings) -> DbResult<Self> {
+        let pool = PgPoolOptions::new()
+            .max_connections(100)
+            .connect(settings.db_uri.as_str())
+            .await
+            .map_err(fix_error)?;
+
+        sqlx::migrate!("./migrations")
+            .run(&pool)
+            .await
+            .map_err(|error| DbError::Other(error.into()))?;
+
+        Ok(Self { pool })
+    }
+
+    #[instrument(skip_all)]
+    async fn get_session(&self, token: &str) -> DbResult<Session> {
+        sqlx::query_as("select id, user_id, token from sessions where token = $1")
+            .bind(token)
+            .fetch_one(&self.pool)
+            .await
+            .map_err(fix_error)
+            .map(|DbSession(session)| session)
+    }
+
+    #[instrument(skip_all)]
+    async fn get_user(&self, username: &str) -> DbResult<User> {
+        sqlx::query_as("select id, username, email, password from users where username = $1")
+            .bind(username)
+            .fetch_one(&self.pool)
+            .await
+            .map_err(fix_error)
+            .map(|DbUser(user)| user)
+    }
+
+    #[instrument(skip_all)]
+    async fn get_session_user(&self, token: &str) -> DbResult<User> {
+        sqlx::query_as(
+            "select users.id, users.username, users.email, users.password from users 
+            inner join sessions 
+            on users.id = sessions.user_id 
+            and sessions.token = $1",
+        )
+        .bind(token)
+        .fetch_one(&self.pool)
+        .await
+        .map_err(fix_error)
+        .map(|DbUser(user)| user)
+    }
+
+    #[instrument(skip_all)]
+    async fn count_history(&self, user: &User) -> DbResult<i64> {
+        // The cache is new, and the user might not yet have a cache value.
+        // They will have one as soon as they post up some new history, but handle that
+        // edge case.
+
+        let res: (i64,) = sqlx::query_as(
+            "select count(1) from history
+            where user_id = $1",
+        )
+        .bind(user.id)
+        .fetch_one(&self.pool)
+        .await
+        .map_err(fix_error)?;
+
+        Ok(res.0)
+    }
+
+    #[instrument(skip_all)]
+    async fn count_history_cached(&self, user: &User) -> DbResult<i64> {
+        let res: (i32,) = sqlx::query_as(
+            "select total from total_history_count_user
+            where user_id = $1",
+        )
+        .bind(user.id)
+        .fetch_one(&self.pool)
+        .await
+        .map_err(fix_error)?;
+
+        Ok(res.0 as i64)
+    }
+
+    async fn delete_history(&self, user: &User, id: String) -> DbResult<()> {
+        sqlx::query(
+            "update history
+            set deleted_at = $3
+            where user_id = $1
+            and client_id = $2
+            and deleted_at is null", // don't just keep setting it
+        )
+        .bind(user.id)
+        .bind(id)
+        .bind(chrono::Utc::now().naive_utc())
+        .fetch_all(&self.pool)
+        .await
+        .map_err(fix_error)?;
+
+        Ok(())
+    }
+
+    #[instrument(skip_all)]
+    async fn deleted_history(&self, user: &User) -> DbResult<Vec<String>> {
+        // The cache is new, and the user might not yet have a cache value.
+        // They will have one as soon as they post up some new history, but handle that
+        // edge case.
+
+        let res = sqlx::query(
+            "select client_id from history 
+            where user_id = $1
+            and deleted_at is not null",
+        )
+        .bind(user.id)
+        .fetch_all(&self.pool)
+        .await
+        .map_err(fix_error)?;
+
+        let res = res
+            .iter()
+            .map(|row| row.get::<String, _>("client_id"))
+            .collect();
+
+        Ok(res)
+    }
+
+    #[instrument(skip_all)]
+    async fn count_history_range(
+        &self,
+        user: &User,
+        start: chrono::NaiveDateTime,
+        end: chrono::NaiveDateTime,
+    ) -> DbResult<i64> {
+        let res: (i64,) = sqlx::query_as(
+            "select count(1) from history
+            where user_id = $1
+            and timestamp >= $2::date
+            and timestamp < $3::date",
+        )
+        .bind(user.id)
+        .bind(start)
+        .bind(end)
+        .fetch_one(&self.pool)
+        .await
+        .map_err(fix_error)?;
+
+        Ok(res.0)
+    }
+
+    #[instrument(skip_all)]
+    async fn list_history(
+        &self,
+        user: &User,
+        created_after: chrono::NaiveDateTime,
+        since: chrono::NaiveDateTime,
+        host: &str,
+        page_size: i64,
+    ) -> DbResult<Vec<History>> {
+        let res = sqlx::query_as(
+            "select id, client_id, user_id, hostname, timestamp, data, created_at from history 
+            where user_id = $1
+            and hostname != $2
+            and created_at >= $3
+            and timestamp >= $4
+            order by timestamp asc
+            limit $5",
+        )
+        .bind(user.id)
+        .bind(host)
+        .bind(created_after)
+        .bind(since)
+        .bind(page_size)
+        .fetch(&self.pool)
+        .map_ok(|DbHistory(h)| h)
+        .try_collect()
+        .await
+        .map_err(fix_error)?;
+
+        Ok(res)
+    }
+
+    #[instrument(skip_all)]
+    async fn add_history(&self, history: &[NewHistory]) -> DbResult<()> {
+        let mut tx = self.pool.begin().await.map_err(fix_error)?;
+
+        for i in history {
+            let client_id: &str = &i.client_id;
+            let hostname: &str = &i.hostname;
+            let data: &str = &i.data;
+
+            sqlx::query(
+                "insert into history
+                    (client_id, user_id, hostname, timestamp, data) 
+                values ($1, $2, $3, $4, $5)
+                on conflict do nothing
+                ",
+            )
+            .bind(client_id)
+            .bind(i.user_id)
+            .bind(hostname)
+            .bind(i.timestamp)
+            .bind(data)
+            .execute(&mut tx)
+            .await
+            .map_err(fix_error)?;
+        }
+
+        tx.commit().await.map_err(fix_error)?;
+
+        Ok(())
+    }
+
+    #[instrument(skip_all)]
+    async fn delete_user(&self, u: &User) -> DbResult<()> {
+        sqlx::query("delete from sessions where user_id = $1")
+            .bind(u.id)
+            .execute(&self.pool)
+            .await
+            .map_err(fix_error)?;
+
+        sqlx::query("delete from users where id = $1")
+            .bind(u.id)
+            .execute(&self.pool)
+            .await
+            .map_err(fix_error)?;
+
+        sqlx::query("delete from history where user_id = $1")
+            .bind(u.id)
+            .execute(&self.pool)
+            .await
+            .map_err(fix_error)?;
+
+        Ok(())
+    }
+
+    #[instrument(skip_all)]
+    async fn add_user(&self, user: &NewUser) -> DbResult<i64> {
+        let email: &str = &user.email;
+        let username: &str = &user.username;
+        let password: &str = &user.password;
+
+        let res: (i64,) = sqlx::query_as(
+            "insert into users
+                (username, email, password)
+            values($1, $2, $3)
+            returning id",
+        )
+        .bind(username)
+        .bind(email)
+        .bind(password)
+        .fetch_one(&self.pool)
+        .await
+        .map_err(fix_error)?;
+
+        Ok(res.0)
+    }
+
+    #[instrument(skip_all)]
+    async fn add_session(&self, session: &NewSession) -> DbResult<()> {
+        let token: &str = &session.token;
+
+        sqlx::query(
+            "insert into sessions
+                (user_id, token)
+            values($1, $2)",
+        )
+        .bind(session.user_id)
+        .bind(token)
+        .execute(&self.pool)
+        .await
+        .map_err(fix_error)?;
+
+        Ok(())
+    }
+
+    #[instrument(skip_all)]
+    async fn get_user_session(&self, u: &User) -> DbResult<Session> {
+        sqlx::query_as("select id, user_id, token from sessions where user_id = $1")
+            .bind(u.id)
+            .fetch_one(&self.pool)
+            .await
+            .map_err(fix_error)
+            .map(|DbSession(session)| session)
+    }
+
+    #[instrument(skip_all)]
+    async fn oldest_history(&self, user: &User) -> DbResult<History> {
+        sqlx::query_as(
+            "select id, client_id, user_id, hostname, timestamp, data, created_at from history 
+            where user_id = $1
+            order by timestamp asc
+            limit 1",
+        )
+        .bind(user.id)
+        .fetch_one(&self.pool)
+        .await
+        .map_err(fix_error)
+        .map(|DbHistory(h)| h)
+    }
+}
diff --git a/atuin-server-postgres/src/wrappers.rs b/atuin-server-postgres/src/wrappers.rs
new file mode 100644
index 00000000..cb3d5a96
--- /dev/null
+++ b/atuin-server-postgres/src/wrappers.rs
@@ -0,0 +1,42 @@
+use ::sqlx::{FromRow, Result};
+use atuin_server_database::models::{History, Session, User};
+use sqlx::{postgres::PgRow, Row};
+
+pub struct DbUser(pub User);
+pub struct DbSession(pub Session);
+pub struct DbHistory(pub History);
+
+impl<'a> FromRow<'a, PgRow> for DbUser {
+    fn from_row(row: &'a PgRow) -> Result<Self> {
+        Ok(Self(User {
+            id: row.try_get("id")?,
+            username: row.try_get("username")?,
+            email: row.try_get("email")?,
+            password: row.try_get("password")?,
+        }))
+    }
+}
+
+impl<'a> ::sqlx::FromRow<'a, PgRow> for DbSession {
+    fn from_row(row: &'a PgRow) -> ::sqlx::Result<Self> {
+        Ok(Self(Session {
+            id: row.try_get("id")?,
+            user_id: row.try_get("user_id")?,
+            token: row.try_get("token")?,
+        }))
+    }
+}
+
+impl<'a> ::sqlx::FromRow<'a, PgRow> for DbHistory {
+    fn from_row(row: &'a PgRow) -> ::sqlx::Result<Self> {
+        Ok(Self(History {
+            id: row.try_get("id")?,
+            client_id: row.try_get("client_id")?,
+            user_id: row.try_get("user_id")?,
+            hostname: row.try_get("hostname")?,
+            timestamp: row.try_get("timestamp")?,
+            data: row.try_get("data")?,
+            created_at: row.try_get("created_at")?,
+        }))
+    }
+}