Split code into cargo workspaces (#67)

More fixes

- fixed docker builds
- fixed mentions regex test
- fixed DATABASE_URL stuff
- change schema path in diesel.toml

Address review comments

- add jsonb column back into activity table
- remove authors field from cargo.toml
- adjust LEMMY_DATABASE_URL env var usage
- rename all occurences of LEMMY_DATABASE_URL to DATABASE_URL

Decouple utils and db

Split code into cargo workspaces

Co-authored-by: Felix Ableitner <[email protected]>
Reviewed-on: https://yerbamate.dev/LemmyNet/lemmy/pulls/67

docker/dev/Dockerfile

@@ -17,13 +17,20 @@ WORKDIR /app
 RUN sudo chown -R rust:rust .
 RUN USER=root cargo new server
 WORKDIR /app/server
+RUN mkdir -p lemmy_db/src/ lemmy_utils/src/
 COPY server/Cargo.toml server/Cargo.lock ./
+COPY server/lemmy_db/Cargo.toml ./lemmy_db/
+COPY server/lemmy_utils/Cargo.toml ./lemmy_utils/
 RUN sudo chown -R rust:rust .
 RUN mkdir -p ./src/bin \
-  && echo 'fn main() { println!("Dummy") }' > ./src/bin/main.rs 
+   && echo 'fn main() { println!("Dummy") }' > ./src/bin/main.rs \
+   && cp ./src/bin/main.rs ./lemmy_db/src/main.rs \
+   && cp ./src/bin/main.rs ./lemmy_utils/src/main.rs
 RUN cargo build
 RUN rm -f ./target/x86_64-unknown-linux-musl/release/deps/lemmy_server*
 COPY server/src ./src/
+COPY server/lemmy_db ./lemmy_db/
+COPY server/lemmy_utils ./lemmy_utils/
 COPY server/migrations ./migrations/
 
 # Build for debug

docker/prod/Dockerfile

@@ -10,13 +10,19 @@ WORKDIR /app
 RUN sudo chown -R rust:rust .
 RUN USER=root cargo new server
 WORKDIR /app/server
+RUN mkdir -p lemmy_db/src/ lemmy_utils/src/
 COPY --chown=rust:rust server/Cargo.toml server/Cargo.lock ./
-#RUN sudo chown -R rust:rust .
+COPY --chown=rust:rust server/lemmy_db/Cargo.toml ./lemmy_db/
+COPY --chown=rust:rust server/lemmy_utils/Cargo.toml ./lemmy_utils/
 RUN mkdir -p ./src/bin \
-   && echo 'fn main() { println!("Dummy") }' > ./src/bin/main.rs
+   && echo 'fn main() { println!("Dummy") }' > ./src/bin/main.rs \
+   && cp ./src/bin/main.rs ./lemmy_db/src/main.rs \
+   && cp ./src/bin/main.rs ./lemmy_utils/src/main.rs
 RUN cargo build --release
 RUN rm -f ./target/$CARGO_BUILD_TARGET/$RUSTRELEASEDIR/deps/lemmy_server*
 COPY --chown=rust:rust server/src ./src/
+COPY --chown=rust:rust server/lemmy_db ./lemmy_db/
+COPY --chown=rust:rust server/lemmy_utils ./lemmy_utils/
 COPY --chown=rust:rust server/migrations ./migrations/
 
 # build for release

docs/src/contributing_tests.md

@@ -9,7 +9,7 @@ following commands in the `server` subfolder:
 psql -U lemmy -c "DROP SCHEMA public CASCADE; CREATE SCHEMA public;"
 export DATABASE_URL=postgres://lemmy:[email protected]:5432/lemmy
 diesel migration run
-RUST_TEST_THREADS=1 cargo test
+RUST_TEST_THREADS=1 cargo test --workspace
 ```
 
 ### Federation

install.sh

@@ -1,4 +1,4 @@
-#!/bin/sh
+#!/bin/bash
 set -e
 
 # Set the database variable to the default first.

server/Cargo.lock

@@ -1399,12 +1399,6 @@ dependencies = [
  "winapi 0.3.9",
 ]
 
-[[package]]
-name = "htmlescape"
-version = "0.3.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e9025058dae765dee5070ec375f591e2ba14638c63feff74f13805a72e523163"
-
 [[package]]
 name = "http"
 version = "0.2.1"
@@ -1572,6 +1566,21 @@ version = "1.4.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646"
 
+[[package]]
+name = "lemmy_db"
+version = "0.1.0"
+dependencies = [
+ "bcrypt",
+ "chrono",
+ "diesel",
+ "log",
+ "serde 1.0.114",
+ "serde_json",
+ "sha2",
+ "strum",
+ "strum_macros",
+]
+
 [[package]]
 name = "lemmy_server"
 version = "0.0.1"
@@ -1589,27 +1598,23 @@ dependencies = [
  "base64 0.12.3",
  "bcrypt",
  "chrono",
- "comrak",
- "config",
  "diesel",
  "diesel_migrations",
  "dotenv",
  "env_logger",
  "failure",
  "futures",
- "htmlescape",
  "http",
  "http-signature-normalization-actix",
  "itertools",
  "jsonwebtoken",
  "lazy_static",
- "lettre",
- "lettre_email",
+ "lemmy_db",
+ "lemmy_utils",
  "log",
  "openssl",
  "percent-encoding",
  "rand 0.7.3",
- "regex",
  "rss",
  "serde 1.0.114",
  "serde_json",
@@ -1621,6 +1626,26 @@ dependencies = [
  "uuid 0.8.1",
 ]
 
+[[package]]
+name = "lemmy_utils"
+version = "0.1.0"
+dependencies = [
+ "chrono",
+ "comrak",
+ "config",
+ "itertools",
+ "lazy_static",
+ "lettre",
+ "lettre_email",
+ "log",
+ "openssl",
+ "rand 0.7.3",
+ "regex",
+ "serde 1.0.114",
+ "serde_json",
+ "url",
+]
+
 [[package]]
 name = "lettre"
 version = "0.9.3"

server/Cargo.toml

@@ -1,14 +1,21 @@
 [package]
 name = "lemmy_server"
 version = "0.0.1"
-authors = ["Dessalines <[email protected]>"]
 edition = "2018"
 
 [profile.release]
 lto = true
 
+[workspace]
+members = [
+    "lemmy_utils",
+    "lemmy_db"
+]
+
 [dependencies]
-diesel = { version = "1.4.4", features = ["postgres","chrono","r2d2","64-column-tables","serde_json"] }
+lemmy_utils = { path = "./lemmy_utils" }
+lemmy_db = { path = "./lemmy_db" }
+diesel = "1.4.4"
 diesel_migrations = "1.4.0"
 dotenv = "0.15.0"
 activitystreams = "0.6.2"
@@ -31,16 +38,10 @@ rand = "0.7.3"
 strum = "0.18.0"
 strum_macros = "0.18.0"
 jsonwebtoken = "7.0.1"
-regex = "1.3.5"
 lazy_static = "1.3.0"
-lettre = "0.9.3"
-lettre_email = "0.9.4"
 rss = "1.9.0"
-htmlescape = "0.3.1"
 url = { version = "2.1.1", features = ["serde"] }
-config = {version = "0.10.1", default-features = false, features = ["hjson"] }
 percent-encoding = "2.1.0"
-comrak = "0.7"
 openssl = "0.10"
 http = "0.2.1"
 http-signature-normalization-actix = { version = "0.4.0-alpha.0", default-features = false, features = ["sha-2"] }

server/db-init.sh

@@ -1,4 +1,5 @@
-#!/bin/sh
+#!/bin/bash
+set -e
 
 # Default configurations
 username=lemmy

server/diesel.toml

@@ -2,4 +2,4 @@
 # see diesel.rs/guides/configuring-diesel-cli
 
 [print_schema]
-file = "src/schema.rs"
+file = "lemmy_db/src/schema.rs"

server/lemmy_db/Cargo.toml

@@ -0,0 +1,15 @@
+[package]
+name = "lemmy_db"
+version = "0.1.0"
+edition = "2018"
+
+[dependencies]
+diesel = { version = "1.4.4", features = ["postgres","chrono","r2d2","64-column-tables","serde_json"] }
+chrono = { version = "0.4.7", features = ["serde"] }
+serde = { version = "1.0.105", features = ["derive"] }
+serde_json = { version = "1.0.52", features = ["preserve_order"]}
+strum = "0.18.0"
+strum_macros = "0.18.0"
+log = "0.4.0"
+sha2 = "0.9"
+bcrypt = "0.8.0"

server/lemmy_utils/Cargo.toml

@@ -0,0 +1,22 @@
+[package]
+name = "lemmy_utils"
+version = "0.1.0"
+edition = "2018"
+
+# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
+
+[dependencies]
+regex = "1.3.5"
+config = { version = "0.10.1", default-features = false, features = ["hjson"] }
+chrono = { version = "0.4.7", features = ["serde"] }
+lettre = "0.9.3"
+lettre_email = "0.9.4"
+log = "0.4.0"
+itertools = "0.9.0"
+rand = "0.7.3"
+serde = { version = "1.0.105", features = ["derive"] }
+serde_json = { version = "1.0.52", features = ["preserve_order"]}
+comrak = "0.7"
+lazy_static = "1.3.0"
+openssl = "0.10"
+url = { version = "2.1.1", features = ["serde"] }

server/lemmy_utils/src/lib.rs

@@ -0,0 +1,324 @@
+#[macro_use]
+pub extern crate lazy_static;
+pub extern crate comrak;
+pub extern crate lettre;
+pub extern crate lettre_email;
+pub extern crate openssl;
+pub extern crate rand;
+pub extern crate regex;
+pub extern crate serde_json;
+pub extern crate url;
+
+pub mod settings;
+
+use crate::settings::Settings;
+use chrono::{DateTime, FixedOffset, Local, NaiveDateTime, Utc};
+use itertools::Itertools;
+use lettre::{
+  smtp::{
+    authentication::{Credentials, Mechanism},
+    extension::ClientId,
+    ConnectionReuseParameters,
+  },
+  ClientSecurity,
+  SmtpClient,
+  Transport,
+};
+use lettre_email::Email;
+use openssl::{pkey::PKey, rsa::Rsa};
+use rand::{distributions::Alphanumeric, thread_rng, Rng};
+use regex::{Regex, RegexBuilder};
+use std::io::{Error, ErrorKind};
+use url::Url;
+
+pub fn to_datetime_utc(ndt: NaiveDateTime) -> DateTime<Utc> {
+  DateTime::<Utc>::from_utc(ndt, Utc)
+}
+
+pub fn naive_from_unix(time: i64) -> NaiveDateTime {
+  NaiveDateTime::from_timestamp(time, 0)
+}
+
+pub fn convert_datetime(datetime: NaiveDateTime) -> DateTime<FixedOffset> {
+  let now = Local::now();
+  DateTime::<FixedOffset>::from_utc(datetime, *now.offset())
+}
+
+pub fn is_email_regex(test: &str) -> bool {
+  EMAIL_REGEX.is_match(test)
+}
+
+pub fn remove_slurs(test: &str) -> String {
+  SLUR_REGEX.replace_all(test, "*removed*").to_string()
+}
+
+pub fn slur_check(test: &str) -> Result<(), Vec<&str>> {
+  let mut matches: Vec<&str> = SLUR_REGEX.find_iter(test).map(|mat| mat.as_str()).collect();
+
+  // Unique
+  matches.sort_unstable();
+  matches.dedup();
+
+  if matches.is_empty() {
+    Ok(())
+  } else {
+    Err(matches)
+  }
+}
+
+pub fn slurs_vec_to_str(slurs: Vec<&str>) -> String {
+  let start = "No slurs - ";
+  let combined = &slurs.join(", ");
+  [start, combined].concat()
+}
+
+pub fn generate_random_string() -> String {
+  thread_rng().sample_iter(&Alphanumeric).take(30).collect()
+}
+
+pub fn send_email(
+  subject: &str,
+  to_email: &str,
+  to_username: &str,
+  html: &str,
+) -> Result<(), String> {
+  let email_config = Settings::get().email.ok_or("no_email_setup")?;
+
+  let email = Email::builder()
+    .to((to_email, to_username))
+    .from(email_config.smtp_from_address.to_owned())
+    .subject(subject)
+    .html(html)
+    .build()
+    .unwrap();
+
+  let mailer = if email_config.use_tls {
+    SmtpClient::new_simple(&email_config.smtp_server).unwrap()
+  } else {
+    SmtpClient::new(&email_config.smtp_server, ClientSecurity::None).unwrap()
+  }
+  .hello_name(ClientId::Domain(Settings::get().hostname))
+  .smtp_utf8(true)
+  .authentication_mechanism(Mechanism::Plain)
+  .connection_reuse(ConnectionReuseParameters::ReuseUnlimited);
+  let mailer = if let (Some(login), Some(password)) =
+    (&email_config.smtp_login, &email_config.smtp_password)
+  {
+    mailer.credentials(Credentials::new(login.to_owned(), password.to_owned()))
+  } else {
+    mailer
+  };
+
+  let mut transport = mailer.transport();
+  let result = transport.send(email.into());
+  transport.close();
+
+  match result {
+    Ok(_) => Ok(()),
+    Err(e) => Err(e.to_string()),
+  }
+}
+
+pub fn markdown_to_html(text: &str) -> String {
+  comrak::markdown_to_html(text, &comrak::ComrakOptions::default())
+}
+
+// TODO nothing is done with community / group webfingers yet, so just ignore those for now
+#[derive(Clone, PartialEq, Eq, Hash)]
+pub struct MentionData {
+  pub name: String,
+  pub domain: String,
+}
+
+impl MentionData {
+  pub fn is_local(&self) -> bool {
+    Settings::get().hostname.eq(&self.domain)
+  }
+  pub fn full_name(&self) -> String {
+    format!("@{}@{}", &self.name, &self.domain)
+  }
+}
+
+pub fn scrape_text_for_mentions(text: &str) -> Vec<MentionData> {
+  let mut out: Vec<MentionData> = Vec::new();
+  for caps in MENTIONS_REGEX.captures_iter(text) {
+    out.push(MentionData {
+      name: caps["name"].to_string(),
+      domain: caps["domain"].to_string(),
+    });
+  }
+  out.into_iter().unique().collect()
+}
+
+pub fn is_valid_username(name: &str) -> bool {
+  VALID_USERNAME_REGEX.is_match(name)
+}
+
+pub fn is_valid_community_name(name: &str) -> bool {
+  VALID_COMMUNITY_NAME_REGEX.is_match(name)
+}
+
+#[cfg(test)]
+mod tests {
+  use crate::{
+    is_email_regex,
+    is_valid_community_name,
+    is_valid_username,
+    remove_slurs,
+    scrape_text_for_mentions,
+    slur_check,
+    slurs_vec_to_str,
+  };
+
+  #[test]
+  fn test_mentions_regex() {
+    let text = "Just read a great blog post by [@[email protected]](/u/test). And another by [email protected] . Another [@[email protected]:8540](/u/fish)";
+    let mentions = scrape_text_for_mentions(text);
+
+    assert_eq!(mentions[0].name, "tedu".to_string());
+    assert_eq!(mentions[0].domain, "honk.teduangst.com".to_string());
+    assert_eq!(mentions[1].domain, "lemmy-alpha:8540".to_string());
+  }
+
+  #[test]
+  fn test_email() {
+    assert!(is_email_regex("[email protected]"));
+    assert!(!is_email_regex("nada_neutho"));
+  }
+
+  #[test]
+  fn test_valid_register_username() {
+    assert!(is_valid_username("Hello_98"));
+    assert!(is_valid_username("ten"));
+    assert!(!is_valid_username("Hello-98"));
+    assert!(!is_valid_username("a"));
+    assert!(!is_valid_username(""));
+  }
+
+  #[test]
+  fn test_valid_community_name() {
+    assert!(is_valid_community_name("example"));
+    assert!(is_valid_community_name("example_community"));
+    assert!(!is_valid_community_name("Example"));
+    assert!(!is_valid_community_name("Ex"));
+    assert!(!is_valid_community_name(""));
+  }
+
+  #[test]
+  fn test_slur_filter() {
+    let test =
+      "coons test dindu ladyboy tranny retardeds. Capitalized Niggerz. This is a bunch of other safe text.";
+    let slur_free = "No slurs here";
+    assert_eq!(
+      remove_slurs(&test),
+      "*removed* test *removed* *removed* *removed* *removed*. Capitalized *removed*. This is a bunch of other safe text."
+        .to_string()
+    );
+
+    let has_slurs_vec = vec![
+      "Niggerz",
+      "coons",
+      "dindu",
+      "ladyboy",
+      "retardeds",
+      "tranny",
+    ];
+    let has_slurs_err_str = "No slurs - Niggerz, coons, dindu, ladyboy, retardeds, tranny";
+
+    assert_eq!(slur_check(test), Err(has_slurs_vec));
+    assert_eq!(slur_check(slur_free), Ok(()));
+    if let Err(slur_vec) = slur_check(test) {
+      assert_eq!(&slurs_vec_to_str(slur_vec), has_slurs_err_str);
+    }
+  }
+
+  // These helped with testing
+  // #[test]
+  // fn test_send_email() {
+  //  let result =  send_email("not a subject", "[email protected]", "ur user", "<h1>HI there</h1>");
+  //   assert!(result.is_ok());
+  // }
+}
+
+lazy_static! {
+  static ref EMAIL_REGEX: Regex = Regex::new(r"^[a-zA-Z0-9.!#$%&’*+/=?^_`{|}~-][email protected][a-zA-Z0-9-]+(?:\.[a-zA-Z0-9-]+)*$").unwrap();
+  static ref SLUR_REGEX: Regex = RegexBuilder::new(r"(fag(g|got|tard)?|maricos?|cock\s?sucker(s|ing)?|\bn(i|1)g(\b|g?(a|er)?(s|z)?)\b|dindu(s?)|mudslime?s?|kikes?|mongoloids?|towel\s*heads?|\bspi(c|k)s?\b|\bchinks?|niglets?|beaners?|\bnips?\b|\bcoons?\b|jungle\s*bunn(y|ies?)|jigg?aboo?s?|\bpakis?\b|rag\s*heads?|gooks?|cunts?|bitch(es|ing|y)?|puss(y|ies?)|twats?|feminazis?|whor(es?|ing)|\bslut(s|t?y)?|\btr(a|@)nn?(y|ies?)|ladyboy(s?)|\b(b|re|r)tard(ed)?s?)").case_insensitive(true).build().unwrap();
+  static ref USERNAME_MATCHES_REGEX: Regex = Regex::new(r"/u/[a-zA-Z][0-9a-zA-Z_]*").unwrap();
+  // TODO keep this old one, it didn't work with port well tho
+  // static ref MENTIONS_REGEX: Regex = Regex::new(r"@(?P<name>[\w.]+)@(?P<domain>[a-zA-Z0-9._-]+\.[a-zA-Z0-9_-]+)").unwrap();
+  static ref MENTIONS_REGEX: Regex = Regex::new(r"@(?P<name>[\w.]+)@(?P<domain>[a-zA-Z0-9._:-]+)").unwrap();
+  static ref VALID_USERNAME_REGEX: Regex = Regex::new(r"^[a-zA-Z0-9_]{3,20}$").unwrap();
+  static ref VALID_COMMUNITY_NAME_REGEX: Regex = Regex::new(r"^[a-z0-9_]{3,20}$").unwrap();
+  pub static ref WEBFINGER_COMMUNITY_REGEX: Regex = Regex::new(&format!(
+    "^group:([a-z0-9_]{{3, 20}})@{}$",
+    Settings::get().hostname
+  ))
+  .unwrap();
+  pub static ref WEBFINGER_USER_REGEX: Regex = Regex::new(&format!(
+    "^acct:([a-z0-9_]{{3, 20}})@{}$",
+    Settings::get().hostname
+  ))
+  .unwrap();
+  pub static ref CACHE_CONTROL_REGEX: Regex =
+    Regex::new("^((text|image)/.+|application/javascript)$").unwrap();
+}
+
+pub struct Keypair {
+  pub private_key: String,
+  pub public_key: String,
+}
+
+/// Generate the asymmetric keypair for ActivityPub HTTP signatures.
+pub fn generate_actor_keypair() -> Result<Keypair, Error> {
+  let rsa = Rsa::generate(2048)?;
+  let pkey = PKey::from_rsa(rsa)?;
+  let public_key = pkey.public_key_to_pem()?;
+  let private_key = pkey.private_key_to_pem_pkcs8()?;
+  let key_to_string = |key| match String::from_utf8(key) {
+    Ok(s) => Ok(s),
+    Err(e) => Err(Error::new(
+      ErrorKind::Other,
+      format!("Failed converting key to string: {}", e),
+    )),
+  };
+  Ok(Keypair {
+    private_key: key_to_string(private_key)?,
+    public_key: key_to_string(public_key)?,
+  })
+}
+
+pub enum EndpointType {
+  Community,
+  User,
+  Post,
+  Comment,
+  PrivateMessage,
+}
+
+pub fn get_apub_protocol_string() -> &'static str {
+  if Settings::get().federation.tls_enabled {
+    "https"
+  } else {
+    "http"
+  }
+}
+
+/// Generates the ActivityPub ID for a given object type and ID.
+pub fn make_apub_endpoint(endpoint_type: EndpointType, name: &str) -> Url {
+  let point = match endpoint_type {
+    EndpointType::Community => "c",
+    EndpointType::User => "u",
+    EndpointType::Post => "post",
+    EndpointType::Comment => "comment",
+    EndpointType::PrivateMessage => "private_message",
+  };
+
+  Url::parse(&format!(
+    "{}://{}/{}/{}",
+    get_apub_protocol_string(),
+    Settings::get().hostname,
+    point,
+    name
+  ))
+  .unwrap()
+}

server/src/api/claims.rs

@@ -0,0 +1,73 @@
+use diesel::{result::Error, PgConnection};
+use jsonwebtoken::{decode, encode, DecodingKey, EncodingKey, Header, TokenData, Validation};
+use lemmy_db::{user::User_, Crud};
+use lemmy_utils::{is_email_regex, settings::Settings};
+use serde::{Deserialize, Serialize};
+
+type Jwt = String;
+
+#[derive(Debug, Serialize, Deserialize)]
+pub struct Claims {
+  pub id: i32,
+  pub username: String,
+  pub iss: String,
+  pub show_nsfw: bool,
+  pub theme: String,
+  pub default_sort_type: i16,
+  pub default_listing_type: i16,
+  pub lang: String,
+  pub avatar: Option<String>,
+  pub show_avatars: bool,
+}
+
+impl Claims {
+  pub fn decode(jwt: &str) -> Result<TokenData<Claims>, jsonwebtoken::errors::Error> {
+    let v = Validation {
+      validate_exp: false,
+      ..Validation::default()
+    };
+    decode::<Claims>(
+      &jwt,
+      &DecodingKey::from_secret(Settings::get().jwt_secret.as_ref()),
+      &v,
+    )
+  }
+
+  pub fn jwt(user: User_, hostname: String) -> Jwt {
+    let my_claims = Claims {
+      id: user.id,
+      username: user.name.to_owned(),
+      iss: hostname,
+      show_nsfw: user.show_nsfw,
+      theme: user.theme.to_owned(),
+      default_sort_type: user.default_sort_type,
+      default_listing_type: user.default_listing_type,
+      lang: user.lang.to_owned(),
+      avatar: user.avatar.to_owned(),
+      show_avatars: user.show_avatars.to_owned(),
+    };
+    encode(
+      &Header::default(),
+      &my_claims,
+      &EncodingKey::from_secret(Settings::get().jwt_secret.as_ref()),
+    )
+    .unwrap()
+  }
+
+  // TODO: move these into user?
+  pub fn find_by_email_or_username(
+    conn: &PgConnection,
+    username_or_email: &str,
+  ) -> Result<User_, Error> {
+    if is_email_regex(username_or_email) {
+      User_::find_by_email(conn, username_or_email)
+    } else {
+      User_::find_by_username(conn, username_or_email)
+    }
+  }
+
+  pub fn find_by_jwt(conn: &PgConnection, jwt: &str) -> Result<User_, Error> {
+    let claims: Claims = Claims::decode(&jwt).expect("Invalid token").claims;
+    User_::read(&conn, claims.id)
+  }
+}

server/src/api/comment.rs

@@ -1,28 +1,7 @@
 use crate::{
-  api::{APIError, Oper, Perform},
+  api::{claims::Claims, APIError, Oper, Perform},
   apub::{ApubLikeableType, ApubObjectType},
   blocking,
-  db::{
-    comment::*,
-    comment_view::*,
-    community_view::*,
-    moderator::*,
-    post::*,
-    site_view::*,
-    user::*,
-    user_mention::*,
-    user_view::*,
-    Crud,
-    Likeable,
-    ListingType,
-    Saveable,
-    SortType,
-  },
-  naive_now,
-  remove_slurs,
-  scrape_text_for_mentions,
-  send_email,
-  settings::Settings,
   websocket::{
     server::{JoinCommunityRoom, SendComment},
     UserOperation,
@@ -30,6 +9,31 @@ use crate::{
   },
   DbPool,
   LemmyError,
+};
+use lemmy_db::{
+  comment::*,
+  comment_view::*,
+  community_view::*,
+  moderator::*,
+  naive_now,
+  post::*,
+  site_view::*,
+  user::*,
+  user_mention::*,
+  user_view::*,
+  Crud,
+  Likeable,
+  ListingType,
+  Saveable,
+  SortType,
+};
+use lemmy_utils::{
+  make_apub_endpoint,
+  remove_slurs,
+  scrape_text_for_mentions,
+  send_email,
+  settings::Settings,
+  EndpointType,
   MentionData,
 };
 use log::error;
@@ -155,7 +159,9 @@ impl Perform for Oper<CreateComment> {
 
     let inserted_comment_id = inserted_comment.id;
     let updated_comment: Comment = match blocking(pool, move |conn| {
-      Comment::update_ap_id(&conn, inserted_comment_id)
+      let apub_id =
+        make_apub_endpoint(EndpointType::Comment, &inserted_comment_id.to_string()).to_string();
+      Comment::update_ap_id(&conn, inserted_comment_id, apub_id)
     })
     .await?
     {

server/src/api/community.rs

@@ -1,26 +1,24 @@
 use super::*;
 use crate::{
-  api::{APIError, Oper, Perform},
-  apub::{
-    extensions::signatures::generate_actor_keypair,
-    make_apub_endpoint,
-    ActorType,
-    EndpointType,
-  },
+  api::{claims::Claims, APIError, Oper, Perform},
+  apub::ActorType,
   blocking,
-  db::{Bannable, Crud, Followable, Joinable, SortType},
-  is_valid_community_name,
-  naive_from_unix,
-  naive_now,
-  slur_check,
-  slurs_vec_to_str,
   websocket::{
     server::{JoinCommunityRoom, SendCommunityRoomMessage},
     UserOperation,
     WebsocketInfo,
   },
   DbPool,
-  LemmyError,
+};
+use lemmy_db::{naive_now, Bannable, Crud, Followable, Joinable, SortType};
+use lemmy_utils::{
+  generate_actor_keypair,
+  is_valid_community_name,
+  make_apub_endpoint,
+  naive_from_unix,
+  slur_check,
+  slurs_vec_to_str,
+  EndpointType,
 };
 use serde::{Deserialize, Serialize};
 use std::str::FromStr;

server/src/api/mod.rs

@@ -1,11 +1,8 @@
-use crate::{
-  db::{community::*, community_view::*, moderator::*, site::*, user::*, user_view::*},
-  websocket::WebsocketInfo,
-  DbPool,
-  LemmyError,
-};
+use crate::{websocket::WebsocketInfo, DbPool, LemmyError};
 use actix_web::client::Client;
+use lemmy_db::{community::*, community_view::*, moderator::*, site::*, user::*, user_view::*};
 
+pub mod claims;
 pub mod comment;
 pub mod community;
 pub mod post;

server/src/api/post.rs

@@ -1,27 +1,8 @@
 use crate::{
-  api::{APIError, Oper, Perform},
+  api::{claims::Claims, APIError, Oper, Perform},
   apub::{ApubLikeableType, ApubObjectType},
   blocking,
-  db::{
-    comment_view::*,
-    community_view::*,
-    moderator::*,
-    post::*,
-    post_view::*,
-    site::*,
-    site_view::*,
-    user::*,
-    user_view::*,
-    Crud,
-    Likeable,
-    ListingType,
-    Saveable,
-    SortType,
-  },
   fetch_iframely_and_pictrs_data,
-  naive_now,
-  slur_check,
-  slurs_vec_to_str,
   websocket::{
     server::{JoinCommunityRoom, JoinPostRoom, SendPost},
     UserOperation,
@@ -30,6 +11,24 @@ use crate::{
   DbPool,
   LemmyError,
 };
+use lemmy_db::{
+  comment_view::*,
+  community_view::*,
+  moderator::*,
+  naive_now,
+  post::*,
+  post_view::*,
+  site::*,
+  site_view::*,
+  user::*,
+  user_view::*,
+  Crud,
+  Likeable,
+  ListingType,
+  Saveable,
+  SortType,
+};
+use lemmy_utils::{make_apub_endpoint, slur_check, slurs_vec_to_str, EndpointType};
 use serde::{Deserialize, Serialize};
 use std::str::FromStr;
 
@@ -191,11 +190,16 @@ impl Perform for Oper<CreatePost> {
     };
 
     let inserted_post_id = inserted_post.id;
-    let updated_post =
-      match blocking(pool, move |conn| Post::update_ap_id(conn, inserted_post_id)).await? {
-        Ok(post) => post,
-        Err(_e) => return Err(APIError::err("couldnt_create_post").into()),
-      };
+    let updated_post = match blocking(pool, move |conn| {
+      let apub_id =
+        make_apub_endpoint(EndpointType::Post, &inserted_post_id.to_string()).to_string();
+      Post::update_ap_id(conn, inserted_post_id, apub_id)
+    })
+    .await?
+    {
+      Ok(post) => post,
+      Err(_e) => return Err(APIError::err("couldnt_create_post").into()),
+    };
 
     updated_post.send_create(&user, &self.client, pool).await?;
 

server/src/api/site.rs

@@ -1,31 +1,28 @@
 use super::user::Register;
 use crate::{
-  api::{APIError, Oper, Perform},
+  api::{claims::Claims, APIError, Oper, Perform},
   apub::fetcher::search_by_apub_id,
   blocking,
-  db::{
-    category::*,
-    comment_view::*,
-    community_view::*,
-    moderator::*,
-    moderator_views::*,
-    post_view::*,
-    site::*,
-    site_view::*,
-    user::*,
-    user_view::*,
-    Crud,
-    SearchType,
-    SortType,
-  },
-  naive_now,
-  settings::Settings,
-  slur_check,
-  slurs_vec_to_str,
   websocket::{server::SendAllMessage, UserOperation, WebsocketInfo},
   DbPool,
   LemmyError,
 };
+use lemmy_db::{
+  category::*,
+  comment_view::*,
+  community_view::*,
+  moderator::*,
+  moderator_views::*,
+  naive_now,
+  post_view::*,
+  site::*,
+  site_view::*,
+  user_view::*,
+  Crud,
+  SearchType,
+  SortType,
+};
+use lemmy_utils::{settings::Settings, slur_check, slurs_vec_to_str};
 use log::{debug, info};
 use serde::{Deserialize, Serialize};
 use std::str::FromStr;

server/src/api/user.rs

@@ -1,53 +1,53 @@
 use crate::{
-  api::{APIError, Oper, Perform},
-  apub::{
-    extensions::signatures::generate_actor_keypair,
-    make_apub_endpoint,
-    ApubObjectType,
-    EndpointType,
-  },
+  api::{claims::Claims, APIError, Oper, Perform},
+  apub::ApubObjectType,
   blocking,
-  db::{
-    comment::*,
-    comment_view::*,
-    community::*,
-    community_view::*,
-    moderator::*,
-    password_reset_request::*,
-    post::*,
-    post_view::*,
-    private_message::*,
-    private_message_view::*,
-    site::*,
-    site_view::*,
-    user::*,
-    user_mention::*,
-    user_mention_view::*,
-    user_view::*,
-    Crud,
-    Followable,
-    Joinable,
-    ListingType,
-    SortType,
+  websocket::{
+    server::{JoinUserRoom, SendAllMessage, SendUserRoomMessage},
+    UserOperation,
+    WebsocketInfo,
   },
+  DbPool,
+  LemmyError,
+};
+use bcrypt::verify;
+use lemmy_db::{
+  comment::*,
+  comment_view::*,
+  community::*,
+  community_view::*,
+  moderator::*,
+  naive_now,
+  password_reset_request::*,
+  post::*,
+  post_view::*,
+  private_message::*,
+  private_message_view::*,
+  site::*,
+  site_view::*,
+  user::*,
+  user_mention::*,
+  user_mention_view::*,
+  user_view::*,
+  Crud,
+  Followable,
+  Joinable,
+  ListingType,
+  SortType,
+};
+use lemmy_utils::{
+  generate_actor_keypair,
   generate_random_string,
   is_valid_username,
+  make_apub_endpoint,
   naive_from_unix,
-  naive_now,
   remove_slurs,
   send_email,
   settings::Settings,
   slur_check,
   slurs_vec_to_str,
-  websocket::{
-    server::{JoinUserRoom, SendAllMessage, SendUserRoomMessage},
-    UserOperation,
-    WebsocketInfo,
-  },
-  DbPool,
-  LemmyError,
+  EndpointType,
 };
-use bcrypt::verify;
 use log::error;
 use serde::{Deserialize, Serialize};
 use std::str::FromStr;
@@ -264,7 +264,7 @@ impl Perform for Oper<Login> {
     // Fetch that username / email
     let username_or_email = data.username_or_email.clone();
     let user = match blocking(pool, move |conn| {
-      User_::find_by_email_or_username(conn, &username_or_email)
+      Claims::find_by_email_or_username(conn, &username_or_email)
     })
     .await?
     {
@@ -279,7 +279,9 @@ impl Perform for Oper<Login> {
     }
 
     // Return the jwt
-    Ok(LoginResponse { jwt: user.jwt() })
+    Ok(LoginResponse {
+      jwt: Claims::jwt(user, Settings::get().hostname),
+    })
   }
 }
 
@@ -421,7 +423,7 @@ impl Perform for Oper<Register> {
 
     // Return the jwt
     Ok(LoginResponse {
-      jwt: inserted_user.jwt(),
+      jwt: Claims::jwt(inserted_user, Settings::get().hostname),
     })
   }
 }
@@ -532,7 +534,7 @@ impl Perform for Oper<SaveUserSettings> {
 
     // Return the jwt
     Ok(LoginResponse {
-      jwt: updated_user.jwt(),
+      jwt: Claims::jwt(updated_user, Settings::get().hostname),
     })
   }
 }
@@ -1155,7 +1157,7 @@ impl Perform for Oper<PasswordChange> {
 
     // Return the jwt
     Ok(LoginResponse {
-      jwt: updated_user.jwt(),
+      jwt: Claims::jwt(updated_user, Settings::get().hostname),
     })
   }
 }
@@ -1213,7 +1215,12 @@ impl Perform for Oper<CreatePrivateMessage> {
 
     let inserted_private_message_id = inserted_private_message.id;
     let updated_private_message = match blocking(pool, move |conn| {
-      PrivateMessage::update_ap_id(&conn, inserted_private_message_id)
+      let apub_id = make_apub_endpoint(
+        EndpointType::PrivateMessage,
+        &inserted_private_message_id.to_string(),
+      )
+      .to_string();
+      PrivateMessage::update_ap_id(&conn, inserted_private_message_id, apub_id)
     })
     .await?
     {

server/src/apub/activities.rs

@@ -1,12 +1,18 @@
 use crate::{
-  apub::{extensions::signatures::sign, is_apub_id_valid, ActorType},
-  db::{activity::insert_activity, community::Community, user::User_},
+  apub::{
+    community::do_announce,
+    extensions::signatures::sign,
+    insert_activity,
+    is_apub_id_valid,
+    ActorType,
+  },
   request::retry_custom,
   DbPool,
   LemmyError,
 };
 use activitystreams::{context, object::properties::ObjectProperties, public, Activity, Base};
 use actix_web::client::Client;
+use lemmy_db::{community::Community, user::User_};
 use log::debug;
 use serde::Serialize;
 use std::fmt::Debug;
@@ -43,7 +49,7 @@ where
 
   // if this is a local community, we need to do an announce from the community instead
   if community.local {
-    Community::do_announce(activity, &community, creator, client, pool).await?;
+    do_announce(activity, &community, creator, client, pool).await?;
   } else {
     send_activity(client, &activity, creator, to).await?;
   }

server/src/apub/comment.rs

@@ -17,19 +17,9 @@ use crate::{
     ToApub,
   },
   blocking,
-  convert_datetime,
-  db::{
-    comment::{Comment, CommentForm},
-    community::Community,
-    post::Post,
-    user::User_,
-    Crud,
-  },
   routes::DbPoolParam,
-  scrape_text_for_mentions,
   DbPool,
   LemmyError,
-  MentionData,
 };
 use activitystreams::{
   activity::{Create, Delete, Dislike, Like, Remove, Undo, Update},
@@ -40,6 +30,14 @@ use activitystreams::{
 use activitystreams_new::object::Tombstone;
 use actix_web::{body::Body, client::Client, web::Path, HttpResponse};
 use itertools::Itertools;
+use lemmy_db::{
+  comment::{Comment, CommentForm},
+  community::Community,
+  post::Post,
+  user::User_,
+  Crud,
+};
+use lemmy_utils::{convert_datetime, scrape_text_for_mentions, MentionData};
 use log::debug;
 use serde::Deserialize;
 

server/src/apub/community.rs

@@ -7,20 +7,13 @@ use crate::{
     extensions::group_extensions::GroupExtension,
     fetcher::get_or_fetch_and_upsert_remote_user,
     get_shared_inbox,
+    insert_activity,
     ActorType,
     FromApub,
     GroupExt,
     ToApub,
   },
   blocking,
-  convert_datetime,
-  db::{
-    activity::insert_activity,
-    community::{Community, CommunityForm},
-    community_view::{CommunityFollowerView, CommunityModeratorView},
-    user::User_,
-  },
-  naive_now,
   routes::DbPoolParam,
   DbPool,
   LemmyError,
@@ -44,6 +37,13 @@ use activitystreams_new::{
 };
 use actix_web::{body::Body, client::Client, web, HttpResponse};
 use itertools::Itertools;
+use lemmy_db::{
+  community::{Community, CommunityForm},
+  community_view::{CommunityFollowerView, CommunityModeratorView},
+  naive_now,
+  user::User_,
+};
+use lemmy_utils::convert_datetime;
 use serde::{Deserialize, Serialize};
 use std::{fmt::Debug, str::FromStr};
 
@@ -462,39 +462,37 @@ pub async fn get_apub_community_followers(
   Ok(create_apub_response(&collection))
 }
 
-impl Community {
-  pub async fn do_announce<A>(
-    activity: A,
-    community: &Community,
-    sender: &dyn ActorType,
-    client: &Client,
-    pool: &DbPool,
-  ) -> Result<HttpResponse, LemmyError>
-  where
-    A: Activity + Base + Serialize + Debug,
-  {
-    let mut announce = Announce::default();
-    populate_object_props(
-      &mut announce.object_props,
-      vec![community.get_followers_url()],
-      &format!("{}/announce/{}", community.actor_id, uuid::Uuid::new_v4()),
-    )?;
-    announce
-      .announce_props
-      .set_actor_xsd_any_uri(community.actor_id.to_owned())?
-      .set_object_base_box(BaseBox::from_concrete(activity)?)?;
-
-    insert_activity(community.creator_id, announce.clone(), true, pool).await?;
-
-    // dont send to the instance where the activity originally came from, because that would result
-    // in a database error (same data inserted twice)
-    let mut to = community.get_follower_inboxes(pool).await?;
-
-    // this seems to be the "easiest" stable alternative for remove_item()
-    to.retain(|x| *x != sender.get_shared_inbox_url());
-
-    send_activity(client, &announce, community, to).await?;
-
-    Ok(HttpResponse::Ok().finish())
-  }
+pub async fn do_announce<A>(
+  activity: A,
+  community: &Community,
+  sender: &dyn ActorType,
+  client: &Client,
+  pool: &DbPool,
+) -> Result<HttpResponse, LemmyError>
+where
+  A: Activity + Base + Serialize + Debug,
+{
+  let mut announce = Announce::default();
+  populate_object_props(
+    &mut announce.object_props,
+    vec![community.get_followers_url()],
+    &format!("{}/announce/{}", community.actor_id, uuid::Uuid::new_v4()),
+  )?;
+  announce
+    .announce_props
+    .set_actor_xsd_any_uri(community.actor_id.to_owned())?
+    .set_object_base_box(BaseBox::from_concrete(activity)?)?;
+
+  insert_activity(community.creator_id, announce.clone(), true, pool).await?;
+
+  // dont send to the instance where the activity originally came from, because that would result
+  // in a database error (same data inserted twice)
+  let mut to = community.get_follower_inboxes(pool).await?;
+
+  // this seems to be the "easiest" stable alternative for remove_item()
+  to.retain(|x| *x != sender.get_shared_inbox_url());
+
+  send_activity(client, &announce, community, to).await?;
+
+  Ok(HttpResponse::Ok().finish())
 }

server/src/apub/community_inbox.rs

@@ -2,21 +2,21 @@ use crate::{
   apub::{
     extensions::signatures::verify,
     fetcher::{get_or_fetch_and_upsert_remote_community, get_or_fetch_and_upsert_remote_user},
+    insert_activity,
     ActorType,
   },
   blocking,
-  db::{
-    activity::insert_activity,
-    community::{Community, CommunityFollower, CommunityFollowerForm},
-    user::User_,
-    Followable,
-  },
   routes::{ChatServerParam, DbPoolParam},
   LemmyError,
 };
 use activitystreams::activity::Undo;
 use activitystreams_new::activity::Follow;
 use actix_web::{client::Client, web, HttpRequest, HttpResponse};
+use lemmy_db::{
+  community::{Community, CommunityFollower, CommunityFollowerForm},
+  user::User_,
+  Followable,
+};
 use log::debug;
 use serde::Deserialize;
 use std::fmt::Debug;

server/src/apub/extensions/group_extensions.rs

@@ -1,9 +1,7 @@
-use crate::{
-  db::{category::Category, Crud},
-  LemmyError,
-};
+use crate::LemmyError;
 use activitystreams::{ext::Extension, Actor};
 use diesel::PgConnection;
+use lemmy_db::{category::Category, Crud};
 use serde::{Deserialize, Serialize};
 
 #[derive(Clone, Debug, Default, Deserialize, Serialize)]

server/src/apub/extensions/signatures.rs

@@ -9,7 +9,6 @@ use log::debug;
 use openssl::{
   hash::MessageDigest,
   pkey::PKey,
-  rsa::Rsa,
   sign::{Signer, Verifier},
 };
 use serde::{Deserialize, Serialize};
@@ -19,23 +18,6 @@ lazy_static! {
   static ref HTTP_SIG_CONFIG: Config = Config::new();
 }
 
-pub struct Keypair {
-  pub private_key: String,
-  pub public_key: String,
-}
-
-/// Generate the asymmetric keypair for ActivityPub HTTP signatures.
-pub fn generate_actor_keypair() -> Result<Keypair, LemmyError> {
-  let rsa = Rsa::generate(2048)?;
-  let pkey = PKey::from_rsa(rsa)?;
-  let public_key = pkey.public_key_to_pem()?;
-  let private_key = pkey.private_key_to_pem_pkcs8()?;
-  Ok(Keypair {
-    private_key: String::from_utf8(private_key)?,
-    public_key: String::from_utf8(public_key)?,
-  })
-}
-
 /// Signs request headers with the given keypair.
 pub async fn sign(
   request: ClientRequest,

server/src/apub/fetcher.rs

@@ -1,29 +1,7 @@
 use crate::{
   api::site::SearchResponse,
-  apub::{
-    get_apub_protocol_string,
-    is_apub_id_valid,
-    FromApub,
-    GroupExt,
-    PageExt,
-    PersonExt,
-    APUB_JSON_CONTENT_TYPE,
-  },
+  apub::{is_apub_id_valid, FromApub, GroupExt, PageExt, PersonExt, APUB_JSON_CONTENT_TYPE},
   blocking,
-  db::{
-    comment::{Comment, CommentForm},
-    comment_view::CommentView,
-    community::{Community, CommunityForm, CommunityModerator, CommunityModeratorForm},
-    community_view::CommunityView,
-    post::{Post, PostForm},
-    post_view::PostView,
-    user::{UserForm, User_},
-    user_view::UserView,
-    Crud,
-    Joinable,
-    SearchType,
-  },
-  naive_now,
   request::{retry, RecvError},
   routes::nodeinfo::{NodeInfo, NodeInfoWellKnown},
   DbPool,
@@ -34,6 +12,21 @@ use activitystreams_new::{base::BaseExt, prelude::*, primitives::XsdAnyUri};
 use actix_web::client::Client;
 use chrono::NaiveDateTime;
 use diesel::{result::Error::NotFound, PgConnection};
+use lemmy_db::{