diff --git a/.github/scripts/ignored-files.sh b/.github/scripts/ignored-files.sh new file mode 100755 index 0000000000..344818a410 --- /dev/null +++ b/.github/scripts/ignored-files.sh @@ -0,0 +1,7 @@ +#!/bin/sh +set -euxo pipefail +apk add git +# GHA workspace file ownership is "dubious" +git config --global --add safe.directory /github/workspace +IGNORED=$(git ls-files --cached -i --exclude-standard) +if [[ "$IGNORED" ]]; then echo "Ignored files present:\n$IGNORED\n"; exit 1; fi diff --git a/.github/scripts/prettier.sh b/.github/scripts/prettier.sh new file mode 100755 index 0000000000..416bf1ab96 --- /dev/null +++ b/.github/scripts/prettier.sh @@ -0,0 +1,2 @@ +#!/bin/sh +prettier -c . '!**/volumes' '!**/dist' '!target' '!**/translations' '!api_tests/pnpm-lock.yaml' diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml new file mode 100644 index 0000000000..a1a8ec6cb9 --- /dev/null +++ b/.github/workflows/build.yaml @@ -0,0 +1,78 @@ +name: Create and publish a Docker image +on: + push: + branches: + - lw-0.* + +env: + REGISTRY: ghcr.io + IMAGE_NAME: ${{ github.repository }} + +permissions: {} + +jobs: + build-and-push-image: + name: Build and push + + runs-on: ubuntu-latest + + permissions: + contents: read + packages: write # required to push container image to ghcr + + steps: + - name: Checkout + uses: actions/checkout@v6 + with: + fetch-depth: 0 + persist-credentials: false + submodules: recursive + + - name: Setup Git metadata + id: git-meta + env: + GH_TOKEN: ${{ github.token }} + run: | + # ensure we're always taking upstream tags into account to calculate distance + upstream="$(gh repo view "${{ github.repository }}" --json parent --jq '.parent.owner.login + "/" + .parent.name')" + git remote add upstream "https://github.com/$upstream.git" + git fetch upstream --tags + echo "git-version=$(git describe --tags)" >> "$GITHUB_OUTPUT" + + - name: Docker meta + id: meta + uses: docker/metadata-action@v5 + with: + images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} + tags: | + type=raw,value=${{ steps.git-meta.outputs.git-version }} + type=sha,format=long + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Log in to the Container registry + uses: docker/login-action@v3 + with: + registry: ${{ env.REGISTRY }} + username: ${{ github.repository_owner }} + password: ${{ github.token }} + + - name: Build and push Docker image + uses: docker/build-push-action@v6 + with: + context: . + file: docker/Dockerfile + build-args: | + RUST_RELEASE_MODE=release + CARGO_BUILD_FEATURES=opentelemetry + platforms: linux/amd64 + push: true + tags: ${{ steps.meta.outputs.tags }} + labels: | + ${{ steps.meta.outputs.labels }} + com.datadoghq.tags.service=${{ github.event.repository.name }} + com.datadoghq.tags.version=${{ steps.git-meta.outputs.git-version }} + annotations: ${{ steps.meta.outputs.annotations }} + cache-from: type=gha + cache-to: type=gha,mode=max diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml new file mode 100644 index 0000000000..48bef9717c --- /dev/null +++ b/.github/workflows/test.yaml @@ -0,0 +1,227 @@ +# This workflow loosely replicates the logic in .woodpecker.yml +name: Run Lemmy tests +on: + push: + branches: + - lw-0.* + +env: + CI_RUST_VERSION: "1.81" + CARGO_HOME: .cargo_home + POSTGRES_USER: lemmy + POSTGRES_PASSWORD: password + HOST_DATABASE_URL: postgres://lemmy:password@127.0.0.1:5432/lemmy + CONTAINER_DATABASE_URL: postgres://lemmy:password@database:5432/lemmy + +permissions: {} + +jobs: + test: + name: test + + services: + database: + image: postgres:16-alpine + env: + POSTGRES_USER: lemmy + POSTGRES_PASSWORD: password + options: >- + --health-cmd pg_isready + --health-interval 10s + --health-timeout 5s + --health-retries 5 + ports: + - 5432:5432 + + runs-on: ubuntu-latest + timeout-minutes: 60 + steps: + - name: Checkout + uses: actions/checkout@v6 + with: + persist-credentials: false + submodules: recursive + show-progress: "false" + + - name: Prettier + uses: docker://tmknom/prettier:3.0.0 + with: + # args is rather janky for this, as it doesn't deal well with quotes + # and multiple lines, so it's easier to just put more complex + # commands in a custom script. + args: ./.github/scripts/prettier.sh + + - name: TOML fmt + uses: docker://tamasfe/taplo:0.8.1 + with: + args: format --check + + - name: SQL fmt + uses: docker://backplane/pgformatter:latest + with: + args: ./scripts/sql_format_check.sh + + - name: Get date for cache key + id: get-date + run: | + echo "date=$(/bin/date -u "+%Y%m%d")" >> $GITHUB_OUTPUT + + - name: Install dependencies from apt + run: sudo apt-get update && sudo apt-get install -y bash curl postgresql-client + + - name: Cargo home cache + uses: actions/cache@v5 + with: + path: ${{ env.CARGO_HOME }} + key: rust-cargo-home-${{ env.CI_RUST_VERSION }}-${{ hashFiles('**/Cargo.toml', '**/Cargo.lock') }} + restore-keys: | + rust-cargo-home-${{ env.CI_RUST_VERSION }}- + rust-cargo-home- + + # https://github.com/rust-lang/rustup/issues/2886 + - name: Disable rustup self update + run: rustup set auto-self-update disable + + - name: Set Rust version to ${{ env.CI_RUST_VERSION }} + run: rustup default "$CI_RUST_VERSION" + + - name: Install Rust nightly toolchain + run: rustup toolchain install nightly + + - name: Cargo fmt + run: | + rustup component add --toolchain nightly rustfmt + cargo +nightly fmt -- --check + + # Unlike Lemmy's woodpecker, we have persistent CARGO_HOME. This causes + # some issues with machete, but it doesn't have a way to exclude + # arbitrary paths, so we cheat by temporarily moving it into the + # target dir. + # https://github.com/bnjbvr/cargo-machete/issues/49 + - name: Cargo machete + env: + CARGO_HOME: target/${{ env.CARGO_HOME }} + ORIG_CARGO_HOME: ${{ env.CARGO_HOME }} + run: | + test -d "$ORIG_CARGO_HOME" && mkdir -v target && mv -v "$ORIG_CARGO_HOME" "$CARGO_HOME" || true + cargo +nightly install cargo-machete + cargo +nightly machete --skip-target-dir + mv -v "$CARGO_HOME" "$ORIG_CARGO_HOME" && rmdir -v target + + - name: Ignored files + uses: docker://alpine:3 + with: + args: ./.github/scripts/ignored-files.sh + + - name: check_api_common_default_features + run: cargo check --package lemmy_api_common + + - name: lemmy_api_common_doesnt_depend_on_diesel + run: | + ! cargo tree -p lemmy_api_common --no-default-features -i diesel + + - name: lemmy_api_common_works_with_wasm + run: | + rustup target add wasm32-unknown-unknown + cargo check --target wasm32-unknown-unknown -p lemmy_api_common + + - name: check_defaults_hjson_updated + env: + LEMMY_CONFIG_LOCATION: ./config/config.hjson + run: | + ./scripts/update_config_defaults.sh config/defaults_current.hjson + diff config/defaults.hjson config/defaults_current.hjson + + - name: Install diesel cli + run: cargo install --locked diesel_cli@2.2.8 --no-default-features --features postgres + + - name: Check diesel schema + env: + DATABASE_URL: ${{ env.HOST_DATABASE_URL }} + run: | + set -euxo pipefail + export PATH="$CARGO_HOME/bin:$PATH" + diesel migration run + diesel print-schema --config-file=diesel.toml > tmp.schema + diff tmp.schema crates/db_schema/src/schema.rs + + - name: Check DB performance + env: + DATABASE_URL: ${{ env.HOST_DATABASE_URL }} + LEMMY_CONFIG_LOCATION: ./config/config.hjson + RUST_BACKTRACE: "1" + run: cargo run --package lemmy_db_perf -- --posts 10 --read-post-pages 1 + + - name: Cargo clippy + run: | + rustup component add clippy + cargo clippy --workspace --tests --all-targets --features console -- -D warnings + + - name: Cargo build + run: | + cargo build + mv target/debug/lemmy_server target/lemmy_server + + - name: Cargo test + env: + RUST_BACKTRACE: "1" + LEMMY_CONFIG_LOCATION: ../../config/config.hjson + LEMMY_DATABASE_URL: ${{ env.HOST_DATABASE_URL }} + LEMMY_TEST_FAST_FEDERATION: "1" + run: cargo test --workspace --no-fail-fast + + - name: Check diesel migration + env: + RUST_BACKTRACE: "1" + DATABASE_URL: ${{ env.HOST_DATABASE_URL }} + LEMMY_DATABASE_URL: ${{ env.HOST_DATABASE_URL }} + PGUSER: ${{ env.POSTGRES_USER }} + PGPASSWORD: ${{ env.POSTGRES_PASSWORD }} + PGHOST: 127.0.0.1 + PGDATABASE: lemmy + run: | + set -euxo pipefail + export PATH="$CARGO_HOME/bin:$PATH" + # aliases don't work in non-interactive shells by default + shopt -s expand_aliases + # Run all migrations + diesel migration run + psql -c "DROP SCHEMA IF EXISTS r CASCADE;" + pg_dump --no-owner --no-privileges --no-table-access-method --schema-only --no-sync --restrict-key=empty -f before.sqldump + # Make sure that the newest migration is revertable without the `r` schema + diesel migration redo + # Run schema setup twice, which fails on the 2nd time if `DROP SCHEMA IF EXISTS r CASCADE` drops the wrong things + alias lemmy_schema_setup="target/lemmy_server --disable-scheduled-tasks --disable-http-server --disable-activity-sending" + lemmy_schema_setup + lemmy_schema_setup + # Make sure that the newest migration is revertable with the `r` schema + diesel migration redo + # Check for changes in the schema, which would be caused by an incorrect migration + psql -c "DROP SCHEMA IF EXISTS r CASCADE;" + pg_dump --no-owner --no-privileges --no-table-access-method --schema-only --no-sync --restrict-key=empty -f after.sqldump + diff before.sqldump after.sqldump + + - name: Set up pnpm + uses: pnpm/action-setup@41ff72655975bd51cab0327fa583b6e92b6d3061 # v4.2.0 + with: + run_install: false + package_json_file: api_tests/package.json + + - name: Set up Node 20 + uses: actions/setup-node@v6 + with: + node-version: 20.x + cache: pnpm + cache-dependency-path: api_tests/pnpm-lock.yaml + + - name: Run federation tests + env: + DO_WRITE_HOSTS_FILE: "1" + LEMMY_DATABASE_URL: postgres://${{ env.POSTGRES_USER }}:${{ env.POSTGRES_PASSWORD }}@127.0.0.1:5432 + run: | + set -euxo pipefail + sed -i -e 's#>>/etc/hosts#| sudo tee -a /etc/hosts#' api_tests/prepare-drone-federation-test.sh + bash api_tests/prepare-drone-federation-test.sh + cd api_tests/ + pnpm i + pnpm api-test diff --git a/Cargo.lock b/Cargo.lock index 38cd6b9cc5..9d78f7aa51 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -39,7 +39,7 @@ dependencies = [ "moka", "once_cell", "pin-project-lite", - "rand 0.8.5", + "rand 0.8.6", "regex", "reqwest 0.11.27", "reqwest-middleware 0.2.5", @@ -177,7 +177,7 @@ dependencies = [ "log", "memchr", "mime", - "rand 0.8.5", + "rand 0.8.6", "serde", "serde_json", "serde_plain", @@ -948,7 +948,7 @@ dependencies = [ "hound", "image 0.24.9", "lodepng", - "rand 0.8.5", + "rand 0.8.6", "serde_json", ] @@ -1899,7 +1899,7 @@ checksum = "2e1f6c3800b304a6be0012039e2a45a322a093539c45ab818d9e6895a39c90fe" dependencies = [ "proc-macro2", "quote", - "rand 0.8.5", + "rand 0.8.6", "syn 1.0.109", ] @@ -3243,6 +3243,7 @@ dependencies = [ "actix-web", "anyhow", "bcrypt", + "chrono", "diesel-async", "futures", "lemmy_api_common", @@ -4104,7 +4105,7 @@ dependencies = [ "num-integer", "num-iter", "num-traits", - "rand 0.8.5", + "rand 0.8.6", "smallvec", "zeroize", ] @@ -4350,7 +4351,7 @@ dependencies = [ "opentelemetry 0.21.0", "ordered-float", "percent-encoding", - "rand 0.8.5", + "rand 0.8.6", "thiserror 1.0.69", "tokio", "tokio-stream", @@ -4539,7 +4540,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5d5285893bb5eb82e6aaf5d59ee909a06a16737a8970984dd7746ba9283498d6" dependencies = [ "phf_shared 0.10.0", - "rand 0.8.5", + "rand 0.8.6", ] [[package]] @@ -4549,7 +4550,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3c80231409c20246a13fddb31776fb942c38553c51e871f8cbd687a4cfb5843d" dependencies = [ "phf_shared 0.11.3", - "rand 0.8.5", + "rand 0.8.6", ] [[package]] @@ -5143,9 +5144,9 @@ checksum = "74765f6d916ee2faa39bc8e68e4f3ed8949b48cccdac59983d287a7cb71ce9c5" [[package]] name = "rand" -version = "0.8.5" +version = "0.8.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "34af8d1a0e25924bc5b7c43c079c942339d8f0a8b57c39049bef581b46327404" +checksum = "5ca0ecfa931c29007047d1bc58e623ab12e5590e8c7cc53200d5202b69266d8a" dependencies = [ "libc", "rand_chacha 0.3.1", @@ -5657,7 +5658,7 @@ version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "10e15a1550bf8261ce5cb3b40930bdcb8c38f9a0e930f1546a86f3fd53375c50" dependencies = [ - "rand 0.8.5", + "rand 0.8.6", "rustls 0.23.27", ] @@ -6886,7 +6887,7 @@ dependencies = [ "indexmap 1.9.3", "pin-project", "pin-project-lite", - "rand 0.8.5", + "rand 0.8.6", "slab", "tokio", "tokio-util", diff --git a/config/defaults.hjson b/config/defaults.hjson index 6aebfcb44c..d016ee366f 100644 --- a/config/defaults.hjson +++ b/config/defaults.hjson @@ -137,4 +137,22 @@ cors_origin: "lemmy.tld" # Print logs in JSON format. You can also disable ANSI colors in logs with env var `NO_COLOR`. json_logging: false + # Native automod configuration + # All variables inside are empty/false by default + fhf_automod_config: { + # Username of a local user used as actor for automated moderation actions. + # This user should be an instance moderator, although this is not enforced. + # Federation compatibility with non-admin users is unknown. + # Required for most native automod functionality + actor_username: "automod" + # This enables the scheduled task for resolving any reports about removed content when the + # creator is banned or deleted and the reported content is removed or deleted. + # Reports that were unresolved will not be touched by this. + # Requires actor_username to be set. + resolve_banned_or_deleted_creators_reports: false + # Set this to a number of days to enable automatic bans of users deleting their account within + # that many days of signup. Bans include content removal. + # Requires actor_username to be set. + ban_deleted_persons_created_within_days: 7 + } } diff --git a/crates/api/src/lib.rs b/crates/api/src/lib.rs index 2b8e12d372..811971d61e 100644 --- a/crates/api/src/lib.rs +++ b/crates/api/src/lib.rs @@ -42,6 +42,7 @@ pub mod private_message; pub mod private_message_report; pub mod site; pub mod sitemap; +pub mod vote_analytics; /// Converts the captcha to a base64 encoded wav audio file pub(crate) fn captcha_as_wav_base64(captcha: &Captcha) -> LemmyResult { diff --git a/crates/api/src/site/registration_applications/approve.rs b/crates/api/src/site/registration_applications/approve.rs index 7efee0fa84..0b5c19e8ae 100644 --- a/crates/api/src/site/registration_applications/approve.rs +++ b/crates/api/src/site/registration_applications/approve.rs @@ -91,6 +91,7 @@ pub async fn approve_registration_application( })) } +#[tracing::instrument(skip_all)] async fn send_application_approved_email( user: &LocalUserView, settings: &Settings, @@ -102,6 +103,7 @@ async fn send_application_approved_email( send_email(&subject, email, &user.person.name, &body, settings).await } +#[tracing::instrument(skip_all)] async fn send_application_denied_email( user: &LocalUserView, settings: &Settings, diff --git a/crates/api/src/vote_analytics/given_by_person.rs b/crates/api/src/vote_analytics/given_by_person.rs new file mode 100644 index 0000000000..86d71f869b --- /dev/null +++ b/crates/api/src/vote_analytics/given_by_person.rs @@ -0,0 +1,46 @@ +use activitypub_federation::config::Data; +use actix_web::web::{Json, Query}; +use chrono::{TimeZone, Utc}; +use lemmy_api_common::{context::LemmyContext, person::GetVoteAnalyticsByPerson, utils::is_admin}; +use lemmy_db_views::structs::LocalUserView; +use lemmy_db_views_actor::structs::VoteAnalyticsGivenByPersonView; +use lemmy_utils::{error::LemmyResult, LemmyErrorType}; + +#[tracing::instrument(skip(context))] +pub async fn get_vote_analytics_given_by_person( + data: Query, + context: Data, + local_user_view: LocalUserView, +) -> LemmyResult> { + is_admin(&local_user_view)?; + + let since = match data.start_time { + Some(t) => Some( + Utc + .timestamp_opt(t, 0) + .single() + .ok_or(LemmyErrorType::InvalidUnixTime)?, + ), + _ => None, + }; + let until = match data.end_time { + Some(t) => Some( + Utc + .timestamp_opt(t, 0) + .single() + .ok_or(LemmyErrorType::InvalidUnixTime)?, + ), + _ => None, + }; + + let view = VoteAnalyticsGivenByPersonView::read( + &mut context.pool(), + data.person_id, + since, + until, + data.limit, + ) + .await?; + + Ok(Json(view)) +} diff --git a/crates/api/src/vote_analytics/mod.rs b/crates/api/src/vote_analytics/mod.rs new file mode 100644 index 0000000000..83e74a8b9d --- /dev/null +++ b/crates/api/src/vote_analytics/mod.rs @@ -0,0 +1 @@ +pub mod given_by_person; diff --git a/crates/api_common/src/person.rs b/crates/api_common/src/person.rs index f61f784c20..d8f052491e 100644 --- a/crates/api_common/src/person.rs +++ b/crates/api_common/src/person.rs @@ -441,3 +441,13 @@ pub struct ListMedia { pub struct ListMediaResponse { pub images: Vec, } + +#[derive(Debug, Deserialize)] +#[cfg_attr(feature = "full", derive(TS))] +#[cfg_attr(feature = "full", ts(export))] +pub struct GetVoteAnalyticsByPerson { + pub person_id: PersonId, + pub start_time: Option, + pub end_time: Option, + pub limit: Option, +} diff --git a/crates/api_common/src/utils.rs b/crates/api_common/src/utils.rs index 2923757932..802d88d63a 100644 --- a/crates/api_common/src/utils.rs +++ b/crates/api_common/src/utils.rs @@ -418,6 +418,7 @@ pub fn honeypot_check(honeypot: &Option) -> LemmyResult<()> { } } +#[tracing::instrument(skip_all)] pub async fn send_email_to_user( local_user_view: &LocalUserView, subject: &str, diff --git a/crates/api_crud/Cargo.toml b/crates/api_crud/Cargo.toml index 0c67ff4e1a..f3e3ff1710 100644 --- a/crates/api_crud/Cargo.toml +++ b/crates/api_crud/Cargo.toml @@ -32,5 +32,8 @@ webmention = "0.6.0" accept-language = "3.1.0" diesel-async = { workspace = true, features = ["deadpool", "postgres"] } +# FHF deps +chrono = { workspace = true } + [package.metadata.cargo-machete] ignored = ["futures"] diff --git a/crates/api_crud/src/user/delete.rs b/crates/api_crud/src/user/delete.rs index 363230d836..c02488813d 100644 --- a/crates/api_crud/src/user/delete.rs +++ b/crates/api_crud/src/user/delete.rs @@ -37,10 +37,90 @@ pub async fn delete_account( LoginToken::invalidate_all(&mut context.pool(), local_user_view.local_user.id).await?; ActivityChannel::submit_activity( - SendActivityData::DeleteUser(local_user_view.person, data.delete_content), + SendActivityData::DeleteUser(local_user_view.person.clone(), data.delete_content), &context, ) .await?; + // FHF anti-spam measure + if let Some(account_age_threshold) = context + .settings() + .fhf_automod_config + .ban_deleted_persons_created_within_days + { + if !data.delete_content + && local_user_view.person.published + > lemmy_db_schema::utils::naive_now() - chrono::Days::new(account_age_threshold) + { + tracing::info!( + "[FHF AutoMod][BanLocalDeletedUser] Issuing ban for deletion of recently created user {}", + local_user_view.person.name + ); + + if let Some(automod_username) = &context.settings().fhf_automod_config.actor_username { + // keep these here to minimize risk of merge conflicts with upstream + use lemmy_api_common::utils::remove_user_data; + use lemmy_db_schema::{ + source::{ + moderator::{ModBan, ModBanForm}, + person::PersonUpdateForm, + }, + traits::{ApubActor, Crud}, + }; + use lemmy_utils::error::LemmyErrorExt; + + // todo: this might be better to just log and not return an error + let automod_person = + Person::read_from_name(&mut context.pool(), automod_username.as_str(), false) + .await? + .ok_or(LemmyErrorType::CouldntFindPerson)?; + + // this is more or less the same logic as lemmy_api::local_user::ban_person(), but we can't + // use it here due to circular dependencies. + + let person = Person::update( + &mut context.pool(), + local_user_view.person.id, + &PersonUpdateForm { + banned: Some(true), + ban_expires: Some(None), + ..Default::default() + }, + ) + .await + .with_lemmy_type(LemmyErrorType::CouldntUpdateUser)?; + + remove_user_data(person.id, &context).await?; + + let reason = Some("automod".to_string()); + + let form = ModBanForm { + mod_person_id: automod_person.id, + other_person_id: local_user_view.person.id, + reason: reason.clone(), + banned: Some(true), + expires: None, + }; + + ModBan::create(&mut context.pool(), &form).await?; + + ActivityChannel::submit_activity( + SendActivityData::BanFromSite { + moderator: automod_person, + banned_user: local_user_view.person.clone(), + reason, + remove_data: Some(true), + ban: true, + expires: None, + }, + &context, + ) + .await?; + } else { + tracing::error!("[FHF AutoMod][BanLocalDeletedUser] Unable to ban user: no automod user defined in configuration"); + } + } + } + Ok(Json(SuccessResponse::default())) } diff --git a/crates/apub/src/activities/deletion/mod.rs b/crates/apub/src/activities/deletion/mod.rs index c9d268e749..3e1aeb0d3a 100644 --- a/crates/apub/src/activities/deletion/mod.rs +++ b/crates/apub/src/activities/deletion/mod.rs @@ -266,6 +266,141 @@ async fn receive_delete_action( purge_user_account(person.id, context).await?; } else { Person::delete_account(&mut context.pool(), person.id).await?; + + // FHF anti-spam measure + if let Some(account_age_threshold) = context + .settings() + .fhf_automod_config + .ban_deleted_persons_created_within_days + { + if person.published + > lemmy_db_schema::utils::naive_now() - chrono::Days::new(account_age_threshold) + { + tracing::info!( + "[FHF AutoMod][BanFederatedDeletedUser] Issuing ban for deletion of recently created user {}", + person.actor_id + ); + + if let Some(automod_username) = &context.settings().fhf_automod_config.actor_username { + // keep these here to minimize risk of merge conflicts with upstream + use lemmy_api_common::{ + community::BanFromCommunity, + send_activity::{ActivityChannel, SendActivityData}, + utils::remove_user_data, + }; + use lemmy_db_schema::{ + source::{ + community::{ + CommunityFollower, + CommunityFollowerForm, + CommunityPersonBan, + CommunityPersonBanForm, + }, + moderator::{ModBan, ModBanForm, ModBanFromCommunity, ModBanFromCommunityForm}, + person::PersonUpdateForm, + }, + traits::{Bannable, Followable}, + }; + use lemmy_db_views::structs::LocalUserView; + use lemmy_utils::error::LemmyErrorExt; + + // todo: this might be better to just log and not return an error + let automod_local_user_view = + LocalUserView::read_from_name(&mut context.pool(), automod_username.as_str()) + .await? + .ok_or(LemmyErrorType::CouldntFindPerson)?; + + let person = Person::update( + &mut context.pool(), + person.id, + &PersonUpdateForm { + banned: Some(true), + ban_expires: Some(None), + ..Default::default() + }, + ) + .await + .with_lemmy_type(LemmyErrorType::CouldntUpdateUser)?; + + remove_user_data(person.id, context).await?; + + let reason = Some("automod".to_string()); + + let form = ModBanForm { + mod_person_id: automod_local_user_view.person.id, + other_person_id: person.id, + reason: reason.clone(), + banned: Some(true), + expires: None, + }; + + ModBan::create(&mut context.pool(), &form).await?; + + // this is basically lemmy_api::ban_nonlocal_user_from_local_communities() + let ids = Person::list_local_community_ids(&mut context.pool(), person.id).await?; + + for community_id in ids { + // Ban them from our local communities + let community_user_ban_form = CommunityPersonBanForm { + community_id, + person_id: person.id, + expires: None, + }; + + // Ignore all errors for these + CommunityPersonBan::ban(&mut context.pool(), &community_user_ban_form) + .await + .ok(); + + // Also unsubscribe them from the community, if they are subscribed + let community_follower_form = CommunityFollowerForm { + community_id, + person_id: person.id, + pending: false, + }; + + CommunityFollower::unfollow(&mut context.pool(), &community_follower_form) + .await + .ok(); + + // Mod tables + let form = ModBanFromCommunityForm { + mod_person_id: automod_local_user_view.person.id, + other_person_id: person.id, + community_id, + reason: reason.clone(), + banned: Some(true), + expires: None, + }; + + ModBanFromCommunity::create(&mut context.pool(), &form).await?; + + // Federate the ban from community + let ban_from_community = BanFromCommunity { + community_id, + person_id: person.id, + ban: true, + reason: reason.clone(), + remove_data: Some(true), + expires: None, + }; + + ActivityChannel::submit_activity( + SendActivityData::BanFromCommunity { + moderator: automod_local_user_view.person.clone(), + community_id, + target: person.clone(), + data: ban_from_community, + }, + context, + ) + .await?; + } + } else { + tracing::error!("[FHF AutoMod][BanFederatedDeletedUser] Unable to ban user: no automod user defined in configuration"); + } + } + } } } DeletableObjects::Post(post) => { diff --git a/crates/apub/src/lib.rs b/crates/apub/src/lib.rs index c8506da52b..a128b59d6c 100644 --- a/crates/apub/src/lib.rs +++ b/crates/apub/src/lib.rs @@ -77,7 +77,8 @@ impl UrlVerifier for VerifyUrlData { /// - the correct scheme (either http or https) /// - URL being in the allowlist (if it is active) /// - URL not being in the blocklist (if it is active) -#[tracing::instrument(skip(local_site_data))] +// tracing this generates a very large amount of traces +//#[tracing::instrument(skip(local_site_data))] fn check_apub_id_valid(apub_id: &Url, local_site_data: &LocalSiteData) -> LemmyResult<()> { let domain = apub_id .domain() diff --git a/crates/apub/src/objects/comment.rs b/crates/apub/src/objects/comment.rs index f76590be17..a11335ef7d 100644 --- a/crates/apub/src/objects/comment.rs +++ b/crates/apub/src/objects/comment.rs @@ -64,7 +64,7 @@ impl Object for ApubComment { None } - #[tracing::instrument(skip_all)] + #[tracing::instrument(skip(context))] async fn read_from_id( object_id: Url, context: &Data, diff --git a/crates/apub/src/objects/community.rs b/crates/apub/src/objects/community.rs index da27671adb..58c8c39f96 100644 --- a/crates/apub/src/objects/community.rs +++ b/crates/apub/src/objects/community.rs @@ -73,7 +73,7 @@ impl Object for ApubCommunity { Some(self.last_refreshed_at) } - #[tracing::instrument(skip_all)] + #[tracing::instrument(skip(context))] async fn read_from_id( object_id: Url, context: &Data, diff --git a/crates/apub/src/objects/instance.rs b/crates/apub/src/objects/instance.rs index c67a223e0f..5296fe295a 100644 --- a/crates/apub/src/objects/instance.rs +++ b/crates/apub/src/objects/instance.rs @@ -78,7 +78,7 @@ impl Object for ApubSite { Some(self.last_refreshed_at) } - #[tracing::instrument(skip_all)] + #[tracing::instrument(skip(data))] async fn read_from_id(object_id: Url, data: &Data) -> LemmyResult> { Ok( Site::read_from_apub_id(&mut data.pool(), &object_id.into()) diff --git a/crates/apub/src/objects/person.rs b/crates/apub/src/objects/person.rs index 61ff04622e..16393b2f4f 100644 --- a/crates/apub/src/objects/person.rs +++ b/crates/apub/src/objects/person.rs @@ -75,7 +75,7 @@ impl Object for ApubPerson { Some(self.last_refreshed_at) } - #[tracing::instrument(skip_all)] + #[tracing::instrument(skip(context))] async fn read_from_id( object_id: Url, context: &Data, diff --git a/crates/apub/src/objects/post.rs b/crates/apub/src/objects/post.rs index 18151898ac..e74f6359fd 100644 --- a/crates/apub/src/objects/post.rs +++ b/crates/apub/src/objects/post.rs @@ -80,7 +80,7 @@ impl Object for ApubPost { None } - #[tracing::instrument(skip_all)] + #[tracing::instrument(skip(context))] async fn read_from_id( object_id: Url, context: &Data, diff --git a/crates/apub/src/objects/private_message.rs b/crates/apub/src/objects/private_message.rs index fc96973917..06164496aa 100644 --- a/crates/apub/src/objects/private_message.rs +++ b/crates/apub/src/objects/private_message.rs @@ -59,7 +59,7 @@ impl Object for ApubPrivateMessage { None } - #[tracing::instrument(skip_all)] + #[tracing::instrument(skip(context))] async fn read_from_id( object_id: Url, context: &Data, diff --git a/crates/db_schema/src/impls/community.rs b/crates/db_schema/src/impls/community.rs index eaf35a90d6..894e1257f1 100644 --- a/crates/db_schema/src/impls/community.rs +++ b/crates/db_schema/src/impls/community.rs @@ -50,6 +50,7 @@ use diesel::{ }; use diesel_async::RunQueryDsl; use lemmy_utils::error::{LemmyErrorType, LemmyResult}; +use std::collections::HashMap; #[async_trait] impl Crud for Community { @@ -147,6 +148,26 @@ impl Community { Ok(community_) } + pub async fn read_many( + pool: &mut DbPool<'_>, + community_ids: &[CommunityId], + is_admin: bool, + ) -> Result, Error> { + let conn = &mut get_conn(pool).await?; + let mut query = community::table + .filter(community::id.eq_any(community_ids)) + .into_boxed(); + if !is_admin { + query = query + .filter(community::deleted.eq(false)) + .filter(community::removed.eq(false)); + } + let communities: Vec = query.get_results(conn).await?; + Ok(HashMap::from_iter( + communities.iter().map(|c| (c.id, c.clone())), + )) + } + /// Get the community which has a given moderators or featured url, also return the collection /// type pub async fn get_by_collection_url( diff --git a/crates/db_schema/src/impls/email_verification.rs b/crates/db_schema/src/impls/email_verification.rs index b4951cf733..a0faa3af36 100644 --- a/crates/db_schema/src/impls/email_verification.rs +++ b/crates/db_schema/src/impls/email_verification.rs @@ -34,7 +34,7 @@ impl EmailVerification { let conn = &mut get_conn(pool).await?; email_verification .filter(verification_token.eq(token)) - .filter(published.gt(now.into_sql::() - 7.days())) + .filter(published.gt(now.into_sql::() - 90.days())) .first(conn) .await .optional() diff --git a/crates/db_schema/src/impls/person.rs b/crates/db_schema/src/impls/person.rs index a7802fddb4..ce65e4b5f1 100644 --- a/crates/db_schema/src/impls/person.rs +++ b/crates/db_schema/src/impls/person.rs @@ -21,6 +21,7 @@ use diesel::{ QueryDsl, }; use diesel_async::RunQueryDsl; +use std::collections::HashMap; #[async_trait] impl Crud for Person { @@ -97,6 +98,24 @@ impl Person { .await } + pub async fn read_many( + pool: &mut DbPool<'_>, + person_ids: &[PersonId], + is_admin: bool, + ) -> Result, Error> { + let conn = &mut get_conn(pool).await?; + let mut query = person::table + .filter(person::id.eq_any(person_ids)) + .into_boxed(); + if !is_admin { + query = query.filter(person::deleted.eq(false)); + } + let persons: Vec = query.get_results(conn).await?; + Ok(HashMap::from_iter( + persons.iter().map(|p| (p.id, p.clone())), + )) + } + /// Lists local community ids for all posts and comments for a given creator. pub async fn list_local_community_ids( pool: &mut DbPool<'_>, diff --git a/crates/db_views_actor/src/lib.rs b/crates/db_views_actor/src/lib.rs index e9f8e41890..56db48ebcb 100644 --- a/crates/db_views_actor/src/lib.rs +++ b/crates/db_views_actor/src/lib.rs @@ -19,3 +19,5 @@ pub mod person_mention_view; #[cfg(feature = "full")] pub mod person_view; pub mod structs; +#[cfg(feature = "full")] +mod vote_analytics_given_by_view; diff --git a/crates/db_views_actor/src/structs.rs b/crates/db_views_actor/src/structs.rs index 2356d2be4d..5bd76f335a 100644 --- a/crates/db_views_actor/src/structs.rs +++ b/crates/db_views_actor/src/structs.rs @@ -151,3 +151,43 @@ pub struct PersonView { pub counts: PersonAggregates, pub is_admin: bool, } + +#[derive(Debug, Serialize)] +#[cfg_attr(feature = "full", derive(TS))] +#[cfg_attr(feature = "full", ts(export))] +pub struct VoteAnalyticsByPerson { + pub creator: Person, + pub total_votes: i64, + pub upvotes: i64, + pub downvotes: i64, + pub upvote_percentage: f64, +} + +#[derive(Debug, Serialize)] +#[cfg_attr(feature = "full", derive(TS))] +#[cfg_attr(feature = "full", ts(export))] +pub struct VoteAnalyticsByCommunity { + pub community: Community, + pub total_votes: i64, + pub upvotes: i64, + pub downvotes: i64, + pub upvote_percentage: f64, +} + +#[derive(Debug, Serialize)] +#[cfg_attr(feature = "full", derive(TS))] +#[cfg_attr(feature = "full", ts(export))] +pub struct VoteAnalyticsGivenByPersonView { + pub post_votes_total_votes: i64, + pub post_votes_total_upvotes: i64, + pub post_votes_total_downvotes: i64, + pub post_votes_total_upvote_percentage: f64, + pub post_votes_by_target_user: Vec, + pub post_votes_by_target_community: Vec, + pub comment_votes_total_votes: i64, + pub comment_votes_total_upvotes: i64, + pub comment_votes_total_downvotes: i64, + pub comment_votes_total_upvote_percentage: f64, + pub comment_votes_by_target_user: Vec, + pub comment_votes_by_target_community: Vec, +} diff --git a/crates/db_views_actor/src/vote_analytics_given_by_view.rs b/crates/db_views_actor/src/vote_analytics_given_by_view.rs new file mode 100644 index 0000000000..b7146e1558 --- /dev/null +++ b/crates/db_views_actor/src/vote_analytics_given_by_view.rs @@ -0,0 +1,520 @@ +use crate::structs::{ + VoteAnalyticsByCommunity, + VoteAnalyticsByPerson, + VoteAnalyticsGivenByPersonView, +}; +use chrono::{DateTime, Utc}; +use diesel::{ + dsl::exists, + result::{Error, Error::QueryBuilderError}, + select, + sql_query, + sql_types::{BigInt, Double, Integer, Nullable, Text, Timestamptz}, + QueryDsl, + QueryableByName, +}; +use diesel_async::RunQueryDsl; +use lemmy_db_schema::{ + newtypes::{CommunityId, PersonId}, + schema::person, + source::{community::Community, person::Person}, + utils::{get_conn, DbPool, FETCH_LIMIT_MAX}, +}; +use std::collections::HashMap; + +const VOTE_FETCH_LIMIT_DEFAULT: i64 = 20; +const VOTE_FETCH_LIMIT_MAX: i64 = FETCH_LIMIT_MAX * 2; + +fn fetch_limit(limit: Option) -> Result { + Ok(match limit { + Some(limit) => { + if !(1..=VOTE_FETCH_LIMIT_MAX).contains(&limit) { + return Err(QueryBuilderError( + format!("Vote fetch limit is > {VOTE_FETCH_LIMIT_MAX}").into(), + )); + } + limit + } + None => VOTE_FETCH_LIMIT_DEFAULT, + }) +} + +fn create_person_votes_view( + result: &VotesByTargetResult, + persons: &HashMap, +) -> Result { + if let Some(person_id) = result.target { + return Ok(VoteAnalyticsByPerson { + creator: persons + .get(&PersonId(person_id)) + .ok_or_else(|| Error::NotFound)? + .clone(), + total_votes: result.total_votes, + upvotes: result.upvotes, + downvotes: result.downvotes, + upvote_percentage: result.upvote_percentage, + }); + } + Err(Error::NotFound) +} + +fn create_community_votes_view( + result: &VotesByTargetResult, + communities: &HashMap, +) -> Result { + if let Some(community_id) = result.target { + return Ok(VoteAnalyticsByCommunity { + community: communities + .get(&CommunityId(community_id)) + .ok_or_else(|| Error::NotFound)? + .clone(), + total_votes: result.total_votes, + upvotes: result.upvotes, + downvotes: result.downvotes, + upvote_percentage: result.upvote_percentage, + }); + } + Err(Error::NotFound) +} + +fn extract_person_ids(results: Vec<&VotesByTargetResult>) -> Result, Error> { + // it's possible that this contains duplicates, but that will get deduplicated by postgres + let person_ids = results + .iter() + .map(|&x| x.target.ok_or_else(|| Error::NotFound).map(PersonId)) + .collect::, _>>()?; + Ok(person_ids) +} + +fn extract_community_ids(results: Vec<&VotesByTargetResult>) -> Result, Error> { + // it's possible that this contains duplicates, but that will get deduplicated by postgres + let community_ids = results + .iter() + .map(|&x| x.target.ok_or_else(|| Error::NotFound).map(CommunityId)) + .collect::, _>>()?; + Ok(community_ids) +} + +#[derive(QueryableByName)] +struct VotesByTargetResult { + #[diesel(sql_type = Text)] + target_type: String, + #[diesel(sql_type = Nullable)] + target: Option, + #[diesel(sql_type = BigInt)] + total_votes: i64, + #[diesel(sql_type = BigInt)] + upvotes: i64, + #[diesel(sql_type = BigInt)] + downvotes: i64, + #[diesel(sql_type = Double)] + upvote_percentage: f64, +} + +impl VoteAnalyticsGivenByPersonView { + pub async fn read( + pool: &mut DbPool<'_>, + person_id: PersonId, + start_time: Option>, + end_time: Option>, + limit: Option, + ) -> Result { + let conn = &mut get_conn(pool).await?; + // Ensure person exists, as the other queries do not necessarily return rows that would indicate + // the existence of a user. + let person_exists: bool = select(exists(person::table.find(&person_id))) + .get_result(conn) + .await?; + if !person_exists { + Err(Error::NotFound)? + } + + let limit = fetch_limit(limit)?; + + // This is a rather dangerous workaround; this number must be one above than the highest + // parameter used in the statements below without leaving any space. It could probably be + // improved by implementing QueryFragments. + let mut sql_dynamic_parameter_binding_index = 3u8; + let (sql_since_post, sql_since_comment) = start_time + .map(|_| { + let (s_post, s_comment) = ( + format!("AND post_like.published >= ${sql_dynamic_parameter_binding_index}"), + format!("AND comment_like.published >= ${sql_dynamic_parameter_binding_index}"), + ); + sql_dynamic_parameter_binding_index += 1; + (s_post, s_comment) + }) + .unwrap_or_default(); + let (sql_until_post, sql_until_comment) = end_time + .map(|_| { + let (s_post, s_comment) = ( + format!("AND post_like.published <= ${sql_dynamic_parameter_binding_index}"), + format!("AND comment_like.published <= ${sql_dynamic_parameter_binding_index}"), + ); + sql_dynamic_parameter_binding_index += 1; + (s_post, s_comment) + }) + .unwrap_or_default(); + + let mut post_votes_by_target_query = sql_query(format!( + r#" +WITH post_likes_by_voter AS ( + SELECT post_like.score, + creator.id AS creator, + community.id AS community + FROM person voter + JOIN post_like ON post_like.person_id = voter.id + JOIN post ON post.id = post_like.post_id + JOIN person creator ON creator.id = post.creator_id + JOIN community ON community.id = post.community_id + WHERE voter.id = $1 + AND post_like.score != 0 + AND creator.id != voter.id + {since} + {until} +), post_likes_by_recipient AS ( + SELECT 'person' AS target_type, + creator AS target, + COUNT(*) AS total_votes, + COUNT(score = 1 OR NULL) AS upvotes, + COUNT(score = -1 OR NULL) AS downvotes, + CASE WHEN COUNT(*) > 0 THEN 100::float * COUNT(score = 1 OR NULL) / COUNT(*) ELSE 0::float END AS upvote_percentage + FROM post_likes_by_voter + GROUP BY creator + ORDER BY + total_votes DESC, + creator ASC + LIMIT $2 +), post_likes_by_community AS ( + SELECT 'community' AS target_type, + community AS target, + COUNT(*) AS total_votes, + COUNT(score = 1 OR NULL) AS upvotes, + COUNT(score = -1 OR NULL) AS downvotes, + CASE WHEN COUNT(*) > 0 THEN 100::float * COUNT(score = 1 OR NULL) / COUNT(*) ELSE 0::float END AS upvote_percentage + FROM post_likes_by_voter + GROUP BY community + ORDER BY + total_votes DESC, + community ASC + LIMIT $2 +) + +SELECT 'total' AS target_type, + NULL AS target, + COUNT(*) AS total_votes, + COUNT(score = 1 OR NULL) AS upvotes, + COUNT(score = -1 OR NULL) AS downvotes, + CASE WHEN COUNT(*) > 0 THEN 100::float * COUNT(score = 1 OR NULL) / COUNT(*) ELSE 0::float END AS upvote_percentage +FROM post_likes_by_voter + +UNION ALL +SELECT * FROM post_likes_by_recipient +UNION ALL +SELECT * FROM post_likes_by_community + "#, + since = sql_since_post, + until = sql_until_post, + )).into_boxed() + .bind::(&person_id.0) + .bind::(limit); + // this order must match the order in which the dynamic parameter binding index was generated + if let Some(t) = start_time { + post_votes_by_target_query = post_votes_by_target_query.bind::(t); + } + if let Some(t) = end_time { + post_votes_by_target_query = post_votes_by_target_query.bind::(t); + } + let post_votes_by_target: Vec = + post_votes_by_target_query.get_results(conn).await?; + + let mut comment_votes_by_target_query = sql_query(format!( + r#" +WITH comment_likes_by_voter AS ( + SELECT comment_like.score, + creator.id AS creator, + community.id AS community + FROM person voter + JOIN comment_like ON comment_like.person_id = voter.id + JOIN comment on comment.id = comment_like.comment_id + JOIN person creator ON creator.id = comment.creator_id + JOIN post ON post.id = comment.post_id + JOIN community ON community.id = post.community_id + WHERE voter.id = $1 + AND comment_like.score != 0 + AND creator.id != voter.id + {since} + {until} +), comment_likes_by_recipient AS ( + SELECT 'person' AS target_type, + creator AS target, + COUNT(*) AS total_votes, + COUNT(score = 1 OR NULL) AS upvotes, + COUNT(score = -1 OR NULL) AS downvotes, + CASE WHEN COUNT(*) > 0 THEN 100::float * COUNT(score = 1 OR NULL) / COUNT(*) ELSE 0::float END AS upvote_percentage + FROM comment_likes_by_voter + GROUP BY creator + ORDER BY + total_votes DESC, + creator ASC + LIMIT $2 +), comment_likes_by_community AS ( + SELECT 'community' AS target_type, + community AS target, + COUNT(*) AS total_votes, + COUNT(score = 1 OR NULL) AS upvotes, + COUNT(score = -1 OR NULL) AS downvotes, + CASE WHEN COUNT(*) > 0 THEN 100::float * COUNT(score = 1 OR NULL) / COUNT(*) ELSE 0::float END AS upvote_percentage + FROM comment_likes_by_voter + GROUP BY community + ORDER BY + total_votes DESC, + community ASC + LIMIT $2 +) + +SELECT 'total' AS target_type, + NULL AS target, + COUNT(*) AS total_votes, + COUNT(score = 1 OR NULL) AS upvotes, + COUNT(score = -1 OR NULL) AS downvotes, + CASE WHEN COUNT(*) > 0 THEN 100::float * COUNT(score = 1 OR NULL) / COUNT(*) ELSE 0::float END AS upvote_percentage +FROM comment_likes_by_voter + +UNION ALL +SELECT * FROM comment_likes_by_recipient +UNION ALL +SELECT * FROM comment_likes_by_community + "#, + since = sql_since_comment, + until = sql_until_comment, + )).into_boxed() + .bind::(&person_id.0) + .bind::(limit); + // this order must match the order in which the dynamic parameter binding index was generated + if let Some(t) = start_time { + comment_votes_by_target_query = comment_votes_by_target_query.bind::(t); + } + if let Some(t) = end_time { + comment_votes_by_target_query = comment_votes_by_target_query.bind::(t); + } + let comment_votes_by_target: Vec = + comment_votes_by_target_query.get_results(conn).await?; + + let person_type = "person".to_string(); + let post_votes_by_target_person: Vec<_> = post_votes_by_target + .iter() + .filter(|&x| x.target_type.eq(&person_type)) + .collect(); + let comment_votes_by_target_person: Vec<_> = comment_votes_by_target + .iter() + .filter(|&x| x.target_type.eq(&person_type)) + .collect(); + + let combined_votes_by_target_person: Vec<&VotesByTargetResult> = post_votes_by_target_person + .clone() + .into_iter() + .chain(comment_votes_by_target_person.clone()) + .collect(); + + let person_ids: Vec = extract_person_ids(combined_votes_by_target_person)?; + let persons = Person::read_many(pool, &person_ids, true).await?; + + let post_votes_by_target_person_resolved: Vec = + post_votes_by_target_person + .iter() + .map(|person| create_person_votes_view(person, &persons)) + .collect::>()?; + let comment_votes_by_target_person_resolved: Vec = + comment_votes_by_target_person + .iter() + .map(|person| create_person_votes_view(person, &persons)) + .collect::>()?; + + let community_type = "community".to_string(); + let post_votes_by_target_community: Vec<_> = post_votes_by_target + .iter() + .filter(|&x| x.target_type.eq(&community_type)) + .collect(); + let comment_votes_by_target_community: Vec<_> = comment_votes_by_target + .iter() + .filter(|&x| x.target_type.eq(&community_type)) + .collect(); + + let combined_votes_by_target_community: Vec<&VotesByTargetResult> = + post_votes_by_target_community + .clone() + .into_iter() + .chain(comment_votes_by_target_community.clone()) + .collect(); + + let community_ids = extract_community_ids(combined_votes_by_target_community)?; + let communities = Community::read_many(pool, &community_ids, true).await?; + + let post_votes_by_target_community_resolved: Vec = + post_votes_by_target_community + .iter() + .map(|community| create_community_votes_view(community, &communities)) + .collect::>()?; + let comment_votes_by_target_community_resolved: Vec = + comment_votes_by_target_community + .iter() + .map(|community| create_community_votes_view(community, &communities)) + .collect::>()?; + + let total_type = "total".to_string(); + let post_totals = post_votes_by_target + .iter() + .find(|&x| x.target_type.eq(&total_type)) + .ok_or(Error::NotFound)?; + let comment_totals = comment_votes_by_target + .iter() + .find(|&x| x.target_type.eq(&total_type)) + .ok_or(Error::NotFound)?; + + Ok(VoteAnalyticsGivenByPersonView { + post_votes_total_votes: post_totals.total_votes, + post_votes_total_upvotes: post_totals.upvotes, + post_votes_total_downvotes: post_totals.downvotes, + post_votes_total_upvote_percentage: post_totals.upvote_percentage, + post_votes_by_target_user: post_votes_by_target_person_resolved, + post_votes_by_target_community: post_votes_by_target_community_resolved, + comment_votes_total_votes: comment_totals.total_votes, + comment_votes_total_upvotes: comment_totals.upvotes, + comment_votes_total_downvotes: comment_totals.downvotes, + comment_votes_total_upvote_percentage: comment_totals.upvote_percentage, + comment_votes_by_target_user: comment_votes_by_target_person_resolved, + comment_votes_by_target_community: comment_votes_by_target_community_resolved, + }) + } +} + +#[cfg(test)] +#[allow(clippy::indexing_slicing)] +mod test { + use crate::structs::VoteAnalyticsGivenByPersonView; + use diesel::result::Error; + use lemmy_db_schema::{ + assert_length, + newtypes::PersonId, + source::{ + community::{Community, CommunityInsertForm}, + instance::Instance, + person::{Person, PersonInsertForm}, + post::{Post, PostInsertForm, PostLike, PostLikeForm}, + }, + traits::{Crud, Likeable}, + utils::build_db_pool_for_tests, + }; + use lemmy_utils::error::LemmyResult; + use serial_test::serial; + + #[tokio::test] + #[serial] + async fn test_vote_analytics() -> LemmyResult<()> { + let pool = &build_db_pool_for_tests().await; + let pool = &mut pool.into(); + + let inserted_instance = Instance::read_or_create(pool, "my_domain.tld".to_string()).await?; + let community_form = CommunityInsertForm::builder() + .name("vote_test".to_string()) + .title("vote_test".to_owned()) + .public_key("pubkey".to_string()) + .instance_id(inserted_instance.id) + .build(); + let community = Community::create(pool, &community_form).await?; + + let alice_form = PersonInsertForm { + ..PersonInsertForm::test_form(inserted_instance.id, "alice") + }; + let alice = Person::create(pool, &alice_form).await?; + let mut alice_posts: Vec = vec![]; + for _ in 0..=9 { + let post_form = PostInsertForm::builder() + .name("A test post".into()) + .creator_id(alice.id) + .community_id(community.id) + .build(); + + let post = Post::create(pool, &post_form).await?; + alice_posts.push(post); + } + + let bob_form = PersonInsertForm { + ..PersonInsertForm::test_form(inserted_instance.id, "bob") + }; + let bob = Person::create(pool, &bob_form).await?; + let bob_post_form = PostInsertForm::builder() + .name("A test post".into()) + .creator_id(bob.id) + .community_id(community.id) + .build(); + let bob_post = Post::create(pool, &bob_post_form).await?; + + // readability + #[allow(clippy::needless_range_loop)] + for i in 0..=9 { + let post = alice_posts[i].clone(); + // 2 votes without score, 3 upvotes, 5 downvotes + let score = if i < 2 { + 0 + } else if i < 5 { + 1 + } else { + -1 + }; + let like_form = PostLikeForm { + post_id: post.id, + person_id: bob.id, + score, + }; + PostLike::like(pool, &like_form).await?; + } + let like_form = PostLikeForm { + post_id: bob_post.id, + person_id: bob.id, + score: 1, + }; + PostLike::like(pool, &like_form).await?; + + // Test for non-existing person + let invalid_person_id = PersonId(-1); + let view = + VoteAnalyticsGivenByPersonView::read(pool, invalid_person_id, None, None, None).await; + assert!( + view.is_err_and(|e| e == Error::NotFound), + "query should not match a person", + ); + + // alice exists but hasn't voted on anything + let view = VoteAnalyticsGivenByPersonView::read(pool, alice.id, None, None, None).await?; + assert_eq!(0, view.post_votes_total_votes); + assert_eq!(0, view.post_votes_total_upvotes); + assert_eq!(0, view.post_votes_total_downvotes); + assert_eq!(0.0, view.post_votes_total_upvote_percentage); + assert_length!(0, view.post_votes_by_target_user); + assert_length!(0, view.post_votes_by_target_community); + + let view = VoteAnalyticsGivenByPersonView::read(pool, bob.id, None, None, None).await?; + + assert_eq!(8, view.post_votes_total_votes); + assert_eq!(3, view.post_votes_total_upvotes); + assert_eq!(5, view.post_votes_total_downvotes); + assert_eq!(37.5, view.post_votes_total_upvote_percentage); + assert_length!(1, view.post_votes_by_target_user); + assert_length!(1, view.post_votes_by_target_community); + assert_eq!(alice.id, view.post_votes_by_target_user[0].creator.id); + assert_eq!( + community.id, + view.post_votes_by_target_community[0].community.id + ); + + // TODO: test limits, multiple users, multiple communities, time ranges, comments + + Person::delete(pool, alice.id).await?; + Person::delete(pool, bob.id).await?; + Instance::delete(pool, inserted_instance.id).await?; + + Ok(()) + } +} diff --git a/crates/utils/src/email.rs b/crates/utils/src/email.rs index 7bac7ad672..b2ef280128 100644 --- a/crates/utils/src/email.rs +++ b/crates/utils/src/email.rs @@ -19,6 +19,7 @@ pub mod translations { type AsyncSmtpTransport = lettre::AsyncSmtpTransport; +#[tracing::instrument(skip(html, settings))] pub async fn send_email( subject: &str, to_email: &str, diff --git a/crates/utils/src/settings/structs.rs b/crates/utils/src/settings/structs.rs index b893428bd2..97362318a3 100644 --- a/crates/utils/src/settings/structs.rs +++ b/crates/utils/src/settings/structs.rs @@ -56,6 +56,12 @@ pub struct Settings { cors_origin: Option, /// Print logs in JSON format. You can also disable ANSI colors in logs with env var `NO_COLOR`. pub json_logging: bool, + + /// Native automod configuration + /// All variables inside are empty/false by default + #[default(Default::default())] + #[doku(example = "Default::default()")] + pub fhf_automod_config: FhfAutomodConfig, } impl Settings { @@ -262,3 +268,29 @@ pub struct FederationWorkerConfig { #[default(1)] pub concurrent_sends_per_instance: i8, } + +#[derive(Debug, Deserialize, Serialize, Clone, Document, SmartDefault)] +#[serde(deny_unknown_fields)] +pub struct FhfAutomodConfig { + /// Username of a local user used as actor for automated moderation actions. + /// This user should be an instance moderator, although this is not enforced. + /// Federation compatibility with non-admin users is unknown. + /// Required for most native automod functionality + #[default(None)] + #[doku(example = "automod")] + pub actor_username: Option, + + /// This enables the scheduled task for resolving any reports about removed content when the + /// creator is banned or deleted and the reported content is removed or deleted. + /// Reports that were unresolved will not be touched by this. + /// Requires actor_username to be set. + #[default(false)] + pub resolve_banned_or_deleted_creators_reports: bool, + + /// Set this to a number of days to enable automatic bans of users deleting their account within + /// that many days of signup. Bans include content removal. + /// Requires actor_username to be set. + #[default(None)] + #[doku(example = "7")] + pub ban_deleted_persons_created_within_days: Option, +} diff --git a/migrations/2026-04-13-200409_fhf_post_like_index_person_published/down.sql b/migrations/2026-04-13-200409_fhf_post_like_index_person_published/down.sql new file mode 100644 index 0000000000..09406f447e --- /dev/null +++ b/migrations/2026-04-13-200409_fhf_post_like_index_person_published/down.sql @@ -0,0 +1,2 @@ +DROP INDEX fhf_post_like_person_id_idx; + diff --git a/migrations/2026-04-13-200409_fhf_post_like_index_person_published/up.sql b/migrations/2026-04-13-200409_fhf_post_like_index_person_published/up.sql new file mode 100644 index 0000000000..cb8c32d963 --- /dev/null +++ b/migrations/2026-04-13-200409_fhf_post_like_index_person_published/up.sql @@ -0,0 +1,2 @@ +CREATE INDEX fhf_post_like_person_id_idx ON public.post_like (person_id, published DESC); + diff --git a/src/api_routes_http.rs b/src/api_routes_http.rs index 44fed5120d..875866f5d2 100644 --- a/src/api_routes_http.rs +++ b/src/api_routes_http.rs @@ -89,6 +89,7 @@ use lemmy_api::{ }, }, sitemap::get_sitemap, + vote_analytics::given_by_person::get_vote_analytics_given_by_person, }; use lemmy_api_crud::{ comment::{ @@ -141,6 +142,10 @@ use lemmy_utils::rate_limit::RateLimitCell; pub fn config(cfg: &mut web::ServiceConfig, rate_limit: &RateLimitCell) { cfg.service( web::scope("/api/v3") + .route( + "/lw/vote_analytics_given_by_person", + web::get().to(get_vote_analytics_given_by_person), + ) .route("/image_proxy", web::get().to(image_proxy)) // Site .service( diff --git a/src/code_migrations.rs b/src/code_migrations.rs index 7388f5cad7..a139655655 100644 --- a/src/code_migrations.rs +++ b/src/code_migrations.rs @@ -38,6 +38,7 @@ use lemmy_utils::{error::LemmyResult, settings::structs::Settings}; use tracing::info; use url::Url; +#[tracing::instrument(skip_all)] pub async fn run_advanced_migrations( pool: &mut DbPool<'_>, settings: &Settings, diff --git a/src/lib.rs b/src/lib.rs index daa2aa2739..df71d1ba11 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -50,7 +50,7 @@ use lemmy_utils::{ use prometheus::default_registry; use prometheus_metrics::serve_prometheus; use reqwest_middleware::ClientBuilder; -use reqwest_tracing::TracingMiddleware; +use reqwest_tracing::{SpanBackendWithUrl, TracingMiddleware}; use serde_json::json; use std::{env, ops::Deref, time::Duration}; use tokio::signal::unix::SignalKind; @@ -160,7 +160,7 @@ pub async fn start_lemmy_server(args: CmdArgs) -> LemmyResult<()> { ); let client = ClientBuilder::new(client_builder(&SETTINGS).build()?) - .with(TracingMiddleware::default()) + .with(TracingMiddleware::::new()) .build(); let context = LemmyContext::create( pool.clone(), @@ -300,7 +300,7 @@ fn create_http_server( // Pictrs cannot use proxy let pictrs_client = ClientBuilder::new(client_builder(&SETTINGS).no_proxy().build()?) - .with(TracingMiddleware::default()) + .with(TracingMiddleware::::new()) .build(); // Create Http server diff --git a/src/scheduled_tasks.rs b/src/scheduled_tasks.rs index 5cd07126e2..a1e7f67708 100644 --- a/src/scheduled_tasks.rs +++ b/src/scheduled_tasks.rs @@ -90,6 +90,26 @@ pub async fn setup(context: LemmyContext) -> LemmyResult<()> { } }); + // FHF auto-resolve reports + if context + .settings() + .fhf_automod_config + .resolve_banned_or_deleted_creators_reports + { + if let Some(automod_username) = &context.settings().fhf_automod_config.actor_username { + info!("[FHF AutoMod][ResolveReports] Scheduling automatic resolution of reports"); + let context_1 = context.clone(); + let automod_username_1 = automod_username.clone(); + scheduler.every(CTimeUnits::minutes(1)).run(move || { + let context_2 = context_1.clone(); + let automod_username_2 = automod_username_1.clone(); + async move { + fhf_auto_resolve_reports(&mut context_2.pool(), automod_username_2).await; + } + }); + }; + }; + // Manually run the scheduler in an event loop loop { scheduler.run_pending().await; @@ -99,6 +119,7 @@ pub async fn setup(context: LemmyContext) -> LemmyResult<()> { /// Update the hot_rank columns for the aggregates tables /// Runs in batches until all necessary rows are updated once +#[tracing::instrument(skip_all)] async fn update_hot_ranks(pool: &mut DbPool<'_>) { info!("Updating hot ranks for all history..."); @@ -142,6 +163,7 @@ struct HotRanksUpdateResult { /// In `where_clause` and `set_clause`, "a" will refer to the current aggregates table. /// Locked rows are skipped in order to prevent deadlocks (they will likely get updated on the next /// run) +#[tracing::instrument(skip(conn))] async fn process_ranks_in_batches( conn: &mut AsyncPgConnection, table_name: &str, @@ -198,6 +220,7 @@ async fn process_ranks_in_batches( /// Post aggregates is a special case, since it needs to join to the community_aggregates /// table, to get the active monthly user counts. +#[tracing::instrument(skip_all)] async fn process_post_aggregates_ranks_in_batches(conn: &mut AsyncPgConnection) { let process_start_time: DateTime = Utc .timestamp_opt(0, 0) @@ -246,6 +269,7 @@ async fn process_post_aggregates_ranks_in_batches(conn: &mut AsyncPgConnection) ); } +#[tracing::instrument(skip_all)] async fn delete_expired_captcha_answers(pool: &mut DbPool<'_>) { let conn = get_conn(pool).await; @@ -270,6 +294,7 @@ async fn delete_expired_captcha_answers(pool: &mut DbPool<'_>) { } /// Clear old activities (this table gets very large) +#[tracing::instrument(skip_all)] async fn clear_old_activities(pool: &mut DbPool<'_>) { info!("Clearing old activities..."); let conn = get_conn(pool).await; @@ -300,6 +325,7 @@ async fn clear_old_activities(pool: &mut DbPool<'_>) { } } +#[tracing::instrument(skip_all)] async fn delete_old_denied_users(pool: &mut DbPool<'_>) { LocalUser::delete_old_denied_local_users(pool) .await @@ -311,6 +337,7 @@ async fn delete_old_denied_users(pool: &mut DbPool<'_>) { } /// overwrite posts and comments 30d after deletion +#[tracing::instrument(skip_all)] async fn overwrite_deleted_posts_and_comments(pool: &mut DbPool<'_>) { info!("Overwriting deleted posts..."); let conn = get_conn(pool).await; @@ -377,6 +404,7 @@ struct CommunityAggregatesUpdateResult { } /// Re-calculate the site and community active counts for a given interval +#[tracing::instrument(skip_all)] async fn active_counts(pool: &mut DbPool<'_>, interval: (&str, &str)) { info!( "Updating active site and community aggregates for {}...", @@ -508,6 +536,7 @@ async fn process_community_aggregates(conn: &mut AsyncPgConnection, interval: (& } /// Set banned to false after ban expires +#[tracing::instrument(skip_all)] async fn update_banned_when_expired(pool: &mut DbPool<'_>) { info!("Updating banned column if it expires ..."); let conn = get_conn(pool).await; @@ -545,6 +574,7 @@ async fn update_banned_when_expired(pool: &mut DbPool<'_>) { /// https://github.com/jhass/nodeinfo/blob/main/PROTOCOL.md /// /// TODO: if instance has been dead for a long time, it should be checked less frequently +#[tracing::instrument(skip_all)] async fn update_instance_software( pool: &mut DbPool<'_>, client: &ClientWithMiddleware, @@ -573,6 +603,7 @@ async fn update_instance_software( /// This builds an instance update form, for a given domain. /// If the instance sends a response, but doesn't have a well-known or nodeinfo, /// Then return a default form with only the updated field. +#[tracing::instrument(skip(client))] async fn build_update_instance_form( domain: &str, client: &ClientWithMiddleware, @@ -635,6 +666,131 @@ async fn build_update_instance_form( Some(instance_form) } +/// Mark reports of removed or deleted content by banned or deleted creators as resolved +#[tracing::instrument(skip(pool))] +async fn fhf_auto_resolve_reports(pool: &mut DbPool<'_>, automod_username: String) { + // keep these here to minimize risk of merge conflicts with upstream + use diesel::{BoolExpressionMethods, JoinOnDsl}; + use lemmy_db_schema::{ + newtypes::{CommentReportId, PostReportId, PrivateMessageReportId}, + schema::{comment_report, person, post_report, private_message, private_message_report}, + source::{ + comment_report::CommentReport, + person::Person, + post_report::PostReport, + private_message_report::PrivateMessageReport, + }, + traits::{ApubActor, Reportable}, + }; + + info!("[FHF AutoMod][ResolveReports] Resolving reports of removed content by banned users..."); + let conn = get_conn(pool).await; + + match conn { + Ok(mut conn) => { + if let Ok(Some(automod_person)) = + Person::read_from_name(&mut (&mut conn).into(), automod_username.as_str(), false).await + { + let resolvable_post_reports = post_report::table + .inner_join(post::table) + .inner_join(person::table.on(post::creator_id.eq(person::id))) + .filter(post_report::resolved.eq(false)) + .filter(post_report::resolver_id.is_null()) + .filter(post::removed.eq(true).or(post::deleted.eq(true))) + .filter(person::banned.eq(true).or(person::deleted.eq(true))) + .select(post_report::id) + .load::(&mut conn) + .await; + + if let Ok(resolvable_post_reports) = resolvable_post_reports { + if !resolvable_post_reports.is_empty() { + info!( + "[FHF AutoMod][ResolveReports] Resolving {} post reports", + resolvable_post_reports.len() + ); + } + for resolvable_post_report in resolvable_post_reports { + PostReport::resolve( + &mut (&mut conn).into(), + resolvable_post_report, + automod_person.id, + ) + .await + .ok(); + } + } + + let resolvable_comment_reports = comment_report::table + .inner_join(comment::table) + .inner_join(person::table.on(comment::creator_id.eq(person::id))) + .filter(comment_report::resolved.eq(false)) + .filter(comment_report::resolver_id.is_null()) + .filter(comment::removed.eq(true).or(comment::deleted.eq(true))) + .filter(person::banned.eq(true).or(person::deleted.eq(true))) + .select(comment_report::id) + .load::(&mut conn) + .await; + + if let Ok(resolvable_comment_reports) = resolvable_comment_reports { + if !resolvable_comment_reports.is_empty() { + info!( + "[FHF AutoMod][ResolveReports] Resolving {} comment reports", + resolvable_comment_reports.len() + ); + } + for resolvable_comment_report in resolvable_comment_reports { + CommentReport::resolve( + &mut (&mut conn).into(), + resolvable_comment_report, + automod_person.id, + ) + .await + .ok(); + } + } + + let resolvable_private_message_reports = private_message_report::table + .inner_join(private_message::table) + .inner_join(person::table.on(private_message::creator_id.eq(person::id))) + .filter(private_message_report::resolved.eq(false)) + .filter(private_message_report::resolver_id.is_null()) + .filter( + private_message::removed + .eq(true) + .or(private_message::deleted.eq(true)), + ) + .filter(person::banned.eq(true).or(person::deleted.eq(true))) + .select(private_message_report::id) + .load::(&mut conn) + .await; + + if let Ok(resolvable_private_message_reports) = resolvable_private_message_reports { + if !resolvable_private_message_reports.is_empty() { + info!( + "[FHF AutoMod][ResolveReports] Resolving {} private message reports", + resolvable_private_message_reports.len() + ); + } + for resolvable_private_message_report in resolvable_private_message_reports { + PrivateMessageReport::resolve( + &mut (&mut conn).into(), + resolvable_private_message_report, + automod_person.id, + ) + .await + .ok(); + } + } else { + error!("[FHF AutoMod][ResolveReports] Failed to fetch automod user from DB"); + } + } + } + Err(e) => { + error!("Failed to get connection from pool: {e}"); + } + } +} + #[cfg(test)] #[allow(clippy::indexing_slicing)] mod tests {