diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 339cbcdb..7eaf501f 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -14,11 +14,11 @@ All code contributions must be made under the same license as the project's main ## Setting up a development version -Please see this page on the Tildes Docs for instructions to set up a development version: https://docs.tildes.net/instructions/development-setup +Please see this page on the Tildes Docs for instructions to set up a development version: https://docs.tildes.net/development/initial-setup ## General development information -This page on the Tildes docs contains information about many aspects of Tildes development: https://docs.tildes.net/instructions/development +This page on the Tildes docs contains information about many aspects of Tildes development: https://docs.tildes.net/development/general-development ## Contributing code @@ -48,7 +48,7 @@ Once you've selected an issue to work on, please leave a comment saying so. This After you've finished making your changes, there are a few things to check before proposing your changes as a merge request. -First, ensure that all the checks (tests, mypy and code-style) pass successfully. Merge requests that fail any of the checks will not be accepted. For more information, see this section of the development docs: https://docs.tildes.net/instructions/development#running-checks-on-your-code +First, ensure that all the checks (tests, mypy and code-style) pass successfully. Merge requests that fail any of the checks will not be accepted. For more information, see this section of the development docs: https://docs.tildes.net/development/general-development#running-checks-on-your-code Squash your changes into logical commits. For many changes there should only be a single commit, but some can be broken into multiple logical sections. The commit messages should follow [the formatting described in this article](https://tbaggery.com/2008/04/19/a-note-about-git-commit-messages.html), with the summary line written in the imperative form. The summary line should make sense if you were to use it in a sentence like "If applied, this commit will \_\_\_\_\_\_\_\_". For example, "Add a new X", "Fix a bug with Y", and so on. diff --git a/README.md b/README.md index 4aa8dd50..acd7562a 100644 --- a/README.md +++ b/README.md @@ -16,7 +16,7 @@ For general information about Tildes and its goals, please see [the announcement Known issues and plans for upcoming changes are tracked on GitLab: https://gitlab.com/tildes/tildes/issues -The "board" view is useful as an overview: https://gitlab.com/tildes/tildes/boards +The "board" view is useful as an overview: https://gitlab.com/tildes/tildes/-/boards ## Contributing to Tildes development diff --git a/Vagrantfile b/Vagrantfile index 9dbde1d7..83c46cf4 100644 --- a/Vagrantfile +++ b/Vagrantfile @@ -26,9 +26,6 @@ Vagrant.configure(VAGRANT_CONFIG_VERSION) do |config| salt.run_highstate = true salt.verbose = true salt.log_level = "info" - - salt.install_type = "stable" - salt.version = "3000" end config.vm.provider "virtualbox" do |vb| diff --git a/git_hooks/pre-commit b/git_hooks/pre-commit index c240ebb1..5a83dbe5 100755 --- a/git_hooks/pre-commit +++ b/git_hooks/pre-commit @@ -1,10 +1,6 @@ #!/bin/sh # -# Pre-commit hook script that ensures mypy checks and tests pass +# Pre-commit hook script that ensures type-checking, tests, and fast style checks pass vagrant ssh -c ". activate \ - && echo 'Checking mypy type annotations...' && mypy --no-error-summary . \ - && echo 'Checking if Black would reformat any code...' && black --check . \ - && echo -n 'Running tests: ' && pytest -q \ - && echo 'Checking SCSS style...' && npm run --silent lint:scss \ - && echo 'Checking JS style...' && npm run --silent lint:js" + && invoke type-check test --quiet code-style-check" diff --git a/git_hooks/pre-push b/git_hooks/pre-push index 03f7a8cc..168d670d 100755 --- a/git_hooks/pre-push +++ b/git_hooks/pre-push @@ -1,11 +1,6 @@ #!/bin/sh # -# Pre-push hook script that ensures mypy checks, style checks, and tests pass +# Pre-push hook script that ensures all tests and code checks pass vagrant ssh -c ". activate \ - && echo 'Checking mypy type annotations...' && mypy --no-error-summary . \ - && echo 'Checking if Black would reformat any code...' && black --check . \ - && echo -n 'Running tests: ' && pytest -q \ - && echo 'Checking SCSS style...' && npm run --silent lint:scss \ - && echo 'Checking JS style...' && npm run --silent lint:js \ - && echo 'Checking Python style fully (takes a while)...' && prospector -M" + && invoke type-check test --quiet --full code-style-check --full" diff --git a/salt/salt/boussole.service.jinja2 b/salt/salt/boussole.service.jinja2 index 85c35ae7..05c418ea 100644 --- a/salt/salt/boussole.service.jinja2 +++ b/salt/salt/boussole.service.jinja2 @@ -1,8 +1,10 @@ -{% from 'common.jinja2' import app_dir -%} +{% from 'common.jinja2' import app_dir, app_username -%} [Unit] Description=Boussole - auto-compile SCSS files on change [Service] +User={{ app_username }} +Group={{ app_username }} WorkingDirectory={{ app_dir }} Environment="LC_ALL=C.UTF-8" "LANG=C.UTF-8" ExecStart=/opt/venvs/boussole/bin/boussole watch --backend=yaml --config=boussole.yaml --poll diff --git a/salt/salt/consumers/comment_user_mentions_generator.service.jinja2 b/salt/salt/consumers/comment_user_mentions_generator.service.jinja2 index 66235121..352eb764 100644 --- a/salt/salt/consumers/comment_user_mentions_generator.service.jinja2 +++ b/salt/salt/consumers/comment_user_mentions_generator.service.jinja2 @@ -1,4 +1,4 @@ -{% from 'common.jinja2' import app_dir, bin_dir -%} +{% from 'common.jinja2' import app_dir, app_username, bin_dir -%} [Unit] Description=Comment User Mention Generator (Queue Consumer) Requires=redis.service @@ -6,6 +6,8 @@ After=redis.service PartOf=redis.service [Service] +User={{ app_username }} +Group={{ app_username }} WorkingDirectory={{ app_dir }}/consumers Environment="INI_FILE={{ app_dir }}/{{ pillar['ini_file'] }}" ExecStart={{ bin_dir }}/python comment_user_mentions_generator.py diff --git a/salt/salt/consumers/init.sls b/salt/salt/consumers/init.sls index a05a680a..5c4df936 100644 --- a/salt/salt/consumers/init.sls +++ b/salt/salt/consumers/init.sls @@ -22,6 +22,14 @@ - group: root - mode: 644 +/etc/systemd/system/consumer-post_processing_script_runner.service: + file.managed: + - source: salt://consumers/post_processing_script_runner.service.jinja2 + - template: jinja + - user: root + - group: root + - mode: 644 + consumer-topic_interesting_activity_updater.service: service.running: - enable: True @@ -34,6 +42,10 @@ consumer-comment_user_mentions_generator.service: service.running: - enable: True +consumer-post_processing_script_runner.service: + service.running: + - enable: True + {% if grains['id'] == 'prod' %} /etc/systemd/system/consumer-topic_embedly_extractor.service: file.managed: diff --git a/salt/salt/consumers/post_processing_script_runner.service.jinja2 b/salt/salt/consumers/post_processing_script_runner.service.jinja2 new file mode 100644 index 00000000..b7c0f342 --- /dev/null +++ b/salt/salt/consumers/post_processing_script_runner.service.jinja2 @@ -0,0 +1,18 @@ +{% from 'common.jinja2' import app_dir, app_username, bin_dir -%} +[Unit] +Description=Post Processing Script Runner (Queue Consumer) +Requires=redis.service +After=redis.service +PartOf=redis.service + +[Service] +User={{ app_username }} +Group={{ app_username }} +WorkingDirectory={{ app_dir }}/consumers +Environment="INI_FILE={{ app_dir }}/{{ pillar['ini_file'] }}" +ExecStart={{ bin_dir }}/python post_processing_script_runner.py +Restart=always +RestartSec=5 + +[Install] +WantedBy=multi-user.target diff --git a/salt/salt/consumers/topic_embedly_extractor.service.jinja2 b/salt/salt/consumers/topic_embedly_extractor.service.jinja2 index 06633377..a0061ea5 100644 --- a/salt/salt/consumers/topic_embedly_extractor.service.jinja2 +++ b/salt/salt/consumers/topic_embedly_extractor.service.jinja2 @@ -1,4 +1,4 @@ -{% from 'common.jinja2' import app_dir, bin_dir -%} +{% from 'common.jinja2' import app_dir, app_username, bin_dir -%} [Unit] Description=Topic Embedly Extractor (Queue Consumer) Requires=redis.service @@ -6,6 +6,8 @@ After=redis.service PartOf=redis.service [Service] +User={{ app_username }} +Group={{ app_username }} WorkingDirectory={{ app_dir }}/consumers Environment="INI_FILE={{ app_dir }}/{{ pillar['ini_file'] }}" ExecStart={{ bin_dir }}/python topic_embedly_extractor.py diff --git a/salt/salt/consumers/topic_interesting_activity_updater.service.jinja2 b/salt/salt/consumers/topic_interesting_activity_updater.service.jinja2 index a19c3a3d..0cf8fc5d 100644 --- a/salt/salt/consumers/topic_interesting_activity_updater.service.jinja2 +++ b/salt/salt/consumers/topic_interesting_activity_updater.service.jinja2 @@ -1,4 +1,4 @@ -{% from 'common.jinja2' import app_dir, bin_dir -%} +{% from 'common.jinja2' import app_dir, app_username, bin_dir -%} [Unit] Description=Topic Interesting Activity Updater (Queue Consumer) Requires=redis.service @@ -6,6 +6,8 @@ After=redis.service PartOf=redis.service [Service] +User={{ app_username }} +Group={{ app_username }} WorkingDirectory={{ app_dir }}/consumers Environment="INI_FILE={{ app_dir }}/{{ pillar['ini_file'] }}" ExecStart={{ bin_dir }}/python topic_interesting_activity_updater.py diff --git a/salt/salt/consumers/topic_metadata_generator.service.jinja2 b/salt/salt/consumers/topic_metadata_generator.service.jinja2 index 0545f211..0d20257f 100644 --- a/salt/salt/consumers/topic_metadata_generator.service.jinja2 +++ b/salt/salt/consumers/topic_metadata_generator.service.jinja2 @@ -1,4 +1,4 @@ -{% from 'common.jinja2' import app_dir, bin_dir -%} +{% from 'common.jinja2' import app_dir, app_username, bin_dir -%} [Unit] Description=Topic Metadata Generator (Queue Consumer) Requires=redis.service @@ -6,6 +6,8 @@ After=redis.service PartOf=redis.service [Service] +User={{ app_username }} +Group={{ app_username }} WorkingDirectory={{ app_dir }}/consumers Environment="INI_FILE={{ app_dir }}/{{ pillar['ini_file'] }}" ExecStart={{ bin_dir }}/python topic_metadata_generator.py diff --git a/salt/salt/consumers/topic_youtube_scraper.service.jinja2 b/salt/salt/consumers/topic_youtube_scraper.service.jinja2 index 4fcf5d01..8623e5e0 100644 --- a/salt/salt/consumers/topic_youtube_scraper.service.jinja2 +++ b/salt/salt/consumers/topic_youtube_scraper.service.jinja2 @@ -1,4 +1,4 @@ -{% from 'common.jinja2' import app_dir, bin_dir -%} +{% from 'common.jinja2' import app_dir, app_username, bin_dir -%} [Unit] Description=Topic Youtube Scraper (Queue Consumer) Requires=redis.service @@ -6,6 +6,8 @@ After=redis.service PartOf=redis.service [Service] +User={{ app_username }} +Group={{ app_username }} WorkingDirectory={{ app_dir }}/consumers Environment="INI_FILE={{ app_dir }}/{{ pillar['ini_file'] }}" ExecStart={{ bin_dir }}/python topic_youtube_scraper.py diff --git a/salt/salt/development.sls b/salt/salt/development.sls index 7ea2ffb6..bc4776e0 100644 --- a/salt/salt/development.sls +++ b/salt/salt/development.sls @@ -19,3 +19,9 @@ automatic-activate: file.append: - name: '/home/{{ app_username }}/.bashrc' - text: 'source activate' + +# adds invoke's tab-completion script so that invoke tasks can be completed +invoke-tab-completion: + file.append: + - name: '/home/{{ app_username }}/.bashrc' + - text: 'source <(invoke --print-completion-script bash)' diff --git a/salt/salt/java.sls b/salt/salt/java.sls new file mode 100644 index 00000000..2faa64f2 --- /dev/null +++ b/salt/salt/java.sls @@ -0,0 +1,3 @@ +java-openjdk: + pkg.installed: + - name: openjdk-8-jre diff --git a/salt/salt/nginx/tildes-static-sites.conf.jinja2 b/salt/salt/nginx/tildes-static-sites.conf.jinja2 index ed5cc6d4..869bcf6f 100644 --- a/salt/salt/nginx/tildes-static-sites.conf.jinja2 +++ b/salt/salt/nginx/tildes-static-sites.conf.jinja2 @@ -23,14 +23,6 @@ server { return 302 https://docs.tildes.net/policies/code-of-conduct; } - location /development { - return 302 https://docs.tildes.net/instructions/development; - } - - location /development-setup { - return 302 https://docs.tildes.net/instructions/development-setup; - } - location /mechanics { return 302 https://docs.tildes.net/instructions; } diff --git a/salt/salt/postgresql_redis_bridge.service.jinja2 b/salt/salt/postgresql_redis_bridge.service.jinja2 index d972f0e4..be2a5f35 100644 --- a/salt/salt/postgresql_redis_bridge.service.jinja2 +++ b/salt/salt/postgresql_redis_bridge.service.jinja2 @@ -1,4 +1,4 @@ -{% from 'common.jinja2' import app_dir, bin_dir -%} +{% from 'common.jinja2' import app_dir, app_username, bin_dir -%} [Unit] Description=postgresql_redis_bridge - convert NOTIFY to Redis streams Requires=redis.service @@ -6,6 +6,8 @@ After=redis.service PartOf=redis.service [Service] +User={{ app_username }} +Group={{ app_username }} WorkingDirectory={{ app_dir }}/scripts Environment="INI_FILE={{ app_dir }}/{{ pillar['ini_file'] }}" ExecStart={{ bin_dir }}/python postgresql_redis_bridge.py diff --git a/salt/salt/prometheus/exporters/prometheus_redis_exporter.service b/salt/salt/prometheus/exporters/prometheus_redis_exporter.service index 1b5c03a8..511ad1e9 100644 --- a/salt/salt/prometheus/exporters/prometheus_redis_exporter.service +++ b/salt/salt/prometheus/exporters/prometheus_redis_exporter.service @@ -8,7 +8,7 @@ RemainAfterExit=no WorkingDirectory=/opt/prometheus_redis_exporter User=prometheus Group=prometheus -Environment="REDIS_ADDR=unix:///run/redis/socket,unix:///run/redis_breached_passwords/socket" +Environment="REDIS_ADDR=unix:///run/redis/socket" ExecStart=/opt/prometheus_redis_exporter/redis_exporter [Install] diff --git a/salt/salt/pts-lbsearch.sls b/salt/salt/pts-lbsearch.sls new file mode 100644 index 00000000..d09686ab --- /dev/null +++ b/salt/salt/pts-lbsearch.sls @@ -0,0 +1,12 @@ +compile-pts-lbsearch: + file.managed: + - name: /tmp/pts_lbsearch.c + - source: + - https://raw.githubusercontent.com/pts/pts-line-bisect/2ecd9f59246cfa28cb1aeac7cd8d98a8eea2914f/pts_lbsearch.c + - source_hash: sha256=ef79efc2f1ecde504b6074f9c89bdc71259a833fa2a2dda4538ed5ea3e04aea1 + - creates: /usr/local/bin/pts_lbsearch + cmd.run: + - cwd: /tmp/ + # compilation command taken from the top of the source file + - name: gcc -ansi -W -Wall -Wextra -Werror=missing-declarations -s -O2 -DNDEBUG -o /usr/local/bin/pts_lbsearch pts_lbsearch.c + - creates: /usr/local/bin/pts_lbsearch diff --git a/salt/salt/redis/breached-passwords.sls b/salt/salt/redis/breached-passwords.sls deleted file mode 100644 index 39f48079..00000000 --- a/salt/salt/redis/breached-passwords.sls +++ /dev/null @@ -1,29 +0,0 @@ -/run/redis_breached_passwords: - file.directory: - - user: redis - - group: redis - - mode: 755 - - require: - - user: redis-user - -/etc/redis_breached_passwords.conf: - file.managed: - - source: salt://redis/redis_breached_passwords.conf - - user: redis - - group: redis - - mode: 600 - -/etc/systemd/system/redis_breached_passwords.service: - file.managed: - - source: salt://redis/redis_breached_passwords.service - - user: root - - group: root - - mode: 644 - - require_in: - - service: redis_breached_passwords.service - -redis_breached_passwords.service: - service.running: - - enable: True - - watch: - - file: /etc/redis_breached_passwords.conf diff --git a/salt/salt/redis/modules/rebloom.sls b/salt/salt/redis/modules/rebloom.sls deleted file mode 100644 index ad84ca07..00000000 --- a/salt/salt/redis/modules/rebloom.sls +++ /dev/null @@ -1,18 +0,0 @@ -# Take care if updating this module - Redis Labs changed the license on July 16, 2018 -# to Apache 2 with their "Commons Clause": https://commonsclause.com/ -# The legality and specific implications of that clause are currently unclear, so we -# probably shouldn't update to a version under that license without more research. -rebloom-clone: - git.latest: - - name: https://github.com/RedisLabsModules/rebloom - - rev: 4947c9a75838688df56fc818729b93bf36588400 - - target: /opt/rebloom - -rebloom-make: - cmd.run: - - name: make - - cwd: /opt/rebloom - - onchanges: - - git: rebloom-clone - - require_in: - - service: redis_breached_passwords.service diff --git a/salt/salt/redis/redis_breached_passwords.conf b/salt/salt/redis/redis_breached_passwords.conf deleted file mode 100644 index bf9f781a..00000000 --- a/salt/salt/redis/redis_breached_passwords.conf +++ /dev/null @@ -1,24 +0,0 @@ -loadmodule /opt/rebloom/rebloom.so -bind 127.0.0.1 - -# only listen on unix socket -port 0 - -unixsocket /run/redis_breached_passwords/socket -unixsocketperm 777 - -timeout 0 - -supervised systemd -pidfile /run/redis_breached_passwords/pid - -loglevel notice -logfile "" - -databases 1 -rdbchecksum yes - -dir /var/lib/redis -dbfilename breached_passwords_dump.rdb - -appendonly no diff --git a/salt/salt/redis/redis_breached_passwords.service b/salt/salt/redis/redis_breached_passwords.service deleted file mode 100644 index 9cd57e9b..00000000 --- a/salt/salt/redis/redis_breached_passwords.service +++ /dev/null @@ -1,14 +0,0 @@ -[Unit] -Description=redis breached passwords daemon -After=network.target - -[Service] -PIDFile=/run/redis_breached_passwords/pid -User=redis -Group=redis -RuntimeDirectory=redis_breached_passwords -ExecStart=/usr/local/bin/redis-server /etc/redis_breached_passwords.conf -Restart=always - -[Install] -WantedBy=multi-user.target diff --git a/salt/salt/top.sls b/salt/salt/top.sls index 9f6cb0a0..9ef97a73 100644 --- a/salt/salt/top.sls +++ b/salt/salt/top.sls @@ -9,8 +9,6 @@ base: - postgresql.pgbouncer - python - redis - - redis.breached-passwords - - redis.modules.rebloom - redis.modules.redis-cell - postgresql-redis-bridge - scripts @@ -22,6 +20,7 @@ base: - tildes-wiki - boussole - webassets + - pts-lbsearch - cronjobs - final-setup # keep this state file last 'dev': @@ -30,6 +29,7 @@ base: - development - prometheus - nodejs + - java 'prod': - nginx.shortener-config - nginx.static-sites-config diff --git a/salt/salt/webassets.service.jinja2 b/salt/salt/webassets.service.jinja2 index 69d47ed7..1c3ad7d8 100644 --- a/salt/salt/webassets.service.jinja2 +++ b/salt/salt/webassets.service.jinja2 @@ -1,8 +1,10 @@ -{% from 'common.jinja2' import app_dir, bin_dir -%} +{% from 'common.jinja2' import app_dir, app_username, bin_dir -%} [Unit] Description=Webassets - auto-compile JS files on change [Service] +User={{ app_username }} +Group={{ app_username }} WorkingDirectory={{ app_dir }} ExecStart={{ bin_dir }}/webassets -c webassets.yaml watch Restart=always diff --git a/tildes/alembic/versions/0435c46f64d8_topic_schedule_add_only_new_top_level_.py b/tildes/alembic/versions/0435c46f64d8_topic_schedule_add_only_new_top_level_.py new file mode 100644 index 00000000..7be95568 --- /dev/null +++ b/tildes/alembic/versions/0435c46f64d8_topic_schedule_add_only_new_top_level_.py @@ -0,0 +1,32 @@ +"""topic_schedule: add only_new_top_level_comments_in_latest + +Revision ID: 0435c46f64d8 +Revises: 468cf81f4a6b +Create Date: 2020-07-05 19:33:17.746617 + +""" +from alembic import op +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision = "0435c46f64d8" +down_revision = "468cf81f4a6b" +branch_labels = None +depends_on = None + + +def upgrade(): + op.add_column( + "topic_schedule", + sa.Column( + "only_new_top_level_comments_in_latest", + sa.Boolean(), + server_default="true", + nullable=False, + ), + ) + + +def downgrade(): + op.drop_column("topic_schedule", "only_new_top_level_comments_in_latest") diff --git a/tildes/alembic/versions/28d7ce2c4825_use_enum_for_user_permissions.py b/tildes/alembic/versions/28d7ce2c4825_use_enum_for_user_permissions.py new file mode 100644 index 00000000..dd0d100f --- /dev/null +++ b/tildes/alembic/versions/28d7ce2c4825_use_enum_for_user_permissions.py @@ -0,0 +1,49 @@ +"""Use enum for user permissions + +Revision ID: 28d7ce2c4825 +Revises: 82e9801eb2d6 +Create Date: 2020-08-05 20:32:51.047215 + +""" +from alembic import op +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision = "28d7ce2c4825" +down_revision = "82e9801eb2d6" +branch_labels = None +depends_on = None + + +def upgrade(): + op.execute( + """ + create type userpermission as enum( + 'comment.remove', + 'comment.view_labels', + 'topic.edit_by_generic_user', + 'topic.edit_link', + 'topic.edit_title', + 'topic.lock', + 'topic.move', + 'topic.post', + 'topic.remove', + 'topic.tag', + 'user.ban', + 'user.view_removed_posts', + 'wiki.edit' + ) + """ + ) + op.execute( + """ + alter table user_permissions + alter column permission type userpermission using permission::userpermission + """ + ) + + +def downgrade(): + op.execute("alter table user_permissions alter column permission type text") + op.execute("drop type userpermission") diff --git a/tildes/alembic/versions/468cf81f4a6b_topic_schedule_add_latest_topic_id.py b/tildes/alembic/versions/468cf81f4a6b_topic_schedule_add_latest_topic_id.py new file mode 100644 index 00000000..61f118d2 --- /dev/null +++ b/tildes/alembic/versions/468cf81f4a6b_topic_schedule_add_latest_topic_id.py @@ -0,0 +1,113 @@ +"""topic_schedule: add latest_topic_id + +Revision ID: 468cf81f4a6b +Revises: 4d86b372a8db +Create Date: 2020-06-25 02:53:09.435947 + +""" +from alembic import op +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision = "468cf81f4a6b" +down_revision = "4d86b372a8db" +branch_labels = None +depends_on = None + + +def upgrade(): + op.add_column( + "topic_schedule", sa.Column("latest_topic_id", sa.Integer(), nullable=True) + ) + op.create_foreign_key( + op.f("fk_topic_schedule_latest_topic_id_topics"), + "topic_schedule", + "topics", + ["latest_topic_id"], + ["topic_id"], + ) + + op.execute( + """ + update topic_schedule set latest_topic_id = ( + select topic_id from topics + where schedule_id = topic_schedule.schedule_id + order by created_time desc limit 1 + ) + """ + ) + + op.execute( + """ + create or replace function update_topic_schedule_latest_topic_id() returns trigger as $$ + begin + if (NEW.schedule_id is not null) then + update topic_schedule + set latest_topic_id = ( + select topic_id + from topics + where schedule_id = NEW.schedule_id + and is_deleted = false + and is_removed = false + order by created_time desc limit 1) + where schedule_id = NEW.schedule_id; + end if; + + -- if it was an update that changed schedule_id, need to update the old schedule's + -- latest_topic_id as well (this will probably be extremely uncommon) + if (TG_OP = 'UPDATE' + and OLD.schedule_id is not null + and OLD.schedule_id is distinct from NEW.schedule_id) then + update topic_schedule + set latest_topic_id = ( + select topic_id + from topics + where schedule_id = OLD.schedule_id + and is_deleted = false + and is_removed = false + order by created_time desc limit 1) + where schedule_id = OLD.schedule_id; + end if; + + return null; + end + $$ language plpgsql; + """ + ) + + op.execute( + """ + create trigger update_topic_schedule_latest_topic_id_insert + after insert on topics + for each row + when (NEW.schedule_id is not null) + execute procedure update_topic_schedule_latest_topic_id(); + """ + ) + + op.execute( + """ + create trigger update_topic_schedule_latest_topic_id_update + after update on topics + for each row + when ((OLD.schedule_id is not null or NEW.schedule_id is not null) + and ((OLD.is_deleted is distinct from NEW.is_deleted) + or (OLD.is_removed is distinct from NEW.is_removed) + or (OLD.schedule_id is distinct from NEW.schedule_id))) + execute procedure update_topic_schedule_latest_topic_id(); + """ + ) + + +def downgrade(): + op.execute("drop trigger update_topic_schedule_latest_topic_id_update on topics") + op.execute("drop trigger update_topic_schedule_latest_topic_id_insert on topics") + op.execute("drop function update_topic_schedule_latest_topic_id") + + op.drop_constraint( + op.f("fk_topic_schedule_latest_topic_id_topics"), + "topic_schedule", + type_="foreignkey", + ) + op.drop_column("topic_schedule", "latest_topic_id") diff --git a/tildes/alembic/versions/55f4c1f951d5_add_group_scripts_table.py b/tildes/alembic/versions/55f4c1f951d5_add_group_scripts_table.py new file mode 100644 index 00000000..b5b287f5 --- /dev/null +++ b/tildes/alembic/versions/55f4c1f951d5_add_group_scripts_table.py @@ -0,0 +1,35 @@ +"""Add group_scripts table + +Revision ID: 55f4c1f951d5 +Revises: 28d7ce2c4825 +Create Date: 2020-11-30 19:54:30.731335 + +""" +from alembic import op +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision = "55f4c1f951d5" +down_revision = "28d7ce2c4825" +branch_labels = None +depends_on = None + + +def upgrade(): + op.create_table( + "group_scripts", + sa.Column("script_id", sa.Integer(), nullable=False), + sa.Column("group_id", sa.Integer(), nullable=True), + sa.Column("code", sa.Text(), nullable=False), + sa.ForeignKeyConstraint( + ["group_id"], + ["groups.group_id"], + name=op.f("fk_group_scripts_group_id_groups"), + ), + sa.PrimaryKeyConstraint("script_id", name=op.f("pk_group_scripts")), + ) + + +def downgrade(): + op.drop_table("group_scripts") diff --git a/tildes/alembic/versions/82e9801eb2d6_update_post_topic_permission_to_topic_.py b/tildes/alembic/versions/82e9801eb2d6_update_post_topic_permission_to_topic_.py new file mode 100644 index 00000000..50f2e62c --- /dev/null +++ b/tildes/alembic/versions/82e9801eb2d6_update_post_topic_permission_to_topic_.py @@ -0,0 +1,28 @@ +"""Update post_topic permission to topic.post + +Revision ID: 82e9801eb2d6 +Revises: 0435c46f64d8 +Create Date: 2020-08-05 00:05:46.690188 + +""" +from alembic import op +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision = "82e9801eb2d6" +down_revision = "0435c46f64d8" +branch_labels = None +depends_on = None + + +def upgrade(): + op.execute( + "update user_permissions set permission = 'topic.post' where permission = 'post_topic'" + ) + + +def downgrade(): + op.execute( + "update user_permissions set permission = 'post_topic' where permission = 'topic.post'" + ) diff --git a/tildes/alembic/versions/da5a84d29685_.py b/tildes/alembic/versions/da5a84d29685_.py new file mode 100644 index 00000000..7c8f4f1a --- /dev/null +++ b/tildes/alembic/versions/da5a84d29685_.py @@ -0,0 +1,24 @@ +"""empty message + +Revision ID: da5a84d29685 +Revises: 2c50cb4dd1b8, 55f4c1f951d5 +Create Date: 2021-03-19 05:40:08.664239 + +""" +from alembic import op +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision = "da5a84d29685" +down_revision = ("2c50cb4dd1b8", "55f4c1f951d5") +branch_labels = None +depends_on = None + + +def upgrade(): + pass + + +def downgrade(): + pass diff --git a/tildes/consumers/post_processing_script_runner.py b/tildes/consumers/post_processing_script_runner.py new file mode 100644 index 00000000..52fd2665 --- /dev/null +++ b/tildes/consumers/post_processing_script_runner.py @@ -0,0 +1,73 @@ +# Copyright (c) 2020 Tildes contributors +# SPDX-License-Identifier: AGPL-3.0-or-later + +"""Consumer that runs processing scripts on posts.""" + +from sqlalchemy import desc +from sqlalchemy.sql.expression import or_ + +from tildes.lib.event_stream import EventStreamConsumer, Message +from tildes.lib.lua import SandboxedLua +from tildes.models.comment import Comment +from tildes.models.group import GroupScript +from tildes.models.scripting import CommentScriptingWrapper, TopicScriptingWrapper +from tildes.models.topic import Topic + + +class PostProcessingScriptRunner(EventStreamConsumer): + """Consumer that generates content_metadata for topics.""" + + METRICS_PORT = 25016 + + def process_message(self, message: Message) -> None: + """Process a message from the stream.""" + if "topic_id" in message.fields: + post = ( + self.db_session.query(Topic) + .filter_by(topic_id=message.fields["topic_id"]) + .one() + ) + wrapper_class = TopicScriptingWrapper + group = post.group + elif "comment_id" in message.fields: + post = ( + self.db_session.query(Comment) + .filter_by(comment_id=message.fields["comment_id"]) + .one() + ) + wrapper_class = CommentScriptingWrapper + group = post.topic.group + + if post.is_deleted: + return + + scripts_to_run = ( + self.db_session.query(GroupScript) + .filter(or_(GroupScript.group == None, GroupScript.group == group)) # noqa + .order_by(desc(GroupScript.group_id)) # sort the global script first + .all() + ) + + for script in scripts_to_run: + lua_sandbox = SandboxedLua() + lua_sandbox.run_code(script.code) + + wrapped_post = wrapper_class(post, lua_sandbox) + + try: + if isinstance(post, Topic): + lua_sandbox.run_lua_function("on_topic_post", wrapped_post) + elif isinstance(post, Comment): + lua_sandbox.run_lua_function("on_comment_post", wrapped_post) + except ValueError: + pass + + +if __name__ == "__main__": + PostProcessingScriptRunner( + "post_processing_script_runner", + source_streams=[ + "comments.insert", + "topics.insert", + ], + ).consume_streams() diff --git a/tildes/lua/sandbox.lua b/tildes/lua/sandbox.lua new file mode 100644 index 00000000..dc68fc7a --- /dev/null +++ b/tildes/lua/sandbox.lua @@ -0,0 +1,277 @@ +-- Lua Sandbox +-- From the Splash project: https://github.com/scrapinghub/splash +-- Original version was as of Splash commit 75a5394af310bf07d704c3c05c0e9902d88592f2 +-- +-- Copyright (c) Scrapinghub +-- All rights reserved. +-- +-- Redistribution and use in source and binary forms, with or without modification, +-- are permitted provided that the following conditions are met: +-- +-- 1. Redistributions of source code must retain the above copyright notice, +-- this list of conditions and the following disclaimer. +-- +-- 2. Redistributions in binary form must reproduce the above copyright +-- notice, this list of conditions and the following disclaimer in the +-- documentation and/or other materials provided with the distribution. +-- +-- 3. Neither the name of Splash nor the names of its contributors may be used +-- to endorse or promote products derived from this software without +-- specific prior written permission. +-- +-- THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +-- ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +-- WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +-- DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR +-- ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +-- (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +-- LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON +-- ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +-- (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +-- SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +local sandbox = {} + +sandbox.allowed_require_names = {} + +-- 6.4 String Manipulation +-- http://www.lua.org/manual/5.2/manual.html#6.4 +local _string = { + byte = string.byte, + char = string.char, + find = string.find, + format = string.format, +-- gmatch = string.gmatch, -- can be CPU intensive +-- gsub = string.gsub, -- can be CPU intensive; can result in arbitrary native code execution (in 5.1)? + len = string.len, + lower = string.lower, +-- match = string.match, -- can be CPU intensive +-- rep = string.rep, -- can eat memory + reverse = string.reverse, + sub = string.sub, + upper = string.upper, +} + + +sandbox.env = { + -- + -- 6.1 Basic Functions + -- http://www.lua.org/manual/5.2/manual.html#6.1 + assert = assert, + error = error, + ipairs = ipairs, + next = next, + pairs = pairs, + pcall = pcall, + print = print, -- should we disable it? + select = select, + tonumber = tonumber, + tostring = tostring, -- Mike Pall says it is unsafe; why? See http://lua-users.org/lists/lua-l/2011-02/msg01595.html + type = type, + xpcall = xpcall, + + -- + -- 6.2 Coroutine Manipulation + -- http://www.lua.org/manual/5.2/manual.html#6.2 + -- + -- Disabled because: + -- 1. coroutines are used internally - users shouldn't yield to Splash themselves; + -- 2. debug hooks are per-coroutine in 'standard' Lua (not LuaJIT) - this requires a workaround. + + -- + -- 6.3 Modules + -- http://www.lua.org/manual/5.2/manual.html#6.3 + -- + require = function(name) + if sandbox.allowed_require_names[name] then + local ok, res = pcall(function() return require(name) end) + if ok then + return res + end + end + error("module '" .. name .. "' not found", 2) + end, + + -- + -- 6.4 String Manipulation + -- http://www.lua.org/manual/5.2/manual.html#6.4 + string = _string, + + -- + -- 6.5 Table Manipulation + -- http://www.lua.org/manual/5.2/manual.html#6.5 + table = { + concat = table.concat, + insert = table.insert, + pack = table.pack, + remove = table.remove, +-- sort = table.sort, -- can result in arbitrary native code execution (in 5.1)? + unpack = table.unpack, + }, + + -- + -- 6.6 Mathematical Functions + -- http://www.lua.org/manual/5.2/manual.html#6.6 + math = { + abs = math.abs, + acos = math.acos, + asin = math.asin, + atan = math.atan, + atan2 = math.atan2, + ceil = math.ceil, + cos = math.cos, + cosh = math.cosh, + deg = math.deg, + exp = math.exp, + floor = math.floor, + fmod = math.fmod, + frexp = math.frexp, + huge = math.huge, + ldexp = math.ldexp, + log = math.log, + max = math.max, + min = math.min, + modf = math.modf, + pi = math.pi, + pow = math.pow, + rad = math.rad, + random = math.random, + randomseed = math.randomseed, + sin = math.sin, + sinh = math.sinh, + sqrt = math.sqrt, + tan = math.tan, + tanh = math.tanh, + }, + + -- + -- 6.7 Bitwise Operations + -- http://www.lua.org/manual/5.2/manual.html#6.7 + -- + -- Disabled: if anyone cares we may add them. + + -- + -- 6.8 Input and Output Facilities + -- http://www.lua.org/manual/5.2/manual.html#6.8 + -- + -- Disabled. + + -- + -- 6.9 Operating System Facilities + -- http://www.lua.org/manual/5.2/manual.html#6.9 + os = { + clock = os.clock, +-- date = os.date, -- from wiki: "This can crash on some platforms (undocumented). For example, os.date'%v'. It is reported that this will be fixed in 5.2 or 5.1.3." + difftime = os.difftime, + time = os.time, + }, + + -- + -- 6.10 The Debug Library + -- http://www.lua.org/manual/5.2/manual.html#6.10 + -- + -- Disabled. +} + +------------------------------------------------------------- +-- +-- Fix metatables. Some of the functions are available +-- via metatables of primitive types; disable them all. +-- +sandbox.fix_metatables = function() + -- Fix string metatable: provide common functions + -- from string module. + local mt = {__index={}} + for k, v in pairs(_string) do + mt['__index'][k] = v + end + debug.setmetatable('', mt) + + -- 2. Make sure there are no other metatables: + debug.setmetatable(1, nil) + debug.setmetatable(function() end, nil) + debug.setmetatable(true, nil) +end + + +------------------------------------------------------------- +-- +-- Basic memory and CPU limits. +-- Based on code by Roberto Ierusalimschy. +-- http://lua-users.org/lists/lua-l/2013-12/msg00406.html +-- + +-- maximum memory (in KB) that can be used by Lua script +sandbox.mem_limit = 100000 +sandbox.mem_limit_reached = false + +function sandbox.enable_memory_limit() + if sandbox._memory_tracking_enabled then + return + end + local mt = {__gc = function (u) + if sandbox.mem_limit_reached then + error("script uses too much memory") + end + if collectgarbage("count") > sandbox.mem_limit then + sandbox.mem_limit_reached = true + error("script uses too much memory") + else + -- create a new object for the next GC cycle + setmetatable({}, getmetatable(u)) + end + end } + -- create an empty object which will be collected at next GC cycle + setmetatable({}, mt) + sandbox._memory_tracking_enabled = true +end + + +-- Maximum number of instructions that can be executed. +-- XXX: the slowdown only becomes percievable at ~5m instructions. +sandbox.instruction_limit = 1e7 +sandbox.instruction_count = 0 + +function sandbox.enable_per_instruction_limits() + local function _debug_step(event, line) + sandbox.instruction_count = sandbox.instruction_count + 1 + if sandbox.instruction_count > sandbox.instruction_limit then + error("script uses too much CPU", 2) + end + if sandbox.mem_limit_reached then + error("script uses too much memory") + end + end + debug.sethook(_debug_step, '', 1) +end + + +-- In Lua (but not in LuaJIT) debug hooks are per-coroutine. +-- Use this function as a replacement for `coroutine.create` to ensure +-- instruction limit is enforced in coroutines. +function sandbox.create_coroutine(f, ...) + return coroutine.create(function(...) + sandbox.enable_per_instruction_limits() + return f(...) + end, ...) +end + + +------------------------------------------------------------- +-- +-- Lua 5.2 sandbox. +-- +-- Note that it changes the global state: after the first `sandbox.run` +-- call the runtime becomes restricted in CPU and memory, and +-- "string":methods() like "foo":upper() stop working. +-- +function sandbox.run(untrusted_code) + sandbox.fix_metatables() + sandbox.enable_memory_limit() + sandbox.enable_per_instruction_limits() + local untrusted_function, message = load(untrusted_code, nil, 't', sandbox.env) + if not untrusted_function then return nil, message end + return pcall(untrusted_function) +end + +return sandbox diff --git a/tildes/mypy.ini b/tildes/mypy.ini index 3d54ee76..55b6322d 100644 --- a/tildes/mypy.ini +++ b/tildes/mypy.ini @@ -9,5 +9,9 @@ show_error_context = true warn_redundant_casts = true warn_unused_ignores = true +# invoke crashes if task functions use type annotations, so we can't use them there +[mypy-tasks] +disallow_untyped_defs = false + [mypy-tests.*] disallow_untyped_defs = false diff --git a/tildes/production.ini.example b/tildes/production.ini.example index 17b091c7..835840b3 100644 --- a/tildes/production.ini.example +++ b/tildes/production.ini.example @@ -28,6 +28,15 @@ stripe.recurring_donation_product_id = prod_ProductID tildes.default_user_comment_label_weight = 1.0 tildes.open_registration = true +# Path to the file to use to check for passwords that have been in data breaches, which +# users will be prevented from using as their password. It's recommended to use the +# "Pwned Passwords" list downloaded from https://haveibeenpwned.com/passwords (must be +# the SHA-1 format, "ordered by hash" one), but you can use any file with a compatible +# format: each line starting with a single uppercase SHA-1 hash of a password to block, +# with the entire file sorted in lexographical order. +# Leave this line commented out to allow all passwords. +# tildes.breached_passwords_hash_file_path = /opt/tildes/pwned-passwords-sha1-ordered-by-hash-v6.txt + webassets.auto_build = false webassets.base_dir = %(here)s/static webassets.base_url = / diff --git a/tildes/prospector.yaml b/tildes/prospector.yaml index d6e20ca9..d115a9aa 100644 --- a/tildes/prospector.yaml +++ b/tildes/prospector.yaml @@ -30,7 +30,7 @@ pylint: - bad-continuation # let Black handle line-wrapping - comparison-with-callable # seems to have a lot of false positives - cyclic-import # not sure what's triggering this, doesn't seem to work correctly - - logging-format-interpolation # rather use f-strings than worry about this + - logging-fstring-interpolation # rather use f-strings than worry about this - no-else-return # elif after return - could refactor to enable this check - no-self-use # schemas do this a lot, would be nice to only disable for schemas - too-few-public-methods # plenty of classes that don't need multiple methods diff --git a/tildes/pytest.ini b/tildes/pytest.ini index 6162a493..ededd1ed 100644 --- a/tildes/pytest.ini +++ b/tildes/pytest.ini @@ -1,7 +1,10 @@ [pytest] testpaths = tests -addopts = -p no:cacheprovider +addopts = -p no:cacheprovider --strict-markers filterwarnings = ignore::DeprecationWarning ignore::PendingDeprecationWarning ignore::yaml.YAMLLoadWarning +markers = + html_validation: mark a test as one that validates HTML using the Nu HTML Checker (very slow) + webtest: mark a test as one that uses the WebTest library, which goes through the actual WSGI app and involves using HTTP/HTML (more of a "functional test" than "unit test") diff --git a/tildes/requirements-dev.in b/tildes/requirements-dev.in index 2a508dfb..fee5842d 100644 --- a/tildes/requirements-dev.in +++ b/tildes/requirements-dev.in @@ -1,6 +1,7 @@ -r requirements.in black freezegun +html5validator mypy prospector pyramid-debugtoolbar diff --git a/tildes/requirements-dev.txt b/tildes/requirements-dev.txt index 17f47ce4..38379d22 100644 --- a/tildes/requirements-dev.txt +++ b/tildes/requirements-dev.txt @@ -1,120 +1,126 @@ ago==0.0.93 -alembic==1.3.3 -appdirs==1.4.3 # via black -argon2-cffi==19.2.0 -astroid==2.3.3 # via prospector, pylint, pylint-celery, pylint-flask, requirements-detector -attrs==19.3.0 # via black, pytest -backcall==0.1.0 # via ipython -beautifulsoup4==4.8.2 -black==19.10b0 -bleach==3.1.5 -certifi==2019.11.28 # via requests, sentry-sdk -cffi==1.13.2 # via argon2-cffi, pygit2 -chardet==3.0.4 # via requests -click==7.0 -cornice==4.0.1 -decorator==4.4.1 # via ipython, traitlets -dodgy==0.2.1 # via prospector -freezegun==0.3.14 +alembic==1.4.3 +appdirs==1.4.4 +argon2-cffi==20.1.0 +astroid==2.4.1 +attrs==20.2.0 +backcall==0.2.0 +beautifulsoup4==4.9.3 +black==20.8b1 +bleach==3.2.1 +cached-property==1.5.2 +certifi==2020.6.20 +cffi==1.14.3 +chardet==3.0.4 +click==7.1.2 +cornice==5.0.3 +decorator==4.4.2 +dodgy==0.2.1 +flake8-polyfill==1.0.2 +flake8==3.8.4 +freezegun==1.0.0 gunicorn==20.0.4 -html5lib==1.0.1 -hupper==1.9.1 # via pyramid -idna==2.8 # via requests -ipython-genutils==0.2.0 # via traitlets -ipython==7.12.0 -isort==4.3.21 # via pylint -jedi==0.16.0 # via ipython -jinja2==2.11.1 # via pyramid-jinja2 -lazy-object-proxy==1.4.3 # via astroid -mako==1.1.1 # via alembic, pyramid-mako -markupsafe==1.1.1 # via jinja2, mako, pyramid-jinja2 -marshmallow==3.4.0 -mccabe==0.6.1 # via prospector, pylint -more-itertools==8.2.0 # via pytest -mypy-extensions==0.4.3 # via mypy -mypy==0.761 -packaging==20.1 # via pytest -parso==0.6.1 # via jedi -pastedeploy==2.1.0 # via plaster-pastedeploy -pathspec==0.7.0 # via black -pep8-naming==0.4.1 # via prospector -pexpect==4.8.0 # via ipython -pickleshare==0.7.5 # via ipython -pillow==7.0.0 -pip-tools==4.4.1 -plaster-pastedeploy==0.7 # via pyramid -plaster==1.0 # via plaster-pastedeploy, pyramid -pluggy==0.13.1 # via pytest -prometheus-client==0.7.1 -prompt-toolkit==3.0.3 # via ipython -prospector==1.2.0 -psycopg2==2.8.4 -ptyprocess==0.6.0 # via pexpect +html5lib==1.1 +html5validator==0.3.3 +hupper==1.10.2 +idna==2.10 +iniconfig==1.1.1 +invoke==1.4.1 +ipython-genutils==0.2.0 +ipython==7.19.0 +isort==4.3.21 +jedi==0.17.2 +jinja2==2.11.2 +lazy-object-proxy==1.4.3 +lupa==1.9 +mako==1.1.3 +markupsafe==1.1.1 +marshmallow==3.9.0 +mccabe==0.6.1 +mypy-extensions==0.4.3 +mypy==0.790 +packaging==20.4 +parso==0.7.1 +pastedeploy==2.1.1 +pathspec==0.8.0 +pep8-naming==0.10.0 +pexpect==4.8.0 +pickleshare==0.7.5 +pillow==8.0.1 +pip-tools==5.3.1 +plaster-pastedeploy==0.7 +plaster==1.0 +pluggy==0.13.1 +prometheus-client==0.8.0 +prompt-toolkit==3.0.8 +prospector==1.3.1 +psycopg2==2.8.6 +ptyprocess==0.6.0 publicsuffix2==2.20160818 -py==1.8.1 # via pytest -pycodestyle==2.4.0 # via prospector -pycparser==2.19 # via cffi -pydocstyle==5.0.2 # via prospector -pyflakes==2.1.1 # via prospector -pygit2==1.0.3 -pygments==2.5.2 -pylint-celery==0.3 # via prospector -pylint-django==2.0.12 # via prospector -pylint-flask==0.6 # via prospector -pylint-plugin-utils==0.6 # via prospector, pylint-celery, pylint-django, pylint-flask -pylint==2.4.4 # via prospector, pylint-celery, pylint-django, pylint-flask, pylint-plugin-utils -pyotp==2.3.0 -pyparsing==2.4.6 # via packaging -pyramid-debugtoolbar==4.6 +py==1.9.0 +pycodestyle==2.6.0 +pycparser==2.20 +pydocstyle==5.1.1 +pyflakes==2.2.0 +pygit2==1.3.0 +pygments==2.7.2 +pylint-celery==0.3 +pylint-django==2.1.0 +pylint-flask==0.6 +pylint-plugin-utils==0.6 +pylint==2.5.3 +pyotp==2.4.1 +pyparsing==2.4.7 +pyramid-debugtoolbar==4.8 pyramid-ipython==0.2 pyramid-jinja2==2.8 -pyramid-mako==1.1.0 # via pyramid-debugtoolbar +pyramid-mako==1.1.0 pyramid-session-redis==1.5.0 pyramid-tm==2.4 pyramid-webassets==0.10 pyramid==1.10.4 -pytest-mock==2.0.0 -pytest==5.3.5 +pytest-mock==3.3.1 +pytest==6.1.2 python-dateutil==2.8.1 -python-editor==1.0.4 # via alembic -pyyaml==5.3 +python-editor==1.0.4 +pyyaml==5.3.1 qrcode==6.1 -redis==3.4.1 -regex==2020.1.8 # via black -repoze.lru==0.7 # via pyramid-debugtoolbar -requests==2.22.0 -requirements-detector==0.6 # via prospector -sentry-sdk==0.14.1 -setoptconf==0.2.0 # via prospector -simplejson==3.17.0 # via cornice -six==1.14.0 # via argon2-cffi, astroid, bleach, cornice, freezegun, html5lib, packaging, pip-tools, pyramid-session-redis, pyramid-webassets, python-dateutil, qrcode, sqlalchemy-utils, traitlets, webtest -snowballstemmer==2.0.0 # via pydocstyle -soupsieve==1.9.5 # via beautifulsoup4 -sqlalchemy-utils==0.36.1 -sqlalchemy==1.3.13 -stripe==2.42.0 -testing.common.database==2.0.3 # via testing.redis +redis==3.5.3 +regex==2020.10.28 +repoze.lru==0.7 +requests==2.24.0 +requirements-detector==0.7 +sentry-sdk==0.19.1 +setoptconf==0.2.0 +six==1.15.0 +snowballstemmer==2.0.0 +soupsieve==2.0.1 +sqlalchemy-utils==0.36.8 +sqlalchemy==1.3.20 +stripe==2.55.0 +testing.common.database==2.0.3 testing.redis==1.1.1 -titlecase==0.12.0 -toml==0.10.0 # via black -traitlets==4.3.3 # via ipython -transaction==3.0.0 # via pyramid-tm, zope.sqlalchemy -translationstring==1.3 # via pyramid -typed-ast==1.4.1 # via black, mypy -typing-extensions==3.7.4.1 # via mypy -urllib3==1.25.8 # via requests, sentry-sdk -venusian==3.0.0 # via cornice, pyramid -waitress==1.4.3 # via webtest -wcwidth==0.1.8 # via prompt-toolkit, pytest -webargs==5.5.3 -webassets==2.0 # via pyramid-webassets -webencodings==0.5.1 # via bleach, html5lib -webob==1.8.6 # via pyramid, webtest -webtest==2.0.34 -wrapt==1.11.2 -zope.deprecation==4.4.0 # via pyramid, pyramid-jinja2 -zope.interface==4.7.1 # via pyramid, pyramid-webassets, transaction, zope.sqlalchemy -zope.sqlalchemy==1.2 +titlecase==1.1.1 +toml==0.10.2 +traitlets==5.0.5 +transaction==3.0.0 +translationstring==1.4 +typed-ast==1.4.1 +typing-extensions==3.7.4.3 +urllib3==1.25.11 +venusian==3.0.0 +waitress==1.4.4 +wcwidth==0.2.5 +webargs==6.1.1 +webassets==2.0 +webencodings==0.5.1 +webob==1.8.6 +webtest==2.0.35 +wrapt==1.12.1 +zope.deprecation==4.4.0 +zope.interface==5.1.2 +zope.sqlalchemy==1.3 # The following packages are considered to be unsafe in a requirements file: +# pip # setuptools diff --git a/tildes/requirements.in b/tildes/requirements.in index 88e2784a..1cc2cee5 100644 --- a/tildes/requirements.in +++ b/tildes/requirements.in @@ -7,7 +7,9 @@ click cornice gunicorn html5lib +invoke ipython +lupa marshmallow Pillow pip-tools diff --git a/tildes/requirements.txt b/tildes/requirements.txt index 6c97599a..23a3aeb3 100644 --- a/tildes/requirements.txt +++ b/tildes/requirements.txt @@ -1,43 +1,48 @@ ago==0.0.93 -alembic==1.3.3 -argon2-cffi==19.2.0 -backcall==0.1.0 # via ipython -beautifulsoup4==4.8.2 -bleach==3.1.5 -certifi==2019.11.28 # via requests, sentry-sdk -cffi==1.13.2 # via argon2-cffi, pygit2 -chardet==3.0.4 # via requests -click==7.0 -cornice==4.0.1 -decorator==4.4.1 # via ipython, traitlets +alembic==1.4.3 +argon2-cffi==20.1.0 +backcall==0.2.0 +beautifulsoup4==4.9.3 +bleach==3.2.1 +cached-property==1.5.2 +certifi==2020.6.20 +cffi==1.14.3 +chardet==3.0.4 +click==7.1.2 +cornice==5.0.3 +decorator==4.4.2 gunicorn==20.0.4 -html5lib==1.0.1 -hupper==1.9.1 # via pyramid -idna==2.8 # via requests -ipython-genutils==0.2.0 # via traitlets -ipython==7.12.0 -jedi==0.16.0 # via ipython -jinja2==2.11.1 # via pyramid-jinja2 -mako==1.1.1 # via alembic -markupsafe==1.1.1 # via jinja2, mako, pyramid-jinja2 -marshmallow==3.4.0 -parso==0.6.1 # via jedi -pastedeploy==2.1.0 # via plaster-pastedeploy -pexpect==4.8.0 # via ipython -pickleshare==0.7.5 # via ipython -pillow==7.0.0 -pip-tools==4.4.1 -plaster-pastedeploy==0.7 # via pyramid -plaster==1.0 # via plaster-pastedeploy, pyramid -prometheus-client==0.7.1 -prompt-toolkit==3.0.3 # via ipython -psycopg2==2.8.4 -ptyprocess==0.6.0 # via pexpect +html5lib==1.1 +hupper==1.10.2 +idna==2.10 +invoke==1.4.1 +ipython-genutils==0.2.0 +ipython==7.19.0 +jedi==0.17.2 +jinja2==2.11.2 +lupa==1.9 +mako==1.1.3 +markupsafe==1.1.1 +marshmallow==3.9.0 +packaging==20.4 +parso==0.7.1 +pastedeploy==2.1.1 +pexpect==4.8.0 +pickleshare==0.7.5 +pillow==8.0.1 +pip-tools==5.3.1 +plaster-pastedeploy==0.7 +plaster==1.0 +prometheus-client==0.8.0 +prompt-toolkit==3.0.8 +psycopg2==2.8.6 +ptyprocess==0.6.0 publicsuffix2==2.20160818 -pycparser==2.19 # via cffi -pygit2==1.0.3 -pygments==2.5.2 -pyotp==2.3.0 +pycparser==2.20 +pygit2==1.3.0 +pygments==2.7.2 +pyotp==2.4.1 +pyparsing==2.4.7 pyramid-ipython==0.2 pyramid-jinja2==2.8 pyramid-session-redis==1.5.0 @@ -45,33 +50,34 @@ pyramid-tm==2.4 pyramid-webassets==0.10 pyramid==1.10.4 python-dateutil==2.8.1 -python-editor==1.0.4 # via alembic -pyyaml==5.3 +python-editor==1.0.4 +pyyaml==5.3.1 qrcode==6.1 -redis==3.4.1 -requests==2.22.0 -sentry-sdk==0.14.1 -simplejson==3.17.0 # via cornice -six==1.14.0 # via argon2-cffi, bleach, cornice, html5lib, pip-tools, pyramid-session-redis, pyramid-webassets, python-dateutil, qrcode, sqlalchemy-utils, traitlets -soupsieve==1.9.5 # via beautifulsoup4 -sqlalchemy-utils==0.36.1 -sqlalchemy==1.3.13 -stripe==2.42.0 -titlecase==0.12.0 -traitlets==4.3.3 # via ipython -transaction==3.0.0 # via pyramid-tm, zope.sqlalchemy -translationstring==1.3 # via pyramid -urllib3==1.25.8 # via requests, sentry-sdk -venusian==3.0.0 # via cornice, pyramid -wcwidth==0.1.8 # via prompt-toolkit -webargs==5.5.3 -webassets==2.0 # via pyramid-webassets -webencodings==0.5.1 # via bleach, html5lib -webob==1.8.6 # via pyramid -wrapt==1.11.2 -zope.deprecation==4.4.0 # via pyramid, pyramid-jinja2 -zope.interface==4.7.1 # via pyramid, pyramid-webassets, transaction, zope.sqlalchemy -zope.sqlalchemy==1.2 +redis==3.5.3 +regex==2020.10.28 +requests==2.24.0 +sentry-sdk==0.19.1 +six==1.15.0 +soupsieve==2.0.1 +sqlalchemy-utils==0.36.8 +sqlalchemy==1.3.20 +stripe==2.55.0 +titlecase==1.1.1 +traitlets==5.0.5 +transaction==3.0.0 +translationstring==1.4 +urllib3==1.25.11 +venusian==3.0.0 +wcwidth==0.2.5 +webargs==6.1.1 +webassets==2.0 +webencodings==0.5.1 +webob==1.8.6 +wrapt==1.12.1 +zope.deprecation==4.4.0 +zope.interface==5.1.2 +zope.sqlalchemy==1.3 # The following packages are considered to be unsafe in a requirements file: +# pip # setuptools diff --git a/tildes/scripts/breached_passwords.py b/tildes/scripts/breached_passwords.py deleted file mode 100644 index 7b5d3014..00000000 --- a/tildes/scripts/breached_passwords.py +++ /dev/null @@ -1,170 +0,0 @@ -# Copyright (c) 2018 Tildes contributors -# SPDX-License-Identifier: AGPL-3.0-or-later - -"""Command-line tools for managing a breached-passwords bloom filter. - -This tool will help with creating and updating a bloom filter in Redis (using ReBloom: -https://github.com/RedisLabsModules/rebloom) to hold hashes for passwords that have been -revealed through data breaches (to prevent users from using these passwords here). The -dumps are likely primarily sourced from Troy Hunt's "Pwned Passwords" files: -https://haveibeenpwned.com/Passwords - -Specifically, the commands in this tool allow building the bloom filter somewhere else, -then the RDB file can be transferred to the production server. Note that it is expected -that a separate redis server instance is running solely for holding this bloom filter. -Replacing the RDB file will result in all other keys being lost. - -Expected usage of this tool should look something like: - -On the machine building the bloom filter: - python breached_passwords.py init --estimate 350000000 - python breached_passwords.py addhashes pwned-passwords-1.0.txt - python breached_passwords.py addhashes pwned-passwords-update-1.txt - -Then the RDB file can simply be transferred to the production server, overwriting any -previous RDB file. - -""" - -import subprocess -from typing import Any - -import click -from redis import Redis, ResponseError - -from tildes.lib.password import ( - BREACHED_PASSWORDS_BF_KEY, - BREACHED_PASSWORDS_REDIS_SOCKET, -) - - -REDIS = Redis(unix_socket_path=BREACHED_PASSWORDS_REDIS_SOCKET) - - -def generate_redis_protocol(*elements: Any) -> str: - """Generate a command in the Redis protocol from the specified elements. - - Based on the example Ruby code from - https://redis.io/topics/mass-insert#generating-redis-protocol - """ - command = f"*{len(elements)}\r\n" - - for element in elements: - element = str(element) - command += f"${len(element)}\r\n{element}\r\n" - - return command - - -@click.group() -def cli() -> None: - """Create a functionality-less command group to attach subcommands to.""" - pass - - -def validate_init_error_rate(ctx: Any, param: Any, value: Any) -> float: - """Validate the --error-rate arg for the init command.""" - # pylint: disable=unused-argument - if not 0 < value < 1: - raise click.BadParameter("error rate must be a float between 0 and 1") - - return value - - -@cli.command(help="Initialize a new empty bloom filter") -@click.option( - "--estimate", - required=True, - type=int, - help="Expected number of passwords that will be added", -) -@click.option( - "--error-rate", - default=0.01, - show_default=True, - help="Bloom filter desired false positive ratio", - callback=validate_init_error_rate, -) -@click.confirmation_option( - prompt="Are you sure you want to clear any existing bloom filter?" -) -def init(estimate: int, error_rate: float) -> None: - """Initialize a new bloom filter (destroying any existing one). - - It generally shouldn't be necessary to re-init a new bloom filter very often with - this command, only if the previous one was created with too low of an estimate for - number of passwords, or to change to a different false positive rate. For choosing - an estimate value, according to the ReBloom documentation: "Performance will begin - to degrade after adding more items than this number. The actual degradation will - depend on how far the limit has been exceeded. Performance will degrade linearly as - the number of entries grow exponentially." - """ - REDIS.delete(BREACHED_PASSWORDS_BF_KEY) - - # BF.RESERVE {key} {error_rate} {size} - REDIS.execute_command("BF.RESERVE", BREACHED_PASSWORDS_BF_KEY, error_rate, estimate) - - click.echo( - "Initialized bloom filter with expected size of {:,} and false " - "positive rate of {}%".format(estimate, error_rate * 100) - ) - - -@cli.command(help="Add hashes from a file to the bloom filter") -@click.argument("filename", type=click.Path(exists=True, dir_okay=False)) -def addhashes(filename: str) -> None: - """Add all hashes from a file to the bloom filter. - - This uses the method of generating commands in Redis protocol and feeding them into - an instance of `redis-cli --pipe`, as recommended in - https://redis.io/topics/mass-insert - """ - # make sure the key exists and is a bloom filter - try: - REDIS.execute_command("BF.DEBUG", BREACHED_PASSWORDS_BF_KEY) - except ResponseError: - click.echo("Bloom filter is not set up properly - run init first.") - raise click.Abort - - # call wc to count the number of lines in the file for the progress bar - click.echo("Determining hash count...") - result = subprocess.run(["wc", "-l", filename], stdout=subprocess.PIPE, check=True) - line_count = int(result.stdout.split(b" ")[0]) - - progress_bar: Any = click.progressbar(length=line_count) - update_interval = 100_000 - - click.echo("Adding {:,} hashes to bloom filter...".format(line_count)) - - redis_pipe = subprocess.Popen( - ["redis-cli", "-s", BREACHED_PASSWORDS_REDIS_SOCKET, "--pipe"], - stdin=subprocess.PIPE, - stdout=subprocess.DEVNULL, - encoding="utf-8", - ) - - for count, line in enumerate(open(filename), start=1): - hashval = line.strip().lower() - - # the Pwned Passwords hash lists now have a frequency count for each hash, which - # is separated from the hash with a colon, so we need to handle that if it's - # present - hashval = hashval.split(":")[0] - - command = generate_redis_protocol("BF.ADD", BREACHED_PASSWORDS_BF_KEY, hashval) - redis_pipe.stdin.write(command) - - if count % update_interval == 0: - progress_bar.update(update_interval) - - # call SAVE to update the RDB file - REDIS.save() - - # manually finish the progress bar so it shows 100% and renders properly - progress_bar.finish() - progress_bar.render_progress() - progress_bar.render_finish() - - -if __name__ == "__main__": - cli() diff --git a/tildes/scss/_base.scss b/tildes/scss/_base.scss index ab5ed400..df622a27 100644 --- a/tildes/scss/_base.scss +++ b/tildes/scss/_base.scss @@ -8,8 +8,34 @@ html { font-size: $html-font-size; } +* { + border-color: var(--border-color); +} + a { text-decoration: none; + + color: var(--link-color); + + &:hover { + color: var(--link-hover-color); + } + + &:visited { + color: var(--link-visited-color); + } + + code { + color: var(--link-color); + + &:hover { + text-decoration: underline; + } + } + + &:visited code { + color: var(--link-visited-color); + } } blockquote { @@ -19,6 +45,9 @@ blockquote { border-left: 1px dotted; + border-color: var(--foreground-highlight-color); + background-color: var(--background-secondary-color); + // nested blockquotes need reduced margin/padding > blockquote { margin: 0; @@ -26,6 +55,11 @@ blockquote { padding-top: 0; padding-bottom: 0; } + + code, + pre { + background-color: var(--background-primary-color); + } } body { @@ -33,6 +67,9 @@ body { min-height: 100vh; @include font-shrink-on-mobile(0.8rem); + + color: var(--foreground-primary-color); + background-color: var(--background-secondary-color); } button { @@ -44,6 +81,9 @@ code { font-size: inherit; -moz-tab-size: 4; tab-size: 4; + + color: var(--foreground-highlight-color); + background-color: var(--background-secondary-color); } dl dd { @@ -63,7 +103,7 @@ fieldset { margin: 1rem; margin-right: 0; padding-left: 0.4rem; - border-left: 3px solid; + border-left: 3px solid var(--border-color); } figcaption { @@ -139,6 +179,8 @@ main { overflow: hidden; max-width: 100vw; + background-color: var(--background-primary-color); + @media (min-width: $size-md) { padding: 0.4rem; } @@ -153,6 +195,40 @@ menu { padding: 0; } +meter { + // Crazy styles to get this to work adapted from Spectre.css's _meters.scss + background: var(--background-secondary-color); + + &::-webkit-meter-bar { + background: var(--background-secondary-color); + } + + // For some mysterious reason, none of the below rules can be merged + &::-webkit-meter-optimum-value { + background: var(--success-color); + } + + &:-moz-meter-optimum::-moz-meter-bar { + background: var(--success-color); + } + + &::-webkit-meter-suboptimum-value { + background: var(--warning-color); + } + + &:-moz-meter-sub-optimum::-moz-meter-bar { + background: var(--warning-color); + } + + &::-webkit-meter-even-less-good-value { + background: var(--error-color); + } + + &:-moz-meter-sub-sub-optimum::-moz-meter-bar { + background: var(--error-color); + } +} + // We'll use lists for their semantic value sometimes, so we don't want them to // have the normal list numbering/etc. by default. We'll specifically add that // back in for text-based lists in places where it's needed. @@ -178,6 +254,8 @@ p:last-child { pre { overflow: auto; + color: var(--foreground-highlight-color); + background-color: var(--background-secondary-color); code { display: block; @@ -202,12 +280,16 @@ table { margin-bottom: 1rem; } +tbody tr:nth-of-type(2n + 1) { + background-color: var(--background-secondary-color); +} + td, th { - border: $border-width solid; + border: $border-width solid var(--border-color); padding: $unit-3 $unit-2; } th { - border-bottom-width: $border-width-lg; + border-bottom: $border-width-lg solid var(--foreground-highlight-color); } diff --git a/tildes/scss/modules/_breadcrumbs.scss b/tildes/scss/modules/_breadcrumbs.scss index af360923..7b349d65 100644 --- a/tildes/scss/modules/_breadcrumbs.scss +++ b/tildes/scss/modules/_breadcrumbs.scss @@ -1,3 +1,28 @@ +// Copyright (c) 2019 Tildes contributors +// SPDX-License-Identifier: AGPL-3.0-or-later + +.breadcrumb .breadcrumb-item { + color: var(--foreground-secondary-color); + + &:not(:last-child) { + a { + color: var(--foreground-secondary-color); + } + } + + &:not(:first-child) { + &::before { + color: var(--foreground-secondary-color); + } + } + + &:last-child { + a { + color: var(--link-color); + } + } +} + ol.breadcrumb, ul.breadcrumb { margin-left: 0; diff --git a/tildes/scss/modules/_btn.scss b/tildes/scss/modules/_btn.scss index 54b2e4ab..591a18dd 100644 --- a/tildes/scss/modules/_btn.scss +++ b/tildes/scss/modules/_btn.scss @@ -10,12 +10,30 @@ font-size: 0.6rem; font-weight: bold; + + color: var(--button-color); + background-color: transparent; + border-color: var(--button-color); + + &:hover { + background-color: var(--button-transparent-color); + } } .btn.btn-sm { font-size: 0.6rem; } +.btn.btn-link { + color: var(--link-color); + background-color: transparent; + border-color: transparent; + + &:hover { + color: var(--link-color); + } +} + .btn-link-minimal { display: inline; height: auto; @@ -44,6 +62,10 @@ margin-right: 0.2rem; min-width: 0.8rem; } + + &:hover { + border-color: var(--border-color); + } } .btn-comment-collapse-label::after { @@ -73,8 +95,49 @@ } } +@mixin label-button($color) { + color: $color; + border-color: $color; + + &:hover { + color: $color; + } + + &.btn-used:hover { + background-color: $color; + color: #fff; + } +} + +.btn-comment-label-exemplary { + @include label-button(var(--comment-label-exemplary-color)); +} + +.btn-comment-label-joke { + @include label-button(var(--comment-label-joke-color)); +} + +.btn-comment-label-noise { + @include label-button(var(--comment-label-noise-color)); +} + +.btn-comment-label-offtopic { + @include label-button(var(--comment-label-offtopic-color)); +} + +.btn-comment-label-malice { + @include label-button(var(--comment-label-malice-color)); +} + .btn-light { font-weight: normal; + + color: var(--foreground-secondary-color); + border-color: var(--border-color); + + &:hover { + color: var(--link-color); + } } .btn-post { @@ -122,8 +185,44 @@ line-height: 0.6rem; cursor: pointer; + + color: var(--foreground-secondary-color); + + &:hover { + color: var(--foreground-extreme-color); + } } .btn-post-action-used { text-decoration: underline; + color: var(--button-used-color); +} + +.btn.btn-primary { + color: var(--button-by-brightness-color); + + background-color: var(--button-color); + border-color: var(--button-color); + + &:hover { + color: var(--button-by-brightness-color); + background-color: var(--button-color); + border-color: var(--button-color); + + filter: brightness(90%); + } + + &:visited { + color: var(--button-by-brightness-color); + } +} + +.btn-used { + color: var(--button-used-color); + border-color: var(--button-used-color); + + &:hover { + filter: brightness(95%); + color: #fff; + } } diff --git a/tildes/scss/modules/_chip.scss b/tildes/scss/modules/_chip.scss index 5aa8cf23..ba9c40dc 100644 --- a/tildes/scss/modules/_chip.scss +++ b/tildes/scss/modules/_chip.scss @@ -4,4 +4,26 @@ .chip { padding: 0 0.5rem; height: 1rem; + + background-color: var(--background-secondary-color); + color: var(--foreground-highlight-color); + + &.active { + background-color: var(--button-color); + color: var(--button-by-brightness-color); + + .btn { + color: var(--button-by-brightness-color); + } + } + + &.error { + background-color: var(--error-color); + + color: var(--error-by-brightness-color); + + .btn { + color: var(--error-by-brightness-color); + } + } } diff --git a/tildes/scss/modules/_comment.scss b/tildes/scss/modules/_comment.scss index 88a2003e..37c8a0a4 100644 --- a/tildes/scss/modules/_comment.scss +++ b/tildes/scss/modules/_comment.scss @@ -2,18 +2,16 @@ // SPDX-License-Identifier: AGPL-3.0-or-later .comment { - border-left: 1px solid; - border-color: inherit; + border-left: 1px solid var(--border-color); margin-bottom: 0.4rem; &:target > .comment-itself { - border-left: 3px solid; + border-left: 3px solid var(--stripe-target-color); } } .comment[data-comment-depth="0"] { - border-bottom: 1px solid; - border-color: inherit; + border-bottom: 1px solid var(--border-color); } .comment-header { @@ -28,6 +26,9 @@ @media (min-width: $size-md) { padding: 0.2rem; } + + color: var(--foreground-highlight-color); + background-color: var(--background-secondary-color); } .comment-user-info { @@ -65,6 +66,8 @@ display: none; margin-right: 0.4rem; + + color: var(--foreground-secondary-color); } .comment-exemplary-reasons { @@ -96,6 +99,12 @@ @media (min-width: $size-md) { margin-left: 1rem; } + + color: var(--foreground-secondary-color); + + &:visited { + color: var(--foreground-secondary-color); + } } .comment-tree { @@ -164,6 +173,7 @@ .comment-removed-warning { font-weight: bold; font-size: 0.6rem; + color: var(--warning-color); } .comment-votes { @@ -217,6 +227,7 @@ font-size: 0.6rem; line-height: 0.8rem; white-space: nowrap; + background-color: var(--background-primary-color); } .btn-comment-collapse-label::after { @@ -249,26 +260,31 @@ .is-comment-removed { font-size: 0.7rem; font-style: italic; + color: var(--foreground-secondary-color); } .is-comment-mine { > .comment-itself { margin-left: -2px; - border-left: 3px solid; + border-left: 3px solid var(--stripe-mine-color); } } .is-comment-new { > .comment-itself { margin-left: -2px; - border-left: 3px solid; + border-left: 3px solid var(--alert-color); + } + + .comment-text { + color: var(--foreground-highlight-color); } } .is-comment-exemplary { > .comment-itself { margin-left: -2px; - border-left: 3px solid; + border-left: 3px solid var(--comment-label-exemplary-color); } } diff --git a/tildes/scss/modules/_divider.scss b/tildes/scss/modules/_divider.scss index 0b2d7b32..77734efe 100644 --- a/tildes/scss/modules/_divider.scss +++ b/tildes/scss/modules/_divider.scss @@ -4,4 +4,10 @@ .divider, .divider[data-content] { margin: 1rem; + border-color: var(--border-color); +} + +.divider[data-content]::after { + color: var(--foreground-primary-color); + background-color: var(--background-primary-color); } diff --git a/tildes/scss/modules/_donation.scss b/tildes/scss/modules/_donation.scss index 2bbbbabf..1acf2826 100644 --- a/tildes/scss/modules/_donation.scss +++ b/tildes/scss/modules/_donation.scss @@ -34,6 +34,14 @@ height: 0.4rem; } +.donation-goal-meter-over-goal { + background: var(--comment-label-exemplary-color); + + &::-webkit-meter-bar { + background: var(--comment-label-exemplary-color); + } +} + .donation-goal-progress { display: flex; align-items: center; diff --git a/tildes/scss/modules/_dropdown.scss b/tildes/scss/modules/_dropdown.scss index 1e758d4b..f154b9b0 100644 --- a/tildes/scss/modules/_dropdown.scss +++ b/tildes/scss/modules/_dropdown.scss @@ -8,6 +8,17 @@ .btn-post-action { justify-content: left; width: 100%; + + &:hover { + background-color: var(--background-secondary-color); + } + } + } + + &.dropdown-bottom { + .menu { + top: auto; + bottom: 100%; } } diff --git a/tildes/scss/modules/_empty.scss b/tildes/scss/modules/_empty.scss index 336b8292..2685e0f6 100644 --- a/tildes/scss/modules/_empty.scss +++ b/tildes/scss/modules/_empty.scss @@ -5,3 +5,7 @@ background: inherit; color: inherit; } + +.empty-subtitle { + color: var(--foreground-secondary-color); +} diff --git a/tildes/scss/modules/_form.scss b/tildes/scss/modules/_form.scss index f7f589e9..a247d427 100644 --- a/tildes/scss/modules/_form.scss +++ b/tildes/scss/modules/_form.scss @@ -1,6 +1,16 @@ // Copyright (c) 2018 Tildes contributors // SPDX-License-Identifier: AGPL-3.0-or-later +.form-autocomplete { + .menu { + background-color: var(--background-secondary-color); + } + + .form-autocomplete-input .form-input { + border-color: transparent; + } +} + .form-group { .form-radio { margin-left: 1rem; @@ -15,10 +25,13 @@ } } -select.form-select:not([multiple]) { +select.form-select:not([multiple]):not([size]) { // would be better to implement autoprefixer to do this -moz-appearance: none; -webkit-appearance: none; + + border-color: var(--border-color); + background-color: var(--background-input-color); } .form-listing-options { @@ -59,6 +72,8 @@ select.form-select:not([multiple]) { padding: 0.4rem; border: 1px dashed; overflow: auto; + + border-color: var(--border-color); } .form-buttons { @@ -85,6 +100,26 @@ textarea.form-input { .form-input { max-width: 40rem; + + // error colors for :invalid inputs, using same approach as Spectre + &:not(:placeholder-shown):invalid { + border-color: var(--error-color); + + &:focus { + box-shadow: 0 0 0 1px var(--error-color); + } + + + .form-input-hint { + color: var(--error-color); + } + } +} + +.form-input, +.form-input[readonly] { + color: var(--foreground-primary-color); + background-color: var(--background-input-color); + border-color: var(--border-color); } .form-input-note { diff --git a/tildes/scss/modules/_group.scss b/tildes/scss/modules/_group.scss index 8a578888..afc2d6bb 100644 --- a/tildes/scss/modules/_group.scss +++ b/tildes/scss/modules/_group.scss @@ -28,6 +28,16 @@ line-height: 0.8rem; } +.group-list-item-not-subscribed { + a.link-group { + color: var(--warning-color); + } +} + +.group-sidebar-text { + margin-top: 1rem; +} + .group-subscription { display: flex; align-items: center; diff --git a/tildes/scss/modules/_input.scss b/tildes/scss/modules/_input.scss index b57d9ee0..36652bd1 100644 --- a/tildes/scss/modules/_input.scss +++ b/tildes/scss/modules/_input.scss @@ -3,6 +3,8 @@ .input-group .input-group-addon { border-color: inherit; + background-color: var(--background-secondary-color); + color: var(--foreground-highlight-color); } .input-invite-code { diff --git a/tildes/scss/modules/_label.scss b/tildes/scss/modules/_label.scss index 7ff11648..9f46ab9f 100644 --- a/tildes/scss/modules/_label.scss +++ b/tildes/scss/modules/_label.scss @@ -12,11 +12,72 @@ text-transform: capitalize; } +@mixin theme-special-label($background-color, $foreground-color, $border-color) { + background-color: $background-color; + color: $foreground-color; + border: 1px solid $border-color; + padding: 0 0.2rem; + line-height: 0.9rem; + + a, + a:hover, + a:visited { + color: $foreground-color; + } +} + +.label-comment-exemplary { + @include theme-special-label( + var(--background-label-exemplary-color), + var(--foreground-label-exemplary-color), + var(--comment-label-exemplary-color) + ); +} + +.label-comment-joke { + @include theme-special-label( + var(--background-label-joke-color), + var(--foreground-label-joke-color), + var(--comment-label-joke-color) + ); +} + +.label-comment-noise { + @include theme-special-label( + var(--background-label-noise-color), + var(--foreground-label-noise-color), + var(--comment-label-noise-color) + ); +} + +.label-comment-offtopic { + @include theme-special-label( + var(--background-label-offtopic-color), + var(--foreground-label-offtopic-color), + var(--comment-label-offtopic-color) + ); +} + +.label-comment-malice { + @include theme-special-label( + var(--background-label-malice-color), + var(--foreground-label-malice-color), + var(--comment-label-malice-color) + ); +} + .label-topic-tag { background-color: transparent; margin: 0; word-wrap: break-word; max-width: 100%; + padding: 0; + + a, + a:hover, + a:visited { + color: var(--foreground-primary-color); + } } .label-topic-tag-spoiler, @@ -25,3 +86,21 @@ .label-topic-tag[class*="label-topic-tag-spoiler-"] { font-weight: bold; } + +.label-topic-tag-nsfw, +.label-topic-tag[class*="label-topic-tag-nsfw-"] { + @include theme-special-label( + var(--topic-tag-nsfw-color), + var(--topic-tag-nsfw-foreground-color), + var(--topic-tag-nsfw-border-color) + ); +} + +.label-topic-tag-spoiler, +.label-topic-tag[class*="label-topic-tag-spoiler-"] { + @include theme-special-label( + var(--topic-tag-spoiler-color), + var(--topic-tag-spoiler-foreground-color), + var(--topic-tag-spoiler-border-color) + ); +} diff --git a/tildes/scss/modules/_link.scss b/tildes/scss/modules/_link.scss index ea7b8afd..cbb07a44 100644 --- a/tildes/scss/modules/_link.scss +++ b/tildes/scss/modules/_link.scss @@ -9,4 +9,20 @@ a.link-group { &:hover { text-decoration: underline; } + + &:visited { + color: var(--link-color); + } +} + +.link-no-visited-color:visited { + color: var(--link-color); +} + +a.logged-in-user-alert { + color: var(--alert-color); + + &:visited { + color: var(--alert-color); + } } diff --git a/tildes/scss/modules/_logged-in-user.scss b/tildes/scss/modules/_logged-in-user.scss index 0a4ed28a..5d72ad55 100644 --- a/tildes/scss/modules/_logged-in-user.scss +++ b/tildes/scss/modules/_logged-in-user.scss @@ -19,3 +19,8 @@ font-weight: bold; font-size: 0.5rem; } + +.logged-in-user-username, +.logged-in-user-username:visited { + color: var(--foreground-primary-color); +} diff --git a/tildes/scss/modules/_menu.scss b/tildes/scss/modules/_menu.scss index 96e3f0dd..5caebf37 100644 --- a/tildes/scss/modules/_menu.scss +++ b/tildes/scss/modules/_menu.scss @@ -3,7 +3,9 @@ .menu { box-shadow: none; - border: 1px outset; + border: 1px outset var(--border-color); + + background-color: var(--background-primary-color); .menu-item { > a:hover, diff --git a/tildes/scss/modules/_message.scss b/tildes/scss/modules/_message.scss index 25af6e40..66325f4e 100644 --- a/tildes/scss/modules/_message.scss +++ b/tildes/scss/modules/_message.scss @@ -13,6 +13,9 @@ font-size: 0.7rem; line-height: 0.9rem; + color: var(--foreground-highlight-color); + background-color: var(--background-secondary-color); + .link-user { margin-right: 0.2rem; } @@ -40,5 +43,5 @@ .is-message-mine { margin-left: -2px; - border-left: 3px solid; + border-left: 3px solid var(--stripe-mine-color); } diff --git a/tildes/scss/modules/_nav.scss b/tildes/scss/modules/_nav.scss index 7eeeb85c..ec5ebee6 100644 --- a/tildes/scss/modules/_nav.scss +++ b/tildes/scss/modules/_nav.scss @@ -12,6 +12,7 @@ .nav { margin-left: 0; + margin-top: 0; li { margin-top: 0.2rem; @@ -27,12 +28,18 @@ .nav-item a { cursor: pointer; + color: var(--link-color); &:hover { text-decoration: underline; + color: var(--link-hover-color); } } + .nav-item.active a { + color: var(--link-color); + } + .nav-item .btn-link { height: auto; font-size: 0.8rem; diff --git a/tildes/scss/modules/_settings.scss b/tildes/scss/modules/_settings.scss index 9b1c874b..764ebdcb 100644 --- a/tildes/scss/modules/_settings.scss +++ b/tildes/scss/modules/_settings.scss @@ -2,6 +2,10 @@ // SPDX-License-Identifier: AGPL-3.0-or-later .settings-list { + a:visited { + color: var(--link-color); + } + li { margin-bottom: 1rem; } diff --git a/tildes/scss/modules/_sidebar.scss b/tildes/scss/modules/_sidebar.scss index 61a68bba..a609c2a8 100644 --- a/tildes/scss/modules/_sidebar.scss +++ b/tildes/scss/modules/_sidebar.scss @@ -2,6 +2,15 @@ // SPDX-License-Identifier: AGPL-3.0-or-later #sidebar { + // Prevents the sidebar background from being transparent on extremely old browsers + // that don't support *either* custom properties or the @supports test for them + background-color: $body-bg; + + // stylelint-disable-next-line declaration-block-no-duplicate-properties + background-color: var(--background-primary-color); + + border-left-color: var(--border-color); + .btn { width: 100%; } @@ -43,6 +52,8 @@ padding: 0.2rem 0.4rem; align-items: center; justify-content: space-between; + + background-color: var(--background-secondary-color); } .is-sidebar-displayed { diff --git a/tildes/scss/modules/_site-footer.scss b/tildes/scss/modules/_site-footer.scss index f4a592b8..abd73f06 100644 --- a/tildes/scss/modules/_site-footer.scss +++ b/tildes/scss/modules/_site-footer.scss @@ -10,6 +10,10 @@ font-size: 0.6rem; text-align: center; + + a:visited { + color: var(--link-color); + } } .site-footer-links { diff --git a/tildes/scss/modules/_site-header.scss b/tildes/scss/modules/_site-header.scss index 58ec8924..64c78e36 100644 --- a/tildes/scss/modules/_site-header.scss +++ b/tildes/scss/modules/_site-header.scss @@ -38,6 +38,16 @@ white-space: nowrap; overflow: hidden; text-overflow: ellipsis; + + a, + a:visited { + color: var(--foreground-primary-color); + } + + .toast a, + .toast a:visited { + color: var(--link-color); + } } .site-header-logo { @@ -64,6 +74,12 @@ font-size: 1.2rem; font-weight: bold; + color: var(--foreground-highlight-color); + + &:visited { + color: var(--foreground-highlight-color); + } + &:hover, &:active, &:focus { @@ -87,6 +103,7 @@ font-size: 0.5rem; height: 0.7rem; transform: translate(50%, -40%); + background-color: var(--alert-color); } } } diff --git a/tildes/scss/modules/_syntax-highlighting.scss b/tildes/scss/modules/_syntax-highlighting.scss new file mode 100644 index 00000000..47eb4c21 --- /dev/null +++ b/tildes/scss/modules/_syntax-highlighting.scss @@ -0,0 +1,211 @@ +// Copyright (c) 2020 Tildes contributors +// SPDX-License-Identifier: AGPL-3.0-or-later + +.highlight { + .syntax-c { + color: var(--syntax-comment-color); + } // Comment + .syntax-err { + color: var(--foreground-color); + } // Error + .syntax-g { + color: var(--foreground-color); + } // Generic + .syntax-k { + color: var(--syntax-keyword-color); + } // Keyword + .syntax-l { + color: var(--foreground-color); + } // Literal + .syntax-n { + color: var(--foreground-color); + } // Name + .syntax-o { + color: var(--syntax-comment-color); + } // Operator + .syntax-x { + color: var(--syntax-constant-color); + } // Other + .syntax-p { + color: var(--foreground-color); + } // Punctuation + .syntax-cm { + color: var(--syntax-comment-color); + } // Comment.Multiline + .syntax-cp { + color: var(--syntax-comment-color); + } // Comment.Preproc + .syntax-c1 { + color: var(--syntax-comment-color); + } // Comment.Single + .syntax-cs { + color: var(--syntax-comment-color); + } // Comment.Special + .syntax-gd { + color: var(--syntax-comment-color); + } // Generic.Deleted + .syntax-ge { + color: var(--foreground-color); + font-style: italic; + } // Generic.Emph + .syntax-gr { + color: var(--syntax-constant-color); + } // Generic.Error + .syntax-gh { + color: var(--syntax-constant-color); + } // Generic.Heading + .syntax-gi { + color: var(--syntax-comment-color); + } // Generic.Inserted + .syntax-go { + color: var(--foreground-color); + } // Generic.Output + .syntax-gp { + color: var(--foreground-color); + } // Generic.Prompt + .syntax-gs { + color: var(--foreground-color); + font-weight: bold; + } // Generic.Strong + .syntax-gu { + color: var(--syntax-constant-color); + } // Generic.Subheading + .syntax-gt { + color: var(--foreground-color); + } // Generic.Traceback + .syntax-kc { + color: var(--syntax-constant-color); + } // Keyword.Constant + .syntax-kd { + color: var(--syntax-keyword-color); + } // Keyword.Declaration + .syntax-kn { + color: var(--syntax-comment-color); + } // Keyword.Namespace + .syntax-kp { + color: var(--syntax-comment-color); + } // Keyword.Pseudo + .syntax-kr { + color: var(--syntax-keyword-color); + } // Keyword.Reserved + .syntax-kt { + color: var(--syntax-keyword-color); + } // Keyword.Type + .syntax-ld { + color: var(--foreground-color); + } // Literal.Date + .syntax-m { + color: var(--syntax-comment-color); + } // Literal.Number + .syntax-s { + color: var(--syntax-comment-color); + } // Literal.String + .syntax-na { + color: var(--foreground-color); + } // Name.Attribute + .syntax-nb { + color: var(--syntax-builtin-color); + } // Name.Builtin + .syntax-nc { + color: var(--syntax-keyword-color); + } // Name.Class + .syntax-no { + color: var(--syntax-constant-color); + } // Name.Constant + .syntax-nd { + color: var(--syntax-keyword-color); + } // Name.Decorator + .syntax-ni { + color: var(--syntax-builtin-color); + } // Name.Entity + .syntax-ne { + color: var(--syntax-builtin-color); + } // Name.Exception + .syntax-nf { + color: var(--syntax-builtin-color); + } // Name.Function + .syntax-nl { + color: var(--foreground-color); + } // Name.Label + .syntax-nn { + color: var(--foreground-color); + } // Name.Namespace + .syntax-nx { + color: var(--foreground-color); + } // Name.Other + .syntax-py { + color: var(--foreground-color); + } // Name.Property + .syntax-nt { + color: var(--syntax-keyword-color); + } // Name.Tag + .syntax-nv { + color: var(--syntax-keyword-color); + } // Name.Variable + .syntax-ow { + color: var(--syntax-comment-color); + } // Operator.Word + .syntax-w { + color: var(--foreground-color); + } // Text.Whitespace + .syntax-mf { + color: var(--syntax-literal-color); + } // Literal.Number.Float + .syntax-mh { + color: var(--syntax-literal-color); + } // Literal.Number.Hex + .syntax-mi { + color: var(--syntax-literal-color); + } // Literal.Number.Integer + .syntax-mo { + color: var(--syntax-literal-color); + } // Literal.Number.Oct + .syntax-sb { + color: var(--syntax-string-color); + } // Literal.String.Backtick + .syntax-sc { + color: var(--syntax-string-color); + } // Literal.String.Char + .syntax-sd { + color: var(--syntax-comment-color); + } // Literal.String.Doc + .syntax-s2 { + color: var(--syntax-string-color); + } // Literal.String.Double + .syntax-se { + color: var(--syntax-constant-color); + } // Literal.String.Escape + .syntax-sh { + color: var(--syntax-comment-color); + } // Literal.String.Heredoc + .syntax-si { + color: var(--syntax-string-color); + } // Literal.String.Interpol + .syntax-sx { + color: var(--syntax-string-color); + } // Literal.String.Other + .syntax-sr { + color: var(--syntax-constant-color); + } // Literal.String.Regex + .syntax-s1 { + color: var(--syntax-string-color); + } // Literal.String.Single + .syntax-ss { + color: var(--syntax-string-color); + } // Literal.String.Symbol + .syntax-bp { + color: var(--syntax-keyword-color); + } // Name.Builtin.Pseudo + .syntax-vc { + color: var(--syntax-keyword-color); + } // Name.Variable.Class + .syntax-vg { + color: var(--syntax-keyword-color); + } // Name.Variable.Global + .syntax-vi { + color: var(--syntax-keyword-color); + } // Name.Variable.Instance + .syntax-il { + color: var(--syntax-comment-color); + } // Literal.Number.Integer.Long +} diff --git a/tildes/scss/modules/_tab.scss b/tildes/scss/modules/_tab.scss index 2857afe5..ee767d67 100644 --- a/tildes/scss/modules/_tab.scss +++ b/tildes/scss/modules/_tab.scss @@ -3,6 +3,19 @@ .tab { font-size: 0.6rem; + border-color: var(--border-color); + + .tab-item { + a { + color: var(--foreground-primary-color); + } + + &.active a, + &.active button { + color: var(--link-color); + border-bottom-color: var(--link-color); + } + } } .tab-item { diff --git a/tildes/scss/modules/_table.scss b/tildes/scss/modules/_table.scss index 09fd900b..4a013dbd 100644 --- a/tildes/scss/modules/_table.scss +++ b/tildes/scss/modules/_table.scss @@ -1,6 +1,14 @@ // Copyright (c) 2019 Tildes contributors // SPDX-License-Identifier: AGPL-3.0-or-later +.table td { + border-bottom-color: var(--border-color); +} + +.table th { + border-bottom-color: var(--foreground-highlight-color); +} + .table-financials { max-width: $paragraph-max-width; margin-top: 1rem; diff --git a/tildes/scss/modules/_text.scss b/tildes/scss/modules/_text.scss index b277d595..c4536552 100644 --- a/tildes/scss/modules/_text.scss +++ b/tildes/scss/modules/_text.scss @@ -1,10 +1,22 @@ // Copyright (c) 2018 Tildes contributors // SPDX-License-Identifier: AGPL-3.0-or-later +.text-error { + color: var(--error-color); +} + .text-formatted { @extend %text-container; } +.text-link { + color: var(--link-color); +} + +.text-secondary { + color: var(--foreground-secondary-color); +} + .text-small { font-size: 0.6rem; line-height: 0.9rem; @@ -14,6 +26,10 @@ font-size: 0.6rem; } +.text-warning { + color: var(--warning-color); +} + // special formatting rules for wiki pages .text-wiki { h1, @@ -24,6 +40,7 @@ h6 { a { text-decoration: none; + color: var(--foreground-highlight-color); } } diff --git a/tildes/scss/modules/_toast.scss b/tildes/scss/modules/_toast.scss index 49119ed8..110e7e94 100644 --- a/tildes/scss/modules/_toast.scss +++ b/tildes/scss/modules/_toast.scss @@ -7,6 +7,14 @@ margin: 1rem 0; font-weight: bold; + color: var(--foreground-highlight-color); + background-color: var(--background-secondary-color); + border-color: var(--border-color); + + a { + color: var(--link-color); + } + ul { margin-bottom: 1rem; } @@ -23,3 +31,9 @@ font-weight: bold; } } + +.toast.toast-warning { + border-color: var(--warning-color); + color: var(--warning-foreground-color); + background-color: var(--warning-background-color); +} diff --git a/tildes/scss/modules/_topic.scss b/tildes/scss/modules/_topic.scss index 128e7b03..64241740 100644 --- a/tildes/scss/modules/_topic.scss +++ b/tildes/scss/modules/_topic.scss @@ -3,6 +3,11 @@ .topic-listing > li { margin: 0; + + &:nth-of-type(2n) { + color: var(--foreground-mixed-color); + background-color: var(--background-mixed-color); + } } .topic-listing-filter { @@ -15,7 +20,8 @@ grid-template-areas: "title voting" "metadata voting" - "info actions"; + "info actions" + "action-settings action-settings"; grid-template-columns: 1fr auto; grid-gap: 0.2rem; @@ -32,10 +38,14 @@ position: relative; + // smaller bottom padding to account for the empty .post-action-settings div padding: 0.2rem; + padding-bottom: 0; @media (min-width: $size-sm) { + // smaller bottom padding to account for the empty .post-action-settings div padding: 0.4rem; + padding-bottom: 0.2rem; } font-size: 0.6rem; @@ -44,6 +54,10 @@ grid-area: title; } + .post-action-settings { + grid-area: action-settings; + } + .topic-metadata { grid-area: metadata; } @@ -90,6 +104,11 @@ .btn-post-action { font-weight: normal; + color: var(--link-color); + } + + .btn-post-action-used { + color: var(--link-visited-color); } .dropdown-toggle .icon { @@ -155,6 +174,10 @@ margin-bottom: 0.8rem; } +.topic-log-entry-time { + color: var(--foreground-secondary-color); +} + .topic-metadata { display: flex; flex-wrap: wrap; @@ -194,6 +217,10 @@ &.btn-used { border-color: transparent; + &:hover { + background-color: var(--button-color); + } + .topic-voting-votes { font-weight: bold; } @@ -226,6 +253,8 @@ border-left: 1px solid; padding: 0.2rem 0.5rem; + color: var(--foreground-secondary-color); + h1 { margin: 0 0 0.4rem; } @@ -238,11 +267,13 @@ font-style: italic; content: "Re-collapse topic text"; display: none; + color: var(--foreground-secondary-color); } &[open] { font-style: normal; font-size: 0.8rem; + color: var(--foreground-primary-color); summary { font-size: 0.6rem; @@ -277,6 +308,10 @@ } } +.topic-info-comments-new { + color: var(--alert-color); +} + .topic-info-source { display: flex; align-items: center; @@ -286,6 +321,7 @@ .topic-info-source-scheduled { font-style: italic; + color: var(--foreground-secondary-color); } .topic-full { @@ -301,6 +337,7 @@ .topic-full-byline { margin-bottom: 0.4rem; font-size: 0.6rem; + color: var(--foreground-secondary-color); } .topic-full-content-metadata { @@ -337,9 +374,11 @@ .topic-full-tags { margin-bottom: 0.4rem; font-size: 0.6rem; + color: var(--foreground-secondary-color); a { text-decoration: underline; + color: var(--foreground-secondary-color); &:hover { text-decoration: none; @@ -362,6 +401,7 @@ @extend %text-container; overflow: auto; + padding-bottom: 0.4rem; } .topic-comments-header { @@ -392,18 +432,26 @@ "title voting" "metadata voting" "content voting" - "info actions"; + "info actions" + "action-settings action-settings"; } } .is-topic-mine { - border-left: 3px solid; + border-left: 3px solid var(--stripe-mine-color); margin-left: -3px; } .is-topic-official { - border-left: 3px solid; + border-left: 3px solid var(--alert-color); margin-left: -3px; + + h1 { + a, + a:visited { + color: var(--alert-color); + } + } } .is-topic-pinned { diff --git a/tildes/scss/styles.scss b/tildes/scss/styles.scss index 0cf56c6c..3a937ecc 100644 --- a/tildes/scss/styles.scss +++ b/tildes/scss/styles.scss @@ -35,6 +35,7 @@ @import "modules/site-footer"; @import "modules/site-header"; @import "modules/static-site"; +@import "modules/syntax-highlighting"; @import "modules/tab"; @import "modules/table"; @import "modules/text"; @@ -45,7 +46,7 @@ // Note: if you add a new theme, you may also want to add a new theme-color // meta tag inside the base.jinja2 template, so mobile browsers can match -@import "themes/theme_base"; +@import "themes/theme_mixins"; @import "themes/default"; @import "themes/black"; @import "themes/dracula"; @@ -53,4 +54,5 @@ @import "themes/solarized"; @import "themes/zenburn"; @import "themes/gruvbox"; +@import "themes/love"; @import "themes/motte"; diff --git a/tildes/scss/themes/_atom_one_dark.scss b/tildes/scss/themes/_atom_one_dark.scss index 9774de2c..ac96a3b8 100644 --- a/tildes/scss/themes/_atom_one_dark.scss +++ b/tildes/scss/themes/_atom_one_dark.scss @@ -49,4 +49,8 @@ body.theme-atom-one-dark { @include use-theme($theme-atom-one-dark); } -@include theme-preview-block($theme-atom-one-dark, "atom-one-dark"); +@include theme-preview-block( + "atom-one-dark", + map-get($theme-atom-one-dark, "foreground-primary"), + map-get($theme-atom-one-dark, "background-primary") +); diff --git a/tildes/scss/themes/_black.scss b/tildes/scss/themes/_black.scss index 663fb0fc..b8ab50e8 100644 --- a/tildes/scss/themes/_black.scss +++ b/tildes/scss/themes/_black.scss @@ -28,4 +28,8 @@ body.theme-black { @include use-theme($theme-black); } -@include theme-preview-block($theme-black, "black"); +@include theme-preview-block( + "black", + map-get($theme-black, "foreground-primary"), + map-get($theme-black, "background-primary") +); diff --git a/tildes/scss/themes/_default.scss b/tildes/scss/themes/_default.scss index 3f0ca680..d620171b 100644 --- a/tildes/scss/themes/_default.scss +++ b/tildes/scss/themes/_default.scss @@ -25,12 +25,15 @@ $default-theme: ( "syntax-literal": #2aa198, // Solarized "syntax-string": #2aa198, // Solarized "topic-tag-spoiler": #e66b00, - "warning": #e66b00, + "warning": #e66b00 ); -// define the default theme using the base values body { @include use-theme($default-theme); } -@include theme-preview-block($default-theme, "white"); +@include theme-preview-block( + "white", + map-get($default-theme, "foreground-primary"), + map-get($default-theme, "background-primary") +); diff --git a/tildes/scss/themes/_dracula.scss b/tildes/scss/themes/_dracula.scss index 60665b40..6ed5bc36 100644 --- a/tildes/scss/themes/_dracula.scss +++ b/tildes/scss/themes/_dracula.scss @@ -51,4 +51,8 @@ body.theme-dracula { @include use-theme($theme-dracula); } -@include theme-preview-block($theme-dracula, "dracula"); +@include theme-preview-block( + "dracula", + map-get($theme-dracula, "foreground-primary"), + map-get($theme-dracula, "background-primary") +); diff --git a/tildes/scss/themes/_gruvbox.scss b/tildes/scss/themes/_gruvbox.scss index f26f891a..94b0e7c3 100644 --- a/tildes/scss/themes/_gruvbox.scss +++ b/tildes/scss/themes/_gruvbox.scss @@ -92,47 +92,63 @@ $gruvbox-base: ( ); // Dark theme definition -$gruvbox-dark: ( - "alert": $gb-dm-light-orange, - "background-input": $gb-dm-bg1, - "background-primary": $gb-dm-bg0, - "background-secondary": $gb-dm-bg1, - "border": $gb-dm-bg2, - "error": $gb-dm-light-red, - "foreground-highlight": $gb-dm-fg0, - "foreground-primary": $gb-dm-fg1, - "foreground-secondary": $gb-dm-fg4, - "link": $gb-dm-light-blue, - "link-visited": $dark-purple, - "success": $dark-green, - "warning": $gb-dm-light-yellow, -); +$theme-gruvbox-dark: + map-merge( + $gruvbox-base, + ( + "alert": $gb-dm-light-orange, + "background-input": $gb-dm-bg1, + "background-primary": $gb-dm-bg0, + "background-secondary": $gb-dm-bg1, + "border": $gb-dm-bg2, + "error": $gb-dm-light-red, + "foreground-highlight": $gb-dm-fg0, + "foreground-primary": $gb-dm-fg1, + "foreground-secondary": $gb-dm-fg4, + "link": $gb-dm-light-blue, + "link-visited": $dark-purple, + "success": $dark-green, + "warning": $gb-dm-light-yellow, + ) + ); body.theme-gruvbox-dark { - @include use-theme(map-merge($gruvbox-base, $gruvbox-dark)); + @include use-theme($theme-gruvbox-dark); } -@include theme-preview-block(map-merge($gruvbox-base, $gruvbox-dark), "gruvbox-dark"); +@include theme-preview-block( + "gruvbox-dark", + map-get($theme-gruvbox-dark, "foreground-primary"), + map-get($theme-gruvbox-dark, "background-primary") +); // Light theme definition -$gruvbox-light: ( - "alert": $dark-orange, - "background-input": $gb-lm-bg1, - "background-primary": $gb-lm-bg0, - "background-secondary": $gb-lm-bg1, - "border": $gb-lm-bg2, - "error": $dark-red, - "foreground-highlight": $gb-lm-fg0, - "foreground-primary": $gb-lm-fg1, - "foreground-secondary": $gb-lm-fg4, - "link": $gb-lm-light-blue, - "link-visited": $gb-lm-light-purple, - "success": $dark-green, - "warning": $gb-lm-light-yellow, -); +$theme-gruvbox-light: + map-merge( + $gruvbox-base, + ( + "alert": $dark-orange, + "background-input": $gb-lm-bg1, + "background-primary": $gb-lm-bg0, + "background-secondary": $gb-lm-bg1, + "border": $gb-lm-bg2, + "error": $dark-red, + "foreground-highlight": $gb-lm-fg0, + "foreground-primary": $gb-lm-fg1, + "foreground-secondary": $gb-lm-fg4, + "link": $gb-lm-light-blue, + "link-visited": $gb-lm-light-purple, + "success": $dark-green, + "warning": $gb-lm-light-yellow, + ) + ); body.theme-gruvbox-light { - @include use-theme(map-merge($gruvbox-base, $gruvbox-light)); + @include use-theme($theme-gruvbox-light); } -@include theme-preview-block(map-merge($gruvbox-base, $gruvbox-light), "gruvbox-light"); +@include theme-preview-block( + "gruvbox-light", + map-get($theme-gruvbox-light, "foreground-primary"), + map-get($theme-gruvbox-light, "background-primary") +); diff --git a/tildes/scss/themes/_love.scss b/tildes/scss/themes/_love.scss new file mode 100644 index 00000000..e22eda93 --- /dev/null +++ b/tildes/scss/themes/_love.scss @@ -0,0 +1,81 @@ +// The Love Color Scheme https://love.holllo.cc + +// Love Dark +$theme-love-dark: ( + "alert": #faa56c, + "background-primary": #1f1731, + "background-secondary": #2a2041, + "border": #ababab, + "button": #41c8e5, + "comment-label-exemplary": #41c8e5, + "comment-label-joke": #96c839, + "comment-label-noise": #d2b83a, + "comment-label-offtopic": #3bd18a, + "comment-label-malice": #f99fb1, + "error": #f99fb1, + "foreground-highlight": #e2e2e2, + "foreground-primary": #f2efff, + "foreground-secondary": #e6deff, + "link": #41c8e5, + "link-visited": #d5a6f8, + "stripe-target": #d2b83a, + "success": #3bd18a, + "syntax-builtin": #41c8e5, + "syntax-comment": #ababab, + "syntax-constant": #f99fb1, + "syntax-keyword": #f99add, + "syntax-literal": #3bd18a, + "syntax-string": #d2b83a, + "topic-tag-spoiler": #faa56c, + "warning": #faa56c, +); + +body.theme-love-dark { + @include use-theme($theme-love-dark); +} + +@include theme-preview-block( + "love-dark", + map-get($theme-love-dark, "foreground-primary"), + map-get($theme-love-dark, "background-primary") +); + +// Love Light +$theme-love-light: ( + "alert": #6a3b11, + "background-primary": #f2efff, + "background-secondary": #e6deff, + "border": #474747, + "button": #144d5a, + "comment-label-exemplary": #144d5a, + "comment-label-joke": #384d10, + "comment-label-noise": #514610, + "comment-label-offtopic": #115133, + "comment-label-malice": #8b123c, + "error": #8b123c, + "foreground-highlight": #1b1b1b, + "foreground-primary": #1f1731, + "foreground-secondary": #2a2041, + "link": #144d5a, + "link-visited": #6f1995, + "stripe-target": #514610, + "success": #115133, + "syntax-builtin": #144d5a, + "syntax-comment": #474747, + "syntax-constant": #8b123c, + "syntax-keyword": #81156a, + "syntax-literal": #115133, + "syntax-string": #514610, + "topic-tag-spoiler": #6a3b11, + "warning": #6a3b11, +); + +body.theme-love-light { + @include use-theme($theme-love-light); +} + +@include theme-preview-block( + "love-light", + map-get($theme-love-light, "foreground-primary"), + map-get($theme-love-light, "background-primary") +); diff --git a/tildes/scss/themes/_motte.scss b/tildes/scss/themes/_motte.scss index b8e1c1ba..5d4c899e 100644 --- a/tildes/scss/themes/_motte.scss +++ b/tildes/scss/themes/_motte.scss @@ -31,4 +31,8 @@ body.theme-motte { @include use-theme($motte-theme); } -@include theme-preview-block($motte-theme, "motte"); +@include theme-preview-block( + "motte", + map-get($motte-theme, "foreground-primary"), + map-get($motte-theme, "background-primary") +); diff --git a/tildes/scss/themes/_solarized.scss b/tildes/scss/themes/_solarized.scss index 5fee66e0..c7e1b3b7 100644 --- a/tildes/scss/themes/_solarized.scss +++ b/tildes/scss/themes/_solarized.scss @@ -34,7 +34,7 @@ $fg-light: $base0; $fg-lightest: $base1; // Shared between both "light" and "dark" variants -$solarized-base: ( +$theme-solarized-base: ( "alert": $orange, "comment-label-exemplary": $blue, "comment-label-joke": $green, @@ -55,43 +55,53 @@ $solarized-base: ( ); // Dark theme definition -$solarized-dark: ( - "background-input": #001f27, - "background-primary": $bg-darkest, - "background-secondary": $bg-dark, - "border": #33555e, - "foreground-highlight": $fg-lightest, - "foreground-primary": $fg-light, - "foreground-secondary": $fg-darkest, -); +$theme-solarized-dark: + map-merge( + $theme-solarized-base, + ( + "background-input": #001f27, + "background-primary": $bg-darkest, + "background-secondary": $bg-dark, + "border": #33555e, + "foreground-highlight": $fg-lightest, + "foreground-primary": $fg-light, + "foreground-secondary": $fg-darkest, + ) + ); body.theme-solarized-dark { - @include use-theme(map-merge($solarized-base, $solarized-dark)); + @include use-theme($theme-solarized-dark); } @include theme-preview-block( - map-merge($solarized-base, $solarized-dark), - "solarized-dark" + "solarized-dark", + map-get($theme-solarized-dark, "foreground-primary"), + map-get($theme-solarized-dark, "background-primary") ); // Light theme definition -$solarized-light: ( - "background-input": #fefbf1, - "background-primary": $bg-lightest, - "background-secondary": $bg-light, - "border": #cbc5b6, - "foreground-highlight": $fg-darkest, - "foreground-primary": $fg-dark, - "foreground-secondary": $fg-lightest, - "stripe-target": $yellow, - "warning": $orange, -); +$theme-solarized-light: + map-merge( + $theme-solarized-base, + ( + "background-input": #fefbf1, + "background-primary": $bg-lightest, + "background-secondary": $bg-light, + "border": #cbc5b6, + "foreground-highlight": $fg-darkest, + "foreground-primary": $fg-dark, + "foreground-secondary": $fg-lightest, + "stripe-target": $yellow, + "warning": $orange, + ) + ); body.theme-solarized-light { - @include use-theme(map-merge($solarized-base, $solarized-light)); + @include use-theme($theme-solarized-light); } @include theme-preview-block( - map-merge($solarized-base, $solarized-light), - "solarized-light" + "solarized-light", + map-get($theme-solarized-light, "foreground-primary"), + map-get($theme-solarized-light, "background-primary") ); diff --git a/tildes/scss/themes/_theme_base.scss b/tildes/scss/themes/_theme_base.scss deleted file mode 100644 index 4c75aab2..00000000 --- a/tildes/scss/themes/_theme_base.scss +++ /dev/null @@ -1,1019 +0,0 @@ -// Copyright (c) 2018 Tildes contributors -// SPDX-License-Identifier: AGPL-3.0-or-later - -// This file should only contain rules that need to be affected by all the -// different themes, defined inside the `use-theme` mixin below. -// Note that all rules inside the mixin will be included in the compiled CSS -// once for each theme, so they should be kept as minimal as possible. - -// Each theme should be defined in its own SCSS file, and consist of a SCSS map -// and a unique `body.theme-` selector. -// The `use-theme` mixin is called inside the body.theme- block and takes -// the theme's map as its only argument, applying each defined color available -// in the map. If a color variable is left undefined in the theme's map, it -// will fall back to the default value from `$theme-base` instead. - -@mixin use-theme($theme) { - $theme: init-theme($theme); - - color: map-get($theme, "foreground-primary"); - background-color: map-get($theme, "background-secondary"); - - // set $is-light as a bool for whether the background seems light or dark - $is-light: is-color-bright(map-get($theme, "background-primary")); - - * { - border-color: map-get($theme, "border"); - } - - a { - color: map-get($theme, "link"); - - &:hover { - color: map-get($theme, "link-hover"); - } - - &:visited { - color: map-get($theme, "link-visited"); - } - - code { - color: map-get($theme, "link"); - - &:hover { - text-decoration: underline; - } - } - - &:visited code { - color: map-get($theme, "link-visited"); - } - } - - a.link-user, - a.link-group { - &:visited { - color: map-get($theme, "link"); - } - } - - a.logged-in-user-alert { - color: map-get($theme, "alert"); - - &:visited { - color: map-get($theme, "alert"); - } - } - - @include syntax-highlighting($theme); - - blockquote { - border-color: map-get($theme, "foreground-highlight"); - background-color: map-get($theme, "background-secondary"); - - code, - pre { - background-color: map-get($theme, "background-primary"); - } - } - - code, - pre { - color: map-get($theme, "foreground-highlight"); - background-color: map-get($theme, "background-secondary"); - } - - main { - background-color: map-get($theme, "background-primary"); - } - - meter { - // Crazy styles to get this to work adapted from Spectre.css's _meters.scss - background: map-get($theme, "background-secondary"); - - &::-webkit-meter-bar { - background: map-get($theme, "background-secondary"); - } - - // For some mysterious reason, none of the below rules can be merged - &::-webkit-meter-optimum-value { - background: map-get($theme, "success"); - } - - &:-moz-meter-optimum::-moz-meter-bar { - background: map-get($theme, "success"); - } - - &::-webkit-meter-suboptimum-value { - background: map-get($theme, "warning"); - } - - &:-moz-meter-sub-optimum::-moz-meter-bar { - background: map-get($theme, "warning"); - } - - &::-webkit-meter-even-less-good-value { - background: map-get($theme, "error"); - } - - &:-moz-meter-sub-sub-optimum::-moz-meter-bar { - background: map-get($theme, "error"); - } - } - - tbody tr:nth-of-type(2n + 1) { - background-color: map-get($theme, "background-secondary"); - } - - .table th { - border-bottom-color: map-get($theme, "foreground-highlight"); - } - - .form-autocomplete { - .menu { - background-color: map-get($theme, "background-secondary"); - } - } - - .breadcrumb .breadcrumb-item { - color: map-get($theme, "foreground-secondary"); - - &:not(:last-child) { - a { - color: map-get($theme, "foreground-secondary"); - } - } - - &:not(:first-child) { - &::before { - color: map-get($theme, "foreground-secondary"); - } - } - - &:last-child { - a { - color: map-get($theme, "link"); - } - } - } - - .btn { - color: map-get($theme, "button"); - background-color: transparent; - border-color: map-get($theme, "button"); - - &:hover { - background-color: rgba(map-get($theme, "button"), 0.2); - } - } - - .btn-light { - color: map-get($theme, "foreground-secondary"); - border-color: map-get($theme, "border"); - - &:hover { - color: map-get($theme, "link"); - } - } - - .btn.btn-link { - color: map-get($theme, "link"); - background-color: transparent; - border-color: transparent; - - &:hover { - color: map-get($theme, "link"); - } - } - - .btn.btn-primary { - color: choose-by-brightness(map-get($theme, "button"), #000, #fff); - - background-color: map-get($theme, "button"); - border-color: map-get($theme, "button"); - - &:hover { - background-color: darken(map-get($theme, "button"), 10%); - border-color: darken(map-get($theme, "button"), 10%); - } - - &:visited { - color: choose-by-brightness(map-get($theme, "button"), #000, #fff); - } - } - - .btn-used { - color: map-get($theme, "button-used"); - border-color: darken(map-get($theme, "button-used"), 3%); - - &:hover { - background-color: darken(map-get($theme, "button-used"), 3%); - border-color: darken(map-get($theme, "button-used"), 8%); - color: #fff; - } - } - - .btn-post-action { - color: map-get($theme, "foreground-secondary"); - - &:hover { - color: map-get($theme, "foreground-extreme"); - } - } - - .btn-post-action-used { - color: map-get($theme, "button-used"); - } - - .btn-comment-label-exemplary { - @include labelbutton(map-get($theme, "comment-label-exemplary")); - } - - .btn-comment-label-joke { - @include labelbutton(map-get($theme, "comment-label-joke")); - } - - .btn-comment-label-noise { - @include labelbutton(map-get($theme, "comment-label-noise")); - } - - .btn-comment-label-offtopic { - @include labelbutton(map-get($theme, "comment-label-offtopic")); - } - - .btn-comment-label-malice { - @include labelbutton(map-get($theme, "comment-label-malice")); - } - - .chip { - background-color: map-get($theme, "background-secondary"); - color: map-get($theme, "foreground-highlight"); - - &.active { - background-color: map-get($theme, "button"); - color: choose-by-brightness(map-get($theme, "button"), #000, #fff); - - .btn { - color: choose-by-brightness(map-get($theme, "button"), #000, #fff); - } - } - - &.error { - background-color: map-get($theme, "error"); - - color: choose-by-brightness(map-get($theme, "error"), #000, #fff); - - .btn { - color: choose-by-brightness(map-get($theme, "error"), #000, #fff); - } - } - } - - .comment-branch-counter { - color: map-get($theme, "foreground-secondary"); - } - - .comment-nav-link, - .comment-nav-link:visited { - color: map-get($theme, "foreground-secondary"); - } - - .comment-removed-warning { - color: map-get($theme, "warning"); - } - - .label-comment-exemplary { - @include theme-special-label(map-get($theme, "comment-label-exemplary"), $is-light); - } - - .label-comment-joke { - @include theme-special-label(map-get($theme, "comment-label-joke"), $is-light); - } - - .label-comment-noise { - @include theme-special-label(map-get($theme, "comment-label-noise"), $is-light); - } - - .label-comment-offtopic { - @include theme-special-label(map-get($theme, "comment-label-offtopic"), $is-light); - } - - .label-comment-malice { - @include theme-special-label(map-get($theme, "comment-label-malice"), $is-light); - } - - %collapsed-theme { - .comment-header { - background-color: map-get($theme, "background-primary"); - } - } - - .is-comment-collapsed:not(:target) { - @extend %collapsed-theme; - } - - .is-comment-collapsed-individual:not(:target) { - > .comment-itself { - @extend %collapsed-theme; - } - } - - .comment-header { - color: map-get($theme, "foreground-highlight"); - background-color: map-get($theme, "background-secondary"); - } - - .comment:target > .comment-itself { - border-left-color: map-get($theme, "stripe-target"); - } - - .divider[data-content]::after { - color: map-get($theme, "foreground-primary"); - background-color: map-get($theme, "background-primary"); - } - - .donation-goal-meter-over-goal { - background: map-get($theme, "comment-label-exemplary"); - - &::-webkit-meter-bar { - background: map-get($theme, "comment-label-exemplary"); - } - } - - .dropdown .menu .btn-post-action:hover { - background-color: map-get($theme, "background-secondary"); - } - - .empty-subtitle { - color: map-get($theme, "foreground-secondary"); - } - - .form-autocomplete .form-autocomplete-input .form-input { - border-color: transparent; - } - - .form-input { - color: map-get($theme, "foreground-primary"); - background-color: map-get($theme, "background-input"); - } - - // error colors for :invalid inputs, using same approach as Spectre - .form-input:not(:placeholder-shown):invalid { - border-color: map-get($theme, "error"); - - &:focus { - box-shadow: 0 0 0 1px map-get($theme, "error"); - } - - + .form-input-hint { - color: map-get($theme, "error"); - } - } - - .form-select:not([multiple]):not([size]) { - background-color: map-get($theme, "background-input"); - } - - .group-list-item-not-subscribed { - a.link-group { - color: map-get($theme, "warning"); - } - } - - .input-group-addon { - background-color: map-get($theme, "background-secondary"); - color: map-get($theme, "foreground-highlight"); - } - - .label-topic-tag { - padding: 0; - - a, - a:hover, - a:visited { - color: map-get($theme, "foreground-primary"); - } - } - - .label-topic-tag-nsfw, - .label-topic-tag[class*="label-topic-tag-nsfw-"] { - @include theme-special-label(map-get($theme, "topic-tag-nsfw"), $is-light); - } - - .label-topic-tag-spoiler, - .label-topic-tag[class*="label-topic-tag-spoiler-"] { - @include theme-special-label(map-get($theme, "topic-tag-spoiler"), $is-light); - } - - .link-no-visited-color:visited { - color: map-get($theme, "link"); - } - - .logged-in-user-username, - .logged-in-user-username:visited { - color: map-get($theme, "foreground-primary"); - } - - .menu { - background-color: map-get($theme, "background-primary"); - border-color: map-get($theme, "border"); - } - - .message { - header { - color: map-get($theme, "foreground-highlight"); - background-color: map-get($theme, "background-secondary"); - } - } - - .nav .nav-item { - a { - color: map-get($theme, "link"); - - &:hover { - color: map-get($theme, "link-hover"); - } - } - - &.active a { - color: map-get($theme, "link"); - } - } - - .settings-list { - a:visited { - color: map-get($theme, "link"); - } - } - - .sidebar-controls { - background-color: map-get($theme, "background-secondary"); - } - - #sidebar { - background-color: map-get($theme, "background-primary"); - } - - #site-footer a:visited { - color: map-get($theme, "link"); - } - - .site-header-context { - a, - a:visited { - color: map-get($theme, "foreground-primary"); - } - - .toast a, - .toast a:visited { - color: map-get($theme, "link"); - } - } - - .site-header-logo, - .site-header-logo:visited { - color: map-get($theme, "foreground-highlight"); - } - - .site-header-sidebar-button.badge[data-badge]::after { - background-color: map-get($theme, "alert"); - } - - .tab .tab-item { - a { - color: map-get($theme, "foreground-primary"); - } - - &.active a, - &.active button { - color: map-get($theme, "link"); - border-bottom-color: map-get($theme, "link"); - } - } - - .text-error { - color: map-get($theme, "error"); - } - - .text-link { - color: map-get($theme, "link"); - } - - .text-secondary { - color: map-get($theme, "foreground-secondary"); - } - - .text-warning { - color: map-get($theme, "warning"); - } - - .text-wiki { - h1, - h2, - h3, - h4, - h5, - h6 { - a { - color: map-get($theme, "foreground-highlight"); - } - } - } - - .toast { - color: map-get($theme, "foreground-highlight"); - background-color: map-get($theme, "background-secondary"); - - a { - color: map-get($theme, "link"); - } - } - - // Toasts should have colored border + text for dark themes, instead of a - // brightly colored background - @if ($is-light == false) { - .toast-warning { - border-color: map-get($theme, "warning"); - color: map-get($theme, "warning"); - background-color: transparent; - } - } @else { - .toast-warning { - background-color: rgba(map-get($theme, "warning"), 0.9); - border-color: map-get($theme, "warning"); - - @if (perceived-brightness(map-get($theme, "warning")) > 50) { - color: #000; - } @else { - color: #fff; - } - } - } - - .topic-actions { - .btn-post-action { - color: map-get($theme, "link"); - } - - .btn-post-action-used { - color: map-get($theme, "link-visited"); - } - } - - .topic-listing { - > li:nth-of-type(2n) { - color: - mix( - map-get($theme, "foreground-primary"), - map-get($theme, "foreground-highlight") - ); - background-color: - mix( - map-get($theme, "background-primary"), - map-get($theme, "background-secondary") - ); - } - } - - .topic-full-byline { - color: map-get($theme, "foreground-secondary"); - } - - .topic-full-tags { - color: map-get($theme, "foreground-secondary"); - - a { - color: map-get($theme, "foreground-secondary"); - } - } - - .topic-info-comments-new { - color: map-get($theme, "alert"); - } - - .topic-info-source-scheduled { - color: map-get($theme, "foreground-secondary"); - } - - .topic-log-entry-time { - color: map-get($theme, "foreground-secondary"); - } - - .topic-text-excerpt { - color: map-get($theme, "foreground-secondary"); - border-left-color: map-get($theme, "border"); - - summary::after { - color: map-get($theme, "foreground-secondary"); - } - - &[open] { - color: map-get($theme, "foreground-primary"); - } - } - - .topic-voting.btn-used { - border-color: transparent; - - &:hover { - background-color: darken(map-get($theme, "button"), 3%); - border-color: darken(map-get($theme, "button"), 8%); - } - } - - .is-comment-deleted, - .is-comment-removed { - color: map-get($theme, "foreground-secondary"); - } - - .is-comment-mine > .comment-itself { - border-left-color: map-get($theme, "stripe-mine"); - } - - .is-comment-new { - > .comment-itself { - border-left-color: map-get($theme, "alert"); - } - - .comment-text { - color: map-get($theme, "foreground-highlight"); - } - } - - .is-comment-exemplary { - > .comment-itself { - border-left-color: map-get($theme, "comment-label-exemplary"); - } - } - - .is-message-mine, - .is-topic-mine { - border-left-color: map-get($theme, "stripe-mine"); - } - - .is-topic-official { - border-left-color: map-get($theme, "alert"); - - h1 { - a, - a:visited { - color: map-get($theme, "alert"); - } - } - } -} - -@mixin theme-special-label($color, $is-light) { - padding: 0 0.2rem; - line-height: 0.9rem; - - @if $is-light { - background-color: $color; - color: #fff; - - a, - a:hover, - a:visited { - color: #fff; - } - } @else { - background-color: transparent; - color: $color; - border: 1px solid $color; - - a, - a:hover, - a:visited { - color: $color; - } - } -} - -@mixin labelbutton($color) { - color: $color; - border-color: $color; - - &:hover { - color: $color; - } - - &.btn-used:hover { - background-color: $color; - color: #fff; - } -} - -@mixin syntax-highlighting($theme) { - .highlight { - .syntax-c { - color: map-get($theme, "syntax-comment"); - } // Comment - .syntax-err { - color: map-get($theme, "foreground"); - } // Error - .syntax-g { - color: map-get($theme, "foreground"); - } // Generic - .syntax-k { - color: map-get($theme, "syntax-keyword"); - } // Keyword - .syntax-l { - color: map-get($theme, "foreground"); - } // Literal - .syntax-n { - color: map-get($theme, "foreground"); - } // Name - .syntax-o { - color: map-get($theme, "syntax-comment"); - } // Operator - .syntax-x { - color: map-get($theme, "syntax-constant"); - } // Other - .syntax-p { - color: map-get($theme, "foreground"); - } // Punctuation - .syntax-cm { - color: map-get($theme, "syntax-comment"); - } // Comment.Multiline - .syntax-cp { - color: map-get($theme, "syntax-comment"); - } // Comment.Preproc - .syntax-c1 { - color: map-get($theme, "syntax-comment"); - } // Comment.Single - .syntax-cs { - color: map-get($theme, "syntax-comment"); - } // Comment.Special - .syntax-gd { - color: map-get($theme, "syntax-comment"); - } // Generic.Deleted - .syntax-ge { - color: map-get($theme, "foreground"); - font-style: italic; - } // Generic.Emph - .syntax-gr { - color: map-get($theme, "syntax-constant"); - } // Generic.Error - .syntax-gh { - color: map-get($theme, "syntax-constant"); - } // Generic.Heading - .syntax-gi { - color: map-get($theme, "syntax-comment"); - } // Generic.Inserted - .syntax-go { - color: map-get($theme, "foreground"); - } // Generic.Output - .syntax-gp { - color: map-get($theme, "foreground"); - } // Generic.Prompt - .syntax-gs { - color: map-get($theme, "foreground"); - font-weight: bold; - } // Generic.Strong - .syntax-gu { - color: map-get($theme, "syntax-constant"); - } // Generic.Subheading - .syntax-gt { - color: map-get($theme, "foreground"); - } // Generic.Traceback - .syntax-kc { - color: map-get($theme, "syntax-constant"); - } // Keyword.Constant - .syntax-kd { - color: map-get($theme, "syntax-keyword"); - } // Keyword.Declaration - .syntax-kn { - color: map-get($theme, "syntax-comment"); - } // Keyword.Namespace - .syntax-kp { - color: map-get($theme, "syntax-comment"); - } // Keyword.Pseudo - .syntax-kr { - color: map-get($theme, "syntax-keyword"); - } // Keyword.Reserved - .syntax-kt { - color: map-get($theme, "syntax-keyword"); - } // Keyword.Type - .syntax-ld { - color: map-get($theme, "foreground"); - } // Literal.Date - .syntax-m { - color: map-get($theme, "syntax-comment"); - } // Literal.Number - .syntax-s { - color: map-get($theme, "syntax-comment"); - } // Literal.String - .syntax-na { - color: map-get($theme, "foreground"); - } // Name.Attribute - .syntax-nb { - color: map-get($theme, "syntax-builtin"); - } // Name.Builtin - .syntax-nc { - color: map-get($theme, "syntax-keyword"); - } // Name.Class - .syntax-no { - color: map-get($theme, "syntax-constant"); - } // Name.Constant - .syntax-nd { - color: map-get($theme, "syntax-keyword"); - } // Name.Decorator - .syntax-ni { - color: map-get($theme, "syntax-builtin"); - } // Name.Entity - .syntax-ne { - color: map-get($theme, "syntax-builtin"); - } // Name.Exception - .syntax-nf { - color: map-get($theme, "syntax-builtin"); - } // Name.Function - .syntax-nl { - color: map-get($theme, "foreground"); - } // Name.Label - .syntax-nn { - color: map-get($theme, "foreground"); - } // Name.Namespace - .syntax-nx { - color: map-get($theme, "foreground"); - } // Name.Other - .syntax-py { - color: map-get($theme, "foreground"); - } // Name.Property - .syntax-nt { - color: map-get($theme, "syntax-keyword"); - } // Name.Tag - .syntax-nv { - color: map-get($theme, "syntax-keyword"); - } // Name.Variable - .syntax-ow { - color: map-get($theme, "syntax-comment"); - } // Operator.Word - .syntax-w { - color: map-get($theme, "foreground"); - } // Text.Whitespace - .syntax-mf { - color: map-get($theme, "syntax-literal"); - } // Literal.Number.Float - .syntax-mh { - color: map-get($theme, "syntax-literal"); - } // Literal.Number.Hex - .syntax-mi { - color: map-get($theme, "syntax-literal"); - } // Literal.Number.Integer - .syntax-mo { - color: map-get($theme, "syntax-literal"); - } // Literal.Number.Oct - .syntax-sb { - color: map-get($theme, "syntax-string"); - } // Literal.String.Backtick - .syntax-sc { - color: map-get($theme, "syntax-string"); - } // Literal.String.Char - .syntax-sd { - color: map-get($theme, "syntax-comment"); - } // Literal.String.Doc - .syntax-s2 { - color: map-get($theme, "syntax-string"); - } // Literal.String.Double - .syntax-se { - color: map-get($theme, "syntax-constant"); - } // Literal.String.Escape - .syntax-sh { - color: map-get($theme, "syntax-comment"); - } // Literal.String.Heredoc - .syntax-si { - color: map-get($theme, "syntax-string"); - } // Literal.String.Interpol - .syntax-sx { - color: map-get($theme, "syntax-string"); - } // Literal.String.Other - .syntax-sr { - color: map-get($theme, "syntax-constant"); - } // Literal.String.Regex - .syntax-s1 { - color: map-get($theme, "syntax-string"); - } // Literal.String.Single - .syntax-ss { - color: map-get($theme, "syntax-string"); - } // Literal.String.Symbol - .syntax-bp { - color: map-get($theme, "syntax-keyword"); - } // Name.Builtin.Pseudo - .syntax-vc { - color: map-get($theme, "syntax-keyword"); - } // Name.Variable.Class - .syntax-vg { - color: map-get($theme, "syntax-keyword"); - } // Name.Variable.Global - .syntax-vi { - color: map-get($theme, "syntax-keyword"); - } // Name.Variable.Instance - .syntax-il { - color: map-get($theme, "syntax-comment"); - } // Literal.Number.Integer.Long - } -} - -@function map-get-fallback($map, $preferred-key, $fallback-key) { - // map-get that will fall back to a second key if the first isn't set - @if (map-has-key($map, $preferred-key)) { - @return map-get($map, $preferred-key); - } - - @return map-get($map, $fallback-key); -} - -@function init-theme($theme) { - // check to make sure the theme has all of the essential colors set - $essential-keys: - "alert" - "background-primary" - "background-secondary" - "comment-label-exemplary" - "comment-label-joke" - "comment-label-noise" - "comment-label-offtopic" - "comment-label-malice" - "error" - "foreground-primary" - "foreground-secondary" - "link" - "link-visited" - "success" - "warning"; - - @each $key in $essential-keys { - @if (not map-has-key($theme, $key)) { - @error "Missing essential key in theme: #{$key}"; - } - } - - // colors that simply fall back to another if not defined - $background-input: map-get-fallback($theme, "background-input", "background-primary"); - $border: map-get-fallback($theme, "border", "foreground-secondary"); - $button: map-get-fallback($theme, "button", "link"); - $button-used: map-get-fallback($theme, "button-used", "link-visited"); - // stylelint-disable-next-line - $foreground-highlight: - map-get-fallback($theme, "foreground-highlight", "foreground-primary"); - $stripe-mine: map-get-fallback($theme, "stripe-mine", "link-visited"); - $stripe-target: map-get-fallback($theme, "stripe-target", "warning"); - $syntax-builtin: map-get-fallback($theme, "syntax-builtin", "foreground-primary"); - $syntax-comment: map-get-fallback($theme, "syntax-comment", "foreground-primary"); - $syntax-constant: map-get-fallback($theme, "syntax-constant", "foreground-primary"); - $syntax-keyword: map-get-fallback($theme, "syntax-keyword", "foreground-primary"); - $syntax-literal: map-get-fallback($theme, "syntax-literal", "foreground-primary"); - $syntax-string: map-get-fallback($theme, "syntax-string", "foreground-primary"); - $topic-tag-nsfw: map-get-fallback($theme, "topic-tag-nsfw", "error"); - $topic-tag-spoiler: map-get-fallback($theme, "topic-tag-spoiler", "warning"); - - // foreground-extreme: if not defined, use white on a dark bg and black on a light one - $foreground-extreme: map-get($theme, "foreground-extreme"); - $foreground-extreme: - choose-by-brightness( - map-get($theme, "background-primary"), - #000, - #fff, - ) !default; - - // foreground-middle: if not defined, mix foreground-primary and foreground-secondary - $foreground-middle: map-get($theme, "foreground-middle"); - $foreground-middle: - mix( - map-get($theme, "foreground-primary"), - map-get($theme, "foreground-secondary") - ) !default; - - // link-hover: if not defined, darken the link color slightly - $link-hover: map-get($theme, "link-hover"); - $link-hover: darken(map-get($theme, "link"), 5%) !default; - - @return map-merge($theme, ( - "background-input": $background-input, - "border": $border, - "button": $button, - "button-used": $button-used, - "foreground-extreme": $foreground-extreme, - "foreground-highlight": $foreground-highlight, - "foreground-middle": $foreground-middle, - "link-hover": $link-hover, - "stripe-mine": $stripe-mine, - "stripe-target": $stripe-target, - "syntax-builtin": $syntax-builtin, - "syntax-comment": $syntax-comment, - "syntax-constant": $syntax-constant, - "syntax-keyword": $syntax-keyword, - "syntax-literal": $syntax-literal, - "syntax-string": $syntax-string, - "topic-tag-nsfw": $topic-tag-nsfw, - "topic-tag-spoiler": $topic-tag-spoiler, - )); -} - -@mixin theme-preview-block($theme, $name) { - .theme-preview-block-#{$name} { - background-color: map-get($theme, "background-primary"); - color: map-get($theme, "foreground-primary"); - border: 1px solid; - } -} diff --git a/tildes/scss/themes/_theme_mixins.scss b/tildes/scss/themes/_theme_mixins.scss new file mode 100644 index 00000000..7feac092 --- /dev/null +++ b/tildes/scss/themes/_theme_mixins.scss @@ -0,0 +1,305 @@ +// Copyright (c) 2020 Tildes contributors +// SPDX-License-Identifier: AGPL-3.0-or-later + +@mixin theme-preview-block($name, $foreground, $background) { + .theme-preview-block-#{$name} { + background-color: $background; + color: $foreground; + border: 1px solid; + } +} + +@function map-get-fallback($map, $preferred-key, $fallback-key) { + // map-get that will fall back to a second key if the first isn't set + @if (map-has-key($map, $preferred-key)) { + @return map-get($map, $preferred-key); + } + + @return map-get($map, $fallback-key); +} + +@function init-theme($theme) { + // check to make sure the theme has all of the essential colors set + $essential-keys: + "alert" + "background-primary" + "background-secondary" + "comment-label-exemplary" + "comment-label-joke" + "comment-label-noise" + "comment-label-offtopic" + "comment-label-malice" + "error" + "foreground-primary" + "foreground-secondary" + "link" + "link-visited" + "success" + "warning"; + + @each $key in $essential-keys { + @if (not map-has-key($theme, $key)) { + @error "Missing essential key in theme: #{$key}"; + } + } + + // colors that simply fall back to another if not defined + $background-input: map-get-fallback($theme, "background-input", "background-primary"); + $border: map-get-fallback($theme, "border", "foreground-secondary"); + $button: map-get-fallback($theme, "button", "link"); + $button-used: map-get-fallback($theme, "button-used", "link-visited"); + // stylelint-disable-next-line + $foreground-highlight: map-get-fallback($theme, "foreground-highlight", "foreground-primary"); + $stripe-mine: map-get-fallback($theme, "stripe-mine", "link-visited"); + $stripe-target: map-get-fallback($theme, "stripe-target", "warning"); + $syntax-builtin: map-get-fallback($theme, "syntax-builtin", "foreground-primary"); + $syntax-comment: map-get-fallback($theme, "syntax-comment", "foreground-primary"); + $syntax-constant: map-get-fallback($theme, "syntax-constant", "foreground-primary"); + $syntax-keyword: map-get-fallback($theme, "syntax-keyword", "foreground-primary"); + $syntax-literal: map-get-fallback($theme, "syntax-literal", "foreground-primary"); + $syntax-string: map-get-fallback($theme, "syntax-string", "foreground-primary"); + $topic-tag-nsfw: map-get-fallback($theme, "topic-tag-nsfw", "error"); + $topic-tag-spoiler: map-get-fallback($theme, "topic-tag-spoiler", "warning"); + + // foreground-extreme: if not defined, use white on a dark bg and black on a light one + $foreground-extreme: map-get($theme, "foreground-extreme"); + $foreground-extreme: + choose-by-brightness( + map-get($theme, "background-primary"), + #000, + #fff, + ) !default; + + // foreground-middle: if not defined, mix foreground-primary and foreground-secondary + $foreground-middle: map-get($theme, "foreground-middle"); + $foreground-middle: + mix( + map-get($theme, "foreground-primary"), + map-get($theme, "foreground-secondary") + ) !default; + + // link-hover: if not defined, darken the link color slightly + $link-hover: map-get($theme, "link-hover"); + $link-hover: darken(map-get($theme, "link"), 5%) !default; + $new-theme: + map-merge( + $theme, + ( + "background-input": $background-input, + "border": $border, + "button": $button, + "button-used": $button-used, + "foreground-extreme": $foreground-extreme, + "foreground-highlight": $foreground-highlight, + "foreground-middle": $foreground-middle, + "link-hover": $link-hover, + "stripe-mine": $stripe-mine, + "stripe-target": $stripe-target, + "syntax-builtin": $syntax-builtin, + "syntax-comment": $syntax-comment, + "syntax-constant": $syntax-constant, + "syntax-keyword": $syntax-keyword, + "syntax-literal": $syntax-literal, + "syntax-string": $syntax-string, + "topic-tag-nsfw": $topic-tag-nsfw, + "topic-tag-spoiler": $topic-tag-spoiler, + ) + ); + + @return map-merge($default-theme, $new-theme); +} + +@mixin minimal-hardcoded-theme($theme) { + // Outputs rules with "hardcoded" colors for old browsers with no support for custom + // properties. These rules will be repeated for every theme and will only be used by + // a tiny percentage of users, so something should only be added in here to fix major + // issues - the goal is only to make the themes *usable*, not perfect. + @supports not (--test: green) { + *, + #sidebar { + background-color: map-get($theme, "background-primary"); + border-color: map-get($theme, "border"); + color: map-get($theme, "foreground-primary"); + } + + a, + a:hover, + a:visited, + .btn-link, + .nav-item a, + .tab-item { + color: map-get($theme, "link"); + } + + // "&" represents the element itself + &, + #site-header, + #site-header *, + .comment-header, + .comment-header * { + background-color: map-get($theme, "background-secondary"); + } + + input, + input[readonly], + textarea, + textarea[readonly], + .form-select:not([multiple]):not([size]) { + background-color: map-get($theme, "background-input"); + } + + .btn.btn-primary { + color: $light-color; + } + + .text-secondary { + color: map-get($theme, "foreground-secondary"); + } + + // Prevents the * rule from causing a to override its parent + span { + color: inherit; + } + } +} + +@mixin use-theme($selected-theme) { + $theme: init-theme($selected-theme); + $is-light: is-color-bright(map-get($theme, "background-primary")); + + @include minimal-hardcoded-theme($theme); + + // When creating CSS custom properties and using any of Sass' capabilities + // you'll have to interpolate it with the Sass syntax `#{...}` as seen below. + --alert-color: #{map-get($theme, "alert")}; + + --background-input-color: #{map-get($theme, "background-input")}; + --background-mixed-color: + #{mix( + map-get($theme, "background-primary"), + map-get($theme, "background-secondary") + )}; + --background-primary-color: #{map-get($theme, "background-primary")}; + --background-secondary-color: #{map-get($theme, "background-secondary")}; + + --border-color: #{map-get($theme, "border")}; + + --button-color: #{map-get($theme, "button")}; + --button-by-brightness-color: + #{choose-by-brightness( + map-get($theme, "button"), + #000, + #fff + )}; + --button-transparent-color: #{rgba(map-get($theme, "button"), 0.2)}; + + --button-used-color: #{map-get($theme, "button-used")}; + + --error-color: #{map-get($theme, "error")}; + --error-by-brightness-color: + #{choose-by-brightness( + map-get($theme, "error"), + #000, + #fff + )}; + + --foreground-extreme-color: + #{choose-by-brightness( + map-get($theme, "background-primary"), + #000, + #fff + )}; + --foreground-highlight-color: #{map-get($theme, "foreground-highlight")}; + --foreground-mixed-color: + #{mix( + map-get($theme, "foreground-primary"), + map-get($theme, "foreground-highlight") + )}; + --foreground-primary-color: #{map-get($theme, "foreground-primary")}; + --foreground-secondary-color: #{map-get($theme, "foreground-secondary")}; + + --link-color: #{map-get($theme, "link")}; + --link-hover-color: #{map-get($theme, "link-hover")}; + --link-visited-color: #{map-get($theme, "link-visited")}; + + --stripe-mine-color: #{map-get($theme, "stripe-mine")}; + --stripe-target-color: #{map-get($theme, "stripe-target")}; + + --success-color: #{map-get($theme, "success")}; + + --syntax-builtin-color: #{map-get($theme, "syntax-builtin")}; + --syntax-comment-color: #{map-get($theme, "syntax-comment")}; + --syntax-constant-color: #{map-get($theme, "syntax-constant")}; + --syntax-keyword-color: #{map-get($theme, "syntax-keyword")}; + --syntax-literal-color: #{map-get($theme, "syntax-literal")}; + --syntax-string-color: #{map-get($theme, "syntax-string")}; + + // Colors for the special topic tags + @if $is-light { + --topic-tag-nsfw-color: #{map-get($theme, "topic-tag-nsfw")}; + --topic-tag-nsfw-foreground-color: #fff; + --topic-tag-nsfw-border-color: transparent; + + --topic-tag-spoiler-color: #{map-get($theme, "topic-tag-spoiler")}; + --topic-tag-spoiler-foreground-color: #fff; + --topic-tag-spoiler-border-color: transparent; + } @else { + --topic-tag-nsfw-color: transparent; + --topic-tag-nsfw-foreground-color: #{map-get($theme, "topic-tag-nsfw")}; + --topic-tag-nsfw-border-color: #{map-get($theme, "topic-tag-nsfw")}; + + --topic-tag-spoiler-color: transparent; + --topic-tag-spoiler-foreground-color: #{map-get($theme, "topic-tag-spoiler")}; + --topic-tag-spoiler-border-color: #{map-get($theme, "topic-tag-spoiler")}; + } + + --warning-color: #{map-get($theme, "warning")}; + + // Colors for warning toasts + @if $is-light { + --warning-background-color: #{rgba(map-get($theme, "warning"), 0.9)}; + --warning-foreground-color: #000; + } @else { + --warning-background-color: transparent; + --warning-foreground-color: #{map-get($theme, "warning")}; + } + + // Variables for the comment labels. + @if $is-light { + --background-label-exemplary-color: #{map-get($theme, "comment-label-exemplary")}; + --background-label-joke-color: #{map-get($theme, "comment-label-joke")}; + --background-label-noise-color: #{map-get($theme, "comment-label-noise")}; + --background-label-offtopic-color: #{map-get($theme, "comment-label-offtopic")}; + --background-label-malice-color: #{map-get($theme, "comment-label-malice")}; + + --comment-label-exemplary-color: #{map-get($theme, "comment-label-exemplary")}; + --comment-label-joke-color: #{map-get($theme, "comment-label-joke")}; + --comment-label-noise-color: #{map-get($theme, "comment-label-noise")}; + --comment-label-offtopic-color: #{map-get($theme, "comment-label-offtopic")}; + --comment-label-malice-color: #{map-get($theme, "comment-label-malice")}; + + --foreground-label-exemplary-color: #fff; + --foreground-label-joke-color: #fff; + --foreground-label-noise-color: #fff; + --foreground-label-offtopic-color: #fff; + --foreground-label-malice-color: #fff; + } @else { + --background-label-exemplary-color: transparent; + --background-label-joke-color: transparent; + --background-label-noise-color: transparent; + --background-label-offtopic-color: transparent; + --background-label-malice-color: transparent; + + --comment-label-exemplary-color: #{map-get($theme, "comment-label-exemplary")}; + --comment-label-joke-color: #{map-get($theme, "comment-label-joke")}; + --comment-label-noise-color: #{map-get($theme, "comment-label-noise")}; + --comment-label-offtopic-color: #{map-get($theme, "comment-label-offtopic")}; + --comment-label-malice-color: #{map-get($theme, "comment-label-malice")}; + + --foreground-label-exemplary-color: #{map-get($theme, "comment-label-exemplary")}; + --foreground-label-joke-color: #{map-get($theme, "comment-label-joke")}; + --foreground-label-noise-color: #{map-get($theme, "comment-label-noise")}; + --foreground-label-offtopic-color: #{map-get($theme, "comment-label-offtopic")}; + --foreground-label-malice-color: #{map-get($theme, "comment-label-malice")}; + } +} diff --git a/tildes/scss/themes/_zenburn.scss b/tildes/scss/themes/_zenburn.scss index 3ebf9b86..b53ea641 100644 --- a/tildes/scss/themes/_zenburn.scss +++ b/tildes/scss/themes/_zenburn.scss @@ -41,4 +41,8 @@ body.theme-zenburn { @include use-theme($theme-zenburn); } -@include theme-preview-block($theme-zenburn, "zenburn"); +@include theme-preview-block( + "zenburn", + map-get($default-theme, "foreground-primary"), + map-get($default-theme, "background-primary") +); diff --git a/tildes/sql/init/triggers/topics/topic_schedule.sql b/tildes/sql/init/triggers/topics/topic_schedule.sql new file mode 100644 index 00000000..54089706 --- /dev/null +++ b/tildes/sql/init/triggers/topics/topic_schedule.sql @@ -0,0 +1,53 @@ +-- Copyright (c) 2020 Tildes contributors +-- SPDX-License-Identifier: AGPL-3.0-or-later + +create or replace function update_topic_schedule_latest_topic_id() returns trigger as $$ +begin + if (NEW.schedule_id is not null) then + update topic_schedule + set latest_topic_id = ( + select topic_id + from topics + where schedule_id = NEW.schedule_id + and is_deleted = false + and is_removed = false + order by created_time desc limit 1) + where schedule_id = NEW.schedule_id; + end if; + + -- if it was an update that changed schedule_id, need to update the old schedule's + -- latest_topic_id as well (this will probably be extremely uncommon) + if (TG_OP = 'UPDATE' + and OLD.schedule_id is not null + and OLD.schedule_id is distinct from NEW.schedule_id) then + update topic_schedule + set latest_topic_id = ( + select topic_id + from topics + where schedule_id = OLD.schedule_id + and is_deleted = false + and is_removed = false + order by created_time desc limit 1) + where schedule_id = OLD.schedule_id; + end if; + + return null; +end +$$ language plpgsql; + + +create trigger update_topic_schedule_latest_topic_id_insert + after insert on topics + for each row + when (NEW.schedule_id is not null) + execute procedure update_topic_schedule_latest_topic_id(); + + +create trigger update_topic_schedule_latest_topic_id_update + after update on topics + for each row + when ((OLD.schedule_id is not null or NEW.schedule_id is not null) + and ((OLD.is_deleted is distinct from NEW.is_deleted) + or (OLD.is_removed is distinct from NEW.is_removed) + or (OLD.schedule_id is distinct from NEW.schedule_id))) + execute procedure update_topic_schedule_latest_topic_id(); diff --git a/tildes/static/js/behaviors/autocomplete-input.js b/tildes/static/js/behaviors/autocomplete-input.js index 52f4d396..fb68f6dd 100644 --- a/tildes/static/js/behaviors/autocomplete-input.js +++ b/tildes/static/js/behaviors/autocomplete-input.js @@ -2,7 +2,7 @@ // SPDX-License-Identifier: AGPL-3.0-or-later $.onmount("[data-js-autocomplete-input]", function() { - function addChip($input) { + function addChip($input, userTypedComma) { var $autocompleteContainer = $input .parents("[data-js-autocomplete-container]") .first(); @@ -11,7 +11,8 @@ $.onmount("[data-js-autocomplete-input]", function() { .first(); var $tagsHiddenInput = $("[data-js-autocomplete-hidden-input]"); - $input + var remaining = ""; + var inputTags = $input .val() .split(",") .map(function(tag) { @@ -19,36 +20,43 @@ $.onmount("[data-js-autocomplete-input]", function() { }) .filter(function(tag) { return tag !== ""; - }) - .forEach(function(tag) { - if (!$tagsHiddenInput.val().match(new RegExp("(^|,)" + tag + ","))) { - var clearIcon = document.createElement("a"); - clearIcon.classList.add("btn"); - clearIcon.classList.add("btn-clear"); - clearIcon.setAttribute("data-js-autocomplete-chip-clear", ""); - clearIcon.setAttribute("aria-label", "Close"); - clearIcon.setAttribute("role", "button"); - clearIcon.setAttribute("tabindex", $chips.children().length); - - var $chip = $(document.createElement("div")); - $chip.addClass("chip"); - $chip.html(tag); - $chip.append(clearIcon); - - if (!tag.match(/^[\w .]+$/)) { - $chip.addClass("error"); - $chip.attr( - "title", - "Tags may only contain letters, numbers, and spaces." - ); - } + }); - $chips.append($chip); + // process only first tag, to left of comma, if user typed comma + if (userTypedComma && $input.val().indexOf(",") !== -1) { + remaining = inputTags.slice(1).join(", "); + inputTags = inputTags.slice(0, 1); + } - $tagsHiddenInput.val($tagsHiddenInput.val() + tag + ","); + inputTags.forEach(function(tag) { + if (!$tagsHiddenInput.val().match(new RegExp("(^|,)" + tag + ","))) { + var clearIcon = document.createElement("a"); + clearIcon.classList.add("btn"); + clearIcon.classList.add("btn-clear"); + clearIcon.setAttribute("data-js-autocomplete-chip-clear", ""); + clearIcon.setAttribute("aria-label", "Close"); + clearIcon.setAttribute("role", "button"); + clearIcon.setAttribute("tabindex", $chips.children().length); + + var $chip = $(document.createElement("div")); + $chip.addClass("chip"); + $chip.html(tag); + $chip.append(clearIcon); + + if (!tag.match(/^[\w .]+$/)) { + $chip.addClass("error"); + $chip.attr( + "title", + "Tags may only contain letters, numbers, and spaces." + ); } - }); - $autocompleteContainer.find("[data-js-autocomplete-input]").val(""); + + $chips.append($chip); + + $tagsHiddenInput.val($tagsHiddenInput.val() + tag + ","); + } + }); + $autocompleteContainer.find("[data-js-autocomplete-input]").val(remaining); $autocompleteContainer.find("[data-js-autocomplete-suggestions]").html(""); $.onmount(); @@ -102,6 +110,7 @@ $.onmount("[data-js-autocomplete-input]", function() { }); $(this).keydown(function(event) { + var $this = $(this); var $autocompleteMenu = $("[data-js-autocomplete-menu]").first(); var $nextActiveItem = null; @@ -111,6 +120,11 @@ $.onmount("[data-js-autocomplete-input]", function() { $(this).blur(); break; case ",": + // wait for comma to be added to text so addChip() sees it + setTimeout(function() { + addChip($this, true); + }, 100); + break; case "Enter": event.preventDefault(); addChip($(this)); diff --git a/tildes/static/js/behaviors/dropdown-toggle.js b/tildes/static/js/behaviors/dropdown-toggle.js index b1cd5a66..7c200482 100644 --- a/tildes/static/js/behaviors/dropdown-toggle.js +++ b/tildes/static/js/behaviors/dropdown-toggle.js @@ -32,7 +32,20 @@ $.onmount(".dropdown-toggle", function() { .parent() .toggleClass( "dropdown-right", - $this.offset().left + $this.width() - $menu.width() > 0 + $this.offset().left + $this.outerWidth() - $menu.outerWidth() > 0 + ); + + // If the menu extends past the bottom of the viewport, or the site footer + // overlaps the menu, push the menu above the button instead. + var menuBottom = $this.offset().top + $this.outerHeight() + $menu.outerHeight(); + var viewportHeight = $(window).height(); + var scrollTop = $(document).scrollTop(); + var footerTop = $("#site-footer").offset().top; + $this + .parent() + .toggleClass( + "dropdown-bottom", + menuBottom > viewportHeight + scrollTop || menuBottom > footerTop ); }); diff --git a/tildes/static/js/behaviors/markdown-preview-tab.js b/tildes/static/js/behaviors/markdown-preview-tab.js index 723550d6..e3c32b9e 100644 --- a/tildes/static/js/behaviors/markdown-preview-tab.js +++ b/tildes/static/js/behaviors/markdown-preview-tab.js @@ -9,9 +9,13 @@ $.onmount("[data-js-markdown-preview-tab]", function() { var $previewDiv = $(this) .closest("form") .find(".form-markdown-preview"); + var $previewErrors = $(this) + .closest("form") + .find(".text-status-message.text-error"); $editTextarea.addClass("d-none"); $previewDiv.removeClass("d-none"); + $previewErrors.remove(); }); $(this).on("after.success.ic success.ic", function(event) { diff --git a/tildes/static/js/scripts.js b/tildes/static/js/scripts.js index 9678546e..39ab25fc 100644 --- a/tildes/static/js/scripts.js +++ b/tildes/static/js/scripts.js @@ -10,6 +10,15 @@ $(function() { var token = $("meta[name='csrftoken']").attr("content"); ajaxSetup.headers["X-CSRF-Token"] = token; + // Remove the ic-current-url param - we aren't using it, and there are some + // overzealous content blockers reacting to phrases like "_show_ads_" in it. + // All browsers that don't support this API also don't have content blockers + if ("URLSearchParams" in window) { + var params = new URLSearchParams(ajaxSetup.data); + params.delete("ic-current-url"); + ajaxSetup.data = params.toString(); + } + // This is pretty ugly - adds an X-IC-Trigger-Name header for DELETE // requests since the POST params are not accessible if (ajaxSetup.headers["X-HTTP-Method-Override"] === "DELETE") { diff --git a/tildes/tasks.py b/tildes/tasks.py new file mode 100644 index 00000000..d403b63e --- /dev/null +++ b/tildes/tasks.py @@ -0,0 +1,145 @@ +# Copyright (c) 2020 Tildes contributors +# SPDX-License-Identifier: AGPL-3.0-or-later + +"""Contains tasks that can be run through the invoke tool.""" + +from pathlib import Path + +from invoke import task +from invoke.exceptions import Exit + + +def output(string): + """Output a string without a line ending and flush immediately.""" + print(string, end="", flush=True) + + +@task +def web_server_reload(context): + """Reload the web server, in order to apply config updates.""" + context.run("sudo systemctl reload nginx.service") + + +@task(help={"full": "Include all checks (very slow)"}) +def code_style_check(context, full=False): + """Run the various utilities to check code style. + + By default, runs checks that return relatively quickly. To run the full set of + checks (which will be quite slow), add the --full flag. + """ + output("Checking if Black would reformat any Python code... ") + context.run("black --check .") + + print("Checking SCSS style...", flush=True) + context.run("npm run --silent lint:scss") + + print("Checking JS style...", flush=True) + context.run("npm run --silent lint:js") + + if full: + output("Checking Python style fully (takes a couple minutes)... ") + + # -M flag hides the "summary information" + context.run("prospector -M") + + +@task +def pip_requirements_update(context): + """Use pip-tools to update package versions in the requirements files.""" + + for filename in ("requirements.in", "requirements-dev.in"): + print(f"Updating package versions from {filename}") + context.run( + f"pip-compile --no-header --no-annotate --quiet --upgrade {filename}" + ) + + +@task +def shell(context): + """Start an IPython shell inside the app environment. + + Will use the settings in production.ini if that file exists, otherwise will fall + back to using development.ini. + """ + if Path("production.ini").exists(): + context.run("pshell production.ini", pty=True) + else: + context.run("pshell development.ini", pty=True) + + +@task( + help={ + "full": "Include all tests", + "html-validation": "Include HTML validation (very slow, includes webtests)", + "quiet": "Reduce verbosity", + "webtests": "Include webtests (a little slow)", + } +) +def test(context, full=False, quiet=False, webtests=False, html_validation=False): + """Run the tests. + + By default, webtests (ones that make actual HTTP requests to the app) and HTML + validation tests (checking the validity of the HTML on some of the site's pages) are + not run because they are slow, but you can include them with the appropriate flag. + """ + # webtests are required as part of HTML validation + if html_validation: + webtests = True + + pytest_args = [] + excluded_markers = [] + + if not full: + if not webtests: + excluded_markers.append("webtest") + if not html_validation: + excluded_markers.append("html_validation") + + if excluded_markers: + excluded_marker_str = " or ".join(excluded_markers) + pytest_args.append(f'-m "not ({excluded_marker_str})"') + + if quiet: + output("Running tests... ") + + pytest_args.append("-q") + result = context.run("pytest " + " ".join(pytest_args), hide=True) + + # only output the final line of pytest's stdout (test count + runtime) + print(result.stdout.splitlines()[-1]) + else: + context.run("pytest " + " ".join(pytest_args), pty=True) + + +@task( + help={ + "domain": "Domain to obtain a cert for (can be specified multiple times)", + }, + iterable=["domain"], + post=[web_server_reload], +) +def tls_certificate_renew(context, domain, wildcard=True): + """Renew the TLS certificate for the specified domain(s).""" + if not domain: + raise Exit("No domains specified") + + domains = [] + for dom in domain: + domains.append(dom) + if wildcard: + domains.append(f"*.{dom}") + + domain_args = " ".join([f"-d {dom}" for dom in domains]) + + context.run( + f"sudo certbot certonly --manual {domain_args} " + "--preferred-challenges dns-01 " + "--server https://acme-v02.api.letsencrypt.org/directory" + ) + + +@task +def type_check(context): + """Run static type checking on the Python code.""" + output("Running static type checking... ") + context.run("mypy .") diff --git a/tildes/tests/conftest.py b/tildes/tests/conftest.py index 50cfd0e1..fe87bdaa 100644 --- a/tildes/tests/conftest.py +++ b/tildes/tests/conftest.py @@ -2,11 +2,10 @@ # SPDX-License-Identifier: AGPL-3.0-or-later import os -from http.cookiejar import CookieJar from pyramid import testing from pyramid.paster import get_app, get_appsettings -from pytest import fixture +from pytest import fixture, mark from redis import Redis from sqlalchemy import create_engine from sqlalchemy.engine.url import make_url @@ -210,7 +209,7 @@ def session_factory(): @fixture(scope="session") def webtest(base_app): """Create a webtest TestApp and log in as the SessionUser account in it.""" - app = TestApp(base_app, extra_environ=WEBTEST_EXTRA_ENVIRON, cookiejar=CookieJar()) + app = TestApp(base_app, extra_environ=WEBTEST_EXTRA_ENVIRON) # fetch the login page, fill in the form, and submit it (sets the cookie) login_page = app.get("/login") @@ -221,7 +220,16 @@ def webtest(base_app): yield app -@fixture(scope="session") +@fixture(scope="function") def webtest_loggedout(base_app): - """Create a logged-out webtest TestApp (no cookies retained).""" + """Create a logged-out webtest TestApp (function scope, so no state is retained).""" yield TestApp(base_app, extra_environ=WEBTEST_EXTRA_ENVIRON) + + +def pytest_collection_modifyitems(items): + """Add "webtest" marker to any tests that use either of the WebTest fixtures.""" + webtest_fixture_names = ("webtest", "webtest_loggedout") + + for item in items: + if any([fixture in item.fixturenames for fixture in webtest_fixture_names]): + item.add_marker(mark.webtest) diff --git a/tildes/tests/test_metrics.py b/tildes/tests/test_metrics.py index db73ae99..ae6c862d 100644 --- a/tildes/tests/test_metrics.py +++ b/tildes/tests/test_metrics.py @@ -1,12 +1,12 @@ # Copyright (c) 2018 Tildes contributors # SPDX-License-Identifier: AGPL-3.0-or-later -from tildes.metrics import _COUNTERS, _HISTOGRAMS +from tildes.metrics import _COUNTERS, _HISTOGRAMS, _SUMMARIES def test_all_metric_names_prefixed(): """Ensure all metric names have the 'tildes_' prefix.""" - for metric_dict in (_COUNTERS, _HISTOGRAMS): + for metric_dict in (_COUNTERS, _HISTOGRAMS, _SUMMARIES): metrics = metric_dict.values() for metric in metrics: # this is ugly, but seems to be the "generic" way to get the name diff --git a/tildes/tests/test_ratelimit.py b/tildes/tests/test_ratelimit.py index 0dffc5e9..a6060328 100644 --- a/tildes/tests/test_ratelimit.py +++ b/tildes/tests/test_ratelimit.py @@ -24,10 +24,15 @@ def test_all_rate_limited_action_names_unique(): seen_names.add(action.name) -def test_action_with_all_types_disabled(): - """Ensure RateLimitedAction can't have both by_user and by_ip disabled.""" - with raises(ValueError): - RateLimitedAction("test", timedelta(hours=1), 5, by_user=False, by_ip=False) +def test_check_global_disabled(): + """Ensure global check is disabled if action is by_user or by_ip.""" + action = RateLimitedAction("test", timedelta(hours=1), 5, by_user=True, by_ip=False) + with raises(RateLimitError): + action.check_global() + + action = RateLimitedAction("test", timedelta(hours=1), 5, by_user=False, by_ip=True) + with raises(RateLimitError): + action.check_global() def test_check_by_user_id_disabled(): @@ -53,6 +58,31 @@ def test_max_burst_with_limit_1(): assert action.max_burst == 1 +def test_simple_global_rate_limiting(redis): + """Ensure simple global rate-limiting is working.""" + limit = 5 + + # define an action with max_burst equal to the full limit + action = RateLimitedAction( + "testaction", + timedelta(hours=1), + limit, + max_burst=limit, + by_user=False, + by_ip=False, + redis=redis, + ) + + # run the action the full number of times, should all be allowed + for _ in range(limit): + result = action.check_global() + assert result.is_allowed + + # try one more time, should be rejected + result = action.check_global() + assert not result.is_allowed + + def test_simple_rate_limiting_by_user_id(redis): """Ensure simple rate-limiting by user_id is working.""" limit = 5 diff --git a/tildes/tests/test_simplestring_field.py b/tildes/tests/test_simplestring_field.py index bdf7e7b9..da675b43 100644 --- a/tildes/tests/test_simplestring_field.py +++ b/tildes/tests/test_simplestring_field.py @@ -77,6 +77,18 @@ def test_control_chars_removed(): assert result == "I can be sneaky and add problemchars." +def test_zero_width_joiners_kept_and_collapsed(): + """"Ensure that multiple zero width joiners are collapsed like spaces.""" + original = "🤷\u200D\u200D\u200D♀\u200d" + assert process_string(original) == "🤷\u200D♀" + + +def test_zero_width_joiners_allowed_inside_emojis_and_not_other_words(): + """"Ensure the zero width joiner char is kept inside emojis.""" + original = "🤷\u200D♀ foo\u200dbar" + assert process_string(original) == "🤷\u200D♀ foobar" + + def test_leading_trailing_spaces_removed(): """Ensure leading/trailing spaces are removed from the string.""" original = " Centered! " diff --git a/tildes/tests/test_title.py b/tildes/tests/test_title.py index f4cccd21..f7afe0cd 100644 --- a/tildes/tests/test_title.py +++ b/tildes/tests/test_title.py @@ -46,12 +46,19 @@ def test_whitespace_trimmed(title_schema): def test_trailing_periods_trimmed(title_schema): - """Ensure trailing periods on a title are removed.""" + """Ensure trailing periods on a single-sentence title are removed.""" title = "This is an interesting story." result = title_schema.load({"title": title}) assert not result["title"].endswith(".") +def test_multisentence_trailing_period_kept(title_schema): + """Ensure a trailing period is kept if the title has multiple sentences.""" + title = "I came. I saw. I conquered." + result = title_schema.load({"title": title}) + assert result["title"].endswith(".") + + def test_consecutive_whitespace_removed(title_schema): """Ensure consecutive whitespace in a title is compressed.""" title = "sure are \n a lot of spaces" @@ -71,3 +78,13 @@ def test_unicode_control_chars_removed(title_schema): title = "nothing\u0000strange\u0085going\u009con\u007fhere" result = title_schema.load({"title": title}) assert result["title"] == "nothingstrangegoingonhere" + + +def test_zero_width_joiner_emojis_kept(title_schema): + """Test that emojis are parsed correctly""" + title = "🤷🤷‍♂️🤷‍♀️🤷🏻🤷🏻‍♀️🤷🏻‍♂️🤷🏼🤷🏼‍♀️🤷🏼‍♂️🤷🏽🤷🏽‍♀️🤷🏽‍♂️🤷🏾🤷🏾‍♀️🤷🏾‍♂️🤷🏿🤷🏿‍♀️🤷🏿‍♂️" + result = title_schema.load({"title": title}) + assert ( + result["title"] + == "🤷🤷‍♂️🤷‍♀️🤷🏻🤷🏻‍♀️🤷🏻‍♂️🤷🏼🤷🏼‍♀️🤷🏼‍♂️🤷🏽🤷🏽‍♀️🤷🏽‍♂️🤷🏾🤷🏾‍♀️🤷🏾‍♂️🤷🏿🤷🏿‍♀️🤷🏿‍♂️" + ) diff --git a/tildes/tests/webtests/test_w3_validator.py b/tildes/tests/webtests/test_w3_validator.py new file mode 100644 index 00000000..a1451cec --- /dev/null +++ b/tildes/tests/webtests/test_w3_validator.py @@ -0,0 +1,33 @@ +# Copyright (c) 2020 Tildes contributors +# SPDX-License-Identifier: AGPL-3.0-or-later + +import subprocess + +from pytest import mark + + +# marks all tests in this module with "html_validation" marker +pytestmark = mark.html_validation + + +def test_homepage_html_loggedout(webtest_loggedout): + """Validate HTML5 on the Tildes homepage, logged out.""" + # For themotte, the homepage redirects to a group. So the group page is the + # "homepage" in this scenario. + homepage = webtest_loggedout.get("/~sessiongroup") + _run_html5validator(homepage.body) + + +# Disable test for themotte because of failed validation: +# - Element "ul" not allowed as child of element "ul" in this context. +# Fix at a later date. +# def test_homepage_html_loggedin(webtest): +# """Validate HTML5 on the Tildes homepage, logged in.""" +# homepage = webtest.get("/") +# _run_html5validator(homepage.body) + + +def _run_html5validator(html): + """Raises CalledProcessError on validation error.""" + result = subprocess.run(["html5validator", "-"], input=html) + result.check_returncode() diff --git a/tildes/tildes/__init__.py b/tildes/tildes/__init__.py index a2c4b5ff..c59ae583 100644 --- a/tildes/tildes/__init__.py +++ b/tildes/tildes/__init__.py @@ -33,6 +33,7 @@ def main(global_config: Dict[str, str], **settings: str) -> PrefixMiddleware: config.include("tildes.json") config.include("tildes.request_methods") config.include("tildes.routes") + config.include("tildes.settings") config.include("tildes.tweens") config.add_webasset("javascript", Bundle(output="js/tildes.js")) diff --git a/tildes/tildes/database_models.py b/tildes/tildes/database_models.py index 6b3e1a37..dd9e1b6e 100644 --- a/tildes/tildes/database_models.py +++ b/tildes/tildes/database_models.py @@ -13,7 +13,7 @@ CommentVote, ) from tildes.models.financials import Financials -from tildes.models.group import Group, GroupStat, GroupSubscription +from tildes.models.group import Group, GroupScript, GroupStat, GroupSubscription from tildes.models.log import Log from tildes.models.message import MessageConversation, MessageReply from tildes.models.scraper import ScraperResult diff --git a/tildes/tildes/enums.py b/tildes/tildes/enums.py index 49d80f8e..fbf6a9db 100644 --- a/tildes/tildes/enums.py +++ b/tildes/tildes/enums.py @@ -107,7 +107,8 @@ def display_name(self) -> str: @classmethod def detail_fields_for_content_type( - cls, content_type: "TopicContentType", + cls, + content_type: "TopicContentType", ) -> List["ContentMetadataFields"]: """Return a list of fields to display for detail about a particular type.""" if content_type is TopicContentType.ARTICLE: @@ -292,6 +293,28 @@ class HTMLSanitizationContext(enum.Enum): USER_BIO = enum.auto() +# Enum for the possible user permissions +# (Using functional API for this one because the values aren't valid Python names) +UserPermission = enum.Enum( + "UserPermission", + [ + "comment.remove", + "comment.view_labels", + "topic.edit_by_generic_user", + "topic.edit_link", + "topic.edit_title", + "topic.lock", + "topic.move", + "topic.post", + "topic.remove", + "topic.tag", + "user.ban", + "user.view_removed_posts", + "wiki.edit", + ], +) + + class UserPermissionType(enum.Enum): """Enum for the types of user permissions.""" diff --git a/tildes/tildes/lib/database.py b/tildes/tildes/lib/database.py index ca325736..6faa98dd 100644 --- a/tildes/tildes/lib/database.py +++ b/tildes/tildes/lib/database.py @@ -147,7 +147,7 @@ def lquery(self, other): # type: ignore if isinstance(other, list): return self.op("?")(cast(other, ARRAY(LQUERY))) else: - return self.op("~")(other) + return self.op("~")(cast(other, LQUERY)) class RecurrenceRule(TypeDecorator): diff --git a/tildes/tildes/lib/datetime.py b/tildes/tildes/lib/datetime.py index 2d0f332e..696d76a8 100644 --- a/tildes/tildes/lib/datetime.py +++ b/tildes/tildes/lib/datetime.py @@ -3,6 +3,7 @@ """Functions/classes related to dates and times.""" +from __future__ import annotations import re from datetime import datetime, timedelta, timezone from typing import Any, Optional @@ -30,7 +31,7 @@ def __init__(self, hours: int): raise ValueError("Time period is too large") @classmethod - def from_short_form(cls, short_form: str) -> "SimpleHoursPeriod": + def from_short_form(cls, short_form: str) -> SimpleHoursPeriod: """Initialize a period from a "short form" string (e.g. "2h", "4d").""" if not cls._SHORT_FORM_REGEX.match(short_form): raise ValueError("Invalid time period") diff --git a/tildes/tildes/lib/lua.py b/tildes/tildes/lib/lua.py new file mode 100644 index 00000000..27d420ed --- /dev/null +++ b/tildes/tildes/lib/lua.py @@ -0,0 +1,81 @@ +# Copyright (c) 2020 Tildes contributors +# SPDX-License-Identifier: AGPL-3.0-or-later + +"""Functions and classes related to Lua scripting.""" + +from pathlib import Path +from typing import Any, Callable, Optional + +from lupa import LuaError, LuaRuntime + + +LUA_PACKAGES_PATH = Path("/opt/tildes/lua", "?.lua") + + +def getter_handler(obj: Any, attr_name: str) -> Any: + """Return the value of an object's attr, if scripts are allowed access. + + Depends on a "gettable_attrs" attribute on the object, which should be a list of + attr names that scripts are allowed to access. + """ + gettable_attrs = getattr(obj, "gettable_attrs", []) + + if attr_name not in gettable_attrs: + raise AttributeError(f"{attr_name}") + + return getattr(obj, attr_name) + + +def setter_handler(obj: Any, attr_name: str, value: Any) -> None: + """Set an object's attr to a new value, if scripts are allowed to do so. + + Depends on a "settable_attrs" attribute on the object, which should be a list of + attr names that scripts are allowed to overwrite the value of. + """ + settable_attrs = getattr(obj, "settable_attrs", []) + + if attr_name not in settable_attrs: + raise AttributeError + + setattr(obj, attr_name, value) + + +class SandboxedLua: + """A Lua runtime environment that's restricted to a sandbox. + + The sandbox is mostly implemented in Lua itself, and restricts the capabilities + and data that code will be able to use. There are also some attempts to restrict + resource usage, but I don't know how effective it is (and should probably be done + on the OS level as well). + """ + + def __init__(self) -> None: + """Create a Lua runtime and set up the sandbox environment inside it.""" + self.lua = LuaRuntime( + register_eval=False, + register_builtins=False, + unpack_returned_tuples=True, + attribute_handlers=(getter_handler, setter_handler), + ) + + self.lua.execute(f"package.path = '{LUA_PACKAGES_PATH}'") + self.sandbox = self.lua.eval('require("sandbox")') + + def run_code(self, code: str) -> None: + """Run Lua code inside the sandboxed environment.""" + result = self.sandbox.run(code) + + if result is not True: + raise LuaError(result[1]) + + def get_lua_function(self, name: str) -> Optional[Callable]: + """Return the named Lua function so it can be called on Python data.""" + return self.sandbox.env[name] + + def run_lua_function(self, name: str, *args: Any) -> None: + """Run the named Lua function, passing in the remaining args.""" + function = self.get_lua_function(name) + if not function: + raise ValueError(f"No Lua function named {name} exists") + + function(*args) diff --git a/tildes/tildes/lib/markdown.py b/tildes/tildes/lib/markdown.py index 4f8fbb6b..7e69e404 100644 --- a/tildes/tildes/lib/markdown.py +++ b/tildes/tildes/lib/markdown.py @@ -92,7 +92,7 @@ def allow_syntax_highlighting_classes(tag: str, name: str, value: str) -> bool: "tr", "ul", ) -ALLOWED_LINK_PROTOCOLS = ("http", "https", "mailto") +ALLOWED_LINK_PROTOCOLS = ("gemini", "http", "https", "mailto") ALLOWED_HTML_ATTRIBUTES_DEFAULT = { "a": ["href", "title"], diff --git a/tildes/tildes/lib/password.py b/tildes/tildes/lib/password.py index c52927f9..153c0020 100644 --- a/tildes/tildes/lib/password.py +++ b/tildes/tildes/lib/password.py @@ -3,29 +3,37 @@ """Functions/constants related to user passwords.""" +import subprocess from hashlib import sha1 -from redis import ConnectionError, Redis, ResponseError # noqa +from tildes import settings +from tildes.metrics import summary_timer -# unix socket path for redis server with the breached passwords bloom filter -BREACHED_PASSWORDS_REDIS_SOCKET = "/run/redis_breached_passwords/socket" - -# Key where the bloom filter of password hashes from data breaches is stored -BREACHED_PASSWORDS_BF_KEY = "breached_passwords_bloom" +@summary_timer("breached_password_check") +def is_breached_password(password: str) -> bool: + """Return whether the password is in the breached-passwords list. + Note: this function uses a binary-search utility on the breached-passwords file, so + the file's format is not flexible. Each line of the file must begin with a single + uppercase SHA-1 hash corresponding to a password that should be blocked, and the + lines must be sorted in lexographical order. -def is_breached_password(password: str) -> bool: - """Return whether the password is in the breached-passwords list.""" - redis = Redis(unix_socket_path=BREACHED_PASSWORDS_REDIS_SOCKET) + This is specifically intended for use with a "Pwned Passwords" list downloaded from + https://haveibeenpwned.com/passwords (SHA-1 format, "ordered by hash"), but any + other file with a compatible format will also work. + """ + try: + hash_list_path = settings.INI_FILE_SETTINGS["breached_passwords_hash_file_path"] + except KeyError: + return False - hashed = sha1(password.encode("utf-8")).hexdigest() + hashed = sha1(password.encode("utf-8")).hexdigest().upper() + # call pts_lbsearch in "prefix search" mode - exit code 0 means it found a match try: - return bool( - redis.execute_command("BF.EXISTS", BREACHED_PASSWORDS_BF_KEY, hashed) - ) - except (ConnectionError, ResponseError): - # server isn't running, bloom filter doesn't exist or the key is a different - # data type + subprocess.run(["pts_lbsearch", "-p", hash_list_path, hashed], check=True) + except subprocess.CalledProcessError: return False + + return True diff --git a/tildes/tildes/lib/ratelimit.py b/tildes/tildes/lib/ratelimit.py index 2f67dcaa..9bd7002b 100644 --- a/tildes/tildes/lib/ratelimit.py +++ b/tildes/tildes/lib/ratelimit.py @@ -3,6 +3,7 @@ """Classes and constants related to rate-limited actions.""" +from __future__ import annotations from datetime import timedelta from ipaddress import ip_address from typing import Any, List, Optional, Sequence @@ -58,7 +59,7 @@ def __eq__(self, other: Any) -> bool: ) @classmethod - def unlimited_result(cls) -> "RateLimitResult": + def unlimited_result(cls) -> RateLimitResult: """Return a "blank" result representing an unlimited action.""" return cls( is_allowed=True, @@ -68,7 +69,7 @@ def unlimited_result(cls) -> "RateLimitResult": ) @classmethod - def from_redis_cell_result(cls, result: List[int]) -> "RateLimitResult": + def from_redis_cell_result(cls, result: List[int]) -> RateLimitResult: """Convert the response from CL.THROTTLE command to a RateLimitResult. CL.THROTTLE responds with an array of 5 integers: @@ -98,7 +99,7 @@ def from_redis_cell_result(cls, result: List[int]) -> "RateLimitResult": ) @classmethod - def merged_result(cls, results: Sequence["RateLimitResult"]) -> "RateLimitResult": + def merged_result(cls, results: Sequence["RateLimitResult"]) -> RateLimitResult: """Merge any number of RateLimitResults into a single result. Basically, the merged result should be the "most restrictive" combination of all @@ -184,9 +185,6 @@ def __init__( if max_burst and not 1 <= max_burst <= limit: raise ValueError("max_burst must be at least 1 and <= limit") - if not (by_user or by_ip): - raise ValueError("At least one of by_user or by_ip must be True") - self.name = name self.period = period self.limit = limit @@ -217,9 +215,16 @@ def redis(self, redis_connection: Redis) -> None: """Set the redis connection.""" self._redis = redis_connection - def _build_redis_key(self, by_type: str, value: Any) -> str: + @property + def is_global(self) -> bool: + """Whether the rate limit applies globally, not to particular users or IPs.""" + return not (self.by_user or self.by_ip) + + def _build_redis_key(self, by_type: str, value: Any = None) -> str: """Build the Redis key where this rate limit is maintained.""" - parts = ["ratelimit", self.name, by_type, str(value)] + parts = ["ratelimit", self.name, by_type] + if value: + parts.append(str(value)) return ":".join(parts) @@ -233,6 +238,24 @@ def _call_redis_command(self, key: str) -> List[int]: int(self.period.total_seconds()), ) + def check_global(self) -> RateLimitResult: + """Check a global rate limit to see if anyone can perform this action.""" + if not self.is_global: + raise RateLimitError("check_global called on non-global-limited action") + + key = self._build_redis_key("global") + result = self._call_redis_command(key) + + return RateLimitResult.from_redis_cell_result(result) + + def reset_global(self) -> None: + """Reset the global ratelimit on this action.""" + if not self.is_global: + raise RateLimitError("reset_global called on non-global-limited action") + + key = self._build_redis_key("global") + self.redis.delete(key) + def check_for_user_id(self, user_id: int) -> RateLimitResult: """Check whether a particular user_id can perform this action.""" if not self.by_user: @@ -285,6 +308,9 @@ def reset_for_ip(self, ip_str: str) -> None: RateLimitedAction("topic_post", timedelta(hours=4), 10, max_burst=4), RateLimitedAction("comment_post", timedelta(hours=1), 10, max_burst=5), RateLimitedAction("donate_stripe", timedelta(hours=1), 5, by_user=False), + RateLimitedAction( + "global_donate_stripe", timedelta(hours=1), 20, by_user=False, by_ip=False + ), ) # (public) dict to be able to look up the actions by name diff --git a/tildes/tildes/lib/string.py b/tildes/tildes/lib/string.py index 08bbb934..74e3181c 100644 --- a/tildes/tildes/lib/string.py +++ b/tildes/tildes/lib/string.py @@ -178,7 +178,7 @@ def _sanitize_characters(original: str) -> str: """Process a string and filter/replace problematic unicode.""" final_characters = [] - for char in original: + for index, char in enumerate(original): category = unicodedata.category(char) if category.startswith("Z"): @@ -189,6 +189,18 @@ def _sanitize_characters(original: str) -> str: # newlines, which are replaced with normal spaces if char == "\n": final_characters.append(" ") + + # Keep zero-width joiner only if it's between two symbol characters, so we + # don't break certain emoji variants + if char == "\u200D": + try: + before_category = unicodedata.category(final_characters[-1]) + after_category = unicodedata.category(original[index + 1]) + except IndexError: + continue + + if before_category.startswith("S") and after_category.startswith("S"): + final_characters.append(char) else: # any other type of character, just keep it final_characters.append(char) diff --git a/tildes/tildes/metrics.py b/tildes/tildes/metrics.py index 835542b3..2002db55 100644 --- a/tildes/tildes/metrics.py +++ b/tildes/tildes/metrics.py @@ -9,7 +9,7 @@ from typing import Callable -from prometheus_client import Counter, Histogram +from prometheus_client import Counter, Histogram, Summary _COUNTERS = { @@ -31,9 +31,6 @@ "login_failures": Counter("tildes_login_failures_total", "Login Failures"), "messages": Counter("tildes_messages_total", "Messages", labelnames=["type"]), "registrations": Counter("tildes_registrations_total", "User Registrations"), - "theme_cookie_tween_sets": Counter( - "tildes_theme_cookie_tween_sets_total", "Theme Cookies Set by Tween" - ), "topics": Counter("tildes_topics_total", "Topics", labelnames=["type"]), "subscriptions": Counter("tildes_subscriptions_total", "Subscriptions"), "unsubscriptions": Counter("tildes_unsubscriptions_total", "Unsubscriptions"), @@ -53,6 +50,13 @@ ), } +_SUMMARIES = { + "breached_password_check": Summary( + "tildes_breached_password_check_seconds", + "Time spent checking whether a password is in the breached list", + ), +} + def incr_counter(name: str, amount: int = 1, **labels: str) -> None: """Increment a Prometheus counter.""" @@ -83,3 +87,21 @@ def get_histogram(name: str, **labels: str) -> Histogram: def histogram_timer(name: str) -> Callable: """Return the .time() decorator for a Prometheus histogram.""" return get_histogram(name).time() + + +def get_summary(name: str, **labels: str) -> Summary: + """Return an (optionally labeled) Prometheus summary by name.""" + try: + hist = _SUMMARIES[name] + except KeyError: + raise ValueError("Invalid summary name") + + if labels: + hist = hist.labels(**labels) + + return hist + + +def summary_timer(name: str) -> Callable: + """Return the .time() decorator for a Prometheus summary.""" + return get_summary(name).time() diff --git a/tildes/tildes/models/comment/comment_notification.py b/tildes/tildes/models/comment/comment_notification.py index 4b83026a..4bc6a86b 100644 --- a/tildes/tildes/models/comment/comment_notification.py +++ b/tildes/tildes/models/comment/comment_notification.py @@ -57,10 +57,14 @@ def __init__( self, user: User, comment: Comment, notification_type: CommentNotificationType ): """Create a new notification for a user from a comment.""" - if notification_type in ( - CommentNotificationType.COMMENT_REPLY, - CommentNotificationType.TOPIC_REPLY, - ) and not self.should_create_reply_notification(comment): + if ( + notification_type + in ( + CommentNotificationType.COMMENT_REPLY, + CommentNotificationType.TOPIC_REPLY, + ) + and not self.should_create_reply_notification(comment) + ): raise ValueError("That comment shouldn't create a reply notification") self.user = user diff --git a/tildes/tildes/models/comment/comment_notification_query.py b/tildes/tildes/models/comment/comment_notification_query.py index 911e7641..1c5fc12e 100644 --- a/tildes/tildes/models/comment/comment_notification_query.py +++ b/tildes/tildes/models/comment/comment_notification_query.py @@ -3,6 +3,7 @@ """Contains the CommentNotificationQuery class.""" +from __future__ import annotations from typing import Any from pyramid.request import Request @@ -32,7 +33,7 @@ def _anchor_subquery(self, anchor_id: int) -> Any: .subquery() ) - def _attach_extra_data(self) -> "CommentNotificationQuery": + def _attach_extra_data(self) -> CommentNotificationQuery: """Attach the user's comment votes to the query.""" vote_subquery = ( self.request.query(CommentVote) @@ -45,7 +46,7 @@ def _attach_extra_data(self) -> "CommentNotificationQuery": ) return self.add_columns(vote_subquery) - def join_all_relationships(self) -> "CommentNotificationQuery": + def join_all_relationships(self) -> CommentNotificationQuery: """Eagerly join the comment, topic, and group to the notification.""" # pylint: disable=self-cls-assignment self = self.options( @@ -69,7 +70,7 @@ def _process_result(result: Any) -> CommentNotification: return notification - def get_page(self, per_page: int) -> "CommentNotificationResults": + def get_page(self, per_page: int) -> CommentNotificationResults: """Get a page worth of results from the query (`per page` items).""" return CommentNotificationResults(self, per_page) diff --git a/tildes/tildes/models/comment/comment_query.py b/tildes/tildes/models/comment/comment_query.py index 620f63b8..bcad415c 100644 --- a/tildes/tildes/models/comment/comment_query.py +++ b/tildes/tildes/models/comment/comment_query.py @@ -3,6 +3,7 @@ """Contains the CommentQuery class.""" +from __future__ import annotations from typing import Any from pyramid.request import Request @@ -31,7 +32,7 @@ def __init__(self, request: Request): self._only_bookmarked = False self._only_user_voted = False - def _attach_extra_data(self) -> "CommentQuery": + def _attach_extra_data(self) -> CommentQuery: """Attach the extra user data to the query.""" # pylint: disable=protected-access if not self.request.user: @@ -39,7 +40,7 @@ def _attach_extra_data(self) -> "CommentQuery": return self._attach_vote_data()._attach_bookmark_data() - def _attach_vote_data(self) -> "CommentQuery": + def _attach_vote_data(self) -> CommentQuery: """Join the data related to whether the user has voted on the comment.""" query = self.join( CommentVote, @@ -53,7 +54,7 @@ def _attach_vote_data(self) -> "CommentQuery": return query - def _attach_bookmark_data(self) -> "CommentQuery": + def _attach_bookmark_data(self) -> CommentQuery: """Join the data related to whether the user has bookmarked the comment.""" query = self.join( CommentBookmark, @@ -86,7 +87,7 @@ def _process_result(result: Any) -> Comment: def apply_sort_option( self, sort: CommentSortOption, desc: bool = True - ) -> "CommentQuery": + ) -> CommentQuery: """Apply a CommentSortOption sorting method (generative).""" if sort == CommentSortOption.VOTES: self._sort_column = Comment.num_votes @@ -97,18 +98,18 @@ def apply_sort_option( return self - def search(self, query: str) -> "CommentQuery": + def search(self, query: str) -> CommentQuery: """Restrict the comments to ones that match a search query (generative).""" return self.filter( Comment.search_tsv.op("@@")(func.websearch_to_tsquery(query)) ) - def only_bookmarked(self) -> "CommentQuery": + def only_bookmarked(self) -> CommentQuery: """Restrict the comments to ones that the user has bookmarked (generative).""" self._only_bookmarked = True return self - def only_user_voted(self) -> "CommentQuery": + def only_user_voted(self) -> CommentQuery: """Restrict the comments to ones that the user has voted on (generative).""" self._only_user_voted = True return self diff --git a/tildes/tildes/models/comment/comment_tree.py b/tildes/tildes/models/comment/comment_tree.py index 107d84f7..40be94d8 100644 --- a/tildes/tildes/models/comment/comment_tree.py +++ b/tildes/tildes/models/comment/comment_tree.py @@ -181,9 +181,9 @@ def num_top_level(self) -> int: @property def most_recent_comment(self) -> Optional[Comment]: - """Return the most recent non-deleted Comment in the tree.""" + """Return the most recent Comment in the tree (excluding deleted/removed).""" for comment in reversed(self.comments): - if not comment.is_deleted: + if not (comment.is_deleted or comment.is_removed): return comment return None diff --git a/tildes/tildes/models/group/__init__.py b/tildes/tildes/models/group/__init__.py index 1189c8a0..1831ce29 100644 --- a/tildes/tildes/models/group/__init__.py +++ b/tildes/tildes/models/group/__init__.py @@ -2,6 +2,7 @@ from .group import Group from .group_query import GroupQuery +from .group_script import GroupScript from .group_stat import GroupStat from .group_subscription import GroupSubscription from .group_wiki_page import GroupWikiPage diff --git a/tildes/tildes/models/group/group.py b/tildes/tildes/models/group/group.py index eedbc4c6..dfa9cdc7 100644 --- a/tildes/tildes/models/group/group.py +++ b/tildes/tildes/models/group/group.py @@ -124,15 +124,15 @@ def __acl__(self) -> AclType: # - all groups can be subscribed to by logged-in users acl.append((Allow, Authenticated, "subscribe")) - # post_topic: + # topic.post: # - only users with specifically-granted permission can post topics in groups # that require permission to post # - otherwise, all logged-in users can post if self.requires_permission_to_post_topics: - acl.append((Allow, f"{self.group_id}:post_topic", "post_topic")) - acl.append((Deny, Everyone, "post_topic")) + acl.append((Allow, f"{self.group_id}:topic.post", "topic.post")) + acl.append((Deny, Everyone, "topic.post")) - acl.append((Allow, Authenticated, "post_topic")) + acl.append((Allow, Authenticated, "topic.post")) # wiki_page_create: # - requires being granted the "wiki.edit" permission diff --git a/tildes/tildes/models/group/group_query.py b/tildes/tildes/models/group/group_query.py index 0339d76d..1c4ea771 100644 --- a/tildes/tildes/models/group/group_query.py +++ b/tildes/tildes/models/group/group_query.py @@ -3,6 +3,7 @@ """Contains the GroupQuery class.""" +from __future__ import annotations from typing import Any from pyramid.request import Request @@ -24,14 +25,14 @@ def __init__(self, request: Request): """ super().__init__(Group, request) - def _attach_extra_data(self) -> "GroupQuery": + def _attach_extra_data(self) -> GroupQuery: """Attach the extra user data to the query.""" if not self.request.user: return self return self._attach_subscription_data() - def _attach_subscription_data(self) -> "GroupQuery": + def _attach_subscription_data(self) -> GroupQuery: """Add a subquery to include whether the user is subscribed.""" subscription_subquery = ( self.request.query(GroupSubscription) diff --git a/tildes/tildes/models/group/group_script.py b/tildes/tildes/models/group/group_script.py new file mode 100644 index 00000000..276208b1 --- /dev/null +++ b/tildes/tildes/models/group/group_script.py @@ -0,0 +1,41 @@ +# Copyright (c) 2020 Tildes contributors +# SPDX-License-Identifier: AGPL-3.0-or-later + +"""Contains the GroupScript class.""" + +from typing import Optional + +from pyramid.security import DENY_ALL +from sqlalchemy import Column, ForeignKey, Integer, Text +from sqlalchemy.orm import relationship + +from tildes.models import DatabaseModel +from tildes.typing import AclType + +from .group import Group + + +class GroupScript(DatabaseModel): + """Model for a script in a group, which can be used to process topics/comments.""" + + __tablename__ = "group_scripts" + + script_id: int = Column(Integer, primary_key=True) + group_id: Optional[int] = Column(Integer, ForeignKey("groups.group_id")) + code: str = Column(Text, nullable=False) + + group: Optional[Group] = relationship("Group") + + def __init__(self, group: Optional[Group], code: str): + """Create a new script for a group.""" + self.group = group + self.code = code + + def __acl__(self) -> AclType: + """Pyramid security ACL.""" + acl = [] + + # for now, deny all permissions through the app + acl.append(DENY_ALL) + + return acl diff --git a/tildes/tildes/models/group/group_stat.py b/tildes/tildes/models/group/group_stat.py index c0f61faf..507c66d1 100644 --- a/tildes/tildes/models/group/group_stat.py +++ b/tildes/tildes/models/group/group_stat.py @@ -23,7 +23,10 @@ class GroupStat(DatabaseModel): __tablename__ = "group_stats" group_id: int = Column( - Integer, ForeignKey("groups.group_id"), nullable=False, primary_key=True, + Integer, + ForeignKey("groups.group_id"), + nullable=False, + primary_key=True, ) stat: GroupStatType = Column(ENUM(GroupStatType), nullable=False, primary_key=True) period: DateTimeTZRange = Column(TSTZRANGE, nullable=False, primary_key=True) diff --git a/tildes/tildes/models/model_query.py b/tildes/tildes/models/model_query.py index 1d944f0a..2203772d 100644 --- a/tildes/tildes/models/model_query.py +++ b/tildes/tildes/models/model_query.py @@ -4,6 +4,7 @@ """Contains the ModelQuery class, a specialized SQLAlchemy Query subclass.""" # pylint: disable=self-cls-assignment +from __future__ import annotations from typing import Any, Iterator, TypeVar from pyramid.request import Request @@ -40,11 +41,11 @@ def __iter__(self) -> Iterator[ModelType]: results = super().__iter__() return iter([self._process_result(result) for result in results]) - def _attach_extra_data(self) -> "ModelQuery": + def _attach_extra_data(self) -> ModelQuery: """Override to attach extra data to query before execution.""" return self - def _finalize(self) -> "ModelQuery": + def _finalize(self) -> ModelQuery: """Finalize the query before it's executed.""" # pylint: disable=protected-access @@ -59,7 +60,7 @@ def _finalize(self) -> "ModelQuery": ._filter_removed_if_necessary() ) - def _before_compile_listener(self) -> "ModelQuery": + def _before_compile_listener(self) -> ModelQuery: """Do any final adjustments to the query before it's compiled. Note that this method cannot be overridden by subclasses because of the way it @@ -68,21 +69,21 @@ def _before_compile_listener(self) -> "ModelQuery": """ return self._finalize() - def _filter_deleted_if_necessary(self) -> "ModelQuery": + def _filter_deleted_if_necessary(self) -> ModelQuery: """Filter out deleted rows unless they were explicitly included.""" if not self.filter_deleted: return self return self.filter(self.model_cls.is_deleted == False) # noqa - def _filter_removed_if_necessary(self) -> "ModelQuery": + def _filter_removed_if_necessary(self) -> ModelQuery: """Filter out removed rows unless they were explicitly included.""" if not self.filter_removed: return self return self.filter(self.model_cls.is_removed == False) # noqa - def lock_based_on_request_method(self) -> "ModelQuery": + def lock_based_on_request_method(self) -> ModelQuery: """Lock the rows if request method implies it's needed (generative). Applying this function to a query will cause the database to acquire a row-level @@ -98,19 +99,19 @@ def lock_based_on_request_method(self) -> "ModelQuery": return self - def include_deleted(self) -> "ModelQuery": + def include_deleted(self) -> ModelQuery: """Specify that deleted rows should be included (generative).""" self.filter_deleted = False return self - def include_removed(self) -> "ModelQuery": + def include_removed(self) -> ModelQuery: """Specify that removed rows should be included (generative).""" self.filter_removed = False return self - def join_all_relationships(self) -> "ModelQuery": + def join_all_relationships(self) -> ModelQuery: """Eagerly join all lazy relationships (generative). This is useful for being able to load an item "fully" in a single query and @@ -120,7 +121,7 @@ def join_all_relationships(self) -> "ModelQuery": return self - def undefer_all_columns(self) -> "ModelQuery": + def undefer_all_columns(self) -> ModelQuery: """Undefer all columns (generative).""" self = self.options(undefer("*")) diff --git a/tildes/tildes/models/pagination.py b/tildes/tildes/models/pagination.py index 31cd0fb9..aa92bd47 100644 --- a/tildes/tildes/models/pagination.py +++ b/tildes/tildes/models/pagination.py @@ -3,6 +3,7 @@ """Contains the PaginatedQuery and PaginatedResults classes.""" +from __future__ import annotations from itertools import chain from typing import Any, Iterator, List, Optional, Sequence, TypeVar @@ -85,14 +86,14 @@ def is_reversed(self) -> bool: """ return bool(self.before_id) - def anchor_type(self, anchor_type: str) -> "PaginatedQuery": + def anchor_type(self, anchor_type: str) -> PaginatedQuery: """Set the type of the "anchor" (before/after item) (generative).""" anchor_table_name = anchor_type + "s" self._anchor_table = self.model_cls.metadata.tables.get(anchor_table_name) return self - def after_id36(self, id36: str) -> "PaginatedQuery": + def after_id36(self, id36: str) -> PaginatedQuery: """Restrict the query to results after an id36 (generative).""" if self.before_id: raise ValueError("Can't set both before and after restrictions") @@ -101,7 +102,7 @@ def after_id36(self, id36: str) -> "PaginatedQuery": return self - def before_id36(self, id36: str) -> "PaginatedQuery": + def before_id36(self, id36: str) -> PaginatedQuery: """Restrict the query to results before an id36 (generative).""" if self.after_id: raise ValueError("Can't set both before and after restrictions") @@ -110,7 +111,7 @@ def before_id36(self, id36: str) -> "PaginatedQuery": return self - def _apply_before_or_after(self) -> "PaginatedQuery": + def _apply_before_or_after(self) -> PaginatedQuery: """Apply the "before" or "after" restrictions if necessary.""" # pylint: disable=assignment-from-no-return if not (self.after_id or self.before_id): @@ -165,7 +166,7 @@ def _anchor_subquery(self, anchor_id: int) -> Any: .subquery() ) - def _finalize(self) -> "PaginatedQuery": + def _finalize(self) -> PaginatedQuery: """Finalize the query before execution.""" query = super()._finalize() @@ -185,7 +186,7 @@ def _finalize(self) -> "PaginatedQuery": return query - def get_page(self, per_page: int) -> "PaginatedResults": + def get_page(self, per_page: int) -> PaginatedResults: """Get a page worth of results from the query (`per page` items).""" return PaginatedResults(self, per_page) diff --git a/tildes/tildes/models/scripting.py b/tildes/tildes/models/scripting.py new file mode 100644 index 00000000..f38dd573 --- /dev/null +++ b/tildes/tildes/models/scripting.py @@ -0,0 +1,89 @@ +# Copyright (c) 2020 Tildes contributors +# SPDX-License-Identifier: AGPL-3.0-or-later + +"""Model wrappers that control which data and methods are accessible for scripting. + +Each wrapper class needs to have "gettable_attrs" and/or "settable_attrs" properties +that define which attributes (including methods) are accessible from inside scripts. +""" + +from wrapt import ObjectProxy + +from tildes.lib.lua import SandboxedLua + +from .comment import Comment +from .topic import Topic +from .user import User + + +class UserScriptingWrapper(ObjectProxy): + # pylint: disable=abstract-method + """Wrapper for the User model.""" + + gettable_attrs = {"username"} + + def __init__(self, user: User, lua_sandbox: SandboxedLua): + """Wrap a User.""" + super().__init__(user) + + self._lua = lua_sandbox.lua + + +class TopicScriptingWrapper(ObjectProxy): + # pylint: disable=abstract-method + """Wrapper for the Topic model.""" + + gettable_attrs = { + "is_link_type", + "is_text_type", + "link", + "link_domain", + "markdown", + "remove", + "tags", + "title", + "user", + } + settable_attrs = {"link", "tags", "title"} + + def __init__(self, topic: Topic, lua_sandbox: SandboxedLua): + """Wrap a Topic.""" + super().__init__(topic) + + self._lua = lua_sandbox.lua + + self.user = UserScriptingWrapper(topic.user, lua_sandbox) + + @property + def tags(self): # type: ignore + """Return the topic's tags as a Lua table.""" + return self._lua.table_from(self.__wrapped__.tags) + + @tags.setter + def tags(self, new_tags): # type: ignore + """Set the topic's tags, the new value should be a Lua table.""" + self.__wrapped__.tags = new_tags.values() + + def remove(self) -> None: + """Remove the topic.""" + self.__wrapped__.is_removed = True + + +class CommentScriptingWrapper(ObjectProxy): + # pylint: disable=abstract-method + """Wrapper for the Comment model.""" + + gettable_attrs = {"markdown", "remove", "topic", "user"} + + def __init__(self, comment: Comment, lua_sandbox: SandboxedLua): + """Wrap a Comment.""" + super().__init__(comment) + + self._lua = lua_sandbox.lua + + self.topic = TopicScriptingWrapper(comment.topic, lua_sandbox) + self.user = UserScriptingWrapper(comment.user, lua_sandbox) + + def remove(self) -> None: + """Remove the comment.""" + self.__wrapped__.is_removed = True diff --git a/tildes/tildes/models/topic/topic.py b/tildes/tildes/models/topic/topic.py index 02ff480f..4c7014c1 100644 --- a/tildes/tildes/models/topic/topic.py +++ b/tildes/tildes/models/topic/topic.py @@ -3,6 +3,7 @@ """Contains the Topic class.""" +from __future__ import annotations from datetime import datetime, timedelta from itertools import chain from pathlib import PurePosixPath @@ -65,6 +66,9 @@ class Topic(DatabaseModel): updates to is_deleted in comments. - last_activity_time will be updated by insertions, deletions, and updates to is_deleted in comments. + Outgoing: + - Inserting rows or updating is_deleted/is_removed to change visibility will + update topic_schedule.latest_topic_id if the topic has a schedule_id. Internal: - deleted_time will be set when is_deleted is set to true """ @@ -84,14 +88,20 @@ class Topic(DatabaseModel): Integer, ForeignKey("topic_schedule.schedule_id"), index=True ) created_time: datetime = Column( - TIMESTAMP(timezone=True), nullable=False, server_default=text("NOW()"), + TIMESTAMP(timezone=True), + nullable=False, + server_default=text("NOW()"), ) last_edited_time: Optional[datetime] = Column(TIMESTAMP(timezone=True)) last_activity_time: datetime = Column( - TIMESTAMP(timezone=True), nullable=False, server_default=text("NOW()"), + TIMESTAMP(timezone=True), + nullable=False, + server_default=text("NOW()"), ) last_interesting_activity_time: datetime = Column( - TIMESTAMP(timezone=True), nullable=False, server_default=text("NOW()"), + TIMESTAMP(timezone=True), + nullable=False, + server_default=text("NOW()"), ) is_deleted: bool = Column( Boolean, nullable=False, server_default="false", index=True @@ -128,6 +138,8 @@ class Topic(DatabaseModel): user: User = relationship("User", lazy=False, innerjoin=True) group: Group = relationship("Group", innerjoin=True) + schedule = relationship("TopicSchedule", foreign_keys=[schedule_id]) + # Create specialized indexes __table_args__ = ( Index("ix_topics_tags_gist", tags, postgresql_using="gist"), @@ -207,7 +219,7 @@ def __repr__(self) -> str: return f'' @classmethod - def _create_base_topic(cls, group: Group, author: User, title: str) -> "Topic": + def _create_base_topic(cls, group: Group, author: User, title: str) -> Topic: """Create the "base" for a new topic.""" new_topic = cls() new_topic.group = group @@ -224,7 +236,7 @@ def _create_base_topic(cls, group: Group, author: User, title: str) -> "Topic": @classmethod def create_text_topic( cls, group: Group, author: User, title: str, markdown: str = "" - ) -> "Topic": + ) -> Topic: """Create a new text topic.""" new_topic = cls._create_base_topic(group, author, title) new_topic.topic_type = TopicType.TEXT @@ -235,7 +247,7 @@ def create_text_topic( @classmethod def create_link_topic( cls, group: Group, author: User, title: str, link: str - ) -> "Topic": + ) -> Topic: """Create a new link topic.""" new_topic = cls._create_base_topic(group, author, title) new_topic.topic_type = TopicType.LINK @@ -312,6 +324,9 @@ def __acl__(self) -> AclType: # noqa # comment: # - removed topics can only be commented on by users who can remove # - locked topics can only be commented on by users who can lock + # - topics posted by the scheduler can only be commented in if they're the + # latest topic from that schedule, or only_new_top_level_comments_in_latest + # is False # - otherwise, logged-in users can comment if self.is_removed: acl.extend( @@ -333,6 +348,13 @@ def __acl__(self) -> AclType: # noqa ) acl.append((Deny, Everyone, "comment")) + if ( + self.was_posted_by_scheduler + and self.schedule.only_new_top_level_comments_in_latest + and self.topic_id != self.schedule.latest_topic_id + ): + acl.append((Deny, Everyone, "comment")) + acl.append((Allow, Authenticated, "comment")) # edit: @@ -399,6 +421,11 @@ def permalink(self) -> str: """Return the permalink for this topic.""" return f"/~{self.group.path}/{self.topic_id36}/{self.url_slug}" + @property + def permalink_absolute(self) -> str: + """Return the absolute permalink for this topic (domain included).""" + return f"https://tildes.net{self.permalink}" + @property def is_text_type(self) -> bool: """Return whether this is a text topic.""" diff --git a/tildes/tildes/models/topic/topic_query.py b/tildes/tildes/models/topic/topic_query.py index 690f2759..7deaeefd 100644 --- a/tildes/tildes/models/topic/topic_query.py +++ b/tildes/tildes/models/topic/topic_query.py @@ -3,6 +3,7 @@ """Contains the TopicQuery class.""" +from __future__ import annotations from typing import Any, Sequence from pyramid.request import Request @@ -40,7 +41,7 @@ def __init__(self, request: Request): self.filter_ignored = False - def _attach_extra_data(self) -> "TopicQuery": + def _attach_extra_data(self) -> TopicQuery: """Attach the extra user data to the query.""" if not self.request.user: return self @@ -53,7 +54,7 @@ def _attach_extra_data(self) -> "TopicQuery": ._attach_ignored_data() ) - def _finalize(self) -> "TopicQuery": + def _finalize(self) -> TopicQuery: """Finalize the query before it's executed.""" # pylint: disable=self-cls-assignment self = super()._finalize() @@ -63,7 +64,7 @@ def _finalize(self) -> "TopicQuery": return self - def _attach_vote_data(self) -> "TopicQuery": + def _attach_vote_data(self) -> TopicQuery: """Join the data related to whether the user has voted on the topic.""" query = self.join( TopicVote, @@ -77,7 +78,7 @@ def _attach_vote_data(self) -> "TopicQuery": return query - def _attach_bookmark_data(self) -> "TopicQuery": + def _attach_bookmark_data(self) -> TopicQuery: """Join the data related to whether the user has bookmarked the topic.""" query = self.join( TopicBookmark, @@ -91,7 +92,7 @@ def _attach_bookmark_data(self) -> "TopicQuery": return query - def _attach_visit_data(self) -> "TopicQuery": + def _attach_visit_data(self) -> TopicQuery: """Join the data related to the user's last visit to the topic(s).""" # subquery using LATERAL to select only the newest visit for each topic lateral_subquery = ( @@ -116,7 +117,7 @@ def _attach_visit_data(self) -> "TopicQuery": return query - def _attach_ignored_data(self) -> "TopicQuery": + def _attach_ignored_data(self) -> TopicQuery: """Join the data related to whether the user has ignored the topic.""" query = self.join( TopicIgnore, @@ -160,7 +161,7 @@ def _process_result(result: Any) -> Topic: def apply_sort_option( self, sort: TopicSortOption, is_desc: bool = True - ) -> "TopicQuery": + ) -> TopicQuery: """Apply a TopicSortOption sorting method (generative).""" if sort == TopicSortOption.VOTES: self._sort_column = Topic.num_votes @@ -179,7 +180,7 @@ def apply_sort_option( def inside_groups( self, groups: Sequence[Group], include_subgroups: bool = True - ) -> "TopicQuery": + ) -> TopicQuery: """Restrict the topics to inside specific groups (generative).""" if include_subgroups: query_paths = [group.path for group in groups] @@ -191,7 +192,7 @@ def inside_groups( return self.filter(Topic.group_id.in_(group_ids)) # type: ignore - def inside_time_period(self, period: SimpleHoursPeriod) -> "TopicQuery": + def inside_time_period(self, period: SimpleHoursPeriod) -> TopicQuery: """Restrict the topics to inside a time period (generative).""" # if the time period is too long, this will crash by creating a datetime outside # the valid range - catch that and just don't filter by time period at all if @@ -203,39 +204,44 @@ def inside_time_period(self, period: SimpleHoursPeriod) -> "TopicQuery": return self.filter(Topic.created_time > start_time) - def has_tag(self, tag: str) -> "TopicQuery": + def has_tag(self, tag: str) -> TopicQuery: """Restrict the topics to ones with a specific tag (generative). - Note that this method searches for topics that have any tag that either starts - or ends with the specified tag, not only exact/full matches. + Note that this method searches for topics that have any tag that contains + the specified tag as a subpath, not only exact/full matches. """ - queries = [f"{tag}.*", f"*.{tag}"] + query = f"*.{tag}.*" # pylint: disable=protected-access - return self.filter(Topic.tags.lquery(queries)) # type: ignore + return self.filter(Topic.tags.lquery(query)) # type: ignore - def search(self, query: str) -> "TopicQuery": + def search(self, query: str) -> TopicQuery: """Restrict the topics to ones that match a search query (generative).""" + # Replace "." with space, since tags are stored as space-separated strings + # in the search index. + # URL domains are not indexed, so removing "." is okay for now. + query = query.replace(".", " ") + return self.filter(Topic.search_tsv.op("@@")(func.websearch_to_tsquery(query))) - def only_bookmarked(self) -> "TopicQuery": + def only_bookmarked(self) -> TopicQuery: """Restrict the topics to ones that the user has bookmarked (generative).""" self._only_bookmarked = True return self - def only_user_voted(self) -> "TopicQuery": + def only_user_voted(self) -> TopicQuery: """Restrict the topics to ones that the user has voted on (generative).""" self._only_user_voted = True return self - def only_ignored(self) -> "TopicQuery": + def only_ignored(self) -> TopicQuery: """Restrict the topics to ones that the user has ignored (generative).""" # pylint: disable=self-cls-assignment self._only_ignored = True return self - def exclude_ignored(self) -> "TopicQuery": + def exclude_ignored(self) -> TopicQuery: """Specify that ignored topics should be excluded (generative).""" self.filter_ignored = True diff --git a/tildes/tildes/models/topic/topic_schedule.py b/tildes/tildes/models/topic/topic_schedule.py index 89a0f580..0b858749 100644 --- a/tildes/tildes/models/topic/topic_schedule.py +++ b/tildes/tildes/models/topic/topic_schedule.py @@ -8,8 +8,16 @@ from dateutil.rrule import rrule from jinja2.sandbox import SandboxedEnvironment -from sqlalchemy import CheckConstraint, Column, ForeignKey, Integer, Text, TIMESTAMP -from sqlalchemy.orm import backref, relationship +from sqlalchemy import ( + Boolean, + CheckConstraint, + Column, + ForeignKey, + Integer, + Text, + TIMESTAMP, +) +from sqlalchemy.orm import relationship from sqlalchemy.orm.session import Session from sqlalchemy.sql.expression import text @@ -23,7 +31,13 @@ class TopicSchedule(DatabaseModel): - """Model for scheduled topics (auto-posted, often repeatedly on a schedule).""" + """Model for scheduled topics (auto-posted, often repeatedly on a schedule). + + Trigger behavior: + Incoming: + - latest_topic_id will be set when a new topic is inserted for the schedule, + and updated when a topic from the schedule is deleted or removed. + """ __tablename__ = "topic_schedule" @@ -46,11 +60,14 @@ class TopicSchedule(DatabaseModel): TIMESTAMP(timezone=True), nullable=True, index=True ) recurrence_rule: Optional[rrule] = Column(RecurrenceRule, nullable=True) + only_new_top_level_comments_in_latest: bool = Column( + Boolean, nullable=False, server_default="true" + ) + latest_topic_id: int = Column(Integer, ForeignKey("topics.topic_id"), nullable=True) group: Group = relationship("Group", innerjoin=True) user: Optional[User] = relationship("User") - - topics: List[Topic] = relationship(Topic, backref=backref("schedule")) + latest_topic: Topic = relationship("Topic", foreign_keys=[latest_topic_id]) def __init__( self, diff --git a/tildes/tildes/models/user/user.py b/tildes/tildes/models/user/user.py index b07958c3..8fe0c048 100644 --- a/tildes/tildes/models/user/user.py +++ b/tildes/tildes/models/user/user.py @@ -242,6 +242,13 @@ def password(self, value: str) -> None: def is_correct_password(self, password: str) -> bool: """Check if the password is correct for this user.""" + # Some accounts have no password - the special-purpose "fake" accounts (ones + # with user_id <= 0), and it's possible to happen for a real account as well, in + # a niche case like un-deleting an account that was deleted a long time ago. + # Trying to check a password on those accounts will error, so just always fail. + if not self.password_hash: + return False + return is_match_for_hash(password, self.password_hash) def change_password(self, old_password: str, new_password: str) -> None: diff --git a/tildes/tildes/models/user/user_permissions.py b/tildes/tildes/models/user/user_permissions.py index 76665b41..fc7e76ca 100644 --- a/tildes/tildes/models/user/user_permissions.py +++ b/tildes/tildes/models/user/user_permissions.py @@ -3,11 +3,11 @@ """Contains the UserPermissions class.""" -from sqlalchemy import Column, ForeignKey, Integer, Text +from sqlalchemy import Column, ForeignKey, Integer from sqlalchemy.dialects.postgresql import ENUM from sqlalchemy.orm import relationship -from tildes.enums import UserPermissionType +from tildes.enums import UserPermission, UserPermissionType from tildes.models import DatabaseModel from tildes.models.group import Group from tildes.models.user import User @@ -21,7 +21,7 @@ class UserPermissions(DatabaseModel): permission_id: int = Column(Integer, primary_key=True) user_id: int = Column(Integer, ForeignKey("users.user_id"), nullable=False) group_id: int = Column(Integer, ForeignKey("groups.group_id"), nullable=True) - permission: str = Column(Text, nullable=False) + permission: UserPermission = Column(ENUM(UserPermission), nullable=False) permission_type: UserPermissionType = Column( ENUM(UserPermissionType), nullable=False, server_default="ALLOW" ) @@ -50,6 +50,6 @@ def auth_principal(self) -> str: if self.permission_type == UserPermissionType.DENY: principal += "!" - principal += self.permission + principal += str(self.permission.name) return principal diff --git a/tildes/tildes/request_methods.py b/tildes/tildes/request_methods.py index 049ea676..7274894f 100644 --- a/tildes/tildes/request_methods.py +++ b/tildes/tildes/request_methods.py @@ -27,6 +27,7 @@ def get_redis_connection(request: Request) -> Redis: def is_bot(request: Request) -> bool: """Return whether the request is by a known bot (e.g. search engine crawlers).""" bot_user_agent_substrings = ( + "Amazonbot", "bingbot", "Googlebot", "heritrix", @@ -36,7 +37,9 @@ def is_bot(request: Request) -> bool: "Python-urllib", "Qwantify", "SeznamBot", + "tildee.py", "Tildes Scraper", + "yacybot", "YandexBot", ) @@ -87,6 +90,9 @@ def check_rate_limit(request: Request, action_name: str) -> RateLimitResult: results = [] + if action.is_global: + results.append(action.check_global()) + if action.by_user and request.user: results.append(action.check_for_user_id(request.user.user_id)) diff --git a/tildes/tildes/resources/comment.py b/tildes/tildes/resources/comment.py index a89fcccb..45aefa6d 100644 --- a/tildes/tildes/resources/comment.py +++ b/tildes/tildes/resources/comment.py @@ -5,15 +5,15 @@ from pyramid.httpexceptions import HTTPForbidden, HTTPNotFound from pyramid.request import Request -from webargs.pyramidparser import use_kwargs from tildes.lib.id import id36_to_id from tildes.models.comment import Comment, CommentNotification from tildes.resources import get_resource from tildes.schemas.comment import CommentSchema +from tildes.views.decorators import use_kwargs -@use_kwargs(CommentSchema(only=("comment_id36",)), locations=("matchdict",)) +@use_kwargs(CommentSchema(only=("comment_id36",)), location="matchdict") def comment_by_id36(request: Request, comment_id36: str) -> Comment: """Get a comment specified by {comment_id36} in the route (or 404).""" query = ( @@ -28,7 +28,7 @@ def comment_by_id36(request: Request, comment_id36: str) -> Comment: raise HTTPNotFound("Comment not found (or it was deleted)") -@use_kwargs(CommentSchema(only=("comment_id36",)), locations=("matchdict",)) +@use_kwargs(CommentSchema(only=("comment_id36",)), location="matchdict") def notification_by_comment_id36( request: Request, comment_id36: str ) -> CommentNotification: diff --git a/tildes/tildes/resources/group.py b/tildes/tildes/resources/group.py index 13b26989..b95f9641 100644 --- a/tildes/tildes/resources/group.py +++ b/tildes/tildes/resources/group.py @@ -7,16 +7,16 @@ from pyramid.httpexceptions import HTTPMovedPermanently, HTTPNotFound from pyramid.request import Request from sqlalchemy_utils import Ltree -from webargs.pyramidparser import use_kwargs from tildes.models.group import Group, GroupWikiPage from tildes.resources import get_resource from tildes.schemas.group import GroupSchema +from tildes.views.decorators import use_kwargs @use_kwargs( GroupSchema(only=("path",), context={"fix_path_capitalization": True}), - locations=("matchdict",), + location="matchdict", ) def group_by_path(request: Request, path: str) -> Group: """Get a group specified by {path} in the route (or 404).""" @@ -35,7 +35,7 @@ def group_by_path(request: Request, path: str) -> Group: return get_resource(request, query) -@use_kwargs({"wiki_page_path": String()}, locations=("matchdict",)) +@use_kwargs({"wiki_page_path": String()}, location="matchdict") def group_wiki_page_by_path(request: Request, wiki_page_path: str) -> GroupWikiPage: """Get a group's wiki page by its path (or 404).""" group = group_by_path(request) # pylint: disable=no-value-for-parameter diff --git a/tildes/tildes/resources/message.py b/tildes/tildes/resources/message.py index 25c30dbc..a4ff203b 100644 --- a/tildes/tildes/resources/message.py +++ b/tildes/tildes/resources/message.py @@ -4,16 +4,16 @@ """Root factories for messages.""" from pyramid.request import Request -from webargs.pyramidparser import use_kwargs from tildes.lib.id import id36_to_id from tildes.models.message import MessageConversation from tildes.resources import get_resource from tildes.schemas.message import MessageConversationSchema +from tildes.views.decorators import use_kwargs @use_kwargs( - MessageConversationSchema(only=("conversation_id36",)), locations=("matchdict",) + MessageConversationSchema(only=("conversation_id36",)), location="matchdict" ) def message_conversation_by_id36( request: Request, conversation_id36: str diff --git a/tildes/tildes/resources/topic.py b/tildes/tildes/resources/topic.py index 541ba10d..e5e7ac76 100644 --- a/tildes/tildes/resources/topic.py +++ b/tildes/tildes/resources/topic.py @@ -5,15 +5,15 @@ from pyramid.httpexceptions import HTTPFound, HTTPNotFound from pyramid.request import Request -from webargs.pyramidparser import use_kwargs from tildes.lib.id import id36_to_id from tildes.models.topic import Topic from tildes.resources import get_resource from tildes.schemas.topic import TopicSchema +from tildes.views.decorators import use_kwargs -@use_kwargs(TopicSchema(only=("topic_id36",)), locations=("matchdict",)) +@use_kwargs(TopicSchema(only=("topic_id36",)), location="matchdict") def topic_by_id36(request: Request, topic_id36: str) -> Topic: """Get a topic specified by {topic_id36} in the route (or 404).""" try: diff --git a/tildes/tildes/resources/user.py b/tildes/tildes/resources/user.py index 79b65abc..f80b7354 100644 --- a/tildes/tildes/resources/user.py +++ b/tildes/tildes/resources/user.py @@ -4,14 +4,14 @@ """Root factories for users.""" from pyramid.request import Request -from webargs.pyramidparser import use_kwargs from tildes.models.user import User from tildes.resources import get_resource from tildes.schemas.user import UserSchema +from tildes.views.decorators import use_kwargs -@use_kwargs(UserSchema(only=("username",)), locations=("matchdict",)) +@use_kwargs(UserSchema(only=("username",)), location="matchdict") def user_by_username(request: Request, username: str) -> User: """Get a user specified by {username} in the route or 404 if not found.""" query = request.query(User).include_deleted().filter(User.username == username) diff --git a/tildes/tildes/routes.py b/tildes/tildes/routes.py index 662d8bb6..c9f3abd4 100644 --- a/tildes/tildes/routes.py +++ b/tildes/tildes/routes.py @@ -21,6 +21,8 @@ def includeme(config: Configurator) -> None: # In order to achieve a single-group tildes, the groups route is disabled # and the home and search routes redirect. config.add_route("home", "/") + config.add_route("home_atom", "/topics.atom") + config.add_route("home_rss", "/topics.rss") config.add_route("search", "/search") @@ -39,6 +41,8 @@ def includeme(config: Configurator) -> None: config.add_route("new_topic", "/new_topic", factory=group_by_path) config.add_route("group_topics", "/topics", factory=group_by_path) + config.add_route("group_topics_atom", "/topics.atom", factory=group_by_path) + config.add_route("group_topics_rss", "/topics.rss", factory=group_by_path) config.add_route("group_search", "/search", factory=group_by_path) diff --git a/tildes/tildes/schemas/fields.py b/tildes/tildes/schemas/fields.py index 8cde3ea7..684b81f0 100644 --- a/tildes/tildes/schemas/fields.py +++ b/tildes/tildes/schemas/fields.py @@ -4,6 +4,7 @@ """Custom schema field definitions.""" import enum +import re from typing import Any, Mapping, Optional, Type import sqlalchemy_utils @@ -36,7 +37,11 @@ def _serialize( return value.name.lower() def _deserialize( - self, value: str, attr: Optional[str], data: DataType, **kwargs: Any, + self, + value: str, + attr: Optional[str], + data: DataType, + **kwargs: Any, ) -> enum.Enum: """Deserialize a string to the enum member with that name.""" if not self._enum_class: @@ -63,7 +68,11 @@ class ShortTimePeriod(Field): """ def _deserialize( - self, value: str, attr: Optional[str], data: DataType, **kwargs: Any, + self, + value: str, + attr: Optional[str], + data: DataType, + **kwargs: Any, ) -> Optional[SimpleHoursPeriod]: """Deserialize to a SimpleHoursPeriod object.""" if value == "all": @@ -75,7 +84,11 @@ def _deserialize( raise ValidationError("Invalid time period") def _serialize( - self, value: Optional[SimpleHoursPeriod], attr: str, obj: object, **kwargs: Any, + self, + value: Optional[SimpleHoursPeriod], + attr: str, + obj: object, + **kwargs: Any, ) -> Optional[str]: """Serialize the value to the "short form" string.""" if not value: @@ -104,7 +117,11 @@ def _validate(self, value: str) -> None: raise ValidationError("Cannot be entirely whitespace.") def _deserialize( - self, value: str, attr: Optional[str], data: DataType, **kwargs: Any, + self, + value: str, + attr: Optional[str], + data: DataType, + **kwargs: Any, ) -> str: """Deserialize the string, removing carriage returns in the process.""" value = value.replace("\r", "") @@ -137,7 +154,11 @@ def __init__(self, max_length: Optional[int] = None, **kwargs: Any): super().__init__(validate=Length(min=1, max=max_length), **kwargs) def _deserialize( - self, value: str, attr: Optional[str], data: DataType, **kwargs: Any, + self, + value: str, + attr: Optional[str], + data: DataType, + **kwargs: Any, ) -> str: """Deserialize the string, removing/replacing as necessary.""" return simplify_string(value) @@ -150,6 +171,10 @@ def _serialize(self, value: str, attr: str, obj: object, **kwargs: Any) -> str: class Ltree(Field): """Field for postgresql ltree type.""" + # note that this regex only checks whether all of the chars are individually valid, + # but doesn't verify that the value as a whole is a valid ltree path + VALID_CHARS_REGEX = re.compile("^[A-Za-z0-9_.]+$") + def _serialize( self, value: sqlalchemy_utils.Ltree, attr: str, obj: object, **kwargs: Any ) -> str: @@ -157,12 +182,19 @@ def _serialize( return value.path def _deserialize( - self, value: str, attr: Optional[str], data: DataType, **kwargs: Any, + self, + value: str, + attr: Optional[str], + data: DataType, + **kwargs: Any, ) -> sqlalchemy_utils.Ltree: """Deserialize a string path to an Ltree object.""" # convert to lowercase and replace spaces with underscores value = value.lower().replace(" ", "_") + if not self.VALID_CHARS_REGEX.fullmatch(value): + raise ValidationError("Only letters, numbers, and spaces allowed.") + try: return sqlalchemy_utils.Ltree(value) except (TypeError, ValueError): diff --git a/tildes/tildes/schemas/group.py b/tildes/tildes/schemas/group.py index 2b518243..1659cb89 100644 --- a/tildes/tildes/schemas/group.py +++ b/tildes/tildes/schemas/group.py @@ -47,10 +47,14 @@ def prepare_path(self, data: dict, many: bool, partial: Any) -> dict: if not self.context.get("fix_path_capitalization"): return data - if "path" in data and isinstance(data["path"], str): - data["path"] = data["path"].lower() + if "path" not in data or not isinstance(data["path"], str): + return data + + new_data = data.copy() - return data + new_data["path"] = new_data["path"].lower() + + return new_data @validates("path") def validate_path(self, value: sqlalchemy_utils.Ltree) -> None: @@ -71,11 +75,13 @@ def prepare_sidebar_markdown(self, data: dict, many: bool, partial: Any) -> dict if "sidebar_markdown" not in data: return data + new_data = data.copy() + # if the value is empty, convert it to None - if not data["sidebar_markdown"] or data["sidebar_markdown"].isspace(): - data["sidebar_markdown"] = None + if not new_data["sidebar_markdown"] or new_data["sidebar_markdown"].isspace(): + new_data["sidebar_markdown"] = None - return data + return new_data def is_valid_group_path(path: str) -> bool: diff --git a/tildes/tildes/schemas/listing.py b/tildes/tildes/schemas/listing.py index 5a797e08..c11508f3 100644 --- a/tildes/tildes/schemas/listing.py +++ b/tildes/tildes/schemas/listing.py @@ -46,10 +46,12 @@ def reset_rank_start_on_first_page( if "rank_start" not in self.fields: return data - if not (data.get("before") or data.get("after")): - data["n"] = 1 + new_data = data.copy() - return data + if not (new_data.get("before") or new_data.get("after")): + new_data["n"] = 1 + + return new_data class MixedListingSchema(PaginatedListingSchema): @@ -71,10 +73,12 @@ def set_anchor_type_from_before_or_after( to the topic with ID36 "123". "c-123" also works, for comments. """ # pylint: disable=unused-argument + new_data = data.copy() + keys = ("after", "before") for key in keys: - value = data.get(key) + value = new_data.get(key) if not value: continue @@ -83,12 +87,12 @@ def set_anchor_type_from_before_or_after( continue if type_char == "c": - data["anchor_type"] = "comment" + new_data["anchor_type"] = "comment" elif type_char == "t": - data["anchor_type"] = "topic" + new_data["anchor_type"] = "topic" else: continue - data[key] = id36 + new_data[key] = id36 - return data + return new_data diff --git a/tildes/tildes/schemas/topic.py b/tildes/tildes/schemas/topic.py index c52aa2cb..af8ef882 100644 --- a/tildes/tildes/schemas/topic.py +++ b/tildes/tildes/schemas/topic.py @@ -43,10 +43,18 @@ def prepare_title(self, data: dict, many: bool, partial: Any) -> dict: if "title" not in data: return data - # strip any trailing periods - data["title"] = data["title"].rstrip(".") + new_data = data.copy() - return data + split_title = re.split("[.?!]+", new_data["title"]) + + # the last string in the list will be empty if it ended with punctuation + num_sentences = len([piece for piece in split_title if piece]) + + # strip trailing periods off single-sentence titles + if num_sentences == 1: + new_data["title"] = new_data["title"].rstrip(".") + + return new_data @pre_load def prepare_tags(self, data: dict, many: bool, partial: Any) -> dict: @@ -55,9 +63,11 @@ def prepare_tags(self, data: dict, many: bool, partial: Any) -> dict: if "tags" not in data: return data + new_data = data.copy() + tags: typing.List[str] = [] - for tag in data["tags"]: + for tag in new_data["tags"]: tag = tag.lower() # replace underscores with spaces @@ -84,9 +94,9 @@ def prepare_tags(self, data: dict, many: bool, partial: Any) -> dict: tags.append(tag) - data["tags"] = tags + new_data["tags"] = tags - return data + return new_data @validates("tags") def validate_tags(self, value: typing.List[str]) -> None: @@ -112,11 +122,13 @@ def prepare_markdown(self, data: dict, many: bool, partial: Any) -> dict: if "markdown" not in data: return data + new_data = data.copy() + # if the value is empty, convert it to None - if not data["markdown"] or data["markdown"].isspace(): - data["markdown"] = None + if not new_data["markdown"] or new_data["markdown"].isspace(): + new_data["markdown"] = None - return data + return new_data @pre_load def prepare_link(self, data: dict, many: bool, partial: Any) -> dict: @@ -125,23 +137,25 @@ def prepare_link(self, data: dict, many: bool, partial: Any) -> dict: if "link" not in data: return data + new_data = data.copy() + # remove leading/trailing whitespace - data["link"] = data["link"].strip() + new_data["link"] = new_data["link"].strip() # if the value is empty, convert it to None - if not data["link"]: - data["link"] = None - return data + if not new_data["link"]: + new_data["link"] = None + return new_data # prepend http:// to the link if it doesn't have a scheme - parsed = urlparse(data["link"]) + parsed = urlparse(new_data["link"]) if not parsed.scheme: - data["link"] = "http://" + data["link"] + new_data["link"] = "http://" + new_data["link"] # run the link through the url-transformation process - data["link"] = apply_url_transformations(data["link"]) + new_data["link"] = apply_url_transformations(new_data["link"]) - return data + return new_data @validates_schema def link_or_markdown(self, data: dict, many: bool, partial: Any) -> None: diff --git a/tildes/tildes/schemas/user.py b/tildes/tildes/schemas/user.py index ffe0686e..44bf55f5 100644 --- a/tildes/tildes/schemas/user.py +++ b/tildes/tildes/schemas/user.py @@ -63,10 +63,17 @@ class UserSchema(Schema): def anonymize_username(self, data: dict, many: bool) -> dict: """Hide the username if the dumping context specifies to do so.""" # pylint: disable=unused-argument - if "username" in data and self.context.get("hide_username"): - data["username"] = "" + if not self.context.get("hide_username"): + return data + + if "username" not in data: + return data + + new_data = data.copy() - return data + new_data["username"] = "" + + return new_data @validates_schema def username_pass_not_substrings( @@ -116,9 +123,11 @@ def username_trim_whitespace(self, data: dict, many: bool, partial: Any) -> dict if "username" not in data: return data - data["username"] = data["username"].strip() + new_data = data.copy() + + new_data["username"] = new_data["username"].strip() - return data + return new_data @pre_load def prepare_email_address(self, data: dict, many: bool, partial: Any) -> dict: @@ -127,14 +136,16 @@ def prepare_email_address(self, data: dict, many: bool, partial: Any) -> dict: if "email_address" not in data: return data + new_data = data.copy() + # remove any leading/trailing whitespace - data["email_address"] = data["email_address"].strip() + new_data["email_address"] = new_data["email_address"].strip() # if the value is empty, convert it to None - if not data["email_address"] or data["email_address"].isspace(): - data["email_address"] = None + if not new_data["email_address"] or new_data["email_address"].isspace(): + new_data["email_address"] = None - return data + return new_data @pre_load def prepare_bio_markdown(self, data: dict, many: bool, partial: Any) -> dict: @@ -143,11 +154,13 @@ def prepare_bio_markdown(self, data: dict, many: bool, partial: Any) -> dict: if "bio_markdown" not in data: return data + new_data = data.copy() + # if the value is empty, convert it to None - if not data["bio_markdown"] or data["bio_markdown"].isspace(): - data["bio_markdown"] = None + if not new_data["bio_markdown"] or new_data["bio_markdown"].isspace(): + new_data["bio_markdown"] = None - return data + return new_data def is_valid_username(username: str) -> bool: diff --git a/tildes/tildes/settings.py b/tildes/tildes/settings.py new file mode 100644 index 00000000..146394c9 --- /dev/null +++ b/tildes/tildes/settings.py @@ -0,0 +1,30 @@ +# Copyright (c) 2020 Tildes contributors +# SPDX-License-Identifier: AGPL-3.0-or-later + +"""Global-like settings for the application. + +This module should always be imported as a whole ("from tildes import settings"), not +importing individual names, since that will cause re-initialization. + +Currently, this module only contains a dict with some of the settings defined in the +INI file, specifically ones with the "tildes." prefix. The values in this dict are +initialized during app startup. + +Important note: this module may be a terrible idea and I may regret this. +""" + +from pyramid.config import Configurator + +INI_FILE_SETTINGS = {} + + +def includeme(config: Configurator) -> None: + """Initialize ini_file_settings with all prefixed settings from the INI file.""" + global INI_FILE_SETTINGS # pylint: disable=global-statement + setting_prefix = "tildes." + + INI_FILE_SETTINGS = { + setting[len(setting_prefix) :]: value + for setting, value in config.get_settings().items() + if setting.startswith(setting_prefix) + } diff --git a/tildes/tildes/templates/base.atom.jinja2 b/tildes/tildes/templates/base.atom.jinja2 new file mode 100644 index 00000000..76f2800f --- /dev/null +++ b/tildes/tildes/templates/base.atom.jinja2 @@ -0,0 +1,14 @@ +{# Copyright (c) 2021 Tildes contributors #} +{# SPDX-License-Identifier: AGPL-3.0-or-later -#} + + + + + {% block feed_title %}Tildes Atom feed{% endblock %} + {% block feed_id %}{{ request.current_route_url() }}{% endblock %} + + {% block feed_updated %}{{ current_time.strftime("%Y-%m-%dT%H:%M:%SZ") }}{% endblock %} + + {% block feed_entries %}{% endblock %} + + diff --git a/tildes/tildes/templates/base.jinja2 b/tildes/tildes/templates/base.jinja2 index 8d686ab4..85693285 100644 --- a/tildes/tildes/templates/base.jinja2 +++ b/tildes/tildes/templates/base.jinja2 @@ -32,6 +32,10 @@ {% elif request.current_theme == "gruvbox-dark" %} + {% elif request.current_theme == "love-dark" %} + + {% elif request.current_theme == "love-light" %} + {% endif %} {% assets "css" %} @@ -51,6 +55,9 @@ + {# RSS/Atom feeds #} + {% block link_alternate_feeds %}{% endblock %} + {% block title_full %}{% block title %}{% endblock %} - Tildes{% endblock %} {% block templates %}{% endblock %} @@ -116,6 +123,8 @@ ("zenburn", "Zenburn"), ("gruvbox-light", "Gruvbox Light"), ("gruvbox-dark", "Gruvbox Dark"), + ("love-dark", "Love Dark"), + ("love-light", "Love Light"), ("motte", "The Motte")) %}