themotte · Zenchreal · Jul 5, 2020 · Jul 5, 2020 · Jul 6, 2020 · Jul 6, 2020
diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
@@ -14,11 +14,11 @@ All code contributions must be made under the same license as the project's main
 
 ## Setting up a development version
 
-Please see this page on the Tildes Docs for instructions to set up a development version: https://docs.tildes.net/instructions/development-setup
+Please see this page on the Tildes Docs for instructions to set up a development version: https://docs.tildes.net/development/initial-setup
 
 ## General development information
 
-This page on the Tildes docs contains information about many aspects of Tildes development: https://docs.tildes.net/instructions/development
+This page on the Tildes docs contains information about many aspects of Tildes development: https://docs.tildes.net/development/general-development
 
 ## Contributing code
 
@@ -48,7 +48,7 @@ Once you've selected an issue to work on, please leave a comment saying so. This
 
 After you've finished making your changes, there are a few things to check before proposing your changes as a merge request.
 
-First, ensure that all the checks (tests, mypy and code-style) pass successfully. Merge requests that fail any of the checks will not be accepted. For more information, see this section of the development docs: https://docs.tildes.net/instructions/development#running-checks-on-your-code
+First, ensure that all the checks (tests, mypy and code-style) pass successfully. Merge requests that fail any of the checks will not be accepted. For more information, see this section of the development docs: https://docs.tildes.net/development/general-development#running-checks-on-your-code
 
 Squash your changes into logical commits. For many changes there should only be a single commit, but some can be broken into multiple logical sections. The commit messages should follow [the formatting described in this article](https://tbaggery.com/2008/04/19/a-note-about-git-commit-messages.html), with the summary line written in the imperative form. The summary line should make sense if you were to use it in a sentence like "If applied, this commit will \_\_\_\_\_\_\_\_". For example, "Add a new X", "Fix a bug with Y", and so on.
 

diff --git a/README.md b/README.md
@@ -16,7 +16,7 @@ For general information about Tildes and its goals, please see [the announcement
 
 Known issues and plans for upcoming changes are tracked on GitLab: https://gitlab.com/tildes/tildes/issues
 
-The "board" view is useful as an overview: https://gitlab.com/tildes/tildes/boards
+The "board" view is useful as an overview: https://gitlab.com/tildes/tildes/-/boards
 
 ## Contributing to Tildes development
 

diff --git a/Vagrantfile b/Vagrantfile
@@ -26,9 +26,6 @@ Vagrant.configure(VAGRANT_CONFIG_VERSION) do |config|
       salt.run_highstate = true
       salt.verbose = true
       salt.log_level = "info"
-
-      salt.install_type = "stable"
-      salt.version = "3000"
   end
 
   config.vm.provider "virtualbox" do |vb|

diff --git a/git_hooks/pre-commit b/git_hooks/pre-commit
@@ -1,10 +1,6 @@
 #!/bin/sh
 #
-# Pre-commit hook script that ensures mypy checks and tests pass
+# Pre-commit hook script that ensures type-checking, tests, and fast style checks pass
 
 vagrant ssh -c ". activate \
-    && echo 'Checking mypy type annotations...' && mypy --no-error-summary . \
-    && echo 'Checking if Black would reformat any code...' && black --check . \
-    && echo -n 'Running tests: ' && pytest -q \
-    && echo 'Checking SCSS style...' && npm run --silent lint:scss \
-    && echo 'Checking JS style...' && npm run --silent lint:js"
+    && invoke type-check test --quiet code-style-check"
diff --git a/git_hooks/pre-push b/git_hooks/pre-push
@@ -1,11 +1,6 @@
 #!/bin/sh
 #
-# Pre-push hook script that ensures mypy checks, style checks, and tests pass
+# Pre-push hook script that ensures all tests and code checks pass
 
 vagrant ssh -c ". activate \
-    && echo 'Checking mypy type annotations...' && mypy --no-error-summary . \
-    && echo 'Checking if Black would reformat any code...' && black --check . \
-    && echo -n 'Running tests: ' && pytest -q \
-    && echo 'Checking SCSS style...' && npm run --silent lint:scss \
-    && echo 'Checking JS style...' && npm run --silent lint:js \
-    && echo 'Checking Python style fully (takes a while)...' && prospector -M"
+    && invoke type-check test --quiet --full code-style-check --full"
diff --git a/salt/salt/boussole.service.jinja2 b/salt/salt/boussole.service.jinja2
@@ -1,8 +1,10 @@
-{% from 'common.jinja2' import app_dir -%}
+{% from 'common.jinja2' import app_dir, app_username -%}
 [Unit]
 Description=Boussole - auto-compile SCSS files on change
 
 [Service]
+User={{ app_username }}
+Group={{ app_username }}
 WorkingDirectory={{ app_dir }}
 Environment="LC_ALL=C.UTF-8" "LANG=C.UTF-8"
 ExecStart=/opt/venvs/boussole/bin/boussole watch --backend=yaml --config=boussole.yaml --poll

diff --git a/salt/salt/consumers/comment_user_mentions_generator.service.jinja2 b/salt/salt/consumers/comment_user_mentions_generator.service.jinja2
@@ -1,11 +1,13 @@
-{% from 'common.jinja2' import app_dir, bin_dir -%}
+{% from 'common.jinja2' import app_dir, app_username, bin_dir -%}
 [Unit]
 Description=Comment User Mention Generator (Queue Consumer)
 Requires=redis.service
 After=redis.service
 PartOf=redis.service
 
 [Service]
+User={{ app_username }}
+Group={{ app_username }}
 WorkingDirectory={{ app_dir }}/consumers
 Environment="INI_FILE={{ app_dir }}/{{ pillar['ini_file'] }}"
 ExecStart={{ bin_dir }}/python comment_user_mentions_generator.py

diff --git a/salt/salt/consumers/init.sls b/salt/salt/consumers/init.sls
@@ -22,6 +22,14 @@
     - group: root
     - mode: 644
 
+/etc/systemd/system/consumer-post_processing_script_runner.service:
+  file.managed:
+    - source: salt://consumers/post_processing_script_runner.service.jinja2
+    - template: jinja
+    - user: root
+    - group: root
+    - mode: 644
+
 consumer-topic_interesting_activity_updater.service:
   service.running:
     - enable: True
@@ -34,6 +42,10 @@ consumer-comment_user_mentions_generator.service:
   service.running:
     - enable: True
 
+consumer-post_processing_script_runner.service:
+  service.running:
+    - enable: True
+
 {% if grains['id'] == 'prod' %}
 /etc/systemd/system/consumer-topic_embedly_extractor.service:
   file.managed:

diff --git a/salt/salt/consumers/post_processing_script_runner.service.jinja2 b/salt/salt/consumers/post_processing_script_runner.service.jinja2
@@ -0,0 +1,18 @@
+{% from 'common.jinja2' import app_dir, app_username, bin_dir -%}
+[Unit]
+Description=Post Processing Script Runner (Queue Consumer)
+Requires=redis.service
+After=redis.service
+PartOf=redis.service
+
+[Service]
+User={{ app_username }}
+Group={{ app_username }}
+WorkingDirectory={{ app_dir }}/consumers
+Environment="INI_FILE={{ app_dir }}/{{ pillar['ini_file'] }}"
+ExecStart={{ bin_dir }}/python post_processing_script_runner.py
+Restart=always
+RestartSec=5
+
+[Install]
+WantedBy=multi-user.target
diff --git a/salt/salt/consumers/topic_embedly_extractor.service.jinja2 b/salt/salt/consumers/topic_embedly_extractor.service.jinja2
@@ -1,11 +1,13 @@
-{% from 'common.jinja2' import app_dir, bin_dir -%}
+{% from 'common.jinja2' import app_dir, app_username, bin_dir -%}
 [Unit]
 Description=Topic Embedly Extractor (Queue Consumer)
 Requires=redis.service
 After=redis.service
 PartOf=redis.service
 
 [Service]
+User={{ app_username }}
+Group={{ app_username }}
 WorkingDirectory={{ app_dir }}/consumers
 Environment="INI_FILE={{ app_dir }}/{{ pillar['ini_file'] }}"
 ExecStart={{ bin_dir }}/python topic_embedly_extractor.py

diff --git a/salt/salt/consumers/topic_interesting_activity_updater.service.jinja2 b/salt/salt/consumers/topic_interesting_activity_updater.service.jinja2
@@ -1,11 +1,13 @@
-{% from 'common.jinja2' import app_dir, bin_dir -%}
+{% from 'common.jinja2' import app_dir, app_username, bin_dir -%}
 [Unit]
 Description=Topic Interesting Activity Updater (Queue Consumer)
 Requires=redis.service
 After=redis.service
 PartOf=redis.service
 
 [Service]
+User={{ app_username }}
+Group={{ app_username }}
 WorkingDirectory={{ app_dir }}/consumers
 Environment="INI_FILE={{ app_dir }}/{{ pillar['ini_file'] }}"
 ExecStart={{ bin_dir }}/python topic_interesting_activity_updater.py

diff --git a/salt/salt/consumers/topic_metadata_generator.service.jinja2 b/salt/salt/consumers/topic_metadata_generator.service.jinja2
@@ -1,11 +1,13 @@
-{% from 'common.jinja2' import app_dir, bin_dir -%}
+{% from 'common.jinja2' import app_dir, app_username, bin_dir -%}
 [Unit]
 Description=Topic Metadata Generator (Queue Consumer)
 Requires=redis.service
 After=redis.service
 PartOf=redis.service
 
 [Service]
+User={{ app_username }}
+Group={{ app_username }}
 WorkingDirectory={{ app_dir }}/consumers
 Environment="INI_FILE={{ app_dir }}/{{ pillar['ini_file'] }}"
 ExecStart={{ bin_dir }}/python topic_metadata_generator.py

diff --git a/salt/salt/consumers/topic_youtube_scraper.service.jinja2 b/salt/salt/consumers/topic_youtube_scraper.service.jinja2
@@ -1,11 +1,13 @@
-{% from 'common.jinja2' import app_dir, bin_dir -%}
+{% from 'common.jinja2' import app_dir, app_username, bin_dir -%}
 [Unit]
 Description=Topic Youtube Scraper (Queue Consumer)
 Requires=redis.service
 After=redis.service
 PartOf=redis.service
 
 [Service]
+User={{ app_username }}
+Group={{ app_username }}
 WorkingDirectory={{ app_dir }}/consumers
 Environment="INI_FILE={{ app_dir }}/{{ pillar['ini_file'] }}"
 ExecStart={{ bin_dir }}/python topic_youtube_scraper.py

diff --git a/salt/salt/development.sls b/salt/salt/development.sls
@@ -19,3 +19,9 @@ automatic-activate:
   file.append:
     - name: '/home/{{ app_username }}/.bashrc'
     - text: 'source activate'
+
+# adds invoke's tab-completion script so that invoke tasks can be completed
+invoke-tab-completion:
+  file.append:
+    - name: '/home/{{ app_username }}/.bashrc'
+    - text: 'source <(invoke --print-completion-script bash)'
diff --git a/salt/salt/java.sls b/salt/salt/java.sls
@@ -0,0 +1,3 @@
+java-openjdk:
+  pkg.installed:
+    - name: openjdk-8-jre
diff --git a/salt/salt/nginx/tildes-static-sites.conf.jinja2 b/salt/salt/nginx/tildes-static-sites.conf.jinja2
@@ -23,14 +23,6 @@ server {
         return 302 https://docs.tildes.net/policies/code-of-conduct;
     }
 
-    location /development {
-        return 302 https://docs.tildes.net/instructions/development;
-    }
-
-    location /development-setup {
-        return 302 https://docs.tildes.net/instructions/development-setup;
-    }
-
     location /mechanics {
         return 302 https://docs.tildes.net/instructions;
     }

diff --git a/salt/salt/postgresql_redis_bridge.service.jinja2 b/salt/salt/postgresql_redis_bridge.service.jinja2
@@ -1,11 +1,13 @@
-{% from 'common.jinja2' import app_dir, bin_dir -%}
+{% from 'common.jinja2' import app_dir, app_username, bin_dir -%}
 [Unit]
 Description=postgresql_redis_bridge - convert NOTIFY to Redis streams
 Requires=redis.service
 After=redis.service
 PartOf=redis.service
 
 [Service]
+User={{ app_username }}
+Group={{ app_username }}
 WorkingDirectory={{ app_dir }}/scripts
 Environment="INI_FILE={{ app_dir }}/{{ pillar['ini_file'] }}"
 ExecStart={{ bin_dir }}/python postgresql_redis_bridge.py

diff --git a/salt/salt/prometheus/exporters/prometheus_redis_exporter.service b/salt/salt/prometheus/exporters/prometheus_redis_exporter.service
@@ -8,7 +8,7 @@ RemainAfterExit=no
 WorkingDirectory=/opt/prometheus_redis_exporter
 User=prometheus
 Group=prometheus
-Environment="REDIS_ADDR=unix:///run/redis/socket,unix:///run/redis_breached_passwords/socket"
+Environment="REDIS_ADDR=unix:///run/redis/socket"
 ExecStart=/opt/prometheus_redis_exporter/redis_exporter
 
 [Install]

diff --git a/salt/salt/pts-lbsearch.sls b/salt/salt/pts-lbsearch.sls
@@ -0,0 +1,12 @@
+compile-pts-lbsearch:
+  file.managed:
+    - name: /tmp/pts_lbsearch.c
+    - source:
+      - https://raw.githubusercontent.com/pts/pts-line-bisect/2ecd9f59246cfa28cb1aeac7cd8d98a8eea2914f/pts_lbsearch.c
+    - source_hash: sha256=ef79efc2f1ecde504b6074f9c89bdc71259a833fa2a2dda4538ed5ea3e04aea1
+    - creates: /usr/local/bin/pts_lbsearch
+  cmd.run:
+    - cwd: /tmp/
+    # compilation command taken from the top of the source file
+    - name: gcc -ansi -W -Wall -Wextra -Werror=missing-declarations -s -O2 -DNDEBUG -o /usr/local/bin/pts_lbsearch pts_lbsearch.c
+    - creates: /usr/local/bin/pts_lbsearch
diff --git a/salt/salt/redis/breached-passwords.sls b/salt/salt/redis/breached-passwords.sls
diff --git a/salt/salt/redis/modules/rebloom.sls b/salt/salt/redis/modules/rebloom.sls
diff --git a/salt/salt/redis/redis_breached_passwords.conf b/salt/salt/redis/redis_breached_passwords.conf
diff --git a/salt/salt/redis/redis_breached_passwords.service b/salt/salt/redis/redis_breached_passwords.service
diff --git a/salt/salt/top.sls b/salt/salt/top.sls
@@ -9,8 +9,6 @@ base:
     - postgresql.pgbouncer
     - python
     - redis
-    - redis.breached-passwords
-    - redis.modules.rebloom
     - redis.modules.redis-cell
     - postgresql-redis-bridge
     - scripts
@@ -22,6 +20,7 @@ base:
     - tildes-wiki
     - boussole
     - webassets
+    - pts-lbsearch
     - cronjobs
     - final-setup  # keep this state file last
   'dev':
@@ -30,6 +29,7 @@ base:
     - development
     - prometheus
     - nodejs
+    - java
   'prod':
     - nginx.shortener-config
     - nginx.static-sites-config

diff --git a/salt/salt/webassets.service.jinja2 b/salt/salt/webassets.service.jinja2
@@ -1,8 +1,10 @@
-{% from 'common.jinja2' import app_dir, bin_dir -%}
+{% from 'common.jinja2' import app_dir, app_username, bin_dir -%}
 [Unit]
 Description=Webassets - auto-compile JS files on change
 
 [Service]
+User={{ app_username }}
+Group={{ app_username }}
 WorkingDirectory={{ app_dir }}
 ExecStart={{ bin_dir }}/webassets -c webassets.yaml watch
 Restart=always