diff --git a/ansible/playbook/tasks/files/pmm.ini b/ansible/playbook/tasks/files/pmm.ini new file mode 100644 index 00000000..e8233963 --- /dev/null +++ b/ansible/playbook/tasks/files/pmm.ini @@ -0,0 +1,142 @@ +[unix_http_server] +chmod = 0700 +username = dummy +password = dummy + +[supervisord] +logfile = /srv/logs/supervisord.log +logfile_maxbytes = 5MB +logfile_backups = 1 +loglevel = info +pidfile = /tmp/supervisord.pid +nodaemon = true +nocleanup = false +user = root +strip_ansi = false + +[supervisorctl] +username = dummy +password = dummy + + +; we rewrite autostart to true during update or build. +[program:pmm-update-perform-init] +command = /usr/sbin/pmm-update -run-playbook -playbook=/usr/share/pmm-update/ansible/playbook/tasks/init.yml +directory = / +autorestart = unexpected +priority=-1 +exitcodes = 0 +autostart = false +startretries = 3 +startsecs = 1 +stopsignal = TERM +stopwaitsecs = 300 +stdout_logfile = /srv/logs/pmm-update-perform-init.log +stdout_logfile_maxbytes = 50MB +stdout_logfile_backups = 3 +redirect_stderr = true + +[program:postgresql] +priority = 1 +command = + /usr/pgsql-14/bin/postgres + -D /srv/postgres14 + -c shared_preload_libraries=pg_stat_statements + -c pg_stat_statements.max=10000 + -c pg_stat_statements.track=all + -c pg_stat_statements.save=off +user = postgres +autorestart = true +autostart = true +startretries = 10 +startsecs = 1 +stopsignal = INT ; Fast Shutdown mode +stopwaitsecs = 300 +; postgresql.conf contains settings to log to stdout, +; so we delegate logfile management to supervisord +stdout_logfile = /srv/logs/postgresql14.log +stdout_logfile_maxbytes = 30MB +stdout_logfile_backups = 2 +redirect_stderr = true + +[program:clickhouse] +priority = 2 +command = /usr/bin/clickhouse-server --config-file=/etc/clickhouse-server/config.xml +autorestart = true +autostart = true +startretries = 10 +startsecs = 1 +stopsignal = TERM +stopwaitsecs = 300 +; config.xml contains settings to log to stdout (console), +; so we delegate logfile managemenet to supervisord +stdout_logfile = /srv/logs/clickhouse-server.log +stdout_logfile_maxbytes = 50MB +stdout_logfile_backups = 2 +redirect_stderr = true + +[program:nginx] +priority = 4 +command = nginx +autorestart = true +autostart = true +startretries = 10 +startsecs = 1 +stopsignal = TERM +stopwaitsecs = 10 +; nginx.conf contains settings to log to /dev/sdtout and /dev/stderr, +; which allows supervisord to manage the logs further. +stdout_logfile = /srv/logs/nginx.log +stdout_logfile_maxbytes = 50MB +stdout_logfile_backups = 2 +redirect_stderr = true + +[program:pmm-managed] +priority = 14 +command = + /usr/sbin/pmm-managed + --victoriametrics-config=/etc/victoriametrics-promscrape.yml + --victoriametrics-url=http://127.0.0.1:9090/prometheus + --postgres-name=pmm-managed + --postgres-username=pmm-managed + --postgres-password=pmm-managed + --supervisord-config-dir=/etc/supervisord.d +autorestart = true +autostart = true +startretries = 1000 +startsecs = 1 +stopsignal = TERM +stopwaitsecs = 300 +stdout_logfile = /srv/logs/pmm-managed.log +stdout_logfile_maxbytes = 50MB +stdout_logfile_backups = 2 +redirect_stderr = true + +[program:pmm-agent] +priority = 15 +command = /usr/sbin/pmm-agent --config-file=/usr/local/percona/pmm2/config/pmm-agent.yaml +autorestart = true +autostart = true +startretries = 1000 +startsecs = 1 +stopsignal = TERM +stopwaitsecs = 10 +stdout_logfile = /srv/logs/pmm-agent.log +stdout_logfile_maxbytes = 50MB +stdout_logfile_backups = 2 +redirect_stderr = true + +[program:pmm-update-perform] +command = /usr/sbin/pmm-update -perform -playbook=/usr/share/pmm-update/ansible/playbook/tasks/update.yml +directory = / +autorestart = unexpected +exitcodes = 0 +autostart = false +startretries = 10 +startsecs = 1 +stopsignal = TERM +stopwaitsecs = 300 +stdout_logfile = /srv/logs/pmm-update-perform.log +stdout_logfile_maxbytes = 50MB +stdout_logfile_backups = 3 +redirect_stderr = true diff --git a/ansible/playbook/tasks/roles/initialization/tasks/main.yml b/ansible/playbook/tasks/roles/initialization/tasks/main.yml index 474f5a33..491ba8b3 100644 --- a/ansible/playbook/tasks/roles/initialization/tasks/main.yml +++ b/ansible/playbook/tasks/roles/initialization/tasks/main.yml @@ -39,23 +39,12 @@ path: /srv/backup state: directory -- name: Wait until postgres socket is present before continuing - wait_for: - path: /var/run/postgresql/.s.PGSQL.5432 - -- name: Get postgres encoding - postgresql_query: - db: pmm-managed - query: SHOW SERVER_ENCODING; - register: postgres_encoding - become: yes - become_user: postgres - become_method: su - -- debug: - var: postgres_encoding.query_result - -- name: Change encoding of pmm-managed database +- name: Check if Postgres 11 exists + stat: + path: /srv/postgres + register: is_postgres_11 + +- name: Upgrade Postgres database include_role: - name: postgres-ascii-to-utf8 - when: "'SQL_ASCII' == postgres_encoding.query_result[0].server_encoding" + name: postgres + when: is_postgres_11.stat.exists diff --git a/ansible/playbook/tasks/roles/postgres-ascii-to-utf8/tasks/main.yml b/ansible/playbook/tasks/roles/postgres-ascii-to-utf8/tasks/main.yml deleted file mode 100644 index 42635c1f..00000000 --- a/ansible/playbook/tasks/roles/postgres-ascii-to-utf8/tasks/main.yml +++ /dev/null @@ -1,84 +0,0 @@ ---- -- name: Create Postgres backup dir - file: - path: /srv/backup/postgres - state: directory - owner: postgres - group: postgres - mode: '0700' - -- name: Stop pmm-managed before changing encoding - command: supervisorctl stop pmm-managed - changed_when: True - -- name: Change encoding from SQL_ASCII to UTF-8 - block: - - name: Disable connections to pmm-managed database - postgresql_query: - query: REVOKE CONNECT ON DATABASE "pmm-managed" FROM PUBLIC; - - - name: Terminate all connections to pmm-managed - postgresql_query: - query: SELECT pg_terminate_backend(pid) FROM pg_stat_activity WHERE datname='pmm-managed'; - - - name: Dump pmm-managed database - postgresql_db: - name: pmm-managed - state: dump - target: /srv/backup/postgres/pmm-managed.sql - - - name: Disable template1 as template - postgresql_query: - query: UPDATE pg_database SET datistemplate = FALSE WHERE datname = 'template1'; - - - name: Remove template1 - postgresql_db: - name: template1 - state: absent - - - name: Create new template1 database with UTF-8 - postgresql_db: - encoding: UTF-8 - lc_collate: en_US.UTF-8 - lc_ctype: en_US.UTF-8 - name: template1 - state: present - template: template0 - - - name: Enable template1 as template - postgresql_query: - query: UPDATE pg_database SET datistemplate = FALSE WHERE datname = 'template1'; - - - name: Remove pmm-managed database - postgresql_db: - name: pmm-managed - state: absent - - - name: Recreate pmm-managed database with UTF-8 - postgresql_db: - encoding: UTF-8 - lc_collate: en_US.UTF-8 - lc_ctype: en_US.UTF-8 - name: pmm-managed - state: present - - - name: Restore pmm-managed database - postgresql_db: - encoding: UTF-8 - lc_collate: en_US.UTF-8 - lc_ctype: en_US.UTF-8 - name: pmm-managed - state: restore - target: /srv/backup/postgres/pmm-managed.sql - - - name: Return connections to pmm-managed database - postgresql_query: - query: GRANT CONNECT ON DATABASE "pmm-managed" TO PUBLIC; - - become: true - become_user: postgres - become_method: su - -- name: Start pmm-managed - command: supervisorctl start pmm-managed - changed_when: True diff --git a/ansible/playbook/tasks/roles/postgres/tasks/main.yml b/ansible/playbook/tasks/roles/postgres/tasks/main.yml new file mode 100644 index 00000000..a74b858e --- /dev/null +++ b/ansible/playbook/tasks/roles/postgres/tasks/main.yml @@ -0,0 +1,173 @@ +--- +# Install Postgres +- name: Add PostgreSQL 14 YUM repository + yum_repository: + name: percona-ppg-14 + description: PostgreSQL YUM repository - x86_64 + baseurl: http://repo.percona.com/ppg-14/yum/release/7/RPMS/x86_64 + gpgcheck: yes + enabled: yes + gpgkey: file:///etc/pki/rpm-gpg/PERCONA-PACKAGING-KEY + +- name: Add PostgreSQL 11 YUM repository + yum_repository: + name: percona-ppg-11 + description: PostgreSQL YUM repository - x86_64 + baseurl: http://repo.percona.com/ppg-11/yum/release/7/RPMS/x86_64 + gpgcheck: yes + enabled: yes + gpgkey: file:///etc/pki/rpm-gpg/PERCONA-PACKAGING-KEY + +# we need old postgres binary for upgrade process +- name: Install Postgres + yum: + name: + - percona-postgresql14-server + - percona-postgresql14-contrib + - percona-postgresql14 + - percona-postgresql11-server + - percona-postgresql11-contrib + - percona-postgresql11 + - python-psycopg2 + state: installed + when: not ansible_check_mode + +- name: Create Postgres log file + file: + path: /srv/logs/postgresql14.log + state: touch + force: yes + group: postgres + owner: postgres + mode: 0644 + +- name: Add the user 'postgres' to group 'pmm' + user: + name: postgres + groups: pmm + append: yes + +- name: Check if Postgres 14 directory exists + stat: + path: /srv/postgres14 + register: is_postgres_14 + +- name: Upgrade (or init) postgres + block: + - name: Create Postgres data dir + file: + path: "/srv/postgres14" + state: directory + owner: postgres + group: postgres + + - name: Initialize Postgres database + command: /usr/pgsql-14/bin/initdb -D /srv/postgres14 + become: yes + become_user: postgres + become_method: su + + - name: Check that supervisor socket exists (Upgrade or init) + stat: + path: /var/run/supervisor/supervisor.sock + register: is_upgrade + + - name: Stop pmm-managed and postgres before backup database + supervisorctl: + name: "{{ item }}" + state: stopped + loop: + - pmm-managed + - postgresql + when: is_upgrade.stat.exists + + - name: Run Postgres database without supervisor + command: /usr/pgsql-11/bin/pg_ctl start -D /srv/postgres + become: yes + become_user: postgres + become_method: su + when: is_upgrade.stat.exists + + - name: Dump pmm-managed database + postgresql_db: + name: pmm-managed + state: dump + target: /tmp/pmm-managed-upgrade.sql + when: is_upgrade.stat.exists + + - name: Stop Postgres database without supervisor + command: /usr/pgsql-11/bin/pg_ctl stop -D /srv/postgres + become: yes + become_user: postgres + become_method: su + when: is_upgrade.stat.exists + + - name: Start Postgres 14 database without supervisor + command: /usr/pgsql-14/bin/pg_ctl start -D /srv/postgres14 + become: yes + become_user: postgres + become_method: su + + - name: Create pmm-managed database + postgresql_db: + name: pmm-managed + state: present + when: is_upgrade.stat.exists + + - name: PostgresDB | Create pmm-managed user + postgresql_user: + db: pmm-managed + name: pmm-managed + password: 'md5da757ec3e22c6d86a2bb8e70307fa937' + priv: 'ALL' + expires: infinity + state: present + when: is_upgrade.stat.exists + + - name: Restore pmm-managed database + postgresql_db: + name: pmm-managed + state: restore + target: /tmp/pmm-managed-upgrade.sql + when: is_upgrade.stat.exists + + - name: Check pg_stat_statements extension + postgresql_ext: + db: postgres + name: pg_stat_statements + + - name: Stop Postgres 14 database without supervisor + command: /usr/pgsql-14/bin/pg_ctl stop -D /srv/postgres14 + become: yes + become_user: postgres + become_method: su + + - name: Rename old Postgres directory + copy: + remote_src: true + src: /srv/postgres + dest: /srv/postgres11 + when: is_upgrade.stat.exists + + - name: Remove old Postgres direcroty + file: + path: /srv/postgres + state: absent + when: is_upgrade.stat.exists + + - name: Restart Postgres + command: supervisorctl {{ item }} postgresql + changed_when: True + loop: + - stop + - remove + - add + when: is_upgrade.stat.exists + + - name: Run pmm-managed again + supervisorctl: + name: pmm-managed + state: started + when: is_upgrade.stat.exists + when: not is_postgres_14.stat.exists + diff --git a/ansible/playbook/tasks/update.yml b/ansible/playbook/tasks/update.yml index 3c80db4b..d9b71cff 100644 --- a/ansible/playbook/tasks/update.yml +++ b/ansible/playbook/tasks/update.yml @@ -46,10 +46,31 @@ - percona-grafana state: latest + - name: Create supervisor dir + file: + path: /etc/supervisord.d/ + state: directory + + - name: Upgrade supervisor config + copy: + src: pmm.ini + dest: /etc/supervisord.d/pmm.ini + + - name: Remove supervisord block from pmm.ini + ini_file: + path: /etc/supervisord.d/pmm.ini + section: supervisord + state: absent + when: not is_docker + - name: Create grafana config include_role: name: grafana + - name: Install postgres + include_role: + name: postgres + # Start the services - name: Supervisord start | Start supervisord service for AMI/OVF when: not is_docker @@ -192,11 +213,6 @@ - screen - yum-cron - - name: Check pg_stat_statements extension - postgresql_ext: - db: postgres - name: pg_stat_statements - # https://jira.percona.com/browse/PMM-9298 - name: Copy rezise-xfs file for lvm file: @@ -244,8 +260,7 @@ command: supervisorctl {{ item.1 }} {{ item.0 }} changed_when: True with_nested: - - - postgresql - - alertmanager + - - alertmanager - grafana - qan-api2 - pmm-agent